Merge branch 'code_coverage_support' into 'master'

Test coverage measurement support

See merge request !95

.gitignore

 *.pyc
 *.swp
-/luacov.stats.out

Makefile

@@ -19,7 +19,7 @@ ifeq (,$(findstring .rpl,$(TESTS)))
 TARGETS := $(wildcard $(TESTS)/*.rpl)
 endif
 SOURCES := $(TARGETS)
-TARGETS := $(sort $(patsubst %.rpl,%.out,$(SOURCES)))
+TARGETS := $(sort $(patsubst %.rpl,%.out-qmin,$(SOURCES))) $(sort $(patsubst %.rpl,%.out-noqmin,$(SOURCES)))
 
 # Dependencies
 include platform.mk
@@ -40,14 +40,37 @@ else
 	preload_syms := LD_PRELOAD="$(libfaketime):$(libcwrap)"
 endif
 
+# Test coverage measurement
+# User has to provide own coverage_env.sh to generate environment variables for daemon under test
+ifdef COVERAGE
+ifndef COVERAGE_ENV_SCRIPT
+$(error COVERAGE requires COVERAGE_ENV_SCRIPT with path to scripts/coverage_env.sh for given daemon)
+endif
+ifndef DAEMONSRCDIR
+$(error COVERAGE requires DAEMONSRCDIR pointing to source directory of daemon under test)
+endif
+ifndef COVERAGE_STATSDIR
+$(error COVERAGE requires COVERAGE_STATSDIR pointing to output directory)
+endif
+define set_coverage_env
+$(shell "$(COVERAGE_ENV_SCRIPT)" "$(DAEMONSRCDIR)" "$(COVERAGE_STATSDIR)" "$(1)")
+endef
+endif
+
+
 # Targets
 all: $(TARGETS)
 depend: $(libfaketime) $(libcwrap)
 
 # Generic rule to run test
 $(SOURCES): depend
-%.out: %.rpl
-	@$(preload_syms) $(PYTHON) $(abspath ./deckard.py) $(OPTS) $< one $(DAEMON) $(TEMPLATE) $(CONFIG) -- $(ADDITIONAL)
+%.out-qmin: %.rpl
+	@test "$${QMIN:-true}" = "true" || exit 0 && \
+	$(call set_coverage_env,$@) $(preload_syms) $(PYTHON) $(abspath ./deckard.py) --qmin true $(OPTS) $< one $(DAEMON) $(TEMPLATE) $(CONFIG) -- $(ADDITIONAL)
+
+%.out-noqmin: %.rpl
+	@test "$${QMIN:-false}" = "false" || exit 0 && \
+	$(call set_coverage_env,$@) $(preload_syms) $(PYTHON) $(abspath ./deckard.py) --qmin false $(OPTS) $< one $(DAEMON) $(TEMPLATE) $(CONFIG) -- $(ADDITIONAL)
 
 # Synchronize submodules
 submodules: .gitmodules

ci/compare-pep8.sh

 #!/bin/bash
 set -o nounset -o xtrace -o errexit
-source "$(dirname "$(readlink -f "$0")")/common.sh"
+source "$(dirname "$0")/common.sh"
 
 git diff "${MERGEBASE}..${HEAD}" | ${PYTHON} -m pep8 --ignore=W503 --diff --show-source --max-line-length=100 && echo "OK, no PEP8 errors detected"

ci/compare-pylint.sh

 #!/bin/bash
 set -o nounset -o xtrace -o errexit
-source "$(dirname "$(readlink -f "$0")")/common.sh"
+source "$(dirname "$0")/common.sh"
 
 PYFILES=$(find . \
 	-path ./.git -prune -o \

ci/compare-tests.sh

 #!/bin/bash
 set -o nounset -o xtrace -o errexit
-source "$(dirname "$(readlink -f "$0")")/common.sh"
+source "$(dirname "$0")/common.sh"
 TESTRUNNER="$1"
 : comparing results from test script "${TESTRUNNER}"
 

ci/runlocally.sh

 #!/bin/bash
 set -o nounset -o errexit
-CIDIR="$(dirname "$(readlink -f "$0")")"
+CIDIR="$(dirname "$0")"
 ORIGNAME="$(git symbolic-ref -q --short HEAD || git describe --all --always HEAD)"
 FAILURE_DETECTED="?"
 

getdns_run.sh

@@ -76,5 +76,5 @@ ADDITIONAL=${ADDITIONAL:-"-C getdns.conf"}
 
 export TESTS DAEMON TEMPLATE CONFIG ADDITIONAL
 
-MAKEDIR="$(dirname "$(readlink -f "$0")")"
+MAKEDIR="$(dirname "$0")"
 make -C "${MAKEDIR}"

knotd_master_run.sh

@@ -20,5 +20,5 @@ ADDITIONAL=${ADDITIONAL:-"-c ./knotd.conf"}
 export TESTS DAEMON TEMPLATE CONFIG ADDITIONAL
 
 echo 'INFO: Tests require Knot compiled with ./configure --enable-recvmmsg=no'
-MAKEDIR="$(dirname "$(readlink -f "$0")")"
+MAKEDIR="$(dirname "$0")"
 make -C "${MAKEDIR}"

knotd_slave_run.sh

@@ -20,5 +20,5 @@ ADDITIONAL=${ADDITIONAL:-"-c ./knotd.conf"}
 export TESTS DAEMON TEMPLATE CONFIG ADDITIONAL
 
 echo 'INFO: Tests require Knot compiled with ./configure --enable-recvmmsg=no'
-MAKEDIR="$(dirname "$(readlink -f "$0")")"
+MAKEDIR="$(dirname "$0")"
 make -C "${MAKEDIR}"

kresd_run.sh

@@ -18,10 +18,5 @@ ADDITIONAL=${ADDITIONAL:-"-f 1"}
 
 export TESTS DAEMON TEMPLATE CONFIG ADDITIONAL
 
-MAKEDIR="$(dirname "$(readlink -f "$0")")"
-echo '=== Testing WITH query minimization ==='
-export QMIN="true"
-make -C "${MAKEDIR}"
-echo '=== Testing WITHOUT query minimization ==='
-export QMIN="false"
+MAKEDIR="$(dirname "$0")"
 make -C "${MAKEDIR}"

pdns_run.sh

@@ -22,5 +22,5 @@ IGNORE_EXIT_CODE=1
 
 export TESTS DAEMON TEMPLATE CONFIG ADDITIONAL IGNORE_EXIT_CODE
 
-MAKEDIR="$(dirname "$(readlink -f "$0")")"
+MAKEDIR="$(dirname "$0")"
 make -C "${MAKEDIR}"

template/kresd.j2

--- measure code coverage if the module is available
-local ok, runner = pcall(require, 'luacov.runner')
-if ok then
-	runner.init({
-		statsfile = '{{INSTALL_DIR}}/luacov.stats.out',
-		exclude = {'test', 'tapered', 'lua/5.1'},
-	})
-	jit.off()
-end
-
 net = { '{{SELF_ADDR}}' }
 modules = {'stats', 'policy', 'hints'}
 -- test. domain is used by some tests, allow it

unbound_run.sh

@@ -19,10 +19,5 @@ ADDITIONAL=${ADDITIONAL:-"-d -c unbound.conf"}
 
 export TESTS DAEMON TEMPLATE CONFIG ADDITIONAL
 
-MAKEDIR="$(dirname "$(readlink -f "$0")")"
-echo '=== Testing WITH query minimization ==='
-export QMIN="true"
-make -C "${MAKEDIR}"
-echo '=== Testing WITHOUT query minimization ==='
-export QMIN="false"
+MAKEDIR="$(dirname "$0")"
 make -C "${MAKEDIR}"