Migrate running of Deckard to use py.test

.gitignore

 *.pyc
 *.swp
+/env.sh

.gitlab-ci.yml

@@ -73,10 +73,8 @@ test:comparative:kresd:
     expire_in: '1 hour'
     paths:
       - modified_tests
-      - base.log
-      - base.tests
-      - head.log
-      - head.tests
+      - base.xml
+      - head.xml
   tags:
     - docker
     - linux
@@ -92,7 +90,7 @@ test:latest:kresd:
     - ( cd kresd-local-build ; git submodule update --init --recursive )
     - PREFIX=$(pwd)/.local make -C kresd-local-build -k all
     - PREFIX=$(pwd)/.local make -C kresd-local-build install
-    - TMPDIR=$(pwd) LD_LIBRARY_PATH=$(pwd)/.local/lib DAEMON=$(pwd)/.local/sbin/kresd MAKEFLAGS="-j $(nproc) --keep-going" ./kresd_run.sh
+    - TMPDIR=$(pwd) LD_LIBRARY_PATH=$(pwd)/.local/lib DAEMON=$(pwd)/.local/sbin/kresd ./kresd_run.sh -n $(nproc)
   artifacts:
     when: on_failure
     expire_in: 1 week
@@ -109,8 +107,8 @@ test:latest:kresd:
 test:sanity:unbound:
   stage: test
   script:
-    - TESTS=sets/resolver/iter_hint_lame.rpl TMPDIR=$(pwd) ./unbound_run.sh
-    - TESTS=sets/resolver/iter_lame_root.rpl TMPDIR=$(pwd) ./unbound_run.sh
+    - TMPDIR=$(pwd) ./unbound_run.sh -k sets/resolver/iter_hint_lame.rpl
+    - TMPDIR=$(pwd) ./unbound_run.sh -k sets/resolver/iter_lame_root.rpl
     # these do not work with Unbound 1.5.8 which is in CI container
       #- TESTS=sets/resolver/nsec_wildcard_answer_response.rpl ./unbound_run.sh
       #- TESTS=sets/resolver/world_cz_lidovky_www.rpl ./unbound_run.sh
@@ -130,8 +128,8 @@ test:sanity:unbound:
 test:sanity:pdnsrecursor:
   stage: test
   script:
-    - TESTS=sets/resolver/iter_recurse.rpl TMPDIR=$(pwd) ./pdns_run.sh
-    - TESTS=sets/resolver/iter_tcbit.rpl TMPDIR=$(pwd) ./pdns_run.sh
+    - TMPDIR=$(pwd) ./pdns_run.sh -k sets/resolver/iter_recurse.rpl
+    - TMPDIR=$(pwd) ./pdns_run.sh -k sets/resolver/iter_tcbit.rpl
   artifacts:
     when: on_failure
     expire_in: 1 week

Makefile

 # Defaults
-TESTS ?= sets/resolver
-DAEMON ?= kresd
-TEMPLATE ?= template/kresd.j2
-CONFIG ?= config
-ADDITIONAL ?= -f 1
-OPTS ?=
-
 PYTHON ?= python3
 LIBEXT := .so
 PLATFORM := $(shell uname -s)
@@ -13,14 +6,6 @@ ifeq ($(PLATFORM),Darwin)
 	LIBEXT := .dylib
 endif
 
-# Find all sub-targets
-TARGETS := $(TESTS)
-ifeq (,$(findstring .rpl,$(TESTS)))
-TARGETS := $(wildcard $(TESTS)/*.rpl)
-endif
-SOURCES := $(TARGETS)
-TARGETS := $(sort $(patsubst %.rpl,%.out-qmin,$(SOURCES))) $(sort $(patsubst %.rpl,%.out-noqmin,$(SOURCES)))
-
 # Dependencies
 include platform.mk
 libcwrap_DIR := contrib/libswrap
@@ -40,37 +25,13 @@ else
 	preload_syms := LD_PRELOAD="$(libfaketime):$(libcwrap)"
 endif
 
-# Test coverage measurement
-# User has to provide own coverage_env.sh to generate environment variables for daemon under test
-ifdef COVERAGE
-ifndef COVERAGE_ENV_SCRIPT
-$(error COVERAGE requires COVERAGE_ENV_SCRIPT with path to scripts/coverage_env.sh for given daemon)
-endif
-ifndef DAEMONSRCDIR
-$(error COVERAGE requires DAEMONSRCDIR pointing to source directory of daemon under test)
-endif
-ifndef COVERAGE_STATSDIR
-$(error COVERAGE requires COVERAGE_STATSDIR pointing to output directory)
-endif
-define set_coverage_env
-$(shell "$(COVERAGE_ENV_SCRIPT)" "$(DAEMONSRCDIR)" "$(COVERAGE_STATSDIR)" "$(1)")
-endef
-endif
-
 
 # Targets
-all: $(TARGETS)
+all:
+	@echo "Deckard is now run using *run.sh scripts in its root directory.\n\
+	To build the dependencies (libfaketime and libcwrap) run 'make depend'."
 depend: $(libfaketime) $(libcwrap)
-
-# Generic rule to run test
-$(SOURCES): depend
-%.out-qmin: %.rpl
-	@test "$${QMIN:-true}" = "true" || exit 0 && \
-	$(call set_coverage_env,$@) $(preload_syms) $(PYTHON) $(abspath ./deckard.py) --qmin true $(OPTS) $< one $(DAEMON) $(TEMPLATE) $(CONFIG) -- $(ADDITIONAL)
-
-%.out-noqmin: %.rpl
-	@test "$${QMIN:-false}" = "false" || exit 0 && \
-	$(call set_coverage_env,$@) $(preload_syms) $(PYTHON) $(abspath ./deckard.py) --qmin false $(OPTS) $< one $(DAEMON) $(TEMPLATE) $(CONFIG) -- $(ADDITIONAL)
+	@echo "export $(preload_syms)" > env.sh
 
 # Synchronize submodules
 submodules: .gitmodules
@@ -92,4 +53,4 @@ $(libcwrap): $(libcwrap_cmake_DIR)/Makefile
 
 check:
 	@echo Running unittests using pytest
-	${PYTHON} -m pytest
+	${PYTHON} -m pytest --ignore=tests/test_runner.py

ci/compare-tests.sh

@@ -32,22 +32,11 @@ MODIFIED_TESTS_FILE="/tmp/modified_tests"
 find_modified_tests
 LOGS[0]="${MODIFIED_TESTS_FILE}"
 
-: get results from all tests, including the failing ones
-export MAKEFLAGS="--output-sync=target --keep-going -j$(nproc)"
-
 : get test results from version under test
-PYTHON=${PYTHON} "${TESTRUNNER}" &> /tmp/head.log || :
-LOGS[1]="/tmp/head.log"
-extract_test_results /tmp/head.log | filter_test_results &> /tmp/head.tests || (: "no tests left, huh?" && cat /tmp/head.log)
-LOGS[2]="/tmp/head.tests"
+"${TESTRUNNER}" -n $(nproc) --junit-xml=/tmp/head.xml
 
 : get test results from common ancestor with master branch
 git checkout --force --detach "${MERGEBASE}"
 git clean -xdf
-PYTHON=${PYTHON} "${TESTRUNNER}" &> /tmp/base.log || :
-LOGS[3]="/tmp/base.log"
-extract_test_results /tmp/base.log | filter_test_results &> /tmp/base.tests || (: "no tests left, huh?" && cat /tmp/base.log)
-LOGS[4]="/tmp/base.tests"
-
-: tests which were not modified should produce the same results
-diff -U0 /tmp/base.tests /tmp/head.tests && echo "OK, no differences found"
+"${TESTRUNNER}" -n $(nproc) --junit-xml=/tmp/base.xml
+test -z "$(./junit_compare.py /tmp/head.xml /tmp/base.xml /tmp/modified_tests)" && echo "OK, no differences found"
\ No newline at end of file

ci/junit_compare.py

+#!/usr/bin/python3
+
+import xml.etree.ElementTree as xml
+import sys
+
+
+def parse_xml(filename):
+    results = set()
+    root = xml.parse(filename).getroot()
+    for case in root:
+        if case.find("failure") is not None:  # Because empty XML elements are falsey
+            results.add((case.get("name"), case.get("name").split("'")[1], "FAILED"))
+        elif case.find("skipped") is not None:
+            results.add((case.get("name"), case.get("name").split("'")[1], "SKIPPED"))
+        else:
+            results.add((case.get("name"), case.get("name").split("'")[1], "PASSED"))
+
+    return results
+
+new = sys.argv[1]
+old = sys.argv[2]
+modified_tests = [line.strip() for line in open(sys.argv[3]).readlines()]
+
+for diff in parse_xml(old) ^ parse_xml(new):
+    print(diff[1])
+    if diff[1] not in modified_tests:
+        print(diff)

configs/getdns.yaml

+programs:
+- name: getdns
+  binary: getdns_query
+  additional:
+    - -C
+    - getdns.conf
+  templates:
+    - template/getdns.j2
+    - template/dnssec_getdns.j2
+  configs:
+    - getdns.conf
+    - getdns-root.key

configs/knotd_master.yaml

+programs:
+- name: knotd
+  binary: knotd
+  additional:
+    - -c
+    - ./knotd.conf
+  templates:
+    - "template/knotd_master.j2"
+  configs:
+    - "knotd.conf"

configs/knotd_slave.yaml

+programs:
+- name: knotd
+  binary: knotd
+  additional:
+    - -c
+    - ./knotd.conf 
+  templates:
+    - "template/knotd_slave.j2"
+  configs:
+    - "knotd.conf"
\ No newline at end of file

configs/kresd.yaml

+programs:
+- name: kresd
+  binary: kresd
+  additional:
+    - -f
+    - "1"
+  templates:
+    - template/kresd.j2
+  configs:
+    - config

configs/pdns.yaml

+programs:
+- name: pdns
+  binary: pdns_recursor
+  additional:
+    - --config-dir=./
+  templates:
+    - template/pdns_recursor.j2
+    - template/hints_zone.j2
+    - template/pdns_dnssec.j2
+  configs:
+    - recursor.conf
+    - hints.pdns
+    - dnssec.lua
+   # SIGTERM leads to return code -15 instead of clean 0 so we have to ignore it
+  ignore_exit_code: true

configs/unbound.yaml

+programs:
+- name: unbound
+  binary: unbound
+  additional:
+    - -d
+    - -c
+    - unbound.conf
+  templates:
+    - template/unbound.j2
+    - template/hints_zone.j2
+  configs:
+    - unbound.conf
+    - hints.zone

deckard.py

 #!/usr/bin/env python3
-import argparse
 from datetime import datetime
 import errno
 import logging
@@ -8,14 +7,12 @@ import os
 import shutil
 import socket
 import subprocess
-import sys
 import tempfile
 import time
 
 import jinja2
-import yaml
 
-from pydnstest import scenario, testserver, test
+from pydnstest import scenario, testserver
 
 
 # path to Deckard files
@@ -86,18 +83,6 @@ class IfaceManager(object):
                 for name in self.name2iface}
 
 
-def find_objects(path):
-    """ Recursively scan file/directory for scenarios. """
-    result = []
-    if os.path.isdir(path):
-        for e in os.listdir(path):
-            result += find_objects(os.path.join(path, e))
-    elif os.path.isfile(path):
-        if path.endswith('.rpl'):
-            result.append(path)
-    return result
-
-
 def write_timestamp_file(path, tst):
     time_file = open(path, 'w')
     time_file.write(datetime.fromtimestamp(tst).strftime('@%Y-%m-%d %H:%M:%S'))
@@ -287,11 +272,11 @@ def conncheck_daemon(process, cfg, sockfamily):
     sock.close()
 
 
-def process_file(path, args, prog_cfgs):
+def process_file(path, qmin, prog_cfgs):
     """Parse scenario from a file object and create workdir."""
     # Parse scenario
     case, cfg_text = scenario.parse_file(os.path.realpath(path))
-    cfg_ctx, ta_files = scenario.parse_config(cfg_text, args.qmin, INSTALLDIR)
+    cfg_ctx, ta_files = scenario.parse_config(cfg_text, qmin, INSTALLDIR)
     template_ctx = setup_network(cfg_ctx['_SOCKET_FAMILY'], prog_cfgs)
     # merge variables from scenario with generated network variables (scenario has priority)
     template_ctx.update(cfg_ctx)
@@ -352,143 +337,10 @@ def run_testcase(daemons, case, root_addr, addr_family, prog_under_test_ip):
             with open(daemon['cfg']['log']) as logf:
                 for line in logf:
                     daemon_logger_log.debug(line.strip())
-            ignore_exit = bool(os.environ.get('IGNORE_EXIT_CODE', 0))
+            ignore_exit = daemon["cfg"].get('ignore_exit_code', False)
             if daemon['proc'].returncode != 0 and not ignore_exit:
                 raise ValueError('process %s terminated with return code %s'
                                  % (daemon['cfg']['name'], daemon['proc'].returncode))
     # Do not clear files if the server crashed (for analysis)
     if server.undefined_answers > 0:
         raise ValueError('the scenario does not define all necessary answers (see error log)')
-
-
-def test_platform():
-    if sys.platform == 'windows':
-        raise NotImplementedError('not supported at all on Windows')
-
-
-def deckard():
-    """Entrypoint for script"""
-
-    # auxilitary classes for argparse
-    class ColonSplitter(argparse.Action):  # pylint: disable=too-few-public-methods
-        """Split argument string into list holding items separated by colon."""
-        def __call__(self, parser, namespace, values, option_string=None):
-            setattr(namespace, self.dest, values.split(':'))
-
-    class EnvDefault(argparse.Action):  # pylint: disable=too-few-public-methods
-        """Get default value for parameter from environment variable."""
-        def __init__(self, envvar, required=True, default=None, **kwargs):
-            if envvar and envvar in os.environ:
-                default = os.environ[envvar]
-            if required and default is not None:
-                required = False
-            super(EnvDefault, self).__init__(default=default, required=required, **kwargs)
-
-        def __call__(self, parser, namespace, values, option_string=None):
-            setattr(namespace, self.dest, values)
-
-    def loglevel2number(level):
-        """Convert direct log level number or symbolic name to a number."""
-        try:
-            return int(level)
-        except ValueError:
-            pass  # not a number, try if it is a named constant from logging module
-        try:
-            return getattr(logging, level.upper())
-        except AttributeError:
-            raise ValueError('unknown log level %s' % level)
-
-    test_platform()
-
-    argparser = argparse.ArgumentParser()
-    argparser.add_argument('--qmin', help='query minimization (default: enabled)', default=True,
-                           action=EnvDefault, envvar='QMIN', type=scenario.str2bool)
-    argparser.add_argument('--loglevel', help='verbosity (default: errors + test results)',
-                           action=EnvDefault, envvar='VERBOSE',
-                           type=loglevel2number, required=False)
-    argparser.add_argument('scenario', help='path to test scenario')
-    argparser.add_argument('--noclean', action='store_true',
-                           help='don\'t delete working directory')
-
-    subparsers = argparser.add_subparsers(
-        dest='cmd', title='sub-commands',
-        description='run scenario with one binary specified on command line '
-                    'or multiple binaries specified in config file')
-
-    run_one = subparsers.add_parser('one', help='run single binary inside single scenario')
-    run_one.add_argument('binary', help='executable to test')
-    run_one.add_argument('templates', help='colon-separated list of jinja2 template files',
-                         action=ColonSplitter)
-    run_one.add_argument('configs',
-                         help='colon-separated list of files to be generated from templates',
-                         action=ColonSplitter)
-    run_one.add_argument('additional', help='additional parameters for the binary', nargs='*')
-
-    run_cfg = subparsers.add_parser(
-        'multiple',
-        help='run all binaries specified in YaML file; '
-             'all binaries will be executed inside single scenario')
-    run_cfg.add_argument('yaml', help='YaML specifying binaries and their parameter',
-                         type=open)
-    args = argparser.parse_args()
-
-    if not args.loglevel:
-        # default verbosity: errors + test results
-        args.loglevel = logging.ERROR
-        logging.config.dictConfig(
-            {
-                'version': 1,
-                'incremental': True,
-                'loggers': {
-                    'deckard.hint': {'level': 'INFO'},
-                    'pydnstest.test.Test': {'level': 'INFO'}
-                }
-            })
-
-    if args.loglevel <= logging.DEBUG:  # include message origin
-        logging.basicConfig(level=args.loglevel)
-    else:
-        logging.basicConfig(level=args.loglevel, format='%(message)s')
-    log = logging.getLogger('deckard')
-
-    if args.cmd == 'multiple':
-        config = yaml.load(args.yaml)
-    else:
-        assert args.cmd == 'one'
-        config = {
-            'programs': [{
-                'binary': args.binary,
-                'templates': args.templates,
-                'configs': args.configs,
-                'additional': args.additional,
-                'name': os.path.basename(args.binary),
-            }],
-            'noclean': args.noclean,
-        }
-
-    mandatory_keys = {'name', 'binary', 'templates', 'configs', 'additional'}
-    for cfg in config['programs']:
-        missing_keys = mandatory_keys - set(cfg.keys())
-        if missing_keys:
-            log.critical('Mandatory fields in configuration are missing: %s', missing_keys)
-            sys.exit(1)
-
-        # sanity check templates vs. configs
-        if len(cfg['templates']) != len(cfg['configs']):
-            log.critical('Number of jinja2 template files is not equal '
-                         'to number of config files to be generated for '
-                         'program "%s", i.e. len(templates) != len(configs)',
-                         cfg['name'])
-            sys.exit(1)
-
-    # Scan for scenarios
-    testset = test.Test()
-    objects = find_objects(args.scenario)
-    for path in objects:
-        testset.add(path, process_file, args, config)
-    sys.exit(testset.run())
-
-
-if __name__ == '__main__':
-    # this is done to avoid creating global variables
-    deckard()

getdns_run.sh

@@ -59,22 +59,6 @@ set -o errexit -o nounset
 ## iter_minim_a_nxdomain.rpl  nsec3_wildcard_no_data_response.rpl
 ## iter_minim_a.rpl           val_nsec3_optout_unsec_cache.rpl
 
+RUNDIR="$(dirname "$0")"
+cd $RUNDIR && ./run.sh --config configs/getdns.yaml "$@"
 
-# Path to scenario files
-TESTS=${TESTS:-"sets/resolver"}
-
-# Path to daemon
-DAEMON=${DAEMON:-"getdns_query"}
-
-# Template file name
-TEMPLATE=${TEMPLATE:-"template/getdns.j2:template/dnssec_getdns.j2"}
-
-# Config file name
-CONFIG=${CONFIG:-"getdns.conf:getdns-root.key"}
-
-ADDITIONAL=${ADDITIONAL:-"-C getdns.conf"}
-
-export TESTS DAEMON TEMPLATE CONFIG ADDITIONAL
-
-MAKEDIR="$(dirname "$0")"
-make -C "${MAKEDIR}"

knotd_master_run.sh

 #!/bin/bash
 set -o errexit -o nounset
+echo 'INFO: Tests require Knot compiled with ./configure --enable-recvmmsg=no'
+RUNDIR="$(dirname "$0")"
+cd $RUNDIR && ./run.sh --config configs/knotd_master.yaml "$@"
 
-# Path to scenario files
-TESTS=${TESTS:-"sets/knotd/master"}
-
-# Path to daemon
-DAEMON=${DAEMON:-"knotd"}
-
-# Template file name
-TEMPLATE=${TEMPLATE:-"template/knotd_master.j2"}
-
-# Config file name
-CONFIG=${CONFIG:-"knotd.conf"}
-
-# Additional parameter for knotd
-# it means configuration file can be found in working directory
-ADDITIONAL=${ADDITIONAL:-"-c ./knotd.conf"}
-
-export TESTS DAEMON TEMPLATE CONFIG ADDITIONAL
 
-echo 'INFO: Tests require Knot compiled with ./configure --enable-recvmmsg=no'
-MAKEDIR="$(dirname "$0")"
-make -C "${MAKEDIR}"

knotd_slave_run.sh

 #!/bin/bash
 set -o errexit -o nounset
+echo 'INFO: Tests require Knot compiled with ./configure --enable-recvmmsg=no'
+RUNDIR="$(dirname "$0")"
+cd $RUNDIR && ./run.sh --config configs/knotd_slave.yaml "$@"
 
-# Path to scenario files
-TESTS=${TESTS:-"sets/knotd/slave"}
-
-# Path to daemon
-DAEMON=${DAEMON:-"knotd"}
-
-# Template file name
-TEMPLATE=${TEMPLATE:-"template/knotd_slave.j2"}
-
-# Config file name
-CONFIG=${CONFIG:-"knotd.conf"}
-
-# Additional parameter for knotd
-# it means configuration file can be found in working directory
-ADDITIONAL=${ADDITIONAL:-"-c ./knotd.conf"}
-
-export TESTS DAEMON TEMPLATE CONFIG ADDITIONAL
 
-echo 'INFO: Tests require Knot compiled with ./configure --enable-recvmmsg=no'
-MAKEDIR="$(dirname "$0")"
-make -C "${MAKEDIR}"

kresd_run.sh

 #!/bin/bash
 set -o errexit -o nounset
 
-# Path to scenario files
-TESTS=${TESTS:-"sets/resolver"}
-
-# Path to daemon
-DAEMON=${DAEMON:-"kresd"}
-
-# Template file name
-TEMPLATE=${TEMPLATE:-"template/kresd.j2"}
-
-# Config file name
-CONFIG=${CONFIG:-"config"}
-
-# Additional parameters for kresd - non-interactive mode
-ADDITIONAL=${ADDITIONAL:-"-f 1"}
-
-export TESTS DAEMON TEMPLATE CONFIG ADDITIONAL
-
-MAKEDIR="$(dirname "$0")"
-make -C "${MAKEDIR}"
+RUNDIR="$(dirname "$0")"
+cd $RUNDIR && ./run.sh --config configs/kresd.yaml "$@"

pdns_run.sh

 #!/bin/bash
 set -o errexit -o nounset
 
-# Path to scenario files
-TESTS=${TESTS:-"sets/resolver"}
-
-# Path to daemon
-DAEMON=${DAEMON:-"pdns_recursor"}
-
-# Template file name
-TEMPLATE=${TEMPLATE:-"template/pdns_recursor.j2:template/hints_zone.j2:template/pdns_dnssec.j2"}
-
-# Config file name
-CONFIG=${CONFIG:-"recursor.conf:hints.pdns:dnssec.lua"}
-
-# Additional parameter for pdns_recursor
-# it means configuration file can be found in working directory
-ADDITIONAL=${ADDITIONAL:-"--config-dir=./"}
-
-# SIGTERM leads to return code -15 instead of clean 0 so we have to ignore it
-IGNORE_EXIT_CODE=1
-
-export TESTS DAEMON TEMPLATE CONFIG ADDITIONAL IGNORE_EXIT_CODE
-
-MAKEDIR="$(dirname "$0")"
-make -C "${MAKEDIR}"
+RUNDIR="$(dirname "$0")"
+cd $RUNDIR && ./run.sh --config configs/pdns.yaml "$@"

pydnstest/test.py

-#!/usr/bin/env python3
-import logging
-
-
-class Test:
-    """ Small library to imitate CMocka output. """
-    log = logging.getLogger('pydnstest.test.Test')
-
-    def __init__(self):
-        self.tests = []
-
-    def add(self, name, test, args, config):
-        """ Add named test to set. """
-        self.tests.append((name, test, args, config))
-
-    def run(self):
-        """ Run planned tests. """
-        planned = len(self.tests)
-        passed = 0
-        if planned == 0:
-            return
-
-        for name, test_callback, args, config in self.tests:
-            try:
-                test_callback(name, args, config)
-                passed += 1
-                self.log.info('[  OK  ] %s', name)
-            except Exception as ex:
-                self.log.error('[ FAIL ] %s', name)
-                self.log.exception(ex)
-
-        # Clear test set
-        self.tests = []
-        if passed == planned:
-            return 0
-        else:
-            return 1

pytest.ini

+[pytest]
+log_print = False
\ No newline at end of file

requirements.txt

@@ -2,4 +2,5 @@ dnspython>=1.15
 Jinja2>=2.8
 PyYAML
 python-augeas
-pytest
\ No newline at end of file
+pytest
+pytest-xdist

run.sh

+MAKEDIR="$(dirname "$0")"
+
+LDPRELOAD=$(make depend -C "${MAKEDIR}")
+
+source ${MAKEDIR}/env.sh
+