Skip to content

GitLab

D
deckard
Commit 3cd8f3cf authored by Štěpán Balážik's avatar Štěpán Balážik Committed by Tomas Krizek
Browse files
Options

Migrate running of Deckard to use py.test

parent 94ed593b
Hide whitespace changes
Inline Side-by-side
Showing with 281 additions and 377 deletions
.gitlab-ci.yml
View file @ 3cd8f3cf
......@@ -73,10 +73,8 @@ test:comparative:kresd:
expire_in: '1 hour'
paths:
- modified_tests
- base.log
- base.tests
- head.log
- head.tests
- base.xml
- head.xml
tags:
- docker
- linux
......@@ -92,7 +90,7 @@ test:latest:kresd:
- ( cd kresd-local-build ; git submodule update --init --recursive )
- PREFIX=$(pwd)/.local make -C kresd-local-build -k all
- PREFIX=$(pwd)/.local make -C kresd-local-build install
- TMPDIR=$(pwd) LD_LIBRARY_PATH=$(pwd)/.local/lib DAEMON=$(pwd)/.local/sbin/kresd MAKEFLAGS="-j $(nproc) --keep-going" ./kresd_run.sh
- TMPDIR=$(pwd) LD_LIBRARY_PATH=$(pwd)/.local/lib DAEMON=$(pwd)/.local/sbin/kresd ./kresd_run.sh -n $(nproc)
artifacts:
when: on_failure
expire_in: 1 week
......@@ -109,8 +107,8 @@ test:latest:kresd:
test:sanity:unbound:
stage: test
script:
- TESTS=sets/resolver/iter_hint_lame.rpl TMPDIR=$(pwd) ./unbound_run.sh
- TESTS=sets/resolver/iter_lame_root.rpl TMPDIR=$(pwd) ./unbound_run.sh
- TMPDIR=$(pwd) ./unbound_run.sh -k sets/resolver/iter_hint_lame.rpl
- TMPDIR=$(pwd) ./unbound_run.sh -k sets/resolver/iter_lame_root.rpl
# these do not work with Unbound 1.5.8 which is in CI container
#- TESTS=sets/resolver/nsec_wildcard_answer_response.rpl ./unbound_run.sh
#- TESTS=sets/resolver/world_cz_lidovky_www.rpl ./unbound_run.sh
......@@ -130,8 +128,8 @@ test:sanity:unbound:
test:sanity:pdnsrecursor:
stage: test
script:
- TESTS=sets/resolver/iter_recurse.rpl TMPDIR=$(pwd) ./pdns_run.sh
- TESTS=sets/resolver/iter_tcbit.rpl TMPDIR=$(pwd) ./pdns_run.sh
- TMPDIR=$(pwd) ./pdns_run.sh -k sets/resolver/iter_recurse.rpl
- TMPDIR=$(pwd) ./pdns_run.sh -k sets/resolver/iter_tcbit.rpl
artifacts:
when: on_failure
expire_in: 1 week
......
Makefile
View file @ 3cd8f3cf
# Defaults
TESTS ?= sets/resolver
DAEMON ?= kresd
TEMPLATE ?= template/kresd.j2
CONFIG ?= config
ADDITIONAL ?= -f 1
OPTS ?=
PYTHON ?= python3
LIBEXT := .so
PLATFORM := $(shell uname -s)
......@@ -13,14 +6,6 @@ ifeq ($(PLATFORM),Darwin)
LIBEXT := .dylib
endif
# Find all sub-targets
TARGETS := $(TESTS)
ifeq (,$(findstring .rpl,$(TESTS)))
TARGETS := $(wildcard $(TESTS)/*.rpl)
endif
SOURCES := $(TARGETS)
TARGETS := $(sort $(patsubst %.rpl,%.out-qmin,$(SOURCES))) $(sort $(patsubst %.rpl,%.out-noqmin,$(SOURCES)))
# Dependencies
include platform.mk
libcwrap_DIR := contrib/libswrap
......@@ -40,37 +25,13 @@ else
preload_syms := LD_PRELOAD="$(libfaketime):$(libcwrap)"
endif
# Test coverage measurement
# User has to provide own coverage_env.sh to generate environment variables for daemon under test
ifdef COVERAGE
ifndef COVERAGE_ENV_SCRIPT
$(error COVERAGE requires COVERAGE_ENV_SCRIPT with path to scripts/coverage_env.sh for given daemon)
endif
ifndef DAEMONSRCDIR
$(error COVERAGE requires DAEMONSRCDIR pointing to source directory of daemon under test)
endif
ifndef COVERAGE_STATSDIR
$(error COVERAGE requires COVERAGE_STATSDIR pointing to output directory)
endif
define set_coverage_env
$(shell "$(COVERAGE_ENV_SCRIPT)" "$(DAEMONSRCDIR)" "$(COVERAGE_STATSDIR)" "$(1)")
endef
endif
# Targets
all: $(TARGETS)
all:
@echo "Deckard is now run using *run.sh scripts in its root directory.\n\
To build the dependencies (libfaketime and libcwrap) run 'make depend'."
depend: $(libfaketime) $(libcwrap)
# Generic rule to run test
$(SOURCES): depend
%.out-qmin: %.rpl
@test "$${QMIN:-true}" = "true" || exit 0 && \
$(call set_coverage_env,$@) $(preload_syms) $(PYTHON) $(abspath ./deckard.py) --qmin true $(OPTS) $< one $(DAEMON) $(TEMPLATE) $(CONFIG) -- $(ADDITIONAL)
%.out-noqmin: %.rpl
@test "$${QMIN:-false}" = "false" || exit 0 && \
$(call set_coverage_env,$@) $(preload_syms) $(PYTHON) $(abspath ./deckard.py) --qmin false $(OPTS) $< one $(DAEMON) $(TEMPLATE) $(CONFIG) -- $(ADDITIONAL)
@echo "export $(preload_syms)" > env.sh
# Synchronize submodules
submodules: .gitmodules
......@@ -92,4 +53,4 @@ $(libcwrap): $(libcwrap_cmake_DIR)/Makefile
check:
@echo Running unittests using pytest
${PYTHON} -m pytest
${PYTHON} -m pytest --ignore=tests/test_runner.py
ci/compare-tests.sh
View file @ 3cd8f3cf
......@@ -32,22 +32,11 @@ MODIFIED_TESTS_FILE="/tmp/modified_tests"
find_modified_tests
LOGS[0]="${MODIFIED_TESTS_FILE}"
: get results from all tests, including the failing ones
export MAKEFLAGS="--output-sync=target --keep-going -j$(nproc)"
: get test results from version under test
PYTHON=${PYTHON} "${TESTRUNNER}" &> /tmp/head.log || :
LOGS[1]="/tmp/head.log"
extract_test_results /tmp/head.log | filter_test_results &> /tmp/head.tests || (: "no tests left, huh?" && cat /tmp/head.log)
LOGS[2]="/tmp/head.tests"
"${TESTRUNNER}" -n $(nproc) --junit-xml=/tmp/head.xml
: get test results from common ancestor with master branch
git checkout --force --detach "${MERGEBASE}"
git clean -xdf
PYTHON=${PYTHON} "${TESTRUNNER}" &> /tmp/base.log || :
LOGS[3]="/tmp/base.log"
extract_test_results /tmp/base.log | filter_test_results &> /tmp/base.tests || (: "no tests left, huh?" && cat /tmp/base.log)
LOGS[4]="/tmp/base.tests"
: tests which were not modified should produce the same results
diff -U0 /tmp/base.tests /tmp/head.tests && echo "OK, no differences found"
"${TESTRUNNER}" -n $(nproc) --junit-xml=/tmp/base.xml
test -z "$(./junit_compare.py /tmp/head.xml /tmp/base.xml /tmp/modified_tests)" && echo "OK, no differences found"
\ No newline at end of file
ci/junit_compare.py 0 → 100755
View file @ 3cd8f3cf
#!/usr/bin/python3
import xml.etree.ElementTree as xml
import sys
def parse_xml(filename):
results = set()
root = xml.parse(filename).getroot()
for case in root:
if case.find("failure") is not None: # Because empty XML elements are falsey
results.add((case.get("name"), case.get("name").split("'")[1], "FAILED"))
elif case.find("skipped") is not None:
results.add((case.get("name"), case.get("name").split("'")[1], "SKIPPED"))
else:
results.add((case.get("name"), case.get("name").split("'")[1], "PASSED"))
return results
new = sys.argv[1]
old = sys.argv[2]
modified_tests = [line.strip() for line in open(sys.argv[3]).readlines()]
for diff in parse_xml(old) ^ parse_xml(new):
print(diff[1])
if diff[1] not in modified_tests:
print(diff)
configs/getdns.yaml 0 → 100644
View file @ 3cd8f3cf
programs:
- name: getdns
binary: getdns_query
additional:
- -C
- getdns.conf
templates:
- template/getdns.j2
- template/dnssec_getdns.j2
configs:
- getdns.conf
- getdns-root.key
configs/knotd_master.yaml 0 → 100644
View file @ 3cd8f3cf
programs:
- name: knotd
binary: knotd
additional:
- -c
- ./knotd.conf
templates:
- "template/knotd_master.j2"
configs:
- "knotd.conf"
configs/knotd_slave.yaml 0 → 100644
View file @ 3cd8f3cf
programs:
- name: knotd
binary: knotd
additional:
- -c
- ./knotd.conf
templates:
- "template/knotd_slave.j2"
configs:
- "knotd.conf"
\ No newline at end of file
configs/kresd.yaml 0 → 100644
View file @ 3cd8f3cf
programs:
- name: kresd
binary: kresd
additional:
- -f
- "1"
templates:
- template/kresd.j2
configs:
- config
configs/pdns.yaml 0 → 100644
View file @ 3cd8f3cf
programs:
- name: pdns
binary: pdns_recursor
additional:
- --config-dir=./
templates:
- template/pdns_recursor.j2
- template/hints_zone.j2
- template/pdns_dnssec.j2
configs:
- recursor.conf
- hints.pdns
- dnssec.lua
# SIGTERM leads to return code -15 instead of clean 0 so we have to ignore it
ignore_exit_code: true
configs/unbound.yaml 0 → 100644
View file @ 3cd8f3cf
programs:
- name: unbound
binary: unbound
additional:
- -d
- -c
- unbound.conf
templates:
- template/unbound.j2
- template/hints_zone.j2
configs:
- unbound.conf
- hints.zone
deckard.py
View file @ 3cd8f3cf
#!/usr/bin/env python3
import argparse
from datetime import datetime
import errno
import logging
......@@ -8,14 +7,12 @@ import os
import shutil
import socket
import subprocess
import sys
import tempfile
import time
import jinja2
import yaml
from pydnstest import scenario, testserver, test
from pydnstest import scenario, testserver
# path to Deckard files
......@@ -86,18 +83,6 @@ class IfaceManager(object):
for name in self.name2iface}
def find_objects(path):
""" Recursively scan file/directory for scenarios. """
result = []
if os.path.isdir(path):
for e in os.listdir(path):
result += find_objects(os.path.join(path, e))
elif os.path.isfile(path):
if path.endswith('.rpl'):
result.append(path)
return result
def write_timestamp_file(path, tst):
time_file = open(path, 'w')
time_file.write(datetime.fromtimestamp(tst).strftime('@%Y-%m-%d %H:%M:%S'))
......@@ -287,11 +272,11 @@ def conncheck_daemon(process, cfg, sockfamily):
sock.close()
def process_file(path, args, prog_cfgs):
def process_file(path, qmin, prog_cfgs):
"""Parse scenario from a file object and create workdir."""
# Parse scenario
case, cfg_text = scenario.parse_file(os.path.realpath(path))
cfg_ctx, ta_files = scenario.parse_config(cfg_text, args.qmin, INSTALLDIR)
cfg_ctx, ta_files = scenario.parse_config(cfg_text, qmin, INSTALLDIR)
template_ctx = setup_network(cfg_ctx['_SOCKET_FAMILY'], prog_cfgs)
# merge variables from scenario with generated network variables (scenario has priority)
template_ctx.update(cfg_ctx)
......@@ -352,143 +337,10 @@ def run_testcase(daemons, case, root_addr, addr_family, prog_under_test_ip):
with open(daemon['cfg']['log']) as logf:
for line in logf:
daemon_logger_log.debug(line.strip())
ignore_exit = bool(os.environ.get('IGNORE_EXIT_CODE', 0))
ignore_exit = daemon["cfg"].get('ignore_exit_code', False)
if daemon['proc'].returncode != 0 and not ignore_exit:
raise ValueError('process %s terminated with return code %s'
% (daemon['cfg']['name'], daemon['proc'].returncode))
# Do not clear files if the server crashed (for analysis)
if server.undefined_answers > 0:
raise ValueError('the scenario does not define all necessary answers (see error log)')
def test_platform():
if sys.platform == 'windows':
raise NotImplementedError('not supported at all on Windows')
def deckard():
"""Entrypoint for script"""
# auxilitary classes for argparse
class ColonSplitter(argparse.Action): # pylint: disable=too-few-public-methods
"""Split argument string into list holding items separated by colon."""
def __call__(self, parser, namespace, values, option_string=None):
setattr(namespace, self.dest, values.split(':'))
class EnvDefault(argparse.Action): # pylint: disable=too-few-public-methods
"""Get default value for parameter from environment variable."""
def __init__(self, envvar, required=True, default=None, **kwargs):
if envvar and envvar in os.environ:
default = os.environ[envvar]
if required and default is not None:
required = False
super(EnvDefault, self).__init__(default=default, required=required, **kwargs)
def __call__(self, parser, namespace, values, option_string=None):
setattr(namespace, self.dest, values)
def loglevel2number(level):
"""Convert direct log level number or symbolic name to a number."""
try:
return int(level)
except ValueError:
pass # not a number, try if it is a named constant from logging module
try:
return getattr(logging, level.upper())
except AttributeError:
raise ValueError('unknown log level %s' % level)
test_platform()
argparser = argparse.ArgumentParser()
argparser.add_argument('--qmin', help='query minimization (default: enabled)', default=True,
action=EnvDefault, envvar='QMIN', type=scenario.str2bool)
argparser.add_argument('--loglevel', help='verbosity (default: errors + test results)',
action=EnvDefault, envvar='VERBOSE',
type=loglevel2number, required=False)
argparser.add_argument('scenario', help='path to test scenario')
argparser.add_argument('--noclean', action='store_true',
help='don\'t delete working directory')
subparsers = argparser.add_subparsers(
dest='cmd', title='sub-commands',
description='run scenario with one binary specified on command line '
'or multiple binaries specified in config file')
run_one = subparsers.add_parser('one', help='run single binary inside single scenario')
run_one.add_argument('binary', help='executable to test')
run_one.add_argument('templates', help='colon-separated list of jinja2 template files',
action=ColonSplitter)
run_one.add_argument('configs',
help='colon-separated list of files to be generated from templates',
action=ColonSplitter)
run_one.add_argument('additional', help='additional parameters for the binary', nargs='*')
run_cfg = subparsers.add_parser(
'multiple',
help='run all binaries specified in YaML file; '
'all binaries will be executed inside single scenario')
run_cfg.add_argument('yaml', help='YaML specifying binaries and their parameter',
type=open)
args = argparser.parse_args()
if not args.loglevel:
# default verbosity: errors + test results
args.loglevel = logging.ERROR
logging.config.dictConfig(
{
'version': 1,
'incremental': True,
'loggers': {
'deckard.hint': {'level': 'INFO'},
'pydnstest.test.Test': {'level': 'INFO'}
}
})
if args.loglevel <= logging.DEBUG: # include message origin
logging.basicConfig(level=args.loglevel)
else:
logging.basicConfig(level=args.loglevel, format='%(message)s')
log = logging.getLogger('deckard')
if args.cmd == 'multiple':
config = yaml.load(args.yaml)
else:
assert args.cmd == 'one'
config = {
'programs': [{
'binary': args.binary,
'templates': args.templates,
'configs': args.configs,
'additional': args.additional,
'name': os.path.basename(args.binary),
}],
'noclean': args.noclean,
}
mandatory_keys = {'name', 'binary', 'templates', 'configs', 'additional'}
for cfg in config['programs']:
missing_keys = mandatory_keys - set(cfg.keys())
if missing_keys:
log.critical('Mandatory fields in configuration are missing: %s', missing_keys)
sys.exit(1)
# sanity check templates vs. configs
if len(cfg['templates']) != len(cfg['configs']):
log.critical('Number of jinja2 template files is not equal '
'to number of config files to be generated for '
'program "%s", i.e. len(templates) != len(configs)',
cfg['name'])
sys.exit(1)
# Scan for scenarios
testset = test.Test()
objects = find_objects(args.scenario)
for path in objects:
testset.add(path, process_file, args, config)
sys.exit(testset.run())
if __name__ == '__main__':
# this is done to avoid creating global variables
deckard()
getdns_run.sh
View file @ 3cd8f3cf
......@@ -59,22 +59,6 @@ set -o errexit -o nounset
## iter_minim_a_nxdomain.rpl nsec3_wildcard_no_data_response.rpl
## iter_minim_a.rpl val_nsec3_optout_unsec_cache.rpl
RUNDIR="$(dirname "$0")"
cd $RUNDIR && ./run.sh --config configs/getdns.yaml "$@"
# Path to scenario files
TESTS=${TESTS:-"sets/resolver"}
# Path to daemon
DAEMON=${DAEMON:-"getdns_query"}
# Template file name
TEMPLATE=${TEMPLATE:-"template/getdns.j2:template/dnssec_getdns.j2"}
# Config file name
CONFIG=${CONFIG:-"getdns.conf:getdns-root.key"}
ADDITIONAL=${ADDITIONAL:-"-C getdns.conf"}
export TESTS DAEMON TEMPLATE CONFIG ADDITIONAL
MAKEDIR="$(dirname "$0")"
make -C "${MAKEDIR}"
knotd_master_run.sh
View file @ 3cd8f3cf
#!/bin/bash
set -o errexit -o nounset
echo 'INFO: Tests require Knot compiled with ./configure --enable-recvmmsg=no'
RUNDIR="$(dirname "$0")"
cd $RUNDIR && ./run.sh --config configs/knotd_master.yaml "$@"
# Path to scenario files
TESTS=${TESTS:-"sets/knotd/master"}
# Path to daemon
DAEMON=${DAEMON:-"knotd"}
# Template file name
TEMPLATE=${TEMPLATE:-"template/knotd_master.j2"}
# Config file name
CONFIG=${CONFIG:-"knotd.conf"}
# Additional parameter for knotd
# it means configuration file can be found in working directory
ADDITIONAL=${ADDITIONAL:-"-c ./knotd.conf"}
export TESTS DAEMON TEMPLATE CONFIG ADDITIONAL
echo 'INFO: Tests require Knot compiled with ./configure --enable-recvmmsg=no'
MAKEDIR="$(dirname "$0")"
make -C "${MAKEDIR}"
knotd_slave_run.sh
View file @ 3cd8f3cf
#!/bin/bash
set -o errexit -o nounset
echo 'INFO: Tests require Knot compiled with ./configure --enable-recvmmsg=no'
RUNDIR="$(dirname "$0")"
cd $RUNDIR && ./run.sh --config configs/knotd_slave.yaml "$@"
# Path to scenario files
TESTS=${TESTS:-"sets/knotd/slave"}
# Path to daemon
DAEMON=${DAEMON:-"knotd"}
# Template file name
TEMPLATE=${TEMPLATE:-"template/knotd_slave.j2"}
# Config file name
CONFIG=${CONFIG:-"knotd.conf"}
# Additional parameter for knotd
# it means configuration file can be found in working directory
ADDITIONAL=${ADDITIONAL:-"-c ./knotd.conf"}
export TESTS DAEMON TEMPLATE CONFIG ADDITIONAL
echo 'INFO: Tests require Knot compiled with ./configure --enable-recvmmsg=no'
MAKEDIR="$(dirname "$0")"
make -C "${MAKEDIR}"
kresd_run.sh
View file @ 3cd8f3cf
#!/bin/bash
set -o errexit -o nounset
# Path to scenario files
TESTS=${TESTS:-"sets/resolver"}
# Path to daemon
DAEMON=${DAEMON:-"kresd"}
# Template file name
TEMPLATE=${TEMPLATE:-"template/kresd.j2"}
# Config file name
CONFIG=${CONFIG:-"config"}
# Additional parameters for kresd - non-interactive mode
ADDITIONAL=${ADDITIONAL:-"-f 1"}
export TESTS DAEMON TEMPLATE CONFIG ADDITIONAL
MAKEDIR="$(dirname "$0")"
make -C "${MAKEDIR}"
RUNDIR="$(dirname "$0")"
cd $RUNDIR && ./run.sh --config configs/kresd.yaml "$@"
pdns_run.sh
View file @ 3cd8f3cf
#!/bin/bash
set -o errexit -o nounset
# Path to scenario files
TESTS=${TESTS:-"sets/resolver"}
# Path to daemon
DAEMON=${DAEMON:-"pdns_recursor"}
# Template file name
TEMPLATE=${TEMPLATE:-"template/pdns_recursor.j2:template/hints_zone.j2:template/pdns_dnssec.j2"}
# Config file name
CONFIG=${CONFIG:-"recursor.conf:hints.pdns:dnssec.lua"}
# Additional parameter for pdns_recursor
# it means configuration file can be found in working directory
ADDITIONAL=${ADDITIONAL:-"--config-dir=./"}
# SIGTERM leads to return code -15 instead of clean 0 so we have to ignore it
IGNORE_EXIT_CODE=1
export TESTS DAEMON TEMPLATE CONFIG ADDITIONAL IGNORE_EXIT_CODE
MAKEDIR="$(dirname "$0")"
make -C "${MAKEDIR}"
RUNDIR="$(dirname "$0")"
cd $RUNDIR && ./run.sh --config configs/pdns.yaml "$@"
pydnstest/test.py deleted 100644 → 0
View file @ 94ed593b
#!/usr/bin/env python3
import logging
class Test:
""" Small library to imitate CMocka output. """
log = logging.getLogger('pydnstest.test.Test')
def __init__(self):
self.tests = []
def add(self, name, test, args, config):
""" Add named test to set. """
self.tests.append((name, test, args, config))
def run(self):
""" Run planned tests. """
planned = len(self.tests)
passed = 0
if planned == 0:
return
for name, test_callback, args, config in self.tests:
try:
test_callback(name, args, config)
passed += 1
self.log.info('[ OK ] %s', name)
except Exception as ex:
self.log.error('[ FAIL ] %s', name)
self.log.exception(ex)
# Clear test set
self.tests = []
if passed == planned:
return 0
else:
return 1
pytest.ini 0 → 100644
View file @ 3cd8f3cf
[pytest]
log_print = False
\ No newline at end of file
requirements.txt
View file @ 3cd8f3cf
......@@ -2,4 +2,5 @@ dnspython>=1.15
Jinja2>=2.8
PyYAML
python-augeas
pytest
\ No newline at end of file
pytest
pytest-xdist
run.sh 0 → 100755
View file @ 3cd8f3cf
MAKEDIR="$(dirname "$0")"
LDPRELOAD=$(make depend -C "${MAKEDIR}")
source ${MAKEDIR}/env.sh
py.test -q tests/test_runner.py "$@"
RETVAL=$?
if [[ $RETVAL -eq 4 && "$@" =~ .*-n.* ]]
then
echo "Running on multiple cores failed. Is pytest-xdist module installed?"
fi