diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b831468ae9759ed319aa87a501d325e54e8aa114..813de76c293f236bb7f2d914e5c56f3ec7ba8532 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -206,8 +206,8 @@ respdiff:iter:udp:linux:amd64: - PREFIX=$(pwd)/.local ./ci/respdiff/start-resolvers.sh - ./ci/respdiff/run-respdiff-tests.sh udp - cat results/respdiff.txt - - echo 'test if mismatch rate >= 1 %' - - grep -q '^target disagrees.*0\.[0-9][0-9] %' results/respdiff.txt + - echo 'test if mismatch rate < 0.8 %' + - grep -q '^target disagrees.*0\.[0-7][0-9] %' results/respdiff.txt - killall --wait kresd - PREFIX=$(pwd)/.local MAKEFLAGS="--jobs $(nproc)" make coverage-c coverage-lua COVERAGE_STAGE=gcov-respdiff-iter-udp dependencies: @@ -233,8 +233,8 @@ respdiff:iter:tcp:linux:amd64: - PREFIX=$(pwd)/.local ./ci/respdiff/start-resolvers.sh - ./ci/respdiff/run-respdiff-tests.sh tcp - cat results/respdiff.txt - - echo 'test if mismatch rate >= 1 %' - - grep -q '^target disagrees.*0\.[0-9][0-9] %' results/respdiff.txt + - echo 'test if mismatch rate < 0.8 %' + - grep -q '^target disagrees.*0\.[0-7][0-9] %' results/respdiff.txt - killall --wait kresd - PREFIX=$(pwd)/.local MAKEFLAGS="--jobs $(nproc)" make coverage-c coverage-lua COVERAGE_STAGE=gcov-respdiff-iter-tcp dependencies: @@ -260,8 +260,8 @@ respdiff:iter:tls:linux:amd64: - PREFIX=$(pwd)/.local ./ci/respdiff/start-resolvers.sh - ./ci/respdiff/run-respdiff-tests.sh tls - cat results/respdiff.txt - - echo 'test if mismatch rate >= 1 %' - - grep -q '^target disagrees.*0\.[0-9][0-9] %' results/respdiff.txt + - echo 'test if mismatch rate < 0.8 %' + - grep -q '^target disagrees.*0\.[0-7][0-9] %' results/respdiff.txt - killall --wait kresd - PREFIX=$(pwd)/.local MAKEFLAGS="--jobs $(nproc)" make coverage-c coverage-lua COVERAGE_STAGE=gcov-respdiff-iter-tls dependencies: diff --git a/ci/respdiff/respdiff-tcp.conf b/ci/respdiff/respdiff-tcp.conf index 87ec297f2560bf64928e5b2654425520846ec3aa..52de4fe6ccf1bb9cfaf8e8a89f2d1d6f3cce43d3 100644 --- a/ci/respdiff/respdiff-tcp.conf +++ b/ci/respdiff/respdiff-tcp.conf @@ -19,18 +19,21 @@ ip = 127.0.0.1 port = 5353 transport = tcp graph_color = #00a2e2 +restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-kresd.sh [bind] ip = 127.0.0.1 port = 53533 transport = udp graph_color = #e2a000 +restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-bind.sh [unbound] ip = 127.0.0.1 port = 53535 transport = udp graph_color = #218669 +restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-unbound.sh [diff] # symbolic name of server under test @@ -45,4 +48,3 @@ criteria = opcode, rcode, flags, question, qname, qtype, answertypes, answerrrsi # diffsum reports mismatches in field values in this order # if particular message has multiple mismatches, it is counted only once into category with highest weight field_weights = timeout, opcode, qcase, qtype, rcode, flags, answertypes, answerrrsigs, answer, authority, additional, edns, nsid - diff --git a/ci/respdiff/respdiff-tls.conf b/ci/respdiff/respdiff-tls.conf index fb3b6715a6a2d324ab72f835f0f2f1a647da892c..501f8554a291eb21d912667fda0326fec0f1bf88 100644 --- a/ci/respdiff/respdiff-tls.conf +++ b/ci/respdiff/respdiff-tls.conf @@ -19,18 +19,21 @@ ip = 127.0.0.1 port = 8853 transport = tls graph_color = #00a2e2 +restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-kresd.sh [bind] ip = 127.0.0.1 port = 53533 transport = udp graph_color = #e2a000 +restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-bind.sh [unbound] ip = 127.0.0.1 port = 53535 transport = udp graph_color = #218669 +restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-unbound.sh [diff] # symbolic name of server under test @@ -45,4 +48,3 @@ criteria = opcode, rcode, flags, question, qname, qtype, answertypes, answerrrsi # diffsum reports mismatches in field values in this order # if particular message has multiple mismatches, it is counted only once into category with highest weight field_weights = timeout, opcode, qcase, qtype, rcode, flags, answertypes, answerrrsigs, answer, authority, additional, edns, nsid - diff --git a/ci/respdiff/respdiff-udp.conf b/ci/respdiff/respdiff-udp.conf index 16d68b595c620e06d02b11e3c3211a6309deb726..cf3056de00b32f797fbbadc3c9598576d5a270c6 100644 --- a/ci/respdiff/respdiff-udp.conf +++ b/ci/respdiff/respdiff-udp.conf @@ -19,18 +19,21 @@ ip = 127.0.0.1 port = 5353 transport = udp graph_color = #00a2e2 +restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-kresd.sh [bind] ip = 127.0.0.1 port = 53533 transport = udp graph_color = #e2a000 +restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-bind.sh [unbound] ip = 127.0.0.1 port = 53535 transport = udp graph_color = #218669 +restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-unbound.sh [diff] # symbolic name of server under test @@ -45,4 +48,3 @@ criteria = opcode, rcode, flags, question, qname, qtype, answertypes, answerrrsi # diffsum reports mismatches in field values in this order # if particular message has multiple mismatches, it is counted only once into category with highest weight field_weights = timeout, opcode, qcase, qtype, rcode, flags, answertypes, answerrrsigs, answer, authority, additional, edns, nsid - diff --git a/ci/respdiff/restart-bind.sh b/ci/respdiff/restart-bind.sh new file mode 100755 index 0000000000000000000000000000000000000000..89fd832b0e4ef41eb589bf9feb998a77797199ef --- /dev/null +++ b/ci/respdiff/restart-bind.sh @@ -0,0 +1,2 @@ +#!/bin/sh +service bind9 restart diff --git a/ci/respdiff/restart-kresd.sh b/ci/respdiff/restart-kresd.sh new file mode 100755 index 0000000000000000000000000000000000000000..dfe82cfcad8d947769b48bc721eba685a3aab744 --- /dev/null +++ b/ci/respdiff/restart-kresd.sh @@ -0,0 +1,11 @@ +#!/bin/sh +exec > /dev/null +exec 2>&1 + +PREFIX=$(pwd)/.local +killall -w kresd +rm -f '*.mdb' +LD_LIBRARY_PATH=$PREFIX/lib $PREFIX/sbin/kresd -f 1 -q -c $(pwd)/ci/respdiff/kresd.config &>>kresd.log & + +# wait until socket is receiving connections +sleep 1 diff --git a/ci/respdiff/restart-unbound.sh b/ci/respdiff/restart-unbound.sh new file mode 100755 index 0000000000000000000000000000000000000000..c9525c057a090f1a944ebbe4c8158c4795f58906 --- /dev/null +++ b/ci/respdiff/restart-unbound.sh @@ -0,0 +1,2 @@ +#!/bin/sh +service unbound restart diff --git a/ci/respdiff/run-respdiff-tests.sh b/ci/respdiff/run-respdiff-tests.sh index d314b6d0b464e612007e4b726379b8cdf12443a1..e596e5fc308b9eeb043d3fbba2dbcaed3bceb860 100755 --- a/ci/respdiff/run-respdiff-tests.sh +++ b/ci/respdiff/run-respdiff-tests.sh @@ -3,6 +3,8 @@ # respdiff scripts must be present in /var/opt/respdiff set -o errexit -o nounset -o xtrace +NDIFFREPRO=3 + wget https://gitlab.labs.nic.cz/knot/knot-resolver/snippets/69/raw?inline=false -O /tmp/queries.txt mkdir results rm -rf respdiff.db @@ -11,6 +13,9 @@ CONFIG="$(pwd)/ci/respdiff/respdiff-${1}.conf" /var/opt/respdiff/qprep.py respdiff.db < /tmp/queries.txt time /var/opt/respdiff/orchestrator.py respdiff.db -c "${CONFIG}" time /var/opt/respdiff/msgdiff.py respdiff.db -c "${CONFIG}" +for i in $(seq $NDIFFREPRO); do + time /var/opt/respdiff/diffrepro.py -c "${CONFIG}" respdiff.db +done /var/opt/respdiff/diffsum.py respdiff.db -c "${CONFIG}" > results/respdiff.txt /var/opt/respdiff/histogram.py respdiff.db -c "${CONFIG}" -o results/histogram.svg : minimize LMDB and log size so they can be effectively archived