Merge branch 'ci-diffrepro' into 'master'

ci: respdiff - run diffrepro 3 times to reproduce results

See merge request !572

.gitlab-ci.yml

@@ -206,8 +206,8 @@ respdiff:iter:udp:linux:amd64:
     - PREFIX=$(pwd)/.local ./ci/respdiff/start-resolvers.sh
     - ./ci/respdiff/run-respdiff-tests.sh udp
     - cat results/respdiff.txt
-    - echo 'test if mismatch rate >= 1 %'
-    - grep -q '^target disagrees.*0\.[0-9][0-9] %' results/respdiff.txt
+    - echo 'test if mismatch rate < 0.8 %'
+    - grep -q '^target disagrees.*0\.[0-7][0-9] %' results/respdiff.txt
     - killall --wait kresd
     - PREFIX=$(pwd)/.local MAKEFLAGS="--jobs $(nproc)" make coverage-c coverage-lua COVERAGE_STAGE=gcov-respdiff-iter-udp
   dependencies:
@@ -233,8 +233,8 @@ respdiff:iter:tcp:linux:amd64:
     - PREFIX=$(pwd)/.local ./ci/respdiff/start-resolvers.sh
     - ./ci/respdiff/run-respdiff-tests.sh tcp
     - cat results/respdiff.txt
-    - echo 'test if mismatch rate >= 1 %'
-    - grep -q '^target disagrees.*0\.[0-9][0-9] %' results/respdiff.txt
+    - echo 'test if mismatch rate < 0.8 %'
+    - grep -q '^target disagrees.*0\.[0-7][0-9] %' results/respdiff.txt
     - killall --wait kresd
     - PREFIX=$(pwd)/.local MAKEFLAGS="--jobs $(nproc)" make coverage-c coverage-lua COVERAGE_STAGE=gcov-respdiff-iter-tcp
   dependencies:
@@ -260,8 +260,8 @@ respdiff:iter:tls:linux:amd64:
     - PREFIX=$(pwd)/.local ./ci/respdiff/start-resolvers.sh
     - ./ci/respdiff/run-respdiff-tests.sh tls
     - cat results/respdiff.txt
-    - echo 'test if mismatch rate >= 1 %'
-    - grep -q '^target disagrees.*0\.[0-9][0-9] %' results/respdiff.txt
+    - echo 'test if mismatch rate < 0.8 %'
+    - grep -q '^target disagrees.*0\.[0-7][0-9] %' results/respdiff.txt
     - killall --wait kresd
     - PREFIX=$(pwd)/.local MAKEFLAGS="--jobs $(nproc)" make coverage-c coverage-lua COVERAGE_STAGE=gcov-respdiff-iter-tls
   dependencies:

ci/respdiff/respdiff-tcp.conf

@@ -19,18 +19,21 @@ ip = 127.0.0.1
 port = 5353
 transport = tcp
 graph_color = #00a2e2
+restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-kresd.sh
 
 [bind]
 ip = 127.0.0.1
 port = 53533
 transport = udp
 graph_color = #e2a000
+restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-bind.sh
 
 [unbound]
 ip = 127.0.0.1
 port = 53535
 transport = udp
 graph_color = #218669
+restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-unbound.sh
 
 [diff]
 # symbolic name of server under test
@@ -45,4 +48,3 @@ criteria = opcode, rcode, flags, question, qname, qtype, answertypes, answerrrsi
 # diffsum reports mismatches in field values in this order
 # if particular message has multiple mismatches, it is counted only once into category with highest weight
 field_weights = timeout, opcode, qcase, qtype, rcode, flags, answertypes, answerrrsigs, answer, authority, additional, edns, nsid
-

ci/respdiff/respdiff-tls.conf

@@ -19,18 +19,21 @@ ip = 127.0.0.1
 port = 8853
 transport = tls
 graph_color = #00a2e2
+restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-kresd.sh
 
 [bind]
 ip = 127.0.0.1
 port = 53533
 transport = udp
 graph_color = #e2a000
+restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-bind.sh
 
 [unbound]
 ip = 127.0.0.1
 port = 53535
 transport = udp
 graph_color = #218669
+restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-unbound.sh
 
 [diff]
 # symbolic name of server under test
@@ -45,4 +48,3 @@ criteria = opcode, rcode, flags, question, qname, qtype, answertypes, answerrrsi
 # diffsum reports mismatches in field values in this order
 # if particular message has multiple mismatches, it is counted only once into category with highest weight
 field_weights = timeout, opcode, qcase, qtype, rcode, flags, answertypes, answerrrsigs, answer, authority, additional, edns, nsid
-

ci/respdiff/respdiff-udp.conf

@@ -19,18 +19,21 @@ ip = 127.0.0.1
 port = 5353
 transport = udp
 graph_color = #00a2e2
+restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-kresd.sh
 
 [bind]
 ip = 127.0.0.1
 port = 53533
 transport = udp
 graph_color = #e2a000
+restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-bind.sh
 
 [unbound]
 ip = 127.0.0.1
 port = 53535
 transport = udp
 graph_color = #218669
+restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-unbound.sh
 
 [diff]
 # symbolic name of server under test
@@ -45,4 +48,3 @@ criteria = opcode, rcode, flags, question, qname, qtype, answertypes, answerrrsi
 # diffsum reports mismatches in field values in this order
 # if particular message has multiple mismatches, it is counted only once into category with highest weight
 field_weights = timeout, opcode, qcase, qtype, rcode, flags, answertypes, answerrrsigs, answer, authority, additional, edns, nsid
-

ci/respdiff/restart-bind.sh

+#!/bin/sh
+service bind9 restart

ci/respdiff/restart-kresd.sh

+#!/bin/sh
+exec > /dev/null
+exec 2>&1
+
+PREFIX=$(pwd)/.local
+killall -w kresd
+rm -f '*.mdb'
+LD_LIBRARY_PATH=$PREFIX/lib $PREFIX/sbin/kresd -f 1 -q -c $(pwd)/ci/respdiff/kresd.config &>>kresd.log &
+
+# wait until socket is receiving connections
+sleep 1

ci/respdiff/restart-unbound.sh

+#!/bin/sh
+service unbound restart

ci/respdiff/run-respdiff-tests.sh

@@ -3,6 +3,8 @@
 # respdiff scripts must be present in /var/opt/respdiff
 set -o errexit -o nounset -o xtrace
 
+NDIFFREPRO=3
+
 wget https://gitlab.labs.nic.cz/knot/knot-resolver/snippets/69/raw?inline=false -O /tmp/queries.txt
 mkdir results
 rm -rf respdiff.db
@@ -11,6 +13,9 @@ CONFIG="$(pwd)/ci/respdiff/respdiff-${1}.conf"
 /var/opt/respdiff/qprep.py respdiff.db < /tmp/queries.txt
 time /var/opt/respdiff/orchestrator.py respdiff.db -c "${CONFIG}"
 time /var/opt/respdiff/msgdiff.py respdiff.db -c "${CONFIG}"
+for i in $(seq $NDIFFREPRO); do
+	time /var/opt/respdiff/diffrepro.py -c "${CONFIG}" respdiff.db
+done
 /var/opt/respdiff/diffsum.py respdiff.db -c "${CONFIG}" > results/respdiff.txt
 /var/opt/respdiff/histogram.py respdiff.db -c "${CONFIG}" -o results/histogram.svg
 : minimize LMDB and log size so they can be effectively archived