Merge branch 'release-v20210714' into 'master'

release v20210714

See merge request !54

Dockerfile

 FROM ubuntu:focal
 ENV DEBIAN_FRONTEND=noninteractive
 RUN \
+	apt-get update -qq && \
+	apt-get install -y -qqq software-properties-common && \
+	add-apt-repository -y ppa:dns-oarc/dnsjit && \
 	apt-get update -qq && \
 	apt-get install -y -qqq \
+		dnsjit \
+		dnsjit-dev \
 		python3 \
 		python3-pip \
 		tshark \
 		jq \
-		libck-dev \
-		libluajit-5.1-dev \
-		libpcap-dev \
-		liblmdb-dev \
-		libgnutls28-dev \
 		libnghttp2-dev \
 		luajit \
 		libuv1-dev \
-		libgoogle-perftools-dev \
 		make \
 		automake \
 		libtool \
 		pkg-config \
 		git && \
 	rm -rf /var/lib/apt/lists/*
-RUN \
-	git clone https://github.com/DNS-OARC/dnsjit.git && \
-	cd dnsjit && \
-	./autogen.sh && \
-	./configure && \
-	make && \
-	make install && \
-	cd ..
 
 COPY . /shotgun
 WORKDIR /shotgun

NEWS

+DNS Shotgun v20210714
+=====================
+
+Incompatible changes
+--------------------
+- dnsjit 1.2.0+ is now required
+- dnssim component was moved from dnsjit to this repository and has to be
+  compiled separately (!52)
+
+Improvements
+------------
+- replay: new option `stop_after_s` added to input (!40)
+- tools/plot-packat-rate: new tool to plot packet rate in PCAPs (!46)
+- replay/shotgun: support reading PCAP from stdin (!36)
+- pcap/merge-chunks: utility to merge chunks on-the-fly (!37)
+- tools/plot-response-rate: add `--rcodes-above-pct` option (!43)
+- tools/plot-response-rate: add `--skip-total` option (!47)
+- tools/plot-response-rate: add `--sum-rcodes` option (!47)
+- tools/plot-latency: add `--group` option to plot aggregate data (!53)
+- pcap/cut-pcap: new script to effetively trim a sorted PCAP (!50)
+- configs: plot response rate chart with non-zero rcodes by default (!44)
+- pcap/extract-clients: add --seed option for deterministic splitting (!34)
+- pcap/filter-dnsq: improved malformed packet handling (!31)
+- pcap/extract-clients: detect non-monotonic timestamps (!31)
+- replay/shotgun: use faster mmpcap (!36)
+- tools/plot-response-rate: add markers for plotting rcodes (!41)
+- tools/plot-response-rate: improve color selection (!41, !42)
+
+Bugfixes
+--------
+- pcap/extract-clients: fix imprecise timestamps and off-by-one error (!32)
+- tools/plot-client-distribution: fix handling of empty query count intervals (!38)
+- replay: add workaround to support /32 and /128 addresses (!39)
+
+
+DNS Shotgun v20210203
+=====================
+
+- new user interface
+  - replay.py is now the main tool instead of shotgun.lua
+  - new configuration format to support complex scenarios
+  - mixed-protocol traffic replay is supported
+  - some scripts in `tools/` were renamed
+- fast scripts for processing PCAP data
+  - can be found in `pcap/`
+  - these replaced the now removed pellet.py and scale.py
+- added IPv4 support
+- DoH is now fully supported
+- comprehensive usage documentation is available at https://dns-shotgun.readthedocs.io
+- minimum required dnsjit version is 1.1.0
+
+
+DNS Shotgun v20200914
+=====================
+
+- fully supported UDP, TCP and DNS-over-TLS with
+  [dnsjit](https://github.com/DNS-OARC/dnsjit) 1.0.0
+- fully supported DNS-over-HTTPS with development version of dnsjit
+- traffic can be replayed only over IPv6
+- user interface
+  - may be unstable
+  - only very basic UI available
+  - more complex scenarios are no supported yet
+    (e.g. simultaneously using multiple protocols)
+- pellet.py is functional, but it is very slow and requires python-dpkt from
+  master

docs/installation.md

 # Installation
 
-There are two options for using DNS Shotgun. You can either install the
-dependencies and use the scripts from the repository directly, or use a
-pre-built docker image.
-
-## Using script directly
-
-You can use the toolchain scripts directly from the git repository. You need to
-ensure you have the required dependencies installed. Also make sure to check
-out some tagged version, as the development happens in master branch.
-
-```
-$ git clone https://gitlab.nic.cz/knot/shotgun.git
-$ git checkout v20210203
-```
-
-### Dependencies
-
-When using the scripts directly, the following dependencies are needed. If you
-only wish to process shotgun JSON output (e.g. plot charts), then dnsjit isn't
-required.
-
-- [dnsjit](https://github.com/DNS-OARC/dnsjit): Can be installed from [DNS-OARC
-  repositories](https://dev.dns-oarc.net/packages/).
-- Python 3.6 or later
-- Python dependencies from [requirements.txt](https://gitlab.nic.cz/knot/shotgun/-/blob/master/requirements.txt)
-- (optional) tshark/wireshark for some PCAP pre-processing
+There are two options for using DNS Shotgun. You can either use a pre-built
+docker image, or install the dependencies, compile the dnssim module and use
+the scripts from the repository directly.
 
 ## Docker Image
 
@@ -33,7 +10,7 @@ Pre-built image can be obtained from [CZ.NIC DNS Shotgun
 Registry](https://gitlab.nic.cz/knot/shotgun/container_registry/65).
 
 ```
-$ docker pull registry.nic.cz/knot/shotgun:v20210203
+$ docker pull registry.nic.cz/knot/shotgun:v20210714
 ```
 
 Alternately, you can build the image yourself from Dockerfile in the repository.
@@ -48,6 +25,39 @@ Alternately, you can build the image yourself from Dockerfile in the repository.
 $ docker run \
     --network host \
     -v "$PWD:/mnt" \
-    registry.nic.cz/knot/shotgun:v20210203 \
+    registry.nic.cz/knot/shotgun:v20210714 \
     $COMMAND
 ```
+
+## Using scripts from sources
+
+You can use the toolchain scripts directly from the git repository. You need to
+ensure you have the required dependencies installed and the compile and install
+the dnssim module. Also make sure to check out some tagged version, as the
+development happens in master branch.
+
+```
+$ git clone https://gitlab.nic.cz/knot/shotgun.git
+$ git checkout v20210714
+$ cd shotgun/replay/dnssim
+$ ./autogen.sh
+$ ./configure
+$ make
+$ make install
+```
+
+### Dependencies
+
+When using the scripts directly, the following dependencies are needed.
+
+If you only wish to process shotgun JSON output (e.g. plot charts), then dnsjit
+and compiling the dnssim module isn't required.
+
+- [dnsjit 1.2+](https://github.com/DNS-OARC/dnsjit): Can be installed from [DNS-OARC
+  repositories](https://dev.dns-oarc.net/packages/).
+- libuv
+- libnghttp2
+
+- Python 3.6 or later
+- Python dependencies from [requirements.txt](https://gitlab.nic.cz/knot/shotgun/-/blob/master/requirements.txt)
+- (optional) tshark/wireshark for some PCAP pre-processing

replay/dnssim/CHANGELOG.md

+dnssim v20210714
+================
+- dnssim was moved from dnsjit to shotgun, this implies new
+  lua name: "shotgun.output.dnssim"
+
 dnssim v20210129
 ================
 

replay/dnssim/configure.ac

 AC_PREREQ(2.61)
-AC_INIT([shotgun-output-dnssim], [20210129],,)
+AC_INIT([shotgun-output-dnssim], [20210714],,)
 AM_INIT_AUTOMAKE([-Wall -Werror foreign subdir-objects])
 AC_CONFIG_HEADER([src/config.h])
 
@@ -21,6 +21,10 @@ PKG_CHECK_MODULES([libuv], [libuv])
 PKG_CHECK_MODULES([libnghttp2], [libnghttp2])
 AC_CHECK_LIB([gnutls], [gnutls_init],, [AC_MSG_ERROR([libgnutls not found])])
 
+# Checks for support.
+AC_SEARCH_LIBS([clock_gettime])
+AC_CHECK_FUNCS([clock_nanosleep])
+
 # Output Makefiles
 AC_CONFIG_FILES([
   Makefile

replay/dnssim/src/output/dnssim.c

@@ -2,6 +2,8 @@
  *  SPDX-License-Identifier: GPL-3.0-or-later
  */
 
+#include "config.h"
+
 #include "output/dnssim.h"
 #include "output/dnssim/internal.h"
 #include "output/dnssim/ll.h"
@@ -28,8 +30,7 @@ static uint64_t _now_ms()
     now_ms += ts.tv_nsec / 1000000;
     return now_ms;
 #else
-    mlfatal("clock_gettime() not available");
-    return 0;
+#error "clock_gettime() is not available"
 #endif
 }
 

replay/dnssim/src/output/dnssim.lua

@@ -149,7 +149,7 @@ local C = ffi.C
 
 local DnsSim = {}
 
-local _DNSSIM_VERSION = 20210129
+local _DNSSIM_VERSION = 20210714
 local _DNSSIM_JSON_VERSION = 20200527
 
 -- Create a new DnsSim output for up to max_clients.

replay/shotgun.lua

@@ -4,7 +4,7 @@ local object = require("dnsjit.core.objects")
 local log = require("dnsjit.core.log")
 local dnssim = require("shotgun.output.dnssim")
 
-local DNSSIM_REQ_VERSION = 20210129
+local DNSSIM_REQ_VERSION = 20210714
 local has_check_version, version = pcall(dnssim.check_version, DNSSIM_REQ_VERSION)
 if not has_check_version or version == nil then
 	log.fatal(string.format(
@@ -45,7 +45,7 @@ local function send_thread_main(thr)
 
 	-- output must be global (per thread) to be accesible in loadstring()
 	-- luacheck: globals output, ignore log
-	output = require("dnsjit.output.dnssim").new(thr:pop())
+	output = require("shotgun.output.dnssim").new(thr:pop())
 	local log = output:log(thr:pop())
 
 	output:target(thr:pop(), thr:pop())
@@ -65,9 +65,6 @@ local function send_thread_main(thr)
 	elseif protocol_func == "tls" then
 		cmd = cmd .. "('" .. gnutls_priority .. "')"
 	elseif protocol_func == "https2" then
-		if type(output.https2) ~= "function" then
-			log:fatal("https2 isn't supported with this version of dnsjit")
-		end
 		cmd = cmd .. "({ method = '" .. http_method .. "' }, '" .. gnutls_priority .. "')"
 	else
 		log:fatal("unknown protocol_func: " .. protocol_func)