Merge branch 'shotgun' into 'master'

shotgun prototype

See merge request !1

.gitlab-ci.yml

@@ -26,3 +26,8 @@ test:pylint:
   script:
     - pip3 install -r requirements.txt
     - ./ci/pylint-run.sh
+
+test:luacheck:
+  <<: *debian
+  script:
+    - ./ci/luacheck-run.sh

README.md

@@ -2,6 +2,14 @@
 
 Realistic DNS traffic simulator with many independent clients
 
+## Current status
+
+- under development: active branches unstable, docker containers should work
+- prototype for processing inut PCAPs is functional, but slow and requires
+  python-dpkt from master
+- prototype for sending traffic is able to simulate UDP clients
+- dnsjit extensions are not merged upstream
+
 ## Overview
 
 The idea is to simulate many simultaneous clients with real behaviour, e.g.
@@ -14,8 +22,14 @@ of view.
 
 ### Dependencies
 
+#### pellet.py
+
 - python-dpkt (latest from git, commit 2c6aada35 or newer)
 
+#### shotgun.lua
+
+- dnsjit (with dnssim installed from https://github.com/tomaskrizek/dnsjit/tree/simulator )
+
 ### Input data
 
 To have a realistic simulation of clients, no synthetic queries are created.
@@ -44,4 +58,37 @@ time period.
 Simulating tens thousands of individual clients is challenging, especially with
 TCP. Plans are to support UDP, TCP and TLS.
 
-TODO
+For ease of use, docker container with shotgun is available. Note that running
+``--privileged`` can improve its performance by a few percent, if you don't mind
+the security risk.
+
+```
+docker run registry.labs.nic.cz/knot/shotgun:20191002 --help
+```
+
+#### Shotgun
+
+The machine that will act as the sender of the traffic should have enough IPs
+and ports to avoid their exhaustion. This is especially important for TCP/TLS.
+
+Only IPv6 is supported right now. You can use the fd00::/8 range to create
+unique local addresses and assign multiple of them to a single interface.
+
+It's also a good idea to extend the port range. In my testing with linux
+kernel 5.3.1, it seemed once a half of this range is depleted, creating a new
+socket starts to take a significantly longer time, slowing the tool down. I'd
+recommend planing the expected port usage to utilize no more than half of the
+port range per IP.
+
+```
+sysctl -w net.ipv4.ip_local_port_range="1025 60999"
+```
+
+#### UDP
+
+- On the server, make sure the socket's receive buffer is sufficient.
+  Otherwise, many packets can be lost, resulting in low response rate.
+
+  ```
+  net.core.rmem_default=8192000
+  ```

ci/luacheck-run.sh

+#!/usr/bin/env bash
+set -e
+
+luacheck --codes --formatter TAP .

docker/shotgun/Dockerfile

+FROM ubuntu:18.04
+
+RUN \
+	apt-get update -qq && \
+	apt-get install -y -qqq \
+		libck-dev \
+		libluajit-5.1-dev \
+		libpcap-dev \
+		liblmdb-dev \
+		libgnutls28-dev \
+		luajit \
+		libuv1-dev \
+		libgoogle-perftools-dev \
+		automake \
+		libtool \
+		pkg-config \
+		git && \
+	rm -rf /var/lib/apt/lists/*
+
+RUN \
+	git clone https://github.com/tomaskrizek/dnsjit.git && \
+	cd dnsjit && \
+	git checkout simulator && \
+	./autogen.sh && \
+	./configure && \
+	make && \
+	cd ..
+
+RUN \
+	git clone https://gitlab.labs.nic.cz/knot/shotgun.git && \
+	cd shotgun && \
+	cd ..
+
+ENTRYPOINT ["dnsjit/src/dnsjit", "shotgun/shotgun.lua"]

shotgun.lua

+#!/usr/bin/env dnsjit
+local object = require("dnsjit.core.objects")
+local log = require("dnsjit.core.log")
+local getopt = require("dnsjit.lib.getopt").new({
+	{ "v", "verbose", 2, "Verbosity level (0-4)", "?" },
+	{ "T", "threads", 1, "Number of sender threads", "?" },
+	{ "p", "port", 53, "Target port", "?" },
+	{ "s", "server", "::1", "Target IPv6 address", "?" },
+	{ "t", "timeout", 2, "Timeout for requests", "?" },
+	{ "b", "bind", "", "Source IPv6 bind address pattern (example: 'fd00::%x')", "?" },
+	{ "i", "ips", 1, "Number of source IPs per thread (when -b is set)", "?" },
+	{ "d", "drift", 1.0, "Maximum realtime drift (seconds)", "?" },
+	{ "S", "stats_interval", 100000,
+		"Interval for logging statistics (in packets per thread)", "?" },
+	{ "O", "outdir", ".", "directory for output files (must exist)", "?" },
+})
+local pcap = unpack(getopt:parse())
+if getopt:val("help") then
+	getopt:usage()
+	return
+end
+local v = getopt:val("v")
+if v > 0 then
+	log.enable("warning")
+end
+if v > 1 then
+	log.enable("notice")
+end
+if v > 2 then
+	log.enable("info")
+end
+if v > 3 then
+	log.enable("debug")
+end
+
+if pcap == nil then
+	print("usage: "..arg[1].." <pcap>")
+	return
+end
+
+local SEND_THREADS = getopt:val("T")
+local TARGET_IP = getopt:val("s")
+local TARGET_PORT = getopt:val("p")
+local TIMEOUT = getopt:val("t")
+local BIND_IP_PATTERN = getopt:val("b")
+local NUM_BIND_IP = getopt:val("i")
+local REALTIME_DRIFT = getopt:val("d")
+local LOG_INTERVAL = getopt:val("S")
+local OUTDIR = getopt:val("O")
+local MAX_CLIENTS_DNSSIM = 200000
+local CHANNEL_SIZE = 2048  -- dnsjit default
+local MAX_BATCH_SIZE = 32  -- libuv default
+
+
+local function thread_output(thr)
+	local channel = thr:pop()
+	local output = require("dnsjit.output.dnssim").new(thr:pop())
+	local running
+
+	output:udp_only()
+	output:target(thr:pop(), thr:pop())
+	output:timeout(thr:pop())
+	output:log_interval(thr:pop())
+	output:free_after_use(true)
+
+	local outfile = thr:pop()
+	local batch_size = thr:pop()
+
+	local nbind = thr:pop()
+	for _ = 1, nbind do
+		output:bind(thr:pop())
+	end
+
+	local recv, rctx = output:receive()
+	while true do
+		local obj
+		local i = 0
+
+		-- read available data from channel
+		while i < batch_size do
+			obj = channel:try_get()
+			if obj == nil then break end
+			recv(rctx, obj)
+			i = i + 1
+		end
+
+		-- execute libuv loop
+		running = output:run_nowait()
+
+		-- check if channel is still open
+		if obj == nil and channel.closed == 1 then
+			break
+		end
+	end
+
+	-- finish processing outstanding requests
+	while running ~= 0 do
+		running = output:run_nowait()
+	end
+
+	output:export(outfile)
+end
+
+
+-- setup input
+local input = require("dnsjit.input.fpcap").new()
+local delay = require("dnsjit.filter.timing").new()
+local layer = require("dnsjit.filter.layer").new()
+local split = require("dnsjit.filter.dnssim").new()
+local copy = require("dnsjit.filter.copy").new()
+input:open(pcap)
+delay:realtime(REALTIME_DRIFT)
+delay:producer(input)
+layer:producer(delay)
+
+-- setup threads
+local thread = require("dnsjit.core.thread")
+local channel = require("dnsjit.core.channel")
+local threads = {}
+local channels = {}
+
+-- send threads
+local outname = OUTDIR.."/data_"..os.time().."_%02d.json"
+for i = 1, SEND_THREADS do
+	channels[i] = channel.new(CHANNEL_SIZE)
+	split:receiver(channels[i])
+
+	threads[i] = thread.new()
+	threads[i]:start(thread_output)
+	threads[i]:push(channels[i])
+	threads[i]:push(MAX_CLIENTS_DNSSIM)
+	threads[i]:push(TARGET_IP)
+	threads[i]:push(TARGET_PORT)
+	threads[i]:push(TIMEOUT)
+	threads[i]:push(LOG_INTERVAL)
+	threads[i]:push(string.format(outname, i))
+	threads[i]:push(MAX_BATCH_SIZE)
+	if BIND_IP_PATTERN ~= "" then
+		threads[i]:push(NUM_BIND_IP)
+		for j = 1, NUM_BIND_IP do
+			local addr = string.format(BIND_IP_PATTERN, NUM_BIND_IP*(i-1)+j)
+			threads[i]:push(addr)
+		end
+	else
+		threads[i]:push(0)
+	end
+end
+
+copy:obj_type(object.PAYLOAD)
+copy:obj_type(object.IP6)
+copy:receiver(split)
+
+
+-- process PCAP
+local prod, pctx = layer:produce()
+local recv, rctx = copy:receive()
+while true do
+	local obj = prod(pctx)
+	if obj == nil then break end
+	recv(rctx, obj)
+end
+
+-- teardown
+for i = 1, SEND_THREADS do
+	channels[i]:close()
+end
+for i = 1, SEND_THREADS do
+	threads[i]:stop()
+end
+
+--print outfiles
+for i = 1, SEND_THREADS do
+	local f = assert(io.open(string.format(outname, i), "r"))
+	local content = f:read("*all")
+	f:close()
+	print(content)
+end