Verified Commit dd4a2f77 authored by Pavel Doležal's avatar Pavel Doležal

Merge branch 'devel'

parents 95303cbc 1ce86800
Pipeline #76041 passed with stages
in 7 minutes and 39 seconds
image: registry.nic.cz/adam/dns-probe:latest
variables:
IMAGE_TAG: $CI_REGISTRY_IMAGE:latest
stages:
- build_docker
- build_and_test
- deploy
build_docker_image:
stage: build_docker
image: docker:latest
tags:
- dind
before_script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
script:
- docker build --no-cache -t $IMAGE_TAG .
- docker push $IMAGE_TAG
only:
changes:
- Dockerfile
build_and_test:
stage: build_and_test
image: $IMAGE_TAG
script:
- mkdir build
- cd build
......@@ -16,6 +33,7 @@ build_and_test:
pages:
stage: deploy
image: $IMAGE_TAG
script:
- mkdir build
- cd build
......
cmake_minimum_required(VERSION 3.5)
project("DNS Probe" VERSION 0.7.1)
project("DNS Probe" VERSION 0.8.0)
set(AF_PACKET_BACKEND ON CACHE BOOL "Define backend for packet processing")
set(DPDK_BACKEND OFF CACHE BOOL "Define backend for packet processing")
......@@ -10,6 +10,7 @@ set(BUILD_DOC ON CACHE BOOL "Generate Sphinx and Doxygen documentation")
set(PROBE_CRYPTOPANT ON CACHE BOOL "Enable IP anonymization with cryptopANT library")
set(PROBE_PARQUET ON CACHE BOOL "Enable export to Parquet format with Apache Arrow library")
set(PROBE_CDNS ON CACHE BOOL "Enable export to C-DNS format with C-DNS library")
set(PROBE_DNSTAP ON CACHE BOOL "Enable support for dnstap as input data format")
set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
set(CMAKE_CXX_STANDARD 14)
......@@ -21,6 +22,7 @@ find_package(PCAP REQUIRED)
find_package(Boost REQUIRED)
find_package(OpenSSL REQUIRED)
find_package(Yaml-cpp REQUIRED)
find_package(MaxmindDB REQUIRED)
find_package(Doxygen)
include(CheckCXXCompilerFlag)
......@@ -49,7 +51,7 @@ file(GLOB PROBE_SOURCES CONFIGURE_DEPENDS src/core/Probe.cpp
)
add_library(DNSProbe INTERFACE)
target_link_libraries(DNSProbe INTERFACE ${Boost_LIBRARIES} ${YAML_CPP_LIBRARIES} PCAP::PCAP Threads::Threads OpenSSL::SSL)
target_link_libraries(DNSProbe INTERFACE ${Boost_LIBRARIES} ${YAML_CPP_LIBRARIES} PCAP::PCAP Threads::Threads OpenSSL::SSL MaxmindDB::MaxmindDB)
target_compile_definitions(DNSProbe INTERFACE $<$<CONFIG:Debug>:PRINT_DEBUG>)
target_include_directories(DNSProbe INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/src ${Boost_INCLUDE_DIRS} ${YAML_CPP_INCLUDE_DIR})
target_compile_options(DNSProbe INTERFACE -msse4)
......@@ -85,6 +87,20 @@ if (PROBE_CRYPTOPANT)
target_compile_definitions(DNSProbe INTERFACE PROBE_CRYPTOPANT)
endif()
if (PROBE_DNSTAP)
find_package(Protobuf REQUIRED)
find_package(Fstrm REQUIRED)
execute_process(COMMAND protoc --cpp_out=${CMAKE_CURRENT_BINARY_DIR} --proto_path=${CMAKE_SOURCE_DIR}/src/dnstap dnstap.proto)
add_library(dnstap_proto ${CMAKE_CURRENT_BINARY_DIR}/dnstap.pb.h ${CMAKE_CURRENT_BINARY_DIR}/dnstap.pb.cc)
target_link_libraries(dnstap_proto PUBLIC ${PROTOBUF_LIBRARIES})
target_include_directories(dnstap_proto PUBLIC ${PROTOBUF_INCLUDE_DIRS} ${CMAKE_CURRENT_BINARY_DIR})
target_link_libraries(DNSProbe INTERFACE ${PROTOBUF_LIBRARIES} dnstap_proto Fstrm::Fstrm)
target_include_directories(DNSProbe INTERFACE ${PROTOBUF_INCLUDE_DIRS})
target_compile_definitions(DNSProbe INTERFACE PROBE_DNSTAP)
file(GLOB DNSTAP_HEADERS CONFIGURE_DEPENDS src/dnstap/*.h)
file(GLOB DNSTAP_SOURCES CONFIGURE_DEPENDS src/dnstap/*.cpp)
endif()
# Add warning flags
function(set_warning param)
check_cxx_compiler_flag(-W${param} WARNING_${param})
......@@ -105,7 +121,7 @@ if (AF_PACKET_BACKEND)
file(GLOB AF_PACKET_SOURCES CONFIGURE_DEPENDS src/non-dpdk/*.cpp)
set(AF_FILES ${AF_PACKET_HEADERS} ${AF_PACKET_SOURCES} ${PROBE_HEADERS} ${PROBE_SOURCES}
${PARQUET_HEADERS} ${PARQUET_SOURCES} ${CDNS_HEADERS} ${CDNS_SOURCES})
${PARQUET_HEADERS} ${PARQUET_SOURCES} ${CDNS_HEADERS} ${CDNS_SOURCES} ${DNSTAP_HEADERS} ${DNSTAP_SOURCES})
add_executable(dns-probe-af src/application/dp.cpp ${AF_FILES})
target_link_libraries(dns-probe-af PUBLIC DNSProbe)
install(TARGETS dns-probe-af RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
......@@ -127,7 +143,7 @@ if (DPDK_BACKEND)
file(GLOB DPDK_SOURCES CONFIGURE_DEPENDS src/dpdk/*.cpp)
set(DPDK_FILES ${DPDK_HEADERS} ${DPDK_SOURCES} ${PROBE_HEADERS} ${PROBE_SOURCES}
${PARQUET_HEADERS} ${PARQUET_SOURCES} ${CDNS_HEADERS} ${CDNS_SOURCES})
${PARQUET_HEADERS} ${PARQUET_SOURCES} ${CDNS_HEADERS} ${CDNS_SOURCES} ${DNSTAP_HEADERS} ${DNSTAP_SOURCES})
add_executable(dns-probe-dpdk src/application/ddp.cpp ${DPDK_FILES})
target_link_libraries(dns-probe-dpdk PUBLIC DNSProbe DPDK::DPDK)
target_compile_definitions(dns-probe-dpdk PUBLIC USE_DPDK)
......
FROM debian:10
MAINTAINER Pavel Doležal (pavel.dolezal@nic.cz)
LABEL version="stable"
LABEL description="Debian 10 with pre-installed DNS Probe"
RUN apt-get update -yqq &&\
apt-get install -yqq gnupg curl ca-certificates lsb-release wget &&\
wget https://apache.bintray.com/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-archive-keyring-latest-$(lsb_release --codename --short).deb &&\
apt-get install -yqq -V ./apache-arrow-archive-keyring-latest-$(lsb_release --codename --short).deb &&\
echo 'deb http://download.opensuse.org/repositories/home:/CZ-NIC:/dns-probe/Debian_10/ /' | tee /etc/apt/sources.list.d/dns-probe.list &&\
curl -fsSL https://download.opensuse.org/repositories/home:CZ-NIC:/dns-probe/Debian_10/Release.key | gpg --dearmor | tee /etc/apt/trusted.gpg.d/dns-probe.gpg > /dev/null &&\
apt-get update -yqq &&\
apt-get install -yqq --no-install-recommends \
pkg-config \
git \
g++ \
make \
cmake \
procps \
doxygen \
python3 \
python3-pip \
python3-pandas \
python3-sphinx \
libssl-dev \
libboost-all-dev \
libcdns-dev \
libpcap-dev \
libarrow-dev \
libarrow-python-dev \
libparquet-dev \
libcryptopant-dev \
libyaml-cpp-dev \
libprotobuf-dev \
protobuf-compiler \
libfstrm-dev \
libmaxminddb-dev \
dpdk-dev \
dns-probe-af \
dns-probe-dpdk &&\
pip3 install pyarrow==2.0.* &&\
apt-get -qy autoremove &&\
apt-get -y clean
include(utils)
FindLibrary(libfstrm Fstrm)
\ No newline at end of file
# - Try to find libmaxminddb include dirs and libraries
#
# Usage of this module as follows:
#
# find_package(MaxmindDB)
#
# Variables used by this module, they can change the default behaviour and need
# to be set before calling find_package:
#
# MAXMINDDB_ROOT_DIR Set this variable to the root installation of
# libmaxminddb if the module has problems finding the
# proper installation path.
#
# Variables defined by this module:
#
# MAXMINDDB_FOUND System has libmaxminddb, include and library dirs found
# MAXMINDDB_INCLUDE_DIR The libmaxminddb include directories.
# MAXMINDDB_LIBRARY The libmaxminddb library
find_path(MAXMINDDB_ROOT_DIR NAMES maxminddb.h HINTS include/*/)
find_path(MAXMINDDB_INCLUDE_DIR NAMES maxminddb.h HINTS ${MAXMINDDB_ROOT_DIR}/include/*/)
find_library(MAXMINDDB_LIBRARY NAMES maxminddb HINTS ${MAXMINDDB_ROOT_DIR}/lib/*/ ${MAXMINDDB_ROOT_DIR}/lib64/*/)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(MaxmindDB DEFAULT_MSG MAXMINDDB_ROOT_DIR MAXMINDDB_LIBRARY)
mark_as_advanced(
MAXMINDDB_ROOT_DIR
MAXMINDDB_INCLUDE_DIR
MAXMINDDB_LIBRARY
)
if(MaxmindDB_FOUND)
add_library(MaxmindDB::MaxmindDB INTERFACE IMPORTED)
set_property(TARGET MaxmindDB::MaxmindDB PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${MAXMINDDB_INCLUDE_DIR})
set_property(TARGET MaxmindDB::MaxmindDB PROPERTY INTERFACE_LINK_LIBRARIES ${MAXMINDDB_LIBRARY})
endif()
# Last revision: 2020-09-22
# Last revision: 2021-02-03
#
# Default instance configuration.
# This configuration is always loaded before other configuration specified by given instance's ID.
......@@ -13,6 +13,10 @@ default:
# List of PCAPs to process in addition to PCAPs passed with '-p' command line parameter.
pcap-list: []
# List of unix sockets to process dnstap data from in addition to sockets passed with '-d'
# command line parameter.
dnstap-socket-list: []
# Indicates RAW PCAPs as input in 'pcap-list' or from command line with '-p' parameter.
# Might get overriden by '-r' command line parameter.
# MUST be set to 'false' if 'interface-list' or '-i' command line parameter are used.
......@@ -73,6 +77,7 @@ default:
export-format: 'parquet'
# This sequence indicates which fields from the C-DNS standard schema are included in exported data.
# 3 implementation specific fields are also included (asn, country_code, round_trip_time).
# By default all fields available in DNS Probe are enabled as shown below.
cdns-fields:
- 'transaction_id'
......@@ -98,7 +103,9 @@ default:
- 'query_opt_rdata'
- 'response_additional_sections'
- 'response_size'
- 'response_delay' # TCP RTT
- 'asn' # asn-maxmind-db configuration option also needs to be set
- 'country_code' # country-maxmind-db configuration option also needs to be set
- 'round_trip_time' # TCP RTT
# Maximum number of DNS records in one exported C-DNS block.
cdns-records-per-block: 10000
......@@ -129,6 +136,16 @@ default:
# Valid values are 'all', 'invalid', 'disabled'.
pcap-export: 'disabled'
# Path to Maxmind Country database. If this option is set to a valid database file, the 'country'
# field in exported Parquets or 'country-code' implementation field in exported C-DNS will be
# filled with ISO 3166-1 country code based on client's IP address.
country-maxmind-db: ''
# Path to Maxmind ASN database. If this iption is set to a valid database file, the 'asn'
# implementation field in exported Parquets or C-DNS will be filled with Autonomous System
# Number (ASN) based on client's IP address.
asn-maxmind-db: ''
# [SECTION] Configuration of client IP anonymization in exported data (Parquet or C-DNS).
# The optional PCAP export does NOT get anonymized!!!
ip-anonymization:
......
......@@ -50,6 +50,10 @@ and destination IP addresses, source and destination ports, and
transport protocol. The packet is then assigned to one of the RX queues
based on this hash value.
Unix sockets for `dnstap <https://dnstap.info/>`_ input are uniformly
distributed among available worker threads and all traffic from a given
socket is then processed by its assigned worker thread.
Backends
========
......
......@@ -24,11 +24,14 @@ DNS Probe uses local file in YAML format to load configuration at startup. Its s
+--rw <instance-id>?
+--rw coremask? <uint64>
+--rw dnstap-socket-list?* <string>
+--rw dns-ports?* <uint16>
+--rw export
| +--rw asn-maxmind-db? <string>
| +--rw cdns-blocks-per-file? <uint64>
| +--rw cdns-fields?* <string>
| +--rw cdns-records-per-block? <uint64>
| +--rw country-maxmind-db? <string>
| +--rw export-dir? <string>
| +--rw export-format? <enumeration>
| +--rw file-compression? <boolean>
......@@ -105,6 +108,14 @@ anonymize-ip
If this flag is true, client IP addresses in exported data (Parquet or C-DNS, NOT optional PCAPs) will be anonymized using Crypto-PAn prefix-preserving algorithm.
asn-maxmind-db
^^^^^^^^^^^^^^
:data node: ``<instance-id>/export/asn-maxmind-db``
:default: empty
Path to Maxmind ASN database. If this option is set to a valid database file, the ``asn`` implementation field in exported Parquets or C-DNS will be filled with Autonomous System Number (ASN) based on client's IP address.
.. _cdns-blocks-per-file:
cdns-blocks-per-file
......@@ -157,6 +168,25 @@ Bitmask indicating which CPU cores should DNS Probe use. At least 3 CPU cores ar
The default value of 7 indicates that DNS Probe should use the first 3 CPU cores with IDs of 0, 1 and 2.
country-maxmind-db
^^^^^^^^^^^^^^^^^^
:data node: ``<instance-id>/export/country-maxmind-db``
:default: empty
Path to Maxmind Country database. If this option is set to a valid database file, the ``country`` field in exported Parquets or ``country-code`` implementation field in exported C-DNS will be filled with ISO 3166-1 country code based on client's IP address.
.. _dnstap-socket-list:
dnstap-socket-list
^^^^^^^^^^^^^^^^^^
:data node: ``<instance-id>/dnstap-socket-list``
:default: empty
List of unix sockets to process dnstap data from in addition to sockets passed with '-d'
command line parameter.
.. _dns-ports:
dns-ports
......
This diff is collapsed.
......@@ -89,6 +89,9 @@ distribution repositories:
- libpcap
- yaml-cpp
- OpenSSL (libssl-dev)
- fstrm
- Protocol Buffers (libprotobuf-dev, protobuf-compiler)
- libmaxminddb
- DPDK (only for DPDK version)
Optionally, to build this user documentation (``make doc``) or manual pages (``make man``)
......@@ -164,6 +167,7 @@ DNS Probe
# For disabling DPDK BACKEND remove `-DDPDK_BACKEND=On`
# For building without IP anonymization support add `-DPROBE_CRYPTOPANT=Off`
# For building without support for one of the export formats add `-DPROBE_PARQUET=Off` or `-DPROBE_CDNS=Off`
# For building without support for dnstap input add `-DPROBE_DNSTAP=Off`
cmake <GIT_REPO> -DCMAKE_INSTALL_PREFIX="$DEP_DIR" -DCMAKE_BUILD_TYPE=Release -DAF_PACKET_BACKEND=On -DDPDK_BACKEND=On -DBUILD_COLLECTOR=On
make -j
make install
......@@ -4,7 +4,7 @@ Overview
DNS Probe is a high-speed DNS monitoring software developed as a part of the `ADAM <https://adam.nic.cz/en/>`_ project by CZ.NIC Laboratories in cooperation with Brno University of Technology, Faculty of Information Technology.
DNS Probe is able to extract DNS packets either from live network traffic or `pcap <https://en.wikipedia.org/wiki/Pcap>`_ traces, match client queries with the corresponding server responses and export consolidated records about individual DNS transactions.
DNS Probe is able to extract DNS packets from live network traffic, `pcap <https://en.wikipedia.org/wiki/Pcap>`_ traces or `dnstap <https://dnstap.info/>`_ data supplied by unix sockets, match client queries with the corresponding server responses and export consolidated records about individual DNS transactions.
DNS Probe is typically deployed together with a DNS server (autoritative or recursive), capturing and processing the traffic received and sent by the server.
......
......@@ -15,7 +15,23 @@ For changes in software see `version descriptions <https://gitlab.nic.cz/adam/dn
- Edition
- Segment
- Change description
* - **0.7**
* - **0.8.0**
- **1.0**
- :doc:`Configuration <Configuration>`, :doc:`Exported Data Schema <ExportedDataSchema>`,
:doc:`Default YAML file <YAMLfile>`
- Update ASN, Country Code and RTT fields in exported data schema
* -
-
- :doc:`Installation <Installation>`
- Add libmaxminddb as dependency
* -
-
- :doc:`Architecture <Architecture>`, :doc:`Configuration <Configuration>`,
:doc:`Installation <Installation>`, :doc:`Overview <Overview>`,
:doc:`Default YAML file <YAMLfile>`, :doc:`AF manual pages <manpages/dns-probe-af>`,
:doc:`DPDK manual pages <manpages/dns-probe-dpdk>`
- Add dnstap as another input data format
* - **0.7.0**
- **1.0**
- :doc:`Architecture <Architecture>`, :doc:`Configuration <Configuration>`,
:doc:`Exported Data Schema <ExportedDataSchema>`, :doc:`Glossary <Glossary>`,
......@@ -35,7 +51,7 @@ For changes in software see `version descriptions <https://gitlab.nic.cz/adam/dn
-
- :doc:`Exported Data Schema <ExportedDataSchema>`, YANG module
- Add TCP RTT item to exported data schema
* - **0.6**
* - **0.6.0**
- **1.1**
- :doc:`Exported Data Schema <ExportedDataSchema>`
- Domainname field in export schema is in lowercase
......@@ -65,7 +81,7 @@ For changes in software see `version descriptions <https://gitlab.nic.cz/adam/dn
-
- :doc:`Configuration <Configuration>`, YANG module
- Add IP filtering to YANG module
* - **0.5**
* - **0.5.0**
- **1.1**
- :doc:`index <index>`, :doc:`Installation <Installation>`, YANG module,
:doc:`Record Of Changes <RecordOfChanges>`
......
......@@ -9,7 +9,7 @@ It is also included in the project repository (`data-model/dns-probe.yml <https:
.. code-block:: yaml
# Last revision: 2020-09-22
# Last revision: 2021-02-03
#
# Default instance configuration.
# This configuration is always loaded before other configuration specified by given instance's ID.
......@@ -24,6 +24,10 @@ It is also included in the project repository (`data-model/dns-probe.yml <https:
# List of PCAPs to process in addition to PCAPs passed with '-p' command line parameter.
pcap-list: []
# List of unix sockets to process dnstap data from in addition to sockets passed with '-d'
# command line parameter.
dnstap-socket-list: []
# Indicates RAW PCAPs as input in 'pcap-list' or from command line with '-p' parameter.
# Might get overriden by '-r' command line parameter.
# MUST be set to 'false' if 'interface-list' or '-i' command line parameter are used.
......@@ -84,6 +88,7 @@ It is also included in the project repository (`data-model/dns-probe.yml <https:
export-format: 'parquet'
# This sequence indicates which fields from the C-DNS standard schema are included in exported data.
# 3 implementation specific fields are also included (asn, country_code, round_trip_time).
# By default all fields available in DNS Probe are enabled as shown below.
cdns-fields:
- 'transaction_id'
......@@ -109,7 +114,9 @@ It is also included in the project repository (`data-model/dns-probe.yml <https:
- 'query_opt_rdata'
- 'response_additional_sections'
- 'response_size'
- 'response_delay' # TCP RTT
- 'asn' # asn-maxmind-db configuration option also needs to be set
- 'country_code' # country-maxmind-db configuration option also needs to be set
- 'round_trip_time' # TCP RTT
# Maximum number of DNS records in one exported C-DNS block.
cdns-records-per-block: 10000
......@@ -140,6 +147,16 @@ It is also included in the project repository (`data-model/dns-probe.yml <https:
# Valid values are 'all', 'invalid', 'disabled'.
pcap-export: 'disabled'
# Path to Maxmind Country database. If this option is set to a valid database file, the 'country'
# field in exported Parquets or 'country-code' implementation field in exported C-DNS will be
# filled with ISO 3166-1 country code based on client's IP address.
country-maxmind-db: ''
# Path to Maxmind ASN database. If this iption is set to a valid database file, the 'asn'
# implementation field in exported Parquets or C-DNS will be filled with Autonomous System
# Number (ASN) based on client's IP address.
asn-maxmind-db: ''
# [SECTION] Configuration of client IP anonymization in exported data (Parquet or C-DNS).
# The optional PCAP export does NOT get anonymized!!!
ip-anonymization:
......
......@@ -8,14 +8,14 @@ dns-probe-af
Synopsis
--------
:program:`dns-probe-af` [-i *interface* | -p *pcap* [-r]] [-l *logfile*] [-n *instance*] [-c *config_file*] [-h]
:program:`dns-probe-af` [-i *interface* | -p *pcap* [-r] | -d *socket*] [-l *logfile*] [-n *instance*] [-c *config_file*] [-h]
Description
-----------
:program:`dns-probe-af` is a network traffic probe that captures DNS queries and corresponding responses and exports them as configurable records about individual DNS transactions.
:program:`dns-probe-af` can either listen on an interface or read packets from a PCAP file. The :option:`-i` and :option:`-p` options are mutually incompatible but either of them can be used repeatedly.
:program:`dns-probe-af` can either listen on an interface or read packets from a PCAP file or read dnstap data from a unix socket. The :option:`-i`, :option:`-p` and :option:`-d` options are mutually incompatible but either of them can be used repeatedly.
Depending on the configuration, :program:`dns-probe-af` exports the transaction records in either Parquet or C-DNS format.
......@@ -34,6 +34,10 @@ Options
Indicates raw PCAP format.
.. option:: -d socket
Read dnstap input from given unix socket.
.. option:: -l logfile
Write logging messages to *logfile* instead of standard output.
......
......@@ -8,14 +8,14 @@ dns-probe-dpdk
Synopsis
--------
:program:`dns-probe-dpdk` [-i *interface* | -p *pcap* [-r]] [-l *logfile*] [-n *instance*] [-c *config_file*] [-h]
:program:`dns-probe-dpdk` [-i *interface* | -p *pcap* [-r] | -d *socket*] [-l *logfile*] [-n *instance*] [-c *config_file*] [-h]
Description
-----------
:program:`dns-probe-dpdk` is a network traffic probe that captures DNS queries and corresponding responses and exports them as configurable records about individual DNS transactions.
:program:`dns-probe-dpdk` can either listen on an interface or read packets from a PCAP file. The :option:`-i` and :option:`-p` options are mutually incompatible but either of them can be used repeatedly.
:program:`dns-probe-dpdk` can either listen on an interface or read packets from a PCAP file or read dnstap data from a unix socket. The :option:`-i`, :option:`-p` and :option:`-d` options are mutually incompatible but either of them can be used repeatedly.
Depending on the configuration, :program:`dns-probe-dpdk` exports the transaction records in either Parquet or C-DNS format.
......@@ -34,6 +34,10 @@ Options
Indicates raw PCAP format.
.. option:: -d socket
Read dnstap input from given unix socket.
.. option:: -l logfile
Write logging messages to *logfile* instead of standard output.
......
......@@ -40,6 +40,7 @@
#include "utils/Logger.h"
#include "dpdk/DpdkPort.h"
#include "dpdk/DpdkPcapPort.h"
#include "core/UnixSocket.h"
DDP::LogWriter logwriter;
......@@ -287,6 +288,7 @@ int main(int argc, char** argv)
}
std::vector<std::shared_ptr<DDP::Port>> ready_ports;
std::vector<std::shared_ptr<DDP::Port>> ready_sockets;
try {
// Port initialization
std::set<uint16_t> ports;
......@@ -318,6 +320,10 @@ int main(int argc, char** argv)
ready_ports.emplace_back(new DDP::DPDKPort(port, runner.slaves_cnt() - 1, interface_mempool));
}
for (auto& port : arguments.args.dnstap_sockets) {
ready_sockets.emplace_back(new DDP::UnixSocket(port.c_str()));
}
// Set up signal handlers to print stats on exit
struct sigaction sa{};
sa.sa_handler = &signal_handler;
......@@ -331,7 +337,7 @@ int main(int argc, char** argv)
// Poll on configuration core
try {
auto ret = static_cast<int>(runner.run(ready_ports));
auto ret = static_cast<int>(runner.run(ready_ports, ready_sockets));
try {
unbind_interfaces(arguments.args);
}
......
......@@ -31,6 +31,7 @@
#include "utils/Logger.h"
#include "non-dpdk/PcapPort.h"
#include "non-dpdk/AfPacketPort.h"
#include "core/UnixSocket.h"
constexpr int PCAP_THREADS = 3;
DDP::LogWriter logwriter;
......@@ -66,6 +67,7 @@ int main(int argc, char** argv)
}
std::vector<std::shared_ptr<DDP::Port>> ready_ports;
std::vector<std::shared_ptr<DDP::Port>> ready_sockets;
try {
// Port initialization
uint16_t id = 0;
......@@ -78,6 +80,10 @@ int main(int argc, char** argv)
ready_ports.emplace_back(new DDP::PCAPPort(port.c_str(), runner.slaves_cnt() - 1));
}
for (auto& port : arguments.args.dnstap_sockets) {
ready_sockets.emplace_back(new DDP::UnixSocket(port.c_str()));
}
// Set up signal handlers to print stats on exit
struct sigaction sa = {};
sa.sa_handler = &signal_handler;
......@@ -91,7 +97,7 @@ int main(int argc, char** argv)
// Poll on configuration core
try {
return static_cast<int>(runner.run(ready_ports));
return static_cast<int>(runner.run(ready_ports, ready_sockets));
} catch (std::exception &e) {
logwriter.log_lvl("ERROR", "Uncaught exception: ", e.what());
return static_cast<uint8_t>(DDP::Probe::ReturnValue::UNCAUGHT_ERROR);
......
......@@ -40,6 +40,7 @@ namespace DDP {
Config() : interface_list(),
pcap_list(),
raw_pcap(false),
dnstap_socket_list(),
log_file(),
coremask(0x7),
dns_ports({53}),
......@@ -58,9 +59,11 @@ namespace DDP {
file_rot_size(0),
file_compression(true),
pcap_export(PcapExportCfg::DISABLED),
country_db(),
asn_db(),
export_format(ExportFormat::PARQUET),
parquet_records(5000000),
cdns_fields(0xFFFFFF),
cdns_fields(get_cdns_bitmask()),
cdns_records_per_block(10000),
cdns_blocks_per_file(0),
export_location(ExportLocation::LOCAL),
......@@ -74,6 +77,7 @@ namespace DDP {
ConfigItem<CList<std::string>> interface_list; //!< List of network interfaces to process traffic from
ConfigItem<CList<std::string>> pcap_list; //!< List of PCAP files to process
ConfigItem<bool> raw_pcap; //!< Defines if input PCAP files are without ethernet headers
ConfigItem<CList<std::string>> dnstap_socket_list; //!< List of unix sockets to process dnstap data from
ConfigItem<std::string> log_file; //!< Log file for storing probe's logs
ConfigItem<ThreadManager::MaskType> coremask; //!< Coremask used fo selecting cores where application will be running.
ConfigItem<CList<Port_t>> dns_ports; //!< TCP/UDP port list used for identifying DNS traffic
......@@ -95,6 +99,8 @@ namespace DDP {
ConfigItem<uint64_t> file_rot_size; //!< Exported file size limit in MB
ConfigItem<bool> file_compression; //!< Enable GZIP compression for exported files
ConfigItem<PcapExportCfg> pcap_export; //!< Define what will be in exported PCAPs
ConfigItem<std::string> country_db; //!< Path to Maxmind Country database
ConfigItem<std::string> asn_db; //!< Path to Maxmind ASN database
ConfigItem<ExportFormat> export_format; //!< Specify export format
ConfigItem<uint64_t> parquet_records; //!< Number of records in parquet file
......
......@@ -66,6 +66,12 @@ void DDP::ConfigFile::load_instance(Config& cfg, YAML::Node node)
if (node["raw-pcap"] && node["raw-pcap"].IsScalar())
cfg.raw_pcap.add_value(node["raw-pcap"].as<bool>());
if (node["dnstap-socket-list"] && node["dnstap-socket-list"].IsSequence()) {
for (auto item : node["dnstap-socket-list"]) {
cfg.dnstap_socket_list.add_value(item.as<std::string>());
}
}
if (node["log-file"] && node["log-file"].IsScalar())
cfg.log_file.add_value(node["log-file"].as<std::string>());
......@@ -158,6 +164,12 @@ void DDP::ConfigFile::load_instance(Config& cfg, YAML::Node node)
if (node["export"]["pcap-export"] && node["export"]["pcap-export"].IsScalar())
cfg.pcap_export.add_value(node["export"]["pcap-export"].as<std::string>());
if (node["export"]["country-maxmind-db"] && node["export"]["country-maxmind-db"].IsScalar())
cfg.country_db.add_value(node["export"]["country-maxmind-db"].as<std::string>());
if (node["export"]["asn-maxmind-db"] && node["export"]["asn-maxmind-db"].IsScalar())