Merge branch 'c-dns_1.0.1_schema' into 'devel'

C-DNS 1.0.1 schema

See merge request !10

CMakeLists.txt

@@ -22,6 +22,7 @@ find_package(PCAP REQUIRED)
 find_package(Boost REQUIRED)
 find_package(OpenSSL REQUIRED)
 find_package(Yaml-cpp REQUIRED)
+find_package(MaxmindDB REQUIRED)
 find_package(Doxygen)
 
 include(CheckCXXCompilerFlag)
@@ -50,7 +51,7 @@ file(GLOB PROBE_SOURCES CONFIGURE_DEPENDS src/core/Probe.cpp
         )
 
 add_library(DNSProbe INTERFACE)
-target_link_libraries(DNSProbe INTERFACE ${Boost_LIBRARIES} ${YAML_CPP_LIBRARIES} PCAP::PCAP Threads::Threads OpenSSL::SSL)
+target_link_libraries(DNSProbe INTERFACE ${Boost_LIBRARIES} ${YAML_CPP_LIBRARIES} PCAP::PCAP Threads::Threads OpenSSL::SSL MaxmindDB::MaxmindDB)
 target_compile_definitions(DNSProbe INTERFACE $<$<CONFIG:Debug>:PRINT_DEBUG>)
 target_include_directories(DNSProbe INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/src ${Boost_INCLUDE_DIRS} ${YAML_CPP_INCLUDE_DIR})
 target_compile_options(DNSProbe INTERFACE -msse4)

Dockerfile

@@ -35,6 +35,7 @@ RUN apt-get update -yqq &&\
         libprotobuf-dev \
         protobuf-compiler \
         libfstrm-dev \
+        libmaxminddb-dev \
         dpdk-dev \
         dns-probe-af \
         dns-probe-dpdk &&\

cmake/FindMaxmindDB.cmake

+# - Try to find libmaxminddb include dirs and libraries
+#
+# Usage of this module as follows:
+#
+#     find_package(MaxmindDB)
+#
+# Variables used by this module, they can change the default behaviour and need
+# to be set before calling find_package:
+#
+#  MAXMINDDB_ROOT_DIR        Set this variable to the root installation of
+#                            libmaxminddb if the module has problems finding the
+#                            proper installation path.
+#
+# Variables defined by this module:
+#
+#  MAXMINDDB_FOUND                System has libmaxminddb, include and library dirs found
+#  MAXMINDDB_INCLUDE_DIR          The libmaxminddb include directories.
+#  MAXMINDDB_LIBRARY              The libmaxminddb library
+
+find_path(MAXMINDDB_ROOT_DIR NAMES maxminddb.h HINTS include/*/)
+find_path(MAXMINDDB_INCLUDE_DIR NAMES maxminddb.h HINTS ${MAXMINDDB_ROOT_DIR}/include/*/)
+find_library(MAXMINDDB_LIBRARY NAMES maxminddb HINTS ${MAXMINDDB_ROOT_DIR}/lib/*/ ${MAXMINDDB_ROOT_DIR}/lib64/*/)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(MaxmindDB DEFAULT_MSG MAXMINDDB_ROOT_DIR MAXMINDDB_LIBRARY)
+
+mark_as_advanced(
+        MAXMINDDB_ROOT_DIR
+        MAXMINDDB_INCLUDE_DIR
+        MAXMINDDB_LIBRARY
+)
+
+if(MaxmindDB_FOUND)
+    add_library(MaxmindDB::MaxmindDB INTERFACE IMPORTED)
+    set_property(TARGET MaxmindDB::MaxmindDB PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${MAXMINDDB_INCLUDE_DIR})
+    set_property(TARGET MaxmindDB::MaxmindDB PROPERTY INTERFACE_LINK_LIBRARIES ${MAXMINDDB_LIBRARY})
+endif()

data-model/dns-probe.yml

-# Last revision: 2021-01-27
+# Last revision: 2021-02-03
 #
 # Default instance configuration.
 # This configuration is always loaded before other configuration specified by given instance's ID.
@@ -77,6 +77,7 @@ default:
     export-format: 'parquet'
 
     # This sequence indicates which fields from the C-DNS standard schema are included in exported data.
+    # 3 implementation specific fields are also included (asn, country_code, round_trip_time).
     # By default all fields available in DNS Probe are enabled as shown below.
     cdns-fields:
       - 'transaction_id'
@@ -102,7 +103,9 @@ default:
       - 'query_opt_rdata'
       - 'response_additional_sections'
       - 'response_size'
-      - 'response_delay' # TCP RTT
+      - 'asn' # asn-maxmind-db configuration option also needs to be set
+      - 'country_code' # country-maxmind-db configuration option also needs to be set
+      - 'round_trip_time' # TCP RTT
 
     # Maximum number of DNS records in one exported C-DNS block.
     cdns-records-per-block: 10000
@@ -133,6 +136,16 @@ default:
     # Valid values are 'all', 'invalid', 'disabled'.
     pcap-export: 'disabled'
 
+    # Path to Maxmind Country database. If this option is set to a valid database file, the 'country'
+    # field in exported Parquets or 'country-code' implementation field in exported C-DNS will be
+    # filled with ISO 3166-1 country code based on client's IP address.
+    country-maxmind-db: ''
+
+    # Path to Maxmind ASN database. If this iption is set to a valid database file, the 'asn'
+    # implementation field in exported Parquets or C-DNS will be filled with Autonomous System
+    # Number (ASN) based on client's IP address.
+    asn-maxmind-db: ''
+
   # [SECTION] Configuration of client IP anonymization in exported data (Parquet or C-DNS).
   # The optional PCAP export does NOT get anonymized!!!
   ip-anonymization:

doc/Configuration.rst

@@ -27,9 +27,11 @@ DNS Probe uses local file in YAML format to load configuration at startup. Its s
       +--rw dnstap-socket-list?* <string>
       +--rw dns-ports?* <uint16>
       +--rw export
+      |  +--rw asn-maxmind-db? <string>
       |  +--rw cdns-blocks-per-file? <uint64>
       |  +--rw cdns-fields?* <string>
       |  +--rw cdns-records-per-block? <uint64>
+      |  +--rw country-maxmind-db? <string>
       |  +--rw export-dir? <string>
       |  +--rw export-format? <enumeration>
       |  +--rw file-compression? <boolean>
@@ -106,6 +108,14 @@ anonymize-ip
 
 If this flag is true, client IP addresses in exported data (Parquet or C-DNS, NOT optional PCAPs) will be anonymized using Crypto-PAn prefix-preserving algorithm.
 
+asn-maxmind-db
+^^^^^^^^^^^^^^
+
+:data node: ``<instance-id>/export/asn-maxmind-db``
+:default: empty
+
+Path to Maxmind ASN database. If this option is set to a valid database file, the ``asn`` implementation field in exported Parquets or C-DNS will be filled with Autonomous System Number (ASN) based on client's IP address.
+
 .. _cdns-blocks-per-file:
 
 cdns-blocks-per-file
@@ -158,6 +168,14 @@ Bitmask indicating which CPU cores should DNS Probe use. At least 3 CPU cores ar
 
 The default value of 7 indicates that DNS Probe should use the first 3 CPU cores with IDs of 0, 1 and 2.
 
+country-maxmind-db
+^^^^^^^^^^^^^^^^^^
+
+:data node: ``<instance-id>/export/country-maxmind-db``
+:default: empty
+
+Path to Maxmind Country database. If this option is set to a valid database file, the ``country`` field in exported Parquets or ``country-code`` implementation field in exported C-DNS will be filled with ISO 3166-1 country code based on client's IP address.
+
 .. _dnstap-socket-list:
 
 dnstap-socket-list

doc/Installation.rst

@@ -91,6 +91,7 @@ distribution repositories:
 - OpenSSL (libssl-dev)
 - fstrm
 - Protocol Buffers (libprotobuf-dev, protobuf-compiler)
+- libmaxminddb
 - DPDK (only for DPDK version)
 
 Optionally, to build this user documentation (``make doc``) or manual pages (``make man``)

doc/RecordOfChanges.rst

@@ -16,6 +16,15 @@ For changes in software see `version descriptions <https://gitlab.nic.cz/adam/dn
      - Segment
      - Change description
    * - **0.7.1**
+     - **1.1**
+     - :doc:`Configuration <Configuration>`, :doc:`Exported Data Schema <ExportedDataSchema>`,
+       :doc:`Default YAML file <YAMLfile>`
+     - Update ASN, Country Code and RTT fields in exported data schema
+   * -
+     -
+     - :doc:`Installation <Installation>`
+     - Add libmaxminddb as dependency
+   * -
      - **1.0**
      - :doc:`Architecture <Architecture>`, :doc:`Configuration <Configuration>`,
        :doc:`Installation <Installation>`, :doc:`Overview <Overview>`,

doc/YAMLfile.rst

@@ -9,7 +9,7 @@ It is also included in the project repository (`data-model/dns-probe.yml <https:
 
 .. code-block:: yaml
 
-  # Last revision: 2021-01-27
+  # Last revision: 2021-02-03
   #
   # Default instance configuration.
   # This configuration is always loaded before other configuration specified by given instance's ID.
@@ -88,6 +88,7 @@ It is also included in the project repository (`data-model/dns-probe.yml <https:
       export-format: 'parquet'
 
       # This sequence indicates which fields from the C-DNS standard schema are included in exported data.
+      # 3 implementation specific fields are also included (asn, country_code, round_trip_time).
       # By default all fields available in DNS Probe are enabled as shown below.
       cdns-fields:
         - 'transaction_id'
@@ -113,7 +114,9 @@ It is also included in the project repository (`data-model/dns-probe.yml <https:
         - 'query_opt_rdata'
         - 'response_additional_sections'
         - 'response_size'
-        - 'response_delay' # TCP RTT
+        - 'asn' # asn-maxmind-db configuration option also needs to be set
+        - 'country_code' # country-maxmind-db configuration option also needs to be set
+        - 'round_trip_time' # TCP RTT
 
       # Maximum number of DNS records in one exported C-DNS block.
       cdns-records-per-block: 10000
@@ -144,6 +147,16 @@ It is also included in the project repository (`data-model/dns-probe.yml <https:
       # Valid values are 'all', 'invalid', 'disabled'.
       pcap-export: 'disabled'
 
+      # Path to Maxmind Country database. If this option is set to a valid database file, the 'country'
+      # field in exported Parquets or 'country-code' implementation field in exported C-DNS will be
+      # filled with ISO 3166-1 country code based on client's IP address.
+      country-maxmind-db: ''
+
+      # Path to Maxmind ASN database. If this iption is set to a valid database file, the 'asn'
+      # implementation field in exported Parquets or C-DNS will be filled with Autonomous System
+      # Number (ASN) based on client's IP address.
+      asn-maxmind-db: ''
+
     # [SECTION] Configuration of client IP anonymization in exported data (Parquet or C-DNS).
     # The optional PCAP export does NOT get anonymized!!!
     ip-anonymization:

doc/index.rst

@@ -15,7 +15,7 @@ scanned requests and responses.
       which means that it contains authoring/editing/managing notes
       in addition to the actual content.
 
-:Edition: :doc:`1.0 <RecordOfChanges>`
+:Edition: :doc:`1.1 <RecordOfChanges>`
 
 :Source: The source code of this documentation is open and can be found
          on `GitLab <https://gitlab.nic.cz/adam/dns-probe/-/tree/master/doc>`_.

src/config/Config.h

@@ -59,9 +59,11 @@ namespace DDP {
                    file_rot_size(0),
                    file_compression(true),
                    pcap_export(PcapExportCfg::DISABLED),
+                   country_db(),
+                   asn_db(),
                    export_format(ExportFormat::PARQUET),
                    parquet_records(5000000),
-                   cdns_fields(0xFFFFFF),
+                   cdns_fields(get_cdns_bitmask()),
                    cdns_records_per_block(10000),
                    cdns_blocks_per_file(0),
                    export_location(ExportLocation::LOCAL),
@@ -97,6 +99,8 @@ namespace DDP {
         ConfigItem<uint64_t> file_rot_size; //!< Exported file size limit in MB
         ConfigItem<bool> file_compression; //!< Enable GZIP compression for exported files
         ConfigItem<PcapExportCfg> pcap_export; //!< Define what will be in exported PCAPs
+        ConfigItem<std::string> country_db; //!< Path to Maxmind Country database
+        ConfigItem<std::string> asn_db; //!< Path to Maxmind ASN database
 
         ConfigItem<ExportFormat> export_format; //!< Specify export format
         ConfigItem<uint64_t> parquet_records; //!< Number of records in parquet file

src/config/ConfigFile.cpp

@@ -164,6 +164,12 @@ void DDP::ConfigFile::load_instance(Config& cfg, YAML::Node node)
     if (node["export"]["pcap-export"] && node["export"]["pcap-export"].IsScalar())
         cfg.pcap_export.add_value(node["export"]["pcap-export"].as<std::string>());
 
+    if (node["export"]["country-maxmind-db"] && node["export"]["country-maxmind-db"].IsScalar())
+        cfg.country_db.add_value(node["export"]["country-maxmind-db"].as<std::string>());
+
+    if (node["export"]["asn-maxmind-db"] && node["export"]["asn-maxmind-db"].IsScalar())
+        cfg.asn_db.add_value(node["export"]["asn-maxmind-db"].as<std::string>());
+
     if (node["ip-anonymization"]["anonymize-ip"] && node["ip-anonymization"]["anonymize-ip"].IsScalar())
         cfg.anonymize_ip.add_value(node["ip-anonymization"]["anonymize-ip"].as<bool>());
 

src/config/ConfigTypes.h

@@ -37,7 +37,14 @@ namespace DDP {
     using IPv4_t = uint32_t;
     using IPv6_t = std::array<uint32_t, 4>;
 
-    static constexpr uint8_t CdnsBits = 24; //!< Number of C-DNS fields options
+    static constexpr uint8_t CdnsBits = 26; //!< Number of C-DNS fields options
+    static constexpr uint64_t get_cdns_bitmask() {
+        uint64_t bitmask = 0;
+        for (unsigned i = 0; i < CdnsBits; i++) {
+            bitmask |= 1 << i;
+        }
+        return bitmask;
+    }
 
     /**
      * Available values for PCAP export config.
@@ -91,7 +98,9 @@ namespace DDP {
         QUERY_OPT_RDATA,
         RESPONSE_ADDITIONAL_SECTIONS,
         RESPONSE_SIZE,
-        RESPONSE_DELAY
+        ASN,
+        COUNTRY_CODE,
+        ROUND_TRIP_TIME
     };
 
     static const std::unordered_map<std::string, uint32_t> CdnsFieldsMap = {
@@ -117,7 +126,9 @@ namespace DDP {
         {"query_opt_data",                  static_cast<uint32_t>(CDNSField::QUERY_OPT_RDATA)},
         {"response_additional_sections",    static_cast<uint32_t>(CDNSField::RESPONSE_ADDITIONAL_SECTIONS)},
         {"response_size",                   static_cast<uint32_t>(CDNSField::RESPONSE_SIZE)},
-        {"response_delay",                  static_cast<uint32_t>(CDNSField::RESPONSE_DELAY)},
+        {"asn",                             static_cast<uint32_t>(CDNSField::ASN)},
+        {"country_code",                    static_cast<uint32_t>(CDNSField::COUNTRY_CODE)},
+        {"round_trip_time",                 static_cast<uint32_t>(CDNSField::ROUND_TRIP_TIME)},
     };
 
     /**

src/core/DnsTcpConnection.cpp

@@ -342,7 +342,7 @@ bool DDP::DnsTcpConnection::process_segment(const Packet& packet, const MemView<
                 record.m_dns_len = seg_len - 2;
                 record.m_len = packet.size();
                 if (m_state >= TcpConnectionState::ESTABLISHED) {
-                    record.m_tcp_rtt = m_rtt.getMillis();
+                    record.m_tcp_rtt = m_rtt.getMillisRounded();
                     m_rtt = Time(-1);
                 }
 
@@ -378,7 +378,7 @@ bool DDP::DnsTcpConnection::process_segment(const Packet& packet, const MemView<
                     DnsRecord& msg = parser->get_empty();
                     fill_record_L3_L4(msg, record);
                     if (m_state >= TcpConnectionState::ESTABLISHED) {
-                        msg.m_tcp_rtt = m_rtt.getMillis();
+                        msg.m_tcp_rtt = m_rtt.getMillisRounded();
                         m_rtt = Time(-1);
                     }
 
@@ -474,7 +474,7 @@ bool DDP::DnsTcpConnection::process_segment(const Packet& packet, const MemView<
                         DnsRecord &msg = parser->get_empty();
                         fill_record_L3_L4(msg, record);
                         if (m_state >= TcpConnectionState::ESTABLISHED) {
-                            msg.m_tcp_rtt = m_rtt.getMillis();
+                            msg.m_tcp_rtt = m_rtt.getMillisRounded();
                             m_rtt = Time(-1);
                         }
 

src/core/Probe.cpp

@@ -85,8 +85,15 @@ namespace DDP {
 
 DDP::Probe::Probe() : m_cfg_loaded(false), m_initialized(false), m_running(false), m_poll(), m_cfg(),
                       m_aggregated_timer(nullptr), m_output_timer(nullptr), m_comm_links(),
-                      m_log_link(), m_dns_record_mempool(), m_export_rings(), m_factory_rings(), m_stats(),
-                      m_stopped_workers(0), m_ret_value(ReturnValue::STOP) {}
+                      m_log_link(), m_dns_record_mempool(), m_export_rings(), m_factory_rings(),
+                      m_country(), m_asn(), m_stats(), m_stopped_workers(0),
+                      m_ret_value(ReturnValue::STOP) {}
+
+DDP::Probe::~Probe()
+{
+    MMDB_close(&m_country);
+    MMDB_close(&m_asn);
+}
 
 DDP::ParsedArgs DDP::Probe::process_args(int argc, char** argv)
 {
@@ -290,6 +297,28 @@ void DDP::Probe::init(const Arguments& args)
 #endif
         }
 
+        if (!m_cfg.country_db.value().empty()) {
+            int status = MMDB_open(m_cfg.country_db.value().c_str(), MMDB_MODE_MMAP, &m_country);
+            if (status != MMDB_SUCCESS) {
+                Logger("Probe").warning() << "Couldn't open Maxmind Country database!";
+                m_country.filename = nullptr;
+            }
+        }
+        else {
+            m_country.filename = nullptr;
+        }
+
+        if (!m_cfg.asn_db.value().empty()) {
+            int status = MMDB_open(m_cfg.asn_db.value().c_str(), MMDB_MODE_MMAP, &m_asn);
+            if (status != MMDB_SUCCESS) {
+                Logger("Probe").warning() << "Couldn't open Maxmind ASN database!";
+                m_asn.filename = nullptr;
+            }
+        }
+        else {
+            m_asn.filename = nullptr;
+        }
+
         if (m_cfg.export_location.value() == ExportLocation::REMOTE)
             TlsCtx::getInstance().init(m_cfg.export_ca_cert.value());
 
@@ -313,7 +342,7 @@ DDP::Probe::ReturnValue DDP::Probe::run(std::vector<std::shared_ptr<DDP::Port>>&
         try {
             Worker w(m_cfg, stats, m_factory_rings.at(worker).get_poll_able_ring(), m_comm_links[worker].worker_endpoint(),
                     *m_dns_record_mempool, *m_tcp_connection_mempool, queue, ports, w_sockets,
-                    m_cfg.match_qname, worker);
+                    m_cfg.match_qname, worker, m_country, m_asn);
             Logger logger("Worker");
             logger.info() << "Starting worker on lcore " << ThreadManager::current_lcore() << ".";
             w.run();

src/core/Probe.h

@@ -27,6 +27,7 @@
 #include <functional>
 #include <vector>
 #include <set>
+#include <maxminddb.h>
 
 #include "utils/Poll.h"
 #include "utils/RingFwdDecl.h"
@@ -174,6 +175,11 @@ namespace DDP {
          */
         Probe();
 
+        /**
+         * Destructor
+         */
+        ~Probe();
+
         /**
          * Read all messages from log commlink and writes them to the output.
          */
@@ -195,6 +201,8 @@ namespace DDP {
         std::unique_ptr<Mempool<DnsTcpConnection>> m_tcp_connection_mempool; //!< Mempool for TCP connections.
         std::unordered_map<unsigned, std::unique_ptr<Ring<boost::any>>> m_export_rings; //!< Rings for sending data from workers to exporter.
         std::unordered_map<unsigned, PollAbleRingFactory<boost::any>> m_factory_rings;
+        MMDB_s m_country; //!< Maxmind Country database
+        MMDB_s m_asn; //!< Maxmind ASN database
 
         std::vector<Statistics> m_stats; //!< Statistics structure for workers. One item in vector per worker.
         AggregatedStatistics m_aggregated_stats; //!< Aggregated statistics from workers.

src/core/Worker.h

@@ -24,6 +24,7 @@
 #pragma once
 
 #include <set>
+#include <maxminddb.h>
 #include "export/BaseExport.h"
 #include "export/BaseWriter.h"
 #include "core/DnsRecord.h"
@@ -153,7 +154,8 @@ namespace DDP {
         Worker(Config& cfg, Statistics& stats, PollAbleRing<boost::any> ring,
                CommLink::CommLinkEP& comm_link, Mempool<DnsRecord>& record_mempool,
                Mempool<DnsTcpConnection>& tcp_mempool, unsigned lcore_queue, std::vector<std::shared_ptr<DDP::Port>> ports,
-               std::vector<std::shared_ptr<DDP::Port>> sockets, bool match_qname, unsigned process_id) :
+               std::vector<std::shared_ptr<DDP::Port>> sockets, bool match_qname, unsigned process_id, MMDB_s& country_db,
+               MMDB_s& asn_db) :
                 Process(cfg, stats, comm_link),
                 m_record_mempool(record_mempool),
                 m_tcp_mempool(tcp_mempool),
@@ -174,14 +176,14 @@ namespace DDP {
         {
             if (cfg.export_format.value() == ExportFormat::PARQUET) {
 #ifdef PROBE_PARQUET
-                m_exporter = new ParquetExport(cfg);
+                m_exporter = new ParquetExport(cfg, country_db, asn_db);
 #else
                 throw std::runtime_error("DNS Probe was built without Parquet support!");
 #endif
             }
             else {
 #ifdef PROBE_CDNS
-                m_exporter = new CdnsExport(cfg);
+                m_exporter = new CdnsExport(cfg, country_db, asn_db);
 #else
                 throw std::runtime_error("DNS Probe was built without C-DNS support!");
 #endif

src/export/BaseExport.cpp

+/*
+ *  Copyright (C) 2021 CZ.NIC, z. s. p. o.
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *  In addition, as a special exception, the copyright holders give
+ *  permission to link the code of portions of this program with the
+ *  OpenSSL library under certain conditions as described in each
+ *  individual source file, and distribute linked combinations including
+ *  the two.
+ */
+
+#include "BaseExport.h"
+
+void DDP::BaseExport::fill_asn_country(const in6_addr* addr, int ipv, std::string& asn, std::string& country)
+{
+    if (m_country.filename != nullptr || m_asn.filename != nullptr) {
+        sockaddr_in sa4;
+        sockaddr_in6 sa6;
+        sockaddr* sa = nullptr;
+
+        if (ipv == AF_INET) {
+            sa4.sin_family = AF_INET;
+            sa = reinterpret_cast<sockaddr*>(&sa4);
+            sa4.sin_addr = *reinterpret_cast<const in_addr*>(addr);
+        }
+        else {
+            sa6.sin6_family = AF_INET6;
+            sa6.sin6_addr = *addr;
+            sa = reinterpret_cast<sockaddr*>(&sa6);
+        }
+
+        if (m_country.filename != nullptr) {
+            int err;
+            auto result = MMDB_lookup_sockaddr(&m_country, sa, &err);
+            if (err == MMDB_SUCCESS && result.found_entry) {
+                MMDB_entry_data_s data;
+                int status = MMDB_get_value(&result.entry, &data, "country", "iso_code", NULL);
+                if (status == MMDB_SUCCESS && data.has_data && data.type == MMDB_DATA_TYPE_UTF8_STRING)
+                    country = std::string(data.utf8_string, data.data_size);
+            }
+        }
+
+        if (m_asn.filename != nullptr) {
+            int err;
+            auto result = MMDB_lookup_sockaddr(&m_asn, sa, &err);
+            if (err == MMDB_SUCCESS && result.found_entry) {
+                MMDB_entry_data_s data;
+                int status = MMDB_get_value(&result.entry, &data, "autonomous_system_number", NULL);
+                if (status == MMDB_SUCCESS && data.has_data && data.type == MMDB_DATA_TYPE_UINT32)
+                    asn = std::to_string(data.uint32);
+            }
+        }
+    }
+}

src/export/BaseExport.h

@@ -27,6 +27,7 @@
 #include <cstring>
 #include <arpa/inet.h>
 #include <sys/socket.h>
+#include <maxminddb.h>
 
 #ifdef PROBE_CRYPTOPANT
 #include <cryptopANT.h>
@@ -48,7 +49,8 @@ namespace DDP {
     class BaseExport
     {
     public:
-        explicit BaseExport(bool anonymize_ip) : m_anonymize_ip(anonymize_ip) {}
+        explicit BaseExport(bool anonymize_ip, MMDB_s& country_db, MMDB_s& asn_db)
+        : m_anonymize_ip(anonymize_ip), m_country(country_db), m_asn(asn_db) {}
 
         virtual ~BaseExport() {};
 
@@ -79,6 +81,18 @@ namespace DDP {
         virtual void update_configuration(Config& cfg) = 0;
 
     protected:
+
+        /**
+         * @brief Fill given ASN and Country Code strings from Maxmind databases based on given IP address
+         * @param addr IP address to lookup in Maxmind databases
+         * @param ipv Version of given IP address
+         * @param asn ASN string to fill with IP's ASN
+         * @param country Country Code string to fill with IP's ISO 3166-1 country code
+         */
+        void fill_asn_country(const in6_addr* addr, int ipv, std::string& asn, std::string& country);
+
         bool m_anonymize_ip;
+        MMDB_s& m_country;
+        MMDB_s& m_asn;
     };
 }

src/export/cdns/CdnsExport.cpp

@@ -23,8 +23,9 @@
 
 #include "CdnsExport.h"
 
-DDP::CdnsExport::CdnsExport(Config& cfg)
-    : BaseExport(cfg.anonymize_ip.value()), m_fields(cfg.cdns_fields.value()), m_parameters()
+DDP::CdnsExport::CdnsExport(Config& cfg, MMDB_s& country_db, MMDB_s& asn_db)
+    : BaseExport(cfg.anonymize_ip.value(), country_db, asn_db), m_fields(cfg.cdns_fields.value()),
+      m_parameters()
 {
     m_parameters.storage_parameters.max_block_items = cfg.cdns_records_per_block.value();
     set_cdns_hints(m_parameters.storage_parameters.storage_hints.query_response_hints,
@@ -73,6 +74,9 @@ boost::any DDP::CdnsExport::buffer_record(DnsRecord& record)
         qrs_filled = true;
     }
 
+    std::string country;
+    std::string asn;
+    fill_asn_country(record.client_address(), record.m_addr_family == DnsRecord::AddrFamily::IP4 ? AF_INET : AF_INET6, asn, country);
     if (m_fields[static_cast<uint32_t>(CDNSField::CLIENT_ADDRESS)]) {
         in6_addr* addr = record.client_address();
         if (record.m_addr_family == DnsRecord::AddrFamily::IP4) {
@@ -214,8 +218,15 @@ boost::any DDP::CdnsExport::buffer_record(DnsRecord& record)
     if (m_fields[static_cast<uint32_t>(CDNSField::RESPONSE_SIZE)])
         qr.response_size = record.m_res_dns_len;
 
-    if (m_fields[static_cast<uint32_t>(CDNSField::RESPONSE_DELAY)])
-        qr.response_delay = record.m_tcp_rtt;
+    if (m_fields[static_cast<uint32_t>(CDNSField::ROUND_TRIP_TIME)])
+        // need to convert millisecond m_tcp_rtt to ticks which represent microseconds
+        qr.round_trip_time = record.m_tcp_rtt * 1000;
+
+    if (m_fields[static_cast<uint32_t>(CDNSField::ASN)] && !asn.empty())
+        qr.asn = asn;
+
+    if (m_fields[static_cast<uint32_t>(CDNSField::COUNTRY_CODE)] && !country.empty())
+        qr.country_code = country;
 
     // Add QueryResponseSignature to the QueryResponse
     if (qrs_filled)

src/export/cdns/CdnsExport.h

@@ -39,7 +39,7 @@ namespace DDP {
          * @brief Constructor creates new C-DNS block configured for given C-DNS fields
          * @param cfg Object witch configuration options
          */
-        CdnsExport(Config& cfg);
+        CdnsExport(Config& cfg, MMDB_s& country_db, MMDB_s& asn_db);
 
         /**
          * @brief Store DNS record into C-DNS block

src/export/cdns/CdnsWriter.cpp

@@ -96,9 +96,6 @@ void DDP::set_cdns_hints(uint32_t& qr_hints, uint32_t& qr_sig_hints, std::bitset
 
     if (fields[static_cast<uint32_t>(CDNSField::RESPONSE_SIZE)])
         qr_hints |= CDNS::QueryResponseHintsMask::response_size;
-
-    if (fields[static_cast<uint32_t>(CDNSField::RESPONSE_DELAY)])
-        qr_hints |= CDNS::QueryResponseHintsMask::response_delay;
 }
 
 DDP::CdnsWriter::CdnsWriter(Config& cfg, uint32_t process_id) : BaseWriter(cfg, process_id,

src/export/parquet/ParquetExport.cpp

@@ -35,8 +35,9 @@