Add CBOR output logging matching the CSV fields

README.md

@@ -89,12 +89,34 @@ Packet statistics:
 However, the drop to 600 kq/s does not seem to come from dnscol CPU usage but rather the packet capture
 load on the kernel: the Knot speed drop is the same (to 600 kq/s) with dnscol cpulimited to just 0.5 CPU.
 
+## CBOR output
+
+CBOR output has been introduced in 0.1.3 to adress some of the encoding problems of CSV: representing binary data and structured elements. [CBOR format](http://cbor.io/)
+is defined in [RFC 7049](https://tools.ietf.org/html/rfc7049) and there are implementations for many common languages.
+
+The CBOR output file is a sequence of independent CBOR items. This was preferred to a one large CBOR item per file to avoid having to load the entire file with most (non-streaming) parsers. The file structure resembles a CSV file with a header row:
+
+The first item is an array of column names, corresponding to the present columns in order.
+Each subsequent item is array of attributes for one record. The meanings and types of the attributes are the same as for the CSV files below. The only exceptions are EDNS data
+with inner structure:
+
+| field name | format |
+| ---- | ---- |
+| `req_edns_dau` | array of DAU numbers |
+| `req_edns_dhu` | array of DHU numbers |
+| `req_edns_n3u` | array of N3U numbers |
+| `resp_edns_nsid` | raw NSID as a bytestring |
+| `edns_client_subnet` | *TODO* |
+| `edns_other` | *TODO* |
+
+Compared to CSV, CBOR output uses cca 10% CPU (user time), is 30% smaller uncompressed and 5% smmaller gziped.
+
 ## CSV output
 
 ### Escaping
 
 The output is primarily targetted for Impala import, so the CSV does NOT conform to RFC 4180.
-Instead of quoting, problematic characters are escaped with a backslash.
+Instead of quoting, problematic characters are escaped with a backslash. Every record is one line - all newlines are escaped.
 
 The escaped characters are: backslash (`\\`), newline (`\n`), the configured field separator (e.g. `\,`),
 all non-ASCII and most non-printable characters (`\ooo` with octal notation).

dns-collector.conf

@@ -75,7 +75,7 @@ dnscol {
     ### Note that the pipe command is restarted for every output file.
     output_period 600
 
-    ### Output format and type. Currently only "csv" is supported.
+    ### Output format and type. Currently "csv" and "cbor" are supported.
     output_type csv
 
 
@@ -105,6 +105,10 @@ dnscol {
     csv_fields time delay_us req_dns_len resp_dns_len req_net_len resp_net_len \
                client_addr client_port server_addr server_port net_proto net_ipv net_ttl req_udp_sum \
                id qtype qclass opcode rcode flags qname rr_counts edns
+
+    cbor_fields time delay_us req_dns_len resp_dns_len req_net_len resp_net_len \
+               client_addr client_port server_addr server_port net_proto net_ipv net_ttl req_udp_sum \
+               id qtype qclass opcode rcode flags qname rr_counts edns
 }
 
 ### Logging config

src/common.h

@@ -133,7 +133,7 @@ enum dns_output_field {
     dns_field_req_edns_udp = dns_field_edns,
     dns_field_req_edns_do = dns_field_edns, 
     dns_field_resp_edns_rcode = dns_field_edns, 
-    dns_field_req_edns_ping = dns_field_edns,
+    dns_field_req_edns_ping = dns_field_edns, // Not implemented
     dns_field_req_edns_dau = dns_field_edns,
     dns_field_req_edns_dhu = dns_field_edns,
     dns_field_req_edns_n3u = dns_field_edns,

src/output_cbor.c

@@ -36,7 +36,8 @@
 static size_t
 write_packet(FILE *f, uint32_t fields, dns_packet_t *pkt)
 {
-    uint8_t outbuf[2048];
+    char addrbuf[MAX(INET_ADDRSTRLEN, INET6_ADDRSTRLEN) + 4];
+    uint8_t outbuf[4096];
     CborEncoder ebase, eitem;
 
 // Run cmd and die (with an error meaasge) on any CBOR error
@@ -44,16 +45,24 @@ write_packet(FILE *f, uint32_t fields, dns_packet_t *pkt)
         if (cerr__ != CborNoError) { die("CBOR error: %s", cbor_error_string(cerr__)); } }
 
     cbor_encoder_init(&ebase, outbuf, sizeof(outbuf), 0);
-    CERR(cbor_encoder_create_array(&ebase, &eitem, 1));
+    int noitems = __builtin_popcount(fields);
+    if (fields & (1 << dns_field_flags)) noitems += 6;
+    if (fields & (1 << dns_field_rr_counts)) noitems += 2;
+    if (fields & (1 << dns_field_edns)) noitems += 9;
+    CERR(cbor_encoder_create_array(&ebase, &eitem, noitems));
 
 // Start writing a field conditional on the field flag being set,
 // write just the field name for the header if pkt==NULL
-#define COND(fieldname) \
+// write NONE if secondary is not satisfied
+#define CONDIF(fieldname, secondary) \
         if (fields & (1 << (dns_field_ ## fieldname))) { \
-            if (!pkt) { CERR(cbor_encode_text_stringz(&eitem, #fieldname )); } else { \
+            if (!(pkt)) { CERR(cbor_encode_text_stringz(&eitem, #fieldname )); } else { \
+                if (!(secondary)) { CERR(cbor_encode_null(&eitem)); } else {
+
+#define COND(fieldname) CONDIF(fieldname, 1)
 
 // Closing for COND(...) statement
-#define COND_END  } }
+#define COND_END  } } }
 
     // Time
 
@@ -61,6 +70,214 @@ write_packet(FILE *f, uint32_t fields, dns_packet_t *pkt)
         CERR(cbor_encode_double(&eitem, (double)(pkt->ts) / 1000000.0));
     COND_END
 
+    CONDIF(delay_us, pkt->response) 
+        CERR(cbor_encode_int(&eitem, pkt->response->ts - pkt->ts));
+    COND_END
+
+    // Sizes
+
+    CONDIF(req_dns_len, DNS_PACKET_REQUEST(pkt))
+        CERR(cbor_encode_int(&eitem, DNS_PACKET_REQUEST(pkt)->dns_data_size_orig));
+    COND_END
+
+    CONDIF(resp_dns_len, DNS_PACKET_RESPONSE(pkt)) 
+        CERR(cbor_encode_int(&eitem, DNS_PACKET_RESPONSE(pkt)->dns_data_size_orig));
+    COND_END
+
+    CONDIF(req_net_len, DNS_PACKET_REQUEST(pkt))
+        CERR(cbor_encode_int(&eitem, DNS_PACKET_REQUEST(pkt)->net_size));
+    COND_END
+
+    CONDIF(resp_net_len, DNS_PACKET_RESPONSE(pkt))
+        CERR(cbor_encode_int(&eitem, DNS_PACKET_RESPONSE(pkt)->net_size));
+    COND_END
+
+    // IP stats
+
+    COND(client_addr)
+        inet_ntop(DNS_PACKET_AF(pkt), DNS_PACKET_CLIENT_ADDR(pkt), addrbuf, sizeof(addrbuf));
+        CERR(cbor_encode_text_stringz(&eitem, addrbuf));
+    COND_END
+
+    COND(client_port) 
+        CERR(cbor_encode_int(&eitem, DNS_PACKET_CLIENT_PORT(pkt)));
+    COND_END
+
+    COND(server_addr)
+        inet_ntop(DNS_PACKET_AF(pkt), DNS_PACKET_SERVER_ADDR(pkt), addrbuf, sizeof(addrbuf));
+        CERR(cbor_encode_text_stringz(&eitem, addrbuf));
+    COND_END
+
+    COND(server_port) 
+        CERR(cbor_encode_int(&eitem, DNS_PACKET_SERVER_PORT(pkt)));
+    COND_END
+
+    COND(net_proto) 
+        CERR(cbor_encode_int(&eitem, pkt->net_protocol));
+    COND_END
+
+    COND(net_ipv) 
+        switch (DNS_SOCKADDR_AF(&pkt->src_addr)) {
+        case AF_INET:
+            CERR(cbor_encode_int(&eitem, 4));
+            break;
+        case AF_INET6:
+            CERR(cbor_encode_int(&eitem, 6));
+            break;
+        }
+    COND_END
+
+    CONDIF(net_ttl, pkt->net_ttl > 0) 
+        CERR(cbor_encode_int(&eitem, pkt->net_ttl));
+    COND_END
+
+    CONDIF(req_udp_sum, (pkt->net_protocol == IPPROTO_UDP) && DNS_PACKET_REQUEST(pkt)) 
+        CERR(cbor_encode_int(&eitem, DNS_PACKET_REQUEST(pkt)->net_udp_sum));
+    COND_END
+
+    // DNS header
+
+    COND(id) 
+        CERR(cbor_encode_int(&eitem, knot_wire_get_id(pkt->knot_packet->wire)));
+    COND_END
+
+    COND(qtype) 
+        CERR(cbor_encode_int(&eitem, knot_pkt_qtype(pkt->knot_packet)));
+    COND_END
+
+    COND(qclass) 
+        CERR(cbor_encode_int(&eitem, knot_pkt_qclass(pkt->knot_packet)));
+    COND_END
+
+    COND(opcode)
+        CERR(cbor_encode_int(&eitem, knot_wire_get_opcode(pkt->knot_packet->wire)));
+    COND_END
+
+    CONDIF(rcode, DNS_PACKET_RESPONSE(pkt))
+            CERR(cbor_encode_int(&eitem, knot_wire_get_rcode(DNS_PACKET_RESPONSE(pkt)->knot_packet->wire)));
+    COND_END
+
+#define WRITEFLAG(label, type, name) CONDIF(label, DNS_PACKET_ ## type (pkt)) \
+            CERR(cbor_encode_boolean( &eitem, \
+            !! knot_wire_get_ ## name(DNS_PACKET_ ## type (pkt)->knot_packet->wire))); \
+        COND_END
+
+    WRITEFLAG(resp_aa, RESPONSE, aa);
+    WRITEFLAG(resp_tc, RESPONSE, tc);
+    WRITEFLAG(req_rd, REQUEST, rd);
+    WRITEFLAG(resp_ra, RESPONSE, ra);
+    WRITEFLAG(req_z, REQUEST, z);
+    WRITEFLAG(resp_ad, RESPONSE, ad);
+    WRITEFLAG(req_cd, REQUEST, cd);
+
+#undef WRITEFLAG
+
+    COND(qname) 
+        char qname_buf[1024];
+        char *res = knot_dname_to_str(qname_buf, knot_pkt_qname(pkt->knot_packet), sizeof(qname_buf));
+        if (res) {
+            CERR(cbor_encode_text_stringz(&eitem, res));
+        } else {
+            CERR(cbor_encode_null(&eitem)); 
+        }
+    COND_END
+
+    CONDIF(resp_ancount, DNS_PACKET_RESPONSE(pkt)) 
+        CERR(cbor_encode_int(&eitem, knot_wire_get_ancount(DNS_PACKET_RESPONSE(pkt)->dns_data)));
+    COND_END
+
+    CONDIF(resp_arcount, DNS_PACKET_RESPONSE(pkt))
+        CERR(cbor_encode_int(&eitem, knot_wire_get_arcount(DNS_PACKET_RESPONSE(pkt)->dns_data)));
+    COND_END
+
+    CONDIF(resp_nscount, DNS_PACKET_RESPONSE(pkt)) 
+        CERR(cbor_encode_int(&eitem, knot_wire_get_nscount(DNS_PACKET_RESPONSE(pkt)->dns_data)));
+    COND_END
+
+    // EDNS
+
+    const knot_rrset_t *req_opt_rr = (pkt && DNS_PACKET_REQUEST(pkt)) ? DNS_PACKET_REQUEST(pkt)->knot_packet->opt_rr : NULL;
+    const knot_rrset_t *resp_opt_rr = (pkt && DNS_PACKET_RESPONSE(pkt)) ? DNS_PACKET_RESPONSE(pkt)->knot_packet->opt_rr : NULL;
+
+    CONDIF(req_edns_ver, req_opt_rr)
+        CERR(cbor_encode_int(&eitem, knot_edns_get_version(req_opt_rr)));
+    COND_END
+
+    CONDIF(req_edns_udp, req_opt_rr)
+        CERR(cbor_encode_int(&eitem, knot_edns_get_payload(req_opt_rr)));
+    COND_END
+
+    CONDIF(req_edns_do, req_opt_rr)
+        CERR(cbor_encode_boolean(&eitem, !! knot_edns_do(req_opt_rr)));
+    COND_END
+
+    CONDIF(resp_edns_rcode, resp_opt_rr)
+        CERR(cbor_encode_int(&eitem, knot_edns_get_ext_rcode(resp_opt_rr)));
+    COND_END
+
+#define COND_OPT_RR(label, code, rr) \
+    CONDIF(label, rr) \
+        uint8_t *opt = knot_edns_get_option(rr, code); \
+        if (!opt) { CERR(cbor_encode_null(&eitem)); } else { \
+            int len = knot_edns_opt_get_length(opt);
+
+#define COND_OPT_RR_END \
+    } COND_END
+
+#define WRITE_UNDERSTOOD_LIST(label, code) \
+    COND_OPT_RR(label, code, req_opt_rr) \
+            CborEncoder elist; \
+            CERR(cbor_encoder_create_array(&eitem, &elist, len)); \
+            for (int i = 0; i < knot_edns_opt_get_length(opt); i++) { \
+                CERR(cbor_encode_int(&elist, ((opt + 2 * sizeof(uint16_t) + i)[i]))); \
+            } \
+            CERR(cbor_encoder_close_container_checked(&eitem, &elist)); \
+    COND_OPT_RR_END
+
+    //COND(req_edns_ping)
+        // TODO: distinguish a PING request from DAU,
+        // see https://github.com/SIDN/entrada/blob/b787af190267df148683151638ce94508bd6139e/dnslib4java/src/main/java/nl/sidn/dnslib/message/records/edns0/OPTResourceRecord.java#L146
+    //COND_END
+
+    WRITE_UNDERSTOOD_LIST(req_edns_dau, 5); // DAU
+    WRITE_UNDERSTOOD_LIST(req_edns_dhu, 6); // DHU 
+    WRITE_UNDERSTOOD_LIST(req_edns_n3u, 7); // N3U 
+
+    COND_OPT_RR(resp_edns_nsid, 3, resp_opt_rr) // NSID
+        CERR(cbor_encode_byte_string(&eitem, opt + 2 * sizeof(uint16_t), len));
+    COND_OPT_RR_END
+
+    CONDIF(edns_client_subnet, 0) // TODO: PARSE and WRITE client subnet information
+        uint8_t *opt = NULL;
+        if (resp_opt_rr)
+            opt = knot_edns_get_option(resp_opt_rr, 8); // client subnet from response 
+        if (req_opt_rr && !opt)
+            opt = knot_edns_get_option(req_opt_rr, 8); // client subnet from request 
+        if (opt) {
+            CborEncoder elist;
+            CERR(cbor_encoder_create_array(&eitem, &elist, CborIndefiniteLength));
+            // As in: "[4,'118.71.70',24,0]"
+            // Entrada: (fam == 1? "4,": "6,") + address + "/" + sourcenetmask + "," + scopenetmask;
+            CERR(cbor_encoder_close_container_checked(&eitem, &elist));
+        } else {
+            CERR(cbor_encode_null(&eitem));
+        }
+    COND_END
+
+    CONDIF(edns_other, 0) // TODO: traverse all remaining records
+        CborEncoder elist;
+        CERR(cbor_encoder_create_array(&eitem, &elist, 4));
+        if (req_opt_rr) {
+            // ...
+        }
+        if (resp_opt_rr) {
+            // ...
+        }
+        CERR(cbor_encoder_close_container_checked(&eitem, &elist));
+    COND_END
+
+    // Close the array
+
     CERR(cbor_encoder_close_container_checked(&ebase, &eitem));
     size_t written = cbor_encoder_get_buffer_size(&ebase, outbuf);
     fwrite(outbuf, written, 1, f);