Commit 5d7149da authored by Tomas Gavenciak's avatar Tomas Gavenciak
Browse files

Add CBOR output logging matching the CSV fields

parent 117330c4
......@@ -89,12 +89,34 @@ Packet statistics:
However, the drop to 600 kq/s does not seem to come from dnscol CPU usage but rather the packet capture
load on the kernel: the Knot speed drop is the same (to 600 kq/s) with dnscol cpulimited to just 0.5 CPU.
## CBOR output
CBOR output has been introduced in 0.1.3 to adress some of the encoding problems of CSV: representing binary data and structured elements. [CBOR format](http://cbor.io/)
is defined in [RFC 7049](https://tools.ietf.org/html/rfc7049) and there are implementations for many common languages.
The CBOR output file is a sequence of independent CBOR items. This was preferred to a one large CBOR item per file to avoid having to load the entire file with most (non-streaming) parsers. The file structure resembles a CSV file with a header row:
The first item is an array of column names, corresponding to the present columns in order.
Each subsequent item is array of attributes for one record. The meanings and types of the attributes are the same as for the CSV files below. The only exceptions are EDNS data
with inner structure:
| field name | format |
| ---- | ---- |
| `req_edns_dau` | array of DAU numbers |
| `req_edns_dhu` | array of DHU numbers |
| `req_edns_n3u` | array of N3U numbers |
| `resp_edns_nsid` | raw NSID as a bytestring |
| `edns_client_subnet` | *TODO* |
| `edns_other` | *TODO* |
Compared to CSV, CBOR output uses cca 10% CPU (user time), is 30% smaller uncompressed and 5% smmaller gziped.
## CSV output
### Escaping
The output is primarily targetted for Impala import, so the CSV does NOT conform to RFC 4180.
Instead of quoting, problematic characters are escaped with a backslash.
Instead of quoting, problematic characters are escaped with a backslash. Every record is one line - all newlines are escaped.
The escaped characters are: backslash (`\\`), newline (`\n`), the configured field separator (e.g. `\,`),
all non-ASCII and most non-printable characters (`\ooo` with octal notation).
......
......@@ -75,7 +75,7 @@ dnscol {
### Note that the pipe command is restarted for every output file.
output_period 600
### Output format and type. Currently only "csv" is supported.
### Output format and type. Currently "csv" and "cbor" are supported.
output_type csv
......@@ -105,6 +105,10 @@ dnscol {
csv_fields time delay_us req_dns_len resp_dns_len req_net_len resp_net_len \
client_addr client_port server_addr server_port net_proto net_ipv net_ttl req_udp_sum \
id qtype qclass opcode rcode flags qname rr_counts edns
cbor_fields time delay_us req_dns_len resp_dns_len req_net_len resp_net_len \
client_addr client_port server_addr server_port net_proto net_ipv net_ttl req_udp_sum \
id qtype qclass opcode rcode flags qname rr_counts edns
}
### Logging config
......
......@@ -133,7 +133,7 @@ enum dns_output_field {
dns_field_req_edns_udp = dns_field_edns,
dns_field_req_edns_do = dns_field_edns,
dns_field_resp_edns_rcode = dns_field_edns,
dns_field_req_edns_ping = dns_field_edns,
dns_field_req_edns_ping = dns_field_edns, // Not implemented
dns_field_req_edns_dau = dns_field_edns,
dns_field_req_edns_dhu = dns_field_edns,
dns_field_req_edns_n3u = dns_field_edns,
......
......@@ -36,7 +36,8 @@
static size_t
write_packet(FILE *f, uint32_t fields, dns_packet_t *pkt)
{
uint8_t outbuf[2048];
char addrbuf[MAX(INET_ADDRSTRLEN, INET6_ADDRSTRLEN) + 4];
uint8_t outbuf[4096];
CborEncoder ebase, eitem;
// Run cmd and die (with an error meaasge) on any CBOR error
......@@ -44,16 +45,24 @@ write_packet(FILE *f, uint32_t fields, dns_packet_t *pkt)
if (cerr__ != CborNoError) { die("CBOR error: %s", cbor_error_string(cerr__)); } }
cbor_encoder_init(&ebase, outbuf, sizeof(outbuf), 0);
CERR(cbor_encoder_create_array(&ebase, &eitem, 1));
int noitems = __builtin_popcount(fields);
if (fields & (1 << dns_field_flags)) noitems += 6;
if (fields & (1 << dns_field_rr_counts)) noitems += 2;
if (fields & (1 << dns_field_edns)) noitems += 9;
CERR(cbor_encoder_create_array(&ebase, &eitem, noitems));
// Start writing a field conditional on the field flag being set,
// write just the field name for the header if pkt==NULL
#define COND(fieldname) \
// write NONE if secondary is not satisfied
#define CONDIF(fieldname, secondary) \
if (fields & (1 << (dns_field_ ## fieldname))) { \
if (!pkt) { CERR(cbor_encode_text_stringz(&eitem, #fieldname )); } else { \
if (!(pkt)) { CERR(cbor_encode_text_stringz(&eitem, #fieldname )); } else { \
if (!(secondary)) { CERR(cbor_encode_null(&eitem)); } else {
#define COND(fieldname) CONDIF(fieldname, 1)
// Closing for COND(...) statement
#define COND_END } }
#define COND_END } } }
// Time
......@@ -61,6 +70,214 @@ write_packet(FILE *f, uint32_t fields, dns_packet_t *pkt)
CERR(cbor_encode_double(&eitem, (double)(pkt->ts) / 1000000.0));
COND_END
CONDIF(delay_us, pkt->response)
CERR(cbor_encode_int(&eitem, pkt->response->ts - pkt->ts));
COND_END
// Sizes
CONDIF(req_dns_len, DNS_PACKET_REQUEST(pkt))
CERR(cbor_encode_int(&eitem, DNS_PACKET_REQUEST(pkt)->dns_data_size_orig));
COND_END
CONDIF(resp_dns_len, DNS_PACKET_RESPONSE(pkt))
CERR(cbor_encode_int(&eitem, DNS_PACKET_RESPONSE(pkt)->dns_data_size_orig));
COND_END
CONDIF(req_net_len, DNS_PACKET_REQUEST(pkt))
CERR(cbor_encode_int(&eitem, DNS_PACKET_REQUEST(pkt)->net_size));
COND_END
CONDIF(resp_net_len, DNS_PACKET_RESPONSE(pkt))
CERR(cbor_encode_int(&eitem, DNS_PACKET_RESPONSE(pkt)->net_size));
COND_END
// IP stats
COND(client_addr)
inet_ntop(DNS_PACKET_AF(pkt), DNS_PACKET_CLIENT_ADDR(pkt), addrbuf, sizeof(addrbuf));
CERR(cbor_encode_text_stringz(&eitem, addrbuf));
COND_END
COND(client_port)
CERR(cbor_encode_int(&eitem, DNS_PACKET_CLIENT_PORT(pkt)));
COND_END
COND(server_addr)
inet_ntop(DNS_PACKET_AF(pkt), DNS_PACKET_SERVER_ADDR(pkt), addrbuf, sizeof(addrbuf));
CERR(cbor_encode_text_stringz(&eitem, addrbuf));
COND_END
COND(server_port)
CERR(cbor_encode_int(&eitem, DNS_PACKET_SERVER_PORT(pkt)));
COND_END
COND(net_proto)
CERR(cbor_encode_int(&eitem, pkt->net_protocol));
COND_END
COND(net_ipv)
switch (DNS_SOCKADDR_AF(&pkt->src_addr)) {
case AF_INET:
CERR(cbor_encode_int(&eitem, 4));
break;
case AF_INET6:
CERR(cbor_encode_int(&eitem, 6));
break;
}
COND_END
CONDIF(net_ttl, pkt->net_ttl > 0)
CERR(cbor_encode_int(&eitem, pkt->net_ttl));
COND_END
CONDIF(req_udp_sum, (pkt->net_protocol == IPPROTO_UDP) && DNS_PACKET_REQUEST(pkt))
CERR(cbor_encode_int(&eitem, DNS_PACKET_REQUEST(pkt)->net_udp_sum));
COND_END
// DNS header
COND(id)
CERR(cbor_encode_int(&eitem, knot_wire_get_id(pkt->knot_packet->wire)));
COND_END
COND(qtype)
CERR(cbor_encode_int(&eitem, knot_pkt_qtype(pkt->knot_packet)));
COND_END
COND(qclass)
CERR(cbor_encode_int(&eitem, knot_pkt_qclass(pkt->knot_packet)));
COND_END
COND(opcode)
CERR(cbor_encode_int(&eitem, knot_wire_get_opcode(pkt->knot_packet->wire)));
COND_END
CONDIF(rcode, DNS_PACKET_RESPONSE(pkt))
CERR(cbor_encode_int(&eitem, knot_wire_get_rcode(DNS_PACKET_RESPONSE(pkt)->knot_packet->wire)));
COND_END
#define WRITEFLAG(label, type, name) CONDIF(label, DNS_PACKET_ ## type (pkt)) \
CERR(cbor_encode_boolean( &eitem, \
!! knot_wire_get_ ## name(DNS_PACKET_ ## type (pkt)->knot_packet->wire))); \
COND_END
WRITEFLAG(resp_aa, RESPONSE, aa);
WRITEFLAG(resp_tc, RESPONSE, tc);
WRITEFLAG(req_rd, REQUEST, rd);
WRITEFLAG(resp_ra, RESPONSE, ra);
WRITEFLAG(req_z, REQUEST, z);
WRITEFLAG(resp_ad, RESPONSE, ad);
WRITEFLAG(req_cd, REQUEST, cd);
#undef WRITEFLAG
COND(qname)
char qname_buf[1024];
char *res = knot_dname_to_str(qname_buf, knot_pkt_qname(pkt->knot_packet), sizeof(qname_buf));
if (res) {
CERR(cbor_encode_text_stringz(&eitem, res));
} else {
CERR(cbor_encode_null(&eitem));
}
COND_END
CONDIF(resp_ancount, DNS_PACKET_RESPONSE(pkt))
CERR(cbor_encode_int(&eitem, knot_wire_get_ancount(DNS_PACKET_RESPONSE(pkt)->dns_data)));
COND_END
CONDIF(resp_arcount, DNS_PACKET_RESPONSE(pkt))
CERR(cbor_encode_int(&eitem, knot_wire_get_arcount(DNS_PACKET_RESPONSE(pkt)->dns_data)));
COND_END
CONDIF(resp_nscount, DNS_PACKET_RESPONSE(pkt))
CERR(cbor_encode_int(&eitem, knot_wire_get_nscount(DNS_PACKET_RESPONSE(pkt)->dns_data)));
COND_END
// EDNS
const knot_rrset_t *req_opt_rr = (pkt && DNS_PACKET_REQUEST(pkt)) ? DNS_PACKET_REQUEST(pkt)->knot_packet->opt_rr : NULL;
const knot_rrset_t *resp_opt_rr = (pkt && DNS_PACKET_RESPONSE(pkt)) ? DNS_PACKET_RESPONSE(pkt)->knot_packet->opt_rr : NULL;
CONDIF(req_edns_ver, req_opt_rr)
CERR(cbor_encode_int(&eitem, knot_edns_get_version(req_opt_rr)));
COND_END
CONDIF(req_edns_udp, req_opt_rr)
CERR(cbor_encode_int(&eitem, knot_edns_get_payload(req_opt_rr)));
COND_END
CONDIF(req_edns_do, req_opt_rr)
CERR(cbor_encode_boolean(&eitem, !! knot_edns_do(req_opt_rr)));
COND_END
CONDIF(resp_edns_rcode, resp_opt_rr)
CERR(cbor_encode_int(&eitem, knot_edns_get_ext_rcode(resp_opt_rr)));
COND_END
#define COND_OPT_RR(label, code, rr) \
CONDIF(label, rr) \
uint8_t *opt = knot_edns_get_option(rr, code); \
if (!opt) { CERR(cbor_encode_null(&eitem)); } else { \
int len = knot_edns_opt_get_length(opt);
#define COND_OPT_RR_END \
} COND_END
#define WRITE_UNDERSTOOD_LIST(label, code) \
COND_OPT_RR(label, code, req_opt_rr) \
CborEncoder elist; \
CERR(cbor_encoder_create_array(&eitem, &elist, len)); \
for (int i = 0; i < knot_edns_opt_get_length(opt); i++) { \
CERR(cbor_encode_int(&elist, ((opt + 2 * sizeof(uint16_t) + i)[i]))); \
} \
CERR(cbor_encoder_close_container_checked(&eitem, &elist)); \
COND_OPT_RR_END
//COND(req_edns_ping)
// TODO: distinguish a PING request from DAU,
// see https://github.com/SIDN/entrada/blob/b787af190267df148683151638ce94508bd6139e/dnslib4java/src/main/java/nl/sidn/dnslib/message/records/edns0/OPTResourceRecord.java#L146
//COND_END
WRITE_UNDERSTOOD_LIST(req_edns_dau, 5); // DAU
WRITE_UNDERSTOOD_LIST(req_edns_dhu, 6); // DHU
WRITE_UNDERSTOOD_LIST(req_edns_n3u, 7); // N3U
COND_OPT_RR(resp_edns_nsid, 3, resp_opt_rr) // NSID
CERR(cbor_encode_byte_string(&eitem, opt + 2 * sizeof(uint16_t), len));
COND_OPT_RR_END
CONDIF(edns_client_subnet, 0) // TODO: PARSE and WRITE client subnet information
uint8_t *opt = NULL;
if (resp_opt_rr)
opt = knot_edns_get_option(resp_opt_rr, 8); // client subnet from response
if (req_opt_rr && !opt)
opt = knot_edns_get_option(req_opt_rr, 8); // client subnet from request
if (opt) {
CborEncoder elist;
CERR(cbor_encoder_create_array(&eitem, &elist, CborIndefiniteLength));
// As in: "[4,'118.71.70',24,0]"
// Entrada: (fam == 1? "4,": "6,") + address + "/" + sourcenetmask + "," + scopenetmask;
CERR(cbor_encoder_close_container_checked(&eitem, &elist));
} else {
CERR(cbor_encode_null(&eitem));
}
COND_END
CONDIF(edns_other, 0) // TODO: traverse all remaining records
CborEncoder elist;
CERR(cbor_encoder_create_array(&eitem, &elist, 4));
if (req_opt_rr) {
// ...
}
if (resp_opt_rr) {
// ...
}
CERR(cbor_encoder_close_container_checked(&eitem, &elist));
COND_END
// Close the array
CERR(cbor_encoder_close_container_checked(&ebase, &eitem));
size_t written = cbor_encoder_get_buffer_size(&ebase, outbuf);
fwrite(outbuf, written, 1, f);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment