Commit 117330c4 authored by Tomas Gavenciak's avatar Tomas Gavenciak
Browse files

Improve mult. output types support, add CBOR output base

parent 1a651341
.PHONY: all clean veryclean docs libucw install prog
all: prog
veryclean:: clean
LDLIBS?=
LDFLAGS?=
CFLAGS?=-O2 -g -pedantic
## Options
# USE_TCMALLOC=1
## Docs
......@@ -27,24 +35,39 @@ $(LIBUCW_LIBRARY):
veryclean::
cd $(LIBUCW_DIR)/ && make clean
LDLIBS+= $(LIBUCW_LIBRARY)
CFLAGS+= -I$(LIBUCW_DIR)/run/include/
## tinycbor
TINYCBOR_DIR?=./tinycbor
TINYCBOR_LIBRARY?=$(TINYCBOR_DIR)/lib/libtinycbor-freestanding.a
tinycbor: $(TINYCBOR_LIBRARY)
$(TINYCBOR_LIBRARY):
cd $(TINYCBOR_DIR)/ && make lib/libtinycbor-freestanding.a
veryclean::
cd $(TINYCBOR_DIR)/ && make clean
LDLIBS+= $(TINYCBOR_LIBRARY)
CFLAGS+= -I$(TINYCBOR_DIR)/src/
## dns-collector
LDLIBS?=
LDFLAGS?=
CFLAGS?=-O2 -g -pedantic
WARNS?=-Wall -Wcast-align -Wunused-parameter -Wno-variadic-macros
ifeq ($(CC),clang)
WARNS+=-Wno-gnu-statement-expression -Wno-language-extension-token
endif
CFLAGS+=$(WARNS) -rdynamic -pthread -std=gnu11 -I$(LIBUCW_DIR)/run/include/
LDLIBS+=-lknot -ltrace -lpthread $(LIBUCW_DIR)/run/lib/libucw-6.5.a
CFLAGS+= $(WARNS) -rdynamic -pthread -std=gnu11
LDLIBS+= -lknot -ltrace -lpthread
ifdef USE_TCMALLOC
CFLAGS+=-fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free
LDLIBS+=-ltcmalloc
CFLAGS+= -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free
LDLIBS+= -ltcmalloc
endif
PROG=./dns-collector
......@@ -55,7 +78,6 @@ include src/Makefile
## install
PREFIX?=/usr/local
DESTDIR?=$(PREFIX)
CONFIG=./dns-collector.conf
......
......@@ -4,7 +4,7 @@ here=./src
DEPS=$(wildcard $(here)/*.h)
SRCS=$(here)/common.c $(here)/input.c $(here)/frame_queue.c $(here)/packet_frame.c \
$(here)/worker_frame_logger.c $(here)/main.c $(here)/dump.c $(here)/output.c \
$(here)/worker_frame_logger.c $(here)/main.c $(here)/dump.c $(here)/output.c $(here)/output_cbor.c \
$(here)/output_csv.c $(here)/packet.c $(here)/worker_packet_matcher.c \
$(here)/packet_hash.c $(here)/config.c
......@@ -13,10 +13,10 @@ OBJS=$(sort $(SRCS:.c=.o))
clean::
rm -f $(OBJS) $(PROG)
%.o: %.c $(DEPS) libucw
%.o: %.c $(DEPS) libucw tinycbor
$(CC) $(CFLAGS) -c $< -o $@
$(PROG): $(OBJS) libucw
$(PROG): $(OBJS) libucw tinycbor
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(OBJS) $(LDLIBS)
......@@ -48,7 +48,7 @@ dns_collector_conf_init(void *data)
conf->match_window_sec = 5.0;
// General output
conf->output_type = "csv";
conf->output_type = DNS_OUTPUT_TYPE_CSV;
conf->output_path_fmt = "";
conf->output_pipe_cmd = "";
conf->output_period_sec = 0;
......@@ -58,7 +58,9 @@ dns_collector_conf_init(void *data)
conf->csv_inline_header = 0;
conf->csv_external_header_path_fmt = "";
conf->csv_fields = 0; // No fields by default
// conf->csv_fields = (1 << dns_of_LAST) - 1; // All fields by default
// CBOR output
conf->cbor_fields = 0; // No fields by default
return NULL;
}
......@@ -73,16 +75,22 @@ dns_collector_conf_commit(void *data)
return "'max_frame_duration_sec' too small, minimum 0.001 sec";
if (conf->max_queue_len < 1)
return "'max_queue_len' must be at least 1";
if (strcasecmp(conf->output_type, "csv") == 0) {
if (strlen(conf->csv_separator) != 1)
return "'csv_separator' needs to be exactly one character";
if (isalnum(conf->csv_separator[0]) || strchr(".-#_\\\n/", conf->csv_separator[0])
|| (conf->csv_separator[0] > 0x7f))
return "You shold use non-alphanumeric ASCII character for 'csv_separator', avoiding \".-#_\\/\" and newline.";
if (conf->csv_fields == 0)
return "'csv_fields' must have at least one field";
} else {
return "only 'output_type csv' currently supported";
switch (conf->output_type) {
case DNS_OUTPUT_TYPE_CSV:
if (strlen(conf->csv_separator) != 1)
return "'csv_separator' needs to be exactly one character";
if (isalnum(conf->csv_separator[0]) || strchr(".-#_\\\n/", conf->csv_separator[0])
|| (conf->csv_separator[0] > 0x7f))
return "You shold use non-alphanumeric ASCII character for 'csv_separator', avoiding \".-#_\\/\" and newline.";
if (conf->csv_fields == 0)
return "'csv_fields' must have at least one field";
break;
case DNS_OUTPUT_TYPE_CBOR:
if (conf->cbor_fields == 0)
return "'cbor_fields' must have at least one field";
break;
default:
return "only output types 'csv' and 'cbor' currently supported";
}
if (conf->dump_compress_level < 0 || conf->dump_compress_level > 9)
return "'dump_compress_level' must be 0..9";
......@@ -90,6 +98,8 @@ dns_collector_conf_commit(void *data)
return NULL;
}
static const char *dns_output_types[] = {
"csv", "cbor", NULL };
static const char *dns_dump_compress_types[] = {
"none",
......@@ -111,17 +121,17 @@ struct cf_section dns_config_section = {
CF_INIT(dns_collector_conf_init),
CF_COMMIT(dns_collector_conf_commit),
CF_ITEMS {
// Common
CF_DOUBLE("max_frame_duration", PTR_TO(struct dns_config, max_frame_duration_sec)),
CF_INT("max_frame_size", PTR_TO(struct dns_config, max_frame_size)),
CF_INT("max_queue_len", PTR_TO(struct dns_config, max_queue_len)),
CF_INT("report_period", PTR_TO(struct dns_config, report_period_sec)),
// Input
CF_STRING("input_uri", PTR_TO(struct dns_config, input_uri)),
CF_STRING("input_filter", PTR_TO(struct dns_config, input_filter)),
CF_INT("input_snaplen", PTR_TO(struct dns_config, input_snaplen)),
CF_INT("input_promiscuous", PTR_TO(struct dns_config, input_promiscuous)),
// Common
CF_DOUBLE("max_frame_duration", PTR_TO(struct dns_config, max_frame_duration_sec)),
CF_INT("max_frame_size", PTR_TO(struct dns_config, max_frame_size)),
CF_INT("max_queue_len", PTR_TO(struct dns_config, max_queue_len)),
CF_INT("report_period", PTR_TO(struct dns_config, report_period_sec)),
// Input
CF_STRING("input_uri", PTR_TO(struct dns_config, input_uri)),
CF_STRING("input_filter", PTR_TO(struct dns_config, input_filter)),
CF_INT("input_snaplen", PTR_TO(struct dns_config, input_snaplen)),
CF_INT("input_promiscuous", PTR_TO(struct dns_config, input_promiscuous)),
// Packet dump options
CF_STRING("dump_path_fmt", PTR_TO(struct dns_config, dump_path_fmt)),
......@@ -130,20 +140,23 @@ struct cf_section dns_config_section = {
CF_LOOKUP("dump_compress_type", PTR_TO(struct dns_config, dump_compress_type), dns_dump_compress_types),
CF_DOUBLE("dump_rate_limit", PTR_TO(struct dns_config, dump_rate_limit)),
// Matching
CF_DOUBLE("match_window", PTR_TO(struct dns_config, match_window_sec)),
// Matching
CF_DOUBLE("match_window", PTR_TO(struct dns_config, match_window_sec)),
// General output options
CF_LOOKUP("output_type", PTR_TO(struct dns_config, output_type), dns_output_types),
CF_STRING("output_path_fmt", PTR_TO(struct dns_config, output_path_fmt)),
CF_STRING("output_pipe_cmd", PTR_TO(struct dns_config, output_pipe_cmd)),
CF_INT("output_period", PTR_TO(struct dns_config, output_period_sec)),
// General output options
CF_STRING("output_type", PTR_TO(struct dns_config, output_type)),
CF_STRING("output_path_fmt", PTR_TO(struct dns_config, output_path_fmt)),
CF_STRING("output_pipe_cmd", PTR_TO(struct dns_config, output_pipe_cmd)),
CF_INT("output_period", PTR_TO(struct dns_config, output_period_sec)),
// CSV output
CF_STRING("csv_separator", PTR_TO(struct dns_config, csv_separator)),
CF_INT("csv_inline_header", PTR_TO(struct dns_config, csv_inline_header)),
CF_STRING("csv_external_header_path_fmt", PTR_TO(struct dns_config, csv_external_header_path_fmt)),
CF_BITMAP_LOOKUP("csv_fields", PTR_TO(struct dns_config, csv_fields), dns_output_field_flag_names),
// CSV output
CF_STRING("csv_separator", PTR_TO(struct dns_config, csv_separator)),
CF_INT("csv_inline_header", PTR_TO(struct dns_config, csv_inline_header)),
CF_STRING("csv_external_header_path_fmt", PTR_TO(struct dns_config, csv_external_header_path_fmt)),
CF_BITMAP_LOOKUP("csv_fields", PTR_TO(struct dns_config, csv_fields), dns_output_field_flag_names),
// CBOR output
CF_BITMAP_LOOKUP("cbor_fields", PTR_TO(struct dns_config, cbor_fields), dns_output_field_flag_names),
CF_END
}
};
......
......@@ -56,7 +56,7 @@ struct dns_config {
double match_window_sec;
// General output options
char *output_type;
int output_type;
char *output_path_fmt;
char *output_pipe_cmd;
int output_period_sec;
......@@ -66,6 +66,9 @@ struct dns_config {
int csv_inline_header;
char *csv_external_header_path_fmt;
uint32_t csv_fields;
// CBOR output
uint32_t cbor_fields;
};
extern struct cf_section dns_config_section;
......@@ -73,4 +76,7 @@ extern struct cf_section dns_config_section;
/** TRACE_OPTION_COMPRESSTYPE_ corresponding to the values of dump_compress_type */
extern trace_option_compresstype_t dns_dump_compress_types_num[];
#define DNS_OUTPUT_TYPE_CSV 0
#define DNS_OUTPUT_TYPE_CBOR 1
#endif /* DNSCOL_COLLECTOR_CONFIG_H */
......@@ -26,6 +26,7 @@
#include "frame_queue.h"
#include "input.h"
#include "output_csv.h"
#include "output_cbor.h"
#include "packet_frame.h"
#include "worker_frame_logger.h"
#include "worker_packet_matcher.h"
......@@ -134,13 +135,19 @@ int main(int argc UNUSED, char **argv)
struct dns_worker_packet_matcher *w_matcher =
dns_worker_packet_matcher_create(conf, q_input_mathcher, q_matcher_output);
// No other output supported. This is also checked by the config system.
assert(strcasecmp(conf->output_type, "csv") == 0);
struct dns_output_csv *output =
dns_output_csv_create(conf, q_matcher_output);
struct dns_output *output;
switch (conf->output_type) {
case DNS_OUTPUT_TYPE_CSV:
output = (struct dns_output *)dns_output_csv_create(conf, q_matcher_output);
break;
case DNS_OUTPUT_TYPE_CBOR:
output = (struct dns_output *)dns_output_cbor_create(conf, q_matcher_output);
break;
default: die("Invalid output type");
}
dns_worker_packet_matcher_start(w_matcher);
dns_output_csv_start(output);
output->start_output(output);
// Main loop, start inputs
......@@ -167,14 +174,14 @@ int main(int argc UNUSED, char **argv)
dns_input_finish(input);
dns_worker_packet_matcher_finish(w_matcher);
dns_output_csv_finish(output);
output->finish_output(output);
// Dealloc and cleanup
dns_input_destroy(input);
dns_worker_packet_matcher_destroy(w_matcher);
dns_output_csv_destroy(output);
output->finalize_output(output);
free(output);
dns_frame_queue_destroy(q_input_mathcher);
dns_frame_queue_destroy(q_matcher_output);
......
......@@ -67,6 +67,26 @@ struct dns_output {
*/
void (*finish_file)(struct dns_output *out, dns_us_time_t time);
/**
* Hook to start the output thread.
* Normally you want this to call 'dns_output_start'.
*/
void (*start_output)(struct dns_output *out);
/**
* Hook to wait for and join the output thread.
* Normally you want this to call 'dns_output_finish'.
*/
void (*finish_output)(struct dns_output *out);
/**
* Hook to deinitialise the given output, freeing any owned objects.
* Must be called only after thread stopped and dns_output_finish() was called.
* Does NOT dealloc the output struct itself.
* Normally you want this to call 'dns_output_finalize'.
*/
void (*finalize_output)(struct dns_output *out);
/** Output rotation period in seconds. Zero or less means do not rotate.
* The output is rotated whenever sec_since_midnight is divisible by period. */
int period_sec;
......
/*
* Copyright (C) 2016 CZ.NIC, z.s.p.o.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <arpa/inet.h>
#include <assert.h>
#include <inttypes.h>
#include <stdint.h>
#include <time.h>
#include <unistd.h>
#include <cbor.h>
#include "common.h"
#include "output.h"
#include "packet.h"
#include "output_cbor.h"
/**
* Writes a packet to the given file, or writes header if pkt == NULL.
*/
static size_t
write_packet(FILE *f, uint32_t fields, dns_packet_t *pkt)
{
uint8_t outbuf[2048];
CborEncoder ebase, eitem;
// Run cmd and die (with an error meaasge) on any CBOR error
#define CERR(cmd) { CborError cerr__ = (cmd); \
if (cerr__ != CborNoError) { die("CBOR error: %s", cbor_error_string(cerr__)); } }
cbor_encoder_init(&ebase, outbuf, sizeof(outbuf), 0);
CERR(cbor_encoder_create_array(&ebase, &eitem, 1));
// Start writing a field conditional on the field flag being set,
// write just the field name for the header if pkt==NULL
#define COND(fieldname) \
if (fields & (1 << (dns_field_ ## fieldname))) { \
if (!pkt) { CERR(cbor_encode_text_stringz(&eitem, #fieldname )); } else { \
// Closing for COND(...) statement
#define COND_END } }
// Time
COND(time)
CERR(cbor_encode_double(&eitem, (double)(pkt->ts) / 1000000.0));
COND_END
CERR(cbor_encoder_close_container_checked(&ebase, &eitem));
size_t written = cbor_encoder_get_buffer_size(&ebase, outbuf);
fwrite(outbuf, written, 1, f);
#undef COND
#undef COND_END
#undef CERR
return written;
}
/**
* Callback for cbor_output, writes CBOR header.
*/
static void
dns_output_cbor_start_file(struct dns_output *out0, dns_us_time_t UNUSED(time))
{
struct dns_output_cbor *out = (struct dns_output_cbor *) out0;
assert(out && out->base.out_file);
out->base.current_bytes += write_packet(out->base.out_file, out->cbor_fields, NULL);
}
/**
* Callback for cbor_output, writes singe CBOR record.
*/
static dns_ret_t
dns_output_cbor_write_packet(struct dns_output *out0, dns_packet_t *pkt)
{
assert(out0 && pkt);
struct dns_output_cbor *out = (struct dns_output_cbor *) out0;
size_t n = write_packet(out->base.out_file, out->cbor_fields, pkt);
out->base.current_bytes += n;
// Accounting
out->base.current_items ++;
if (!DNS_PACKET_RESPONSE(pkt))
out->base.current_request_only ++;
if (!DNS_PACKET_REQUEST(pkt))
out->base.current_response_only ++;
return DNS_RET_OK;
}
struct dns_output_cbor *
dns_output_cbor_create(struct dns_config *conf, struct dns_frame_queue *in)
{
struct dns_output_cbor *out = xmalloc_zero(sizeof(struct dns_output_cbor));
dns_output_init(&out->base, in, conf->output_path_fmt, conf->output_pipe_cmd, conf->output_period_sec);
out->base.start_file = dns_output_cbor_start_file;
out->base.write_packet = dns_output_cbor_write_packet;
out->base.start_output = dns_output_start;
out->base.finish_output = dns_output_finish;
out->base.finalize_output = dns_output_finalize;
out->cbor_fields = conf->cbor_fields;
msg(L_INFO, "Selected CBOR fields: %#x", out->cbor_fields);
return out;
}
/*
* Copyright (C) 2016 CZ.NIC, z.s.p.o.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef DNSCOL_OUTPUT_CBOR_H
#define DNSCOL_OUTPUT_CBOR_H
/**
* \file output_cbor.h
* Output to CBOR files - configuration and writing.
*/
#include "output.h"
#include "frame_queue.h"
#include "config.h"
/**
* Configuration structure extending `struct dns_output`.
*/
struct dns_output_cbor {
struct dns_output base;
/** Fields to write, bitmask of dns_output_field_flag_names */
uint32_t cbor_fields;
};
struct dns_output_cbor *
dns_output_cbor_create(struct dns_config *conf, struct dns_frame_queue *in);
#endif /* DNSCOL_OUTPUT_CBOR_H */
......@@ -28,56 +28,16 @@
#include "output_csv.h"
// Forward declarations
static void
dns_output_csv_start_file(struct dns_output *out0, dns_us_time_t time);
static dns_ret_t
dns_output_csv_write_packet(struct dns_output *out0, dns_packet_t *pkt);
struct dns_output_csv *
dns_output_csv_create(struct dns_config *conf, struct dns_frame_queue *in)
{
struct dns_output_csv *out = xmalloc_zero(sizeof(struct dns_output_csv));
dns_output_init(&out->base, in, conf->output_path_fmt, conf->output_pipe_cmd, conf->output_period_sec);
out->base.start_file = dns_output_csv_start_file;
out->base.write_packet = dns_output_csv_write_packet;
out->separator = conf->csv_separator[0];
out->inline_header = conf->csv_inline_header;
out->csv_fields = conf->csv_fields;
msg(L_INFO, "Selected fields: %#x", out->csv_fields);
if (conf->csv_external_header_path_fmt)
out->external_header_path_fmt = strdup(conf->csv_external_header_path_fmt);
return out;
}
void
dns_output_csv_start(struct dns_output_csv *out)
{
dns_output_start(&out->base);
}
void
dns_output_csv_finish(struct dns_output_csv *out)
{
dns_output_finish(&out->base);
}
void
dns_output_csv_destroy(struct dns_output_csv *out)
dns_output_csv_finalize(struct dns_output *out0)
{
struct dns_output_csv *out = (struct dns_output_csv *) out0;
dns_output_finalize(&out->base);
if (out->external_header_path_fmt)
free(out->external_header_path_fmt);
free(out);
}
/**
* Writes a packet to the given file, or writes field names if pkt == NULL.
*/
......@@ -412,3 +372,25 @@ dns_output_csv_write_packet(struct dns_output *out0, dns_packet_t *pkt)
return DNS_RET_OK;
}
struct dns_output_csv *
dns_output_csv_create(struct dns_config *conf, struct dns_frame_queue *in)
{
struct dns_output_csv *out = xmalloc_zero(sizeof(struct dns_output_csv));
dns_output_init(&out->base, in, conf->output_path_fmt, conf->output_pipe_cmd, conf->output_period_sec);
out->base.start_file = dns_output_csv_start_file;
out->base.write_packet = dns_output_csv_write_packet;
out->base.start_output = dns_output_start;
out->base.finish_output = dns_output_finish;
out->base.finalize_output = dns_output_csv_finalize;
out->separator = conf->csv_separator[0];
out->inline_header = conf->csv_inline_header;
out->csv_fields = conf->csv_fields;
msg(L_INFO, "Selected CSV fields: %#x", out->csv_fields);
if (conf->csv_external_header_path_fmt)
out->external_header_path_fmt = strdup(conf->csv_external_header_path_fmt);
return out;
}
......@@ -33,33 +33,24 @@
struct dns_output_csv {
struct dns_output base;
/* Separator character */
/** Separator character */
int separator;
/* Write header as the first line of the CSV */
/** Write header as the first line of the CSV */
int inline_header;
/* Write header into a separate file if not NULL,
/** Write header into a separate file if not NULL,
* owned by output */
char *external_header_path_fmt;
/* Bitmask with CSV fields to write */
/** CSV fields to write,
* bitmask of dns_output_field_flag_names */
uint32_t csv_fields;
};
struct dns_output_csv *
dns_output_csv_create(struct dns_config *conf, struct dns_frame_queue *in);
void
dns_output_csv_start(struct dns_output_csv *out);
void
dns_output_csv_finish(struct dns_output_csv *out);
void
dns_output_csv_destroy(struct dns_output_csv *out);
#endif /* DNSCOL_OUTPUT_CSV_H */
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment