diff --git a/src/master/archivist/archivist.pl b/src/master/archivist/archivist.pl index 8f3621b8abe1001b4cd2839c21a296a5d23349a7..921f7985e7c40055fe7e9c9260ea78644e77e41b 100755 --- a/src/master/archivist/archivist.pl +++ b/src/master/archivist/archivist.pl @@ -409,27 +409,6 @@ if (fork == 0) { exit; } -if (fork == 0) { - my $source = connect_db 'source'; - my $destination = connect_db 'destination'; - my ($max_batch) = $destination->selectrow_array('SELECT COALESCE(MAX(batch), TO_TIMESTAMP(0)) FROM nat_counts'); - tprint "Dropping nats from batch $max_batch\n"; - $destination->do('DELETE FROM nat_counts WHERE batch = ?', undef, $max_batch); - tprint "Getting nat records not older than $max_batch\n"; - my $store_nat = $destination->prepare('INSERT INTO nat_counts (from_group, batch, v4direct, v4nat, v6direct, v6nat, total) VALUES(?, ?, ?, ?, ?, ?, ?)'); - my $get_nats = $source->prepare('SELECT in_group, batch, COUNT(CASE WHEN nat_v4 = false THEN true END), COUNT(CASE WHEN nat_v4 = true THEN true END), COUNT(CASE WHEN nat_v6 = false THEN true END), COUNT(CASE WHEN nat_v6 = true THEN true END), COUNT(nats.client) FROM nats JOIN group_members ON nats.client = group_members.client WHERE batch >= ? GROUP BY batch, in_group'); - my $nat_count = -1; - $get_nats->execute($max_batch); - $store_nat->execute_for_fetch(sub { - $nat_count ++; - return $get_nats->fetchrow_arrayref; - }); - tprint "Stored $nat_count nat counts\n"; - $destination->commit; - $source->commit; - exit; -} - if (fork == 0) { my $source = connect_db 'source'; my $destination = connect_db 'destination'; @@ -578,4 +557,4 @@ if (fork == 0) { exit; } -wait for (1..12); +wait for (1..11); diff --git a/src/master/collect-master.conf b/src/master/collect-master.conf index f506a379c5e55dab66417b93f853aa05b25b3000..ff48f47a8f117c692888a4336a1af22e230c4559 100644 --- a/src/master/collect-master.conf +++ b/src/master/collect-master.conf @@ -42,7 +42,6 @@ aggregate_delay: 5 ; How long to wait for answers after sending the query to sto [sniff.main.SniffPlugin] taskers = sniff.cert.Cert sniff.ping.Pinger - sniff.nat.Nat parallel_limit = 20 task_timeout = 1 interval = 1 diff --git a/src/master/dbscripts/initdb b/src/master/dbscripts/initdb index 8eecbbbc144a1ce15828f4e6dd09abe386759559..58c7b99328ffd868e3735c02bbb8b9480cbc2784 100755 --- a/src/master/dbscripts/initdb +++ b/src/master/dbscripts/initdb @@ -30,7 +30,6 @@ DROP TABLE IF EXISTS ssh_commands; DROP TABLE IF EXISTS ssh_sessions; DROP TABLE IF EXISTS refused; DROP TABLE IF EXISTS spoof; -DROP TABLE IF EXISTS nats; DROP TABLE IF EXISTS biflows; DROP TABLE IF EXISTS flow_filters; DROP TABLE IF EXISTS capture_stats; @@ -299,7 +298,7 @@ CREATE SEQUENCE biflow_ids OWNED BY biflows.id; ALTER TABLE biflows ALTER COLUMN id SET DEFAULT NEXTVAL('biflow_ids'); CREATE INDEX ON biflows (tagged_on); -INSERT INTO activity_types (name, plugin) VALUES ('login', NULL), ('logout', NULL), ('buckets', 'Buckets'), ('counts', 'Count'), ('pings', 'Sniff'), ('bandwidth', 'Bandwidth'), ('certs', 'Sniff'), ('flow', 'Flow'), ('nat', 'Sniff'), ('spoof', 'Spoof'), ('refused', 'Refused'), ('fake', 'Fake'); +INSERT INTO activity_types (name, plugin) VALUES ('login', NULL), ('logout', NULL), ('buckets', 'Buckets'), ('counts', 'Count'), ('pings', 'Sniff'), ('bandwidth', 'Bandwidth'), ('certs', 'Sniff'), ('flow', 'Flow'), ('spoof', 'Spoof'), ('refused', 'Refused'), ('fake', 'Fake'); CREATE TABLE starttls_protos ( id SMALLINT NOT NULL PRIMARY KEY, name TEXT NOT NULL, @@ -365,22 +364,13 @@ INSERT INTO count_types (name, description, ord) VALUES ('V6TUNNEL', 'Packets sent over IPv6-tunelled connectivity', 16); INSERT INTO starttls_protos (id, name) VALUES (0, ''), (1, 'smtp'), (2, 'pop3'), (3, 'imap'), (4, 'ftp'), (5, 'xmpp'); -CREATE TABLE nats ( - client INT NOT NULL, - batch TIMESTAMP NOT NULL, - nat_v4 BOOLEAN, - nat_v6 BOOLEAN, - FOREIGN KEY (client) REFERENCES clients(id), - UNIQUE(client, batch) -); - CREATE TABLE config ( plugin TEXT NOT NULL, name TEXT NOT NULL, value TEXT NOT NULL, UNIQUE(plugin, name) ); -INSERT INTO config (plugin, name, value) VALUES ('flow', 'max_flows', '5000'), ('flow', 'timeout', '1800000'), ('flow', 'minpackets', '5'), ('flow', 'version', '1'), ('flow', 'filter', '!(|(i(127.0.0.1,::1),I(127.0.0.1,::1)))'), ('flow', 'filter-diff', 'D(addresses)'), ('sniff', 'nat-interval', '3 days'), ('spoof', 'answer_timeout', '60'), ('spoof', 'dest_addr', 'api.turris.cz'), ('spoof', 'src_addr', '192.0.2.1'), ('spoof', 'interval', '1 day'), ('spoof', 'port', '5678'), ('fwup', 'version', '1'); +INSERT INTO config (plugin, name, value) VALUES ('flow', 'max_flows', '5000'), ('flow', 'timeout', '1800000'), ('flow', 'minpackets', '5'), ('flow', 'version', '1'), ('flow', 'filter', '!(|(i(127.0.0.1,::1),I(127.0.0.1,::1)))'), ('flow', 'filter-diff', 'D(addresses)'), ('spoof', 'answer_timeout', '60'), ('spoof', 'dest_addr', 'api.turris.cz'), ('spoof', 'src_addr', '192.0.2.1'), ('spoof', 'interval', '1 day'), ('spoof', 'port', '5678'), ('fwup', 'version', '1'); CREATE TABLE flow_filters ( filter TEXT NOT NULL, @@ -696,8 +686,6 @@ GRANT INSERT ON cert_chains TO $DBUPDATER; GRANT SELECT ON certs TO $DBUPDATER; GRANT INSERT ON biflows TO $DBUPDATER; GRANT ALL ON biflow_ids TO $DBUPDATER; -GRANT INSERT ON nats TO $DBUPDATER; -GRANT SELECT (batch) ON nats TO $DBUPDATER; GRANT SELECT ON config TO $DBUPDATER; GRANT SELECT ON flow_filters TO $DBUPDATER; GRANT SELECT ON fake_server_names TO $DBUPDATER; @@ -733,7 +721,6 @@ GRANT SELECT (batch) ON pings TO $DBCLEANER; GRANT SELECT (batch, id) ON certs TO $DBCLEANER; GRANT SELECT (cert) ON cert_chains TO $DBCLEANER; GRANT SELECT (start_in, start_out) ON biflows TO $DBCLEANER; -GRANT SELECT (batch) ON nats TO $DBCLEANER; GRANT SELECT (timestamp) ON fake_logs TO $DBCLEANER; GRANT SELECT (batch) ON spoof TO $DBCLEANER; GRANT SELECT (timestamp) ON refused TO $DBCLEANER; @@ -747,7 +734,6 @@ GRANT DELETE ON pings TO $DBCLEANER; GRANT DELETE ON certs TO $DBCLEANER; GRANT DELETE ON cert_chains TO $DBCLEANER; GRANT DELETE ON biflows TO $DBCLEANER; -GRANT DELETE ON nats TO $DBCLEANER; GRANT DELETE ON fake_logs TO $DBCLEANER; GRANT DELETE ON spoof TO $DBCLEANER; GRANT DELETE ON refused TO $DBCLEANER; @@ -775,7 +761,6 @@ GRANT SELECT ON cert_chains TO $DBARCHIVIST; GRANT SELECT ON biflows TO $DBARCHIVIST; GRANT SELECT ON bandwidth TO $DBARCHIVIST; GRANT SELECT ON bandwidth_stats TO $DBARCHIVIST; -GRANT SELECT ON nats TO $DBARCHIVIST; GRANT SELECT ON fake_logs TO $DBARCHIVIST; GRANT SELECT ON spoof TO $DBARCHIVIST; GRANT SELECT ON refused TO $DBARCHIVIST; diff --git a/src/master/dbscripts/purge b/src/master/dbscripts/purge index 3e947faaf7b3a8bf814f7fca4ab5c8d85e0cf547..7ec8606874d6389a0e6074ea76124866df6ba341 100755 --- a/src/master/dbscripts/purge +++ b/src/master/dbscripts/purge @@ -5,7 +5,7 @@ set -ex . ./dbconfig TABLES='activities count_snapshots bandwidth bandwidth_stats refused fake_logs plugin_history' -BATCH_TABLES='pings certs nats spoof' +BATCH_TABLES='pings certs spoof' DATE=$(date -d "$CLEAN_DAYS days ago" "+'%Y-%m-%d'") ( diff --git a/src/master/sniff/nat.py b/src/master/sniff/nat.py deleted file mode 100644 index 3b311552d837ddb4dc11ecf941d15f43d55b497b..0000000000000000000000000000000000000000 --- a/src/master/sniff/nat.py +++ /dev/null @@ -1,72 +0,0 @@ -# -# Ucollect - small utility for real-time analysis of network data -# Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# - -from task import Task -import logging -import database -from activity import log_activity -from twisted.internet import reactor - -logger = logging.getLogger(name='sniff') - -def decode(code): - if code == 'N': - return True - elif code == 'D': - return False - else: - return None - -def submit_data(client, payload, batch_time): - with database.transaction() as t: - t.execute("INSERT INTO nats (batch, client, nat_v4, nat_v6) SELECT %s, clients.id, %s, %s FROM clients WHERE name = %s", (batch_time, decode(payload[0]), decode(payload[1]), client)) - -class NatTask(Task): - def __init__(self): - Task.__init__(self) - with database.transaction() as t: - t.execute("SELECT CURRENT_TIMESTAMP AT TIME ZONE 'UTC'") - (self.__batch_time,) = t.fetchone() - - def name(self): - return 'Nat' - - def message(self, client): - return '' - - def success(self, client, payload): - reactor.callInThread(submit_data, client, payload, self.__batch_time) - log_activity(client, 'nat') - -class Nat: - def __init__(self, config): - pass - - def code(self): - return 'n' - - def check_schedule(self): - with database.transaction() as t: - t.execute("SELECT m.m + i.i <= CURRENT_TIMESTAMP AT TIME ZONE 'UTC' FROM (SELECT COALESCE(MAX(batch), TO_TIMESTAMP(0)) AS m FROM nats) AS m CROSS JOIN (SELECT value::INTERVAL AS i FROM config WHERE plugin = 'sniff' AND name = 'nat-interval') AS i;") - (time_s_up,) = t.fetchone() - if time_s_up: - return [NatTask()] - else: - logger.debug('Not sniffing NAT yet') - return [] diff --git a/src/plugins/sniff/Makefile.dir b/src/plugins/sniff/Makefile.dir index d954c7eef8bab6dc9ea6b416be0afc2849f521be..b679ae3fca48a38b2b737a6d49c83010f9814060 100644 --- a/src/plugins/sniff/Makefile.dir +++ b/src/plugins/sniff/Makefile.dir @@ -1,5 +1,5 @@ LIBRARIES += src/plugins/sniff/libplugin_sniff -SNIFF_TASKS := ping cert nat +SNIFF_TASKS := ping cert libplugin_sniff_MODULES := sniff task fork nop parse $(SNIFF_TASKS) DOCS += src/plugins/sniff/sniff diff --git a/src/plugins/sniff/nat.c b/src/plugins/sniff/nat.c deleted file mode 100644 index b88a1920c4369678d4ae4e832c16eb266907782c..0000000000000000000000000000000000000000 --- a/src/plugins/sniff/nat.c +++ /dev/null @@ -1,90 +0,0 @@ -/* - Ucollect - small utility for real-time analysis of network data - Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -*/ - -#include "nat.h" -#include "fork.h" - -#include "../../core/mem_pool.h" -#include "../../core/context.h" -#include "../../core/util.h" - -#include - -static const char *nat_program = -#include -; - -struct task_data { - bool ok; -}; - -struct task_data *start_nat(struct context *context, struct mem_pool *pool, const uint8_t *message, size_t message_size, int *output, pid_t *pid) { - // We ignore the input parameters, there aren't any for this script. - (void) message; - (void) message_size; - struct task_data *data = mem_pool_alloc(pool, sizeof *data); - char **argv = mem_pool_alloc(context->temp_pool, 6 * sizeof *argv); - argv[0] = "/bin/busybox"; - argv[1] = "ash"; - argv[2] = "-c"; - argv[3] = mem_pool_strdup(context->temp_pool, nat_program); - argv[4] = "sniff-nat"; - argv[5] = NULL; - data->ok = fork_task(context->loop, "/bin/busybox", argv, "nat", output, pid); - return data; -} - -static void parse_family(char *string, uint8_t *output, char family, char *error) { - char *word = strtok(string, "\n"); - *output = '?'; - if (!word) { - ulog(LLOG_ERROR, "Missing nat output for IPv%c\n", family); - *error = 'M'; - return; - } - if (strcmp(word, "NONE") == 0) - *output = '0'; - else if (strcmp(word, "NAT") == 0) - *output = 'N'; - else if (strcmp(word, "DIRECT") == 0) - *output = 'D'; - else { - ulog(LLOG_ERROR, "Unknown nat output for IPv%c: '%s'\n", family, word); - *error = 'U'; - } -} - -const uint8_t *finish_nat(struct context *context, struct task_data *data, uint8_t *output, size_t output_size, size_t *result_size, bool *ok) { - (void) output_size; -#define FAIL(CODE, MESSAGE) do { *result_size = 1; *ok = false; ulog(LLOG_INFO, "Sending error nat response %s: %s\n", CODE, MESSAGE); return (const uint8_t *)(CODE); } while (0) - if (!data->ok) - FAIL("S", "Failed to start"); - uint8_t *result = mem_pool_alloc(context->temp_pool, 2); - *result_size = 2; - char error = '\0'; - parse_family((char *)output, result, '4', &error); - parse_family(NULL, result + 1, '6', &error); - if (error) { - result[0] = error; - result[1] = '\0'; - FAIL(result, "Invalid output"); - } - *ok = true; - return result; -} diff --git a/src/plugins/sniff/nat.h b/src/plugins/sniff/nat.h deleted file mode 100644 index d474e9d38c03ef592bf5159f43fcc6997712869d..0000000000000000000000000000000000000000 --- a/src/plugins/sniff/nat.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - Ucollect - small utility for real-time analysis of network data - Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -*/ - -#ifndef UCOLLECT_SNIFF_NAT_H -#define UCOLLECT_SNIFF_NAT_H - -#include -#include -#include -#include - -struct task_data; -struct context; -struct mem_pool; - -struct task_data *start_nat(struct context *context, struct mem_pool *pool, const uint8_t *message, size_t message_size, int *output, pid_t *pid); -const uint8_t *finish_nat(struct context *context, struct task_data *data, uint8_t *output, size_t output_size, size_t *result_size, bool *ok); - -#endif diff --git a/src/plugins/sniff/task.c b/src/plugins/sniff/task.c index b22760073a75ff2e9ebd60fa28ba43ee8a745727..9d88bce4fcbf34fdfec579950f9f11684c6e29fe 100644 --- a/src/plugins/sniff/task.c +++ b/src/plugins/sniff/task.c @@ -21,7 +21,6 @@ #include "nop.h" #include "ping.h" #include "cert.h" -#include "nat.h" struct task_desc task_descs[] = { { @@ -42,12 +41,6 @@ struct task_desc task_descs[] = { .start = start_cert, .finish = finish_cert }, - { - .name = 'n', - .label = "Nat", - .start = start_nat, - .finish = finish_nat - }, { .name = '\0' }