Verified Commit fb9a4adc authored by Michal 'vorner' Vaner's avatar Michal 'vorner' Vaner
Browse files

Drop the NAT detector support

parent 5d263480
......@@ -409,27 +409,6 @@ if (fork == 0) {
exit;
}
if (fork == 0) {
my $source = connect_db 'source';
my $destination = connect_db 'destination';
my ($max_batch) = $destination->selectrow_array('SELECT COALESCE(MAX(batch), TO_TIMESTAMP(0)) FROM nat_counts');
tprint "Dropping nats from batch $max_batch\n";
$destination->do('DELETE FROM nat_counts WHERE batch = ?', undef, $max_batch);
tprint "Getting nat records not older than $max_batch\n";
my $store_nat = $destination->prepare('INSERT INTO nat_counts (from_group, batch, v4direct, v4nat, v6direct, v6nat, total) VALUES(?, ?, ?, ?, ?, ?, ?)');
my $get_nats = $source->prepare('SELECT in_group, batch, COUNT(CASE WHEN nat_v4 = false THEN true END), COUNT(CASE WHEN nat_v4 = true THEN true END), COUNT(CASE WHEN nat_v6 = false THEN true END), COUNT(CASE WHEN nat_v6 = true THEN true END), COUNT(nats.client) FROM nats JOIN group_members ON nats.client = group_members.client WHERE batch >= ? GROUP BY batch, in_group');
my $nat_count = -1;
$get_nats->execute($max_batch);
$store_nat->execute_for_fetch(sub {
$nat_count ++;
return $get_nats->fetchrow_arrayref;
});
tprint "Stored $nat_count nat counts\n";
$destination->commit;
$source->commit;
exit;
}
if (fork == 0) {
my $source = connect_db 'source';
my $destination = connect_db 'destination';
......@@ -578,4 +557,4 @@ if (fork == 0) {
exit;
}
wait for (1..12);
wait for (1..11);
......@@ -42,7 +42,6 @@ aggregate_delay: 5 ; How long to wait for answers after sending the query to sto
[sniff.main.SniffPlugin]
taskers = sniff.cert.Cert
sniff.ping.Pinger
sniff.nat.Nat
parallel_limit = 20
task_timeout = 1
interval = 1
......
......@@ -30,7 +30,6 @@ DROP TABLE IF EXISTS ssh_commands;
DROP TABLE IF EXISTS ssh_sessions;
DROP TABLE IF EXISTS refused;
DROP TABLE IF EXISTS spoof;
DROP TABLE IF EXISTS nats;
DROP TABLE IF EXISTS biflows;
DROP TABLE IF EXISTS flow_filters;
DROP TABLE IF EXISTS capture_stats;
......@@ -299,7 +298,7 @@ CREATE SEQUENCE biflow_ids OWNED BY biflows.id;
ALTER TABLE biflows ALTER COLUMN id SET DEFAULT NEXTVAL('biflow_ids');
CREATE INDEX ON biflows (tagged_on);
INSERT INTO activity_types (name, plugin) VALUES ('login', NULL), ('logout', NULL), ('buckets', 'Buckets'), ('counts', 'Count'), ('pings', 'Sniff'), ('bandwidth', 'Bandwidth'), ('certs', 'Sniff'), ('flow', 'Flow'), ('nat', 'Sniff'), ('spoof', 'Spoof'), ('refused', 'Refused'), ('fake', 'Fake');
INSERT INTO activity_types (name, plugin) VALUES ('login', NULL), ('logout', NULL), ('buckets', 'Buckets'), ('counts', 'Count'), ('pings', 'Sniff'), ('bandwidth', 'Bandwidth'), ('certs', 'Sniff'), ('flow', 'Flow'), ('spoof', 'Spoof'), ('refused', 'Refused'), ('fake', 'Fake');
CREATE TABLE starttls_protos (
id SMALLINT NOT NULL PRIMARY KEY,
name TEXT NOT NULL,
......@@ -365,22 +364,13 @@ INSERT INTO count_types (name, description, ord) VALUES
('V6TUNNEL', 'Packets sent over IPv6-tunelled connectivity', 16);
INSERT INTO starttls_protos (id, name) VALUES (0, ''), (1, 'smtp'), (2, 'pop3'), (3, 'imap'), (4, 'ftp'), (5, 'xmpp');
CREATE TABLE nats (
client INT NOT NULL,
batch TIMESTAMP NOT NULL,
nat_v4 BOOLEAN,
nat_v6 BOOLEAN,
FOREIGN KEY (client) REFERENCES clients(id),
UNIQUE(client, batch)
);
CREATE TABLE config (
plugin TEXT NOT NULL,
name TEXT NOT NULL,
value TEXT NOT NULL,
UNIQUE(plugin, name)
);
INSERT INTO config (plugin, name, value) VALUES ('flow', 'max_flows', '5000'), ('flow', 'timeout', '1800000'), ('flow', 'minpackets', '5'), ('flow', 'version', '1'), ('flow', 'filter', '!(|(i(127.0.0.1,::1),I(127.0.0.1,::1)))'), ('flow', 'filter-diff', 'D(addresses)'), ('sniff', 'nat-interval', '3 days'), ('spoof', 'answer_timeout', '60'), ('spoof', 'dest_addr', 'api.turris.cz'), ('spoof', 'src_addr', '192.0.2.1'), ('spoof', 'interval', '1 day'), ('spoof', 'port', '5678'), ('fwup', 'version', '1');
INSERT INTO config (plugin, name, value) VALUES ('flow', 'max_flows', '5000'), ('flow', 'timeout', '1800000'), ('flow', 'minpackets', '5'), ('flow', 'version', '1'), ('flow', 'filter', '!(|(i(127.0.0.1,::1),I(127.0.0.1,::1)))'), ('flow', 'filter-diff', 'D(addresses)'), ('spoof', 'answer_timeout', '60'), ('spoof', 'dest_addr', 'api.turris.cz'), ('spoof', 'src_addr', '192.0.2.1'), ('spoof', 'interval', '1 day'), ('spoof', 'port', '5678'), ('fwup', 'version', '1');
CREATE TABLE flow_filters (
filter TEXT NOT NULL,
......@@ -696,8 +686,6 @@ GRANT INSERT ON cert_chains TO $DBUPDATER;
GRANT SELECT ON certs TO $DBUPDATER;
GRANT INSERT ON biflows TO $DBUPDATER;
GRANT ALL ON biflow_ids TO $DBUPDATER;
GRANT INSERT ON nats TO $DBUPDATER;
GRANT SELECT (batch) ON nats TO $DBUPDATER;
GRANT SELECT ON config TO $DBUPDATER;
GRANT SELECT ON flow_filters TO $DBUPDATER;
GRANT SELECT ON fake_server_names TO $DBUPDATER;
......@@ -733,7 +721,6 @@ GRANT SELECT (batch) ON pings TO $DBCLEANER;
GRANT SELECT (batch, id) ON certs TO $DBCLEANER;
GRANT SELECT (cert) ON cert_chains TO $DBCLEANER;
GRANT SELECT (start_in, start_out) ON biflows TO $DBCLEANER;
GRANT SELECT (batch) ON nats TO $DBCLEANER;
GRANT SELECT (timestamp) ON fake_logs TO $DBCLEANER;
GRANT SELECT (batch) ON spoof TO $DBCLEANER;
GRANT SELECT (timestamp) ON refused TO $DBCLEANER;
......@@ -747,7 +734,6 @@ GRANT DELETE ON pings TO $DBCLEANER;
GRANT DELETE ON certs TO $DBCLEANER;
GRANT DELETE ON cert_chains TO $DBCLEANER;
GRANT DELETE ON biflows TO $DBCLEANER;
GRANT DELETE ON nats TO $DBCLEANER;
GRANT DELETE ON fake_logs TO $DBCLEANER;
GRANT DELETE ON spoof TO $DBCLEANER;
GRANT DELETE ON refused TO $DBCLEANER;
......@@ -775,7 +761,6 @@ GRANT SELECT ON cert_chains TO $DBARCHIVIST;
GRANT SELECT ON biflows TO $DBARCHIVIST;
GRANT SELECT ON bandwidth TO $DBARCHIVIST;
GRANT SELECT ON bandwidth_stats TO $DBARCHIVIST;
GRANT SELECT ON nats TO $DBARCHIVIST;
GRANT SELECT ON fake_logs TO $DBARCHIVIST;
GRANT SELECT ON spoof TO $DBARCHIVIST;
GRANT SELECT ON refused TO $DBARCHIVIST;
......
......@@ -5,7 +5,7 @@ set -ex
. ./dbconfig
TABLES='activities count_snapshots bandwidth bandwidth_stats refused fake_logs plugin_history'
BATCH_TABLES='pings certs nats spoof'
BATCH_TABLES='pings certs spoof'
DATE=$(date -d "$CLEAN_DAYS days ago" "+'%Y-%m-%d'")
(
......
#
# Ucollect - small utility for real-time analysis of network data
# Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/)
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
from task import Task
import logging
import database
from activity import log_activity
from twisted.internet import reactor
logger = logging.getLogger(name='sniff')
def decode(code):
if code == 'N':
return True
elif code == 'D':
return False
else:
return None
def submit_data(client, payload, batch_time):
with database.transaction() as t:
t.execute("INSERT INTO nats (batch, client, nat_v4, nat_v6) SELECT %s, clients.id, %s, %s FROM clients WHERE name = %s", (batch_time, decode(payload[0]), decode(payload[1]), client))
class NatTask(Task):
def __init__(self):
Task.__init__(self)
with database.transaction() as t:
t.execute("SELECT CURRENT_TIMESTAMP AT TIME ZONE 'UTC'")
(self.__batch_time,) = t.fetchone()
def name(self):
return 'Nat'
def message(self, client):
return ''
def success(self, client, payload):
reactor.callInThread(submit_data, client, payload, self.__batch_time)
log_activity(client, 'nat')
class Nat:
def __init__(self, config):
pass
def code(self):
return 'n'
def check_schedule(self):
with database.transaction() as t:
t.execute("SELECT m.m + i.i <= CURRENT_TIMESTAMP AT TIME ZONE 'UTC' FROM (SELECT COALESCE(MAX(batch), TO_TIMESTAMP(0)) AS m FROM nats) AS m CROSS JOIN (SELECT value::INTERVAL AS i FROM config WHERE plugin = 'sniff' AND name = 'nat-interval') AS i;")
(time_s_up,) = t.fetchone()
if time_s_up:
return [NatTask()]
else:
logger.debug('Not sniffing NAT yet')
return []
LIBRARIES += src/plugins/sniff/libplugin_sniff
SNIFF_TASKS := ping cert nat
SNIFF_TASKS := ping cert
libplugin_sniff_MODULES := sniff task fork nop parse $(SNIFF_TASKS)
DOCS += src/plugins/sniff/sniff
......
/*
Ucollect - small utility for real-time analysis of network data
Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/)
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#include "nat.h"
#include "fork.h"
#include "../../core/mem_pool.h"
#include "../../core/context.h"
#include "../../core/util.h"
#include <string.h>
static const char *nat_program =
#include <sniff-nat.inc>
;
struct task_data {
bool ok;
};
struct task_data *start_nat(struct context *context, struct mem_pool *pool, const uint8_t *message, size_t message_size, int *output, pid_t *pid) {
// We ignore the input parameters, there aren't any for this script.
(void) message;
(void) message_size;
struct task_data *data = mem_pool_alloc(pool, sizeof *data);
char **argv = mem_pool_alloc(context->temp_pool, 6 * sizeof *argv);
argv[0] = "/bin/busybox";
argv[1] = "ash";
argv[2] = "-c";
argv[3] = mem_pool_strdup(context->temp_pool, nat_program);
argv[4] = "sniff-nat";
argv[5] = NULL;
data->ok = fork_task(context->loop, "/bin/busybox", argv, "nat", output, pid);
return data;
}
static void parse_family(char *string, uint8_t *output, char family, char *error) {
char *word = strtok(string, "\n");
*output = '?';
if (!word) {
ulog(LLOG_ERROR, "Missing nat output for IPv%c\n", family);
*error = 'M';
return;
}
if (strcmp(word, "NONE") == 0)
*output = '0';
else if (strcmp(word, "NAT") == 0)
*output = 'N';
else if (strcmp(word, "DIRECT") == 0)
*output = 'D';
else {
ulog(LLOG_ERROR, "Unknown nat output for IPv%c: '%s'\n", family, word);
*error = 'U';
}
}
const uint8_t *finish_nat(struct context *context, struct task_data *data, uint8_t *output, size_t output_size, size_t *result_size, bool *ok) {
(void) output_size;
#define FAIL(CODE, MESSAGE) do { *result_size = 1; *ok = false; ulog(LLOG_INFO, "Sending error nat response %s: %s\n", CODE, MESSAGE); return (const uint8_t *)(CODE); } while (0)
if (!data->ok)
FAIL("S", "Failed to start");
uint8_t *result = mem_pool_alloc(context->temp_pool, 2);
*result_size = 2;
char error = '\0';
parse_family((char *)output, result, '4', &error);
parse_family(NULL, result + 1, '6', &error);
if (error) {
result[0] = error;
result[1] = '\0';
FAIL(result, "Invalid output");
}
*ok = true;
return result;
}
/*
Ucollect - small utility for real-time analysis of network data
Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/)
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#ifndef UCOLLECT_SNIFF_NAT_H
#define UCOLLECT_SNIFF_NAT_H
#include <stdint.h>
#include <stdbool.h>
#include <stdlib.h>
#include <unistd.h>
struct task_data;
struct context;
struct mem_pool;
struct task_data *start_nat(struct context *context, struct mem_pool *pool, const uint8_t *message, size_t message_size, int *output, pid_t *pid);
const uint8_t *finish_nat(struct context *context, struct task_data *data, uint8_t *output, size_t output_size, size_t *result_size, bool *ok);
#endif
......@@ -21,7 +21,6 @@
#include "nop.h"
#include "ping.h"
#include "cert.h"
#include "nat.h"
struct task_desc task_descs[] = {
{
......@@ -42,12 +41,6 @@ struct task_desc task_descs[] = {
.start = start_cert,
.finish = finish_cert
},
{
.name = 'n',
.label = "Nat",
.start = start_nat,
.finish = finish_nat
},
{
.name = '\0'
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment