Skip to content

GitLab

Commit 8e5493eb authored by Libor Peltan's avatar Libor Peltan
Browse files

journal: added obsolete journal import; definitive configuration for journal

parent a62a293e
Hide whitespace changes
Inline Side-by-side
Showing with 683 additions and 81 deletions
Knot.files
View file @ 8e5493eb
......@@ -258,10 +258,12 @@ src/knot/events/handlers/refresh.c
src/knot/events/handlers/update.c
src/knot/events/replan.c
src/knot/events/replan.h
src/knot/journal/serialization.c
src/knot/journal/serialization.h
src/knot/journal/journal.c
src/knot/journal/journal.h
src/knot/journal/old_journal.c
src/knot/journal/old_journal.h
src/knot/journal/serialization.c
src/knot/journal/serialization.h
src/knot/modules/dnsproxy/dnsproxy.c
src/knot/modules/dnsproxy/dnsproxy.h
src/knot/modules/dnstap/dnstap.c
......
src/Makefile.am
View file @ 8e5493eb
......@@ -337,6 +337,8 @@ libknotd_la_SOURCES = \
knot/server/dthreads.h \
knot/journal/journal.c \
knot/journal/journal.h \
knot/journal/old_journal.c \
knot/journal/old_journal.h \
knot/journal/serialization.c \
knot/journal/serialization.h \
knot/server/server.c \
......
src/knot/conf/conf.c
View file @ 8e5493eb
......@@ -1046,17 +1046,41 @@ char* conf_zonefile_txn(
return get_filename(conf, txn, zone, file);
}
char* conf_journalfile_txn(
char* conf_old_journalfile(
conf_t *conf,
knot_db_txn_t *txn)
const knot_dname_t *zone)
{
conf_val_t val;
if (zone == NULL) {
return NULL;
}
val = conf_default_get_txn(conf, txn, C_STORAGE);
char *storage = conf_abs_path(&val, NULL);
conf_val_t val = conf_zone_get(conf, C_JOURNAL, zone);
const char *journal = conf_str(&val);
val = conf_default_get_txn(conf, txn, C_JOURNAL);
// Use default journalfile name pattern if not specified.
if (journal == NULL) {
journal = "%s.db";
} else {
CONF_LOG_ZONE(LOG_NOTICE, zone, "obsolete configuration 'journal', "
"use 'template.journal-db'' instead");
}
val = conf_zone_get(conf, C_MAX_JOURNAL_SIZE, zone);
if (val.code == KNOT_EOK) {
CONF_LOG_ZONE(LOG_NOTICE, zone, "obsolete configuration 'max-journal-size', "
"use 'max-journal-usage' and 'template.journal-db' instead");
}
return get_filename(conf, &conf->read_txn, zone, journal);
}
char* conf_journalfile_txn(
conf_t *conf,
knot_db_txn_t *txn)
{
conf_val_t val = conf_default_get_txn(conf, txn, C_STORAGE);
char *storage = conf_abs_path(&val, NULL);
val = conf_default_get_txn(conf, txn, C_JOURNAL_DB);
char *journaldir = conf_abs_path(&val, storage);
free(storage);
......
src/knot/conf/conf.h
View file @ 8e5493eb
......@@ -580,11 +580,11 @@ static inline char* conf_zonefile(
*
* \param[in] conf Configuration.
* \param[in] txn Configuration DB transaction.
* \param[in] zone Zone name.
*
* \return Absolute journal file path string pointer.
*/
char* conf_journalfile_txn(conf_t *conf,
char* conf_journalfile_txn(
conf_t *conf,
knot_db_txn_t *txn);
static inline char* conf_journalfile(
conf_t *conf)
......@@ -592,6 +592,11 @@ static inline char* conf_journalfile(
return conf_journalfile_txn(conf, &conf->read_txn);
}
char* conf_old_journalfile(
conf_t *conf,
const knot_dname_t *zone
);
/*!
* Gets the configured number of UDP threads.
*
......
src/knot/conf/scheme.c
View file @ 8e5493eb
......@@ -45,11 +45,14 @@
#define HOURS(x) ((x) * 3600)
#define DAYS(x) ((x) * HOURS(24))
#define GIGA (1024LLU * 1024 * 1024)
#define TERA (1024 * GIGA)
#define KILO(x) (1024LLU * (x))
#define MEGA(x) (KILO(1024) * (x))
#define GIGA(x) (MEGA(1024) * (x))
#define TERA(x) (GIGA(1024) * (x))
#define VIRT_MEM_TOP_32BIT (2 * GIGA)
#define VIRT_MEM_LIMIT(x) (((sizeof(void *) < 8) && ((x) > VIRT_MEM_TOP_32BIT)) ? VIRT_MEM_TOP_32BIT : (x))
#define VIRT_MEM_TOP_32BIT GIGA(2)
#define VIRT_MEM_LIMIT(x) (((sizeof(void *) < 8) && ((x) > VIRT_MEM_TOP_32BIT)) \
? VIRT_MEM_TOP_32BIT : (x))
#define FMOD (YP_FMULTI | CONF_IO_FRLD_MOD | CONF_IO_FRLD_ZONES)
......@@ -242,11 +245,9 @@ static const yp_item_t desc_remote[] = {
{ C_DISABLE_ANY, YP_TBOOL, YP_VNONE }, \
{ C_ZONEFILE_SYNC, YP_TINT, YP_VINT = { -1, INT32_MAX, 0, YP_STIME } }, \
{ C_IXFR_DIFF, YP_TBOOL, YP_VNONE }, \
{ C_MAX_ZONE_SIZE, YP_TINT, YP_VINT = { 0, INT64_MAX, INT64_MAX, YP_SSIZE }, \
FLAGS }, \
{ C_MAX_JOURNAL_USAGE, YP_TINT, YP_VINT = { 40 * 1024, INT64_MAX, 100 * 1024 * 1024, \
YP_SSIZE } }, \
{ C_MAX_JOURNAL_DEPTH, YP_TINT, YP_VINT = { 2, INT64_MAX, INT64_MAX, YP_SSIZE } }, \
{ C_MAX_ZONE_SIZE, YP_TINT, YP_VINT = { 0, INT64_MAX, INT64_MAX, YP_SSIZE }, FLAGS }, \
{ C_MAX_JOURNAL_USAGE, YP_TINT, YP_VINT = { KILO(40), INT64_MAX, MEGA(100), YP_SSIZE } }, \
{ C_MAX_JOURNAL_DEPTH, YP_TINT, YP_VINT = { 2, INT64_MAX, INT64_MAX } }, \
{ C_KASP_DB, YP_TSTR, YP_VSTR = { "keys" }, FLAGS }, \
{ C_DNSSEC_SIGNING, YP_TBOOL, YP_VNONE, FLAGS }, \
{ C_DNSSEC_POLICY, YP_TREF, YP_VREF = { C_POLICY }, FLAGS, { check_ref_dflt } }, \
......@@ -254,18 +255,21 @@ static const yp_item_t desc_remote[] = {
{ C_REQUEST_EDNS_OPTION, YP_TDATA, YP_VDATA = { 0, NULL, edns_opt_to_bin, edns_opt_to_txt } }, \
{ C_MODULE, YP_TDATA, YP_VDATA = { 0, NULL, mod_id_to_bin, mod_id_to_txt }, \
YP_FMULTI | FLAGS, { check_modref } }, \
{ C_COMMENT, YP_TSTR, YP_VNONE },
{ C_COMMENT, YP_TSTR, YP_VNONE }, \
/* Obsolete, old journal items. */ \
{ C_JOURNAL, YP_TSTR, YP_VNONE, FLAGS }, \
{ C_MAX_JOURNAL_SIZE, YP_TINT, YP_VINT = { 0, INT64_MAX, INT64_MAX, YP_SSIZE }, FLAGS }, \
static const yp_item_t desc_template[] = {
{ C_ID, YP_TSTR, YP_VNONE, CONF_IO_FREF },
ZONE_ITEMS(CONF_IO_FRLD_ZONES)
{ C_TIMER_DB, YP_TSTR, YP_VSTR = { "timers" }, CONF_IO_FRLD_ZONES }, \
{ C_GLOBAL_MODULE, YP_TDATA, YP_VDATA = { 0, NULL, mod_id_to_bin, mod_id_to_txt }, \
YP_FMULTI | CONF_IO_FRLD_MOD, { check_modref } }, \
{ C_JOURNAL, YP_TSTR, YP_VSTR = { "journal.db" }, CONF_IO_FRLD_SRV }, \
{ C_MAX_JOURNAL_SIZE, YP_TINT, YP_VINT = { 1024 * 1024, VIRT_MEM_LIMIT(100 * TERA), \
VIRT_MEM_LIMIT(20 * GIGA), YP_SSIZE }, \
CONF_IO_FRLD_SRV }, \
{ C_TIMER_DB, YP_TSTR, YP_VSTR = { "timers" }, CONF_IO_FRLD_ZONES },
{ C_GLOBAL_MODULE, YP_TDATA, YP_VDATA = { 0, NULL, mod_id_to_bin, mod_id_to_txt },
YP_FMULTI | CONF_IO_FRLD_MOD, { check_modref } },
{ C_JOURNAL_DB, YP_TSTR, YP_VSTR = { "journal" }, CONF_IO_FRLD_SRV },
{ C_MAX_JOURNAL_DB_SIZE, YP_TINT, YP_VINT = { JOURNAL_MIN_FSLIMIT, VIRT_MEM_LIMIT(TERA(100)),
VIRT_MEM_LIMIT(GIGA(20)), YP_SSIZE },
CONF_IO_FRLD_SRV },
{ NULL }
};
......
src/knot/conf/scheme.h
View file @ 8e5493eb
......@@ -53,7 +53,8 @@
#define C_IDENT "\x08""identity"
#define C_INCL "\x07""include"
#define C_IXFR_DIFF "\x15""ixfr-from-differences"
#define C_JOURNAL "\x07""journal"
#define C_JOURNAL "\x07""journal" /* obsolete, old journal compat */
#define C_JOURNAL_DB "\x0A""journal-db"
#define C_KASP_DB "\x07""kasp-db"
#define C_KEY "\x03""key"
#define C_KEYSTORE "\x08""keystore"
......@@ -62,7 +63,8 @@
#define C_LOG "\x03""log"
#define C_MANUAL "\x06""manual"
#define C_MASTER "\x06""master"
#define C_MAX_JOURNAL_SIZE "\x10""max-journal-size"
#define C_MAX_JOURNAL_SIZE "\x10""max-journal-size" /* obsolete, old journal compat */
#define C_MAX_JOURNAL_DB_SIZE "\x13""max-journal-db-size"
#define C_MAX_JOURNAL_USAGE "\x11""max-journal-usage"
#define C_MAX_JOURNAL_DEPTH "\x11""max-journal-depth"
#define C_MAX_TCP_CLIENTS "\x0F""max-tcp-clients"
......
src/knot/conf/tools.c
View file @ 8e5493eb
......@@ -495,42 +495,19 @@ int check_template(
return KNOT_EOK;
}
// Check global-module.
conf_val_t g_module = conf_rawid_get_txn(args->conf, args->txn, C_TPL,
C_GLOBAL_MODULE, args->id,
args->id_len);
if (g_module.code == KNOT_EOK) {
args->err_str = "global module in non-default template";
return KNOT_EINVAL;
}
// Check timer-db.
conf_val_t timer_db = conf_rawid_get_txn(args->conf, args->txn, C_TPL,
C_TIMER_DB, args->id, args->id_len);
if (timer_db.code == KNOT_EOK) {
args->err_str = "timer database location in non-default template";
return KNOT_EINVAL;
}
// Check journal.
conf_val_t journal = conf_rawid_get_txn(args->conf, args->txn, C_TPL,
C_JOURNAL, args->id, args->id_len);
if (journal.code == KNOT_EOK) {
args->err_str = "journal location in non-default template";
return KNOT_EINVAL;
}
// Check max-journal-size.
conf_val_t max_journal_size = conf_rawid_get_txn(args->conf, args->txn, C_TPL,
C_MAX_JOURNAL_SIZE, args->id, args->id_len);
conf_val_t val;
#define CHECK_DFLT(item, name) \
val = conf_rawid_get_txn(args->conf, args->txn, C_TPL, item, \
args->id, args->id_len); \
if (val.code == KNOT_EOK) { \
args->err_str = name " in non-default template"; \
return KNOT_EINVAL; \
}
if (max_journal_size.code == KNOT_EOK) {
args->err_str = "journal size in non-default template";
return KNOT_EINVAL;
}
CHECK_DFLT(C_TIMER_DB, "timer database");
CHECK_DFLT(C_GLOBAL_MODULE, "global module");
CHECK_DFLT(C_JOURNAL_DB, "journal database path");
CHECK_DFLT(C_MAX_JOURNAL_DB_SIZE, "journal database maximum size");
return KNOT_EOK;
}
......
src/knot/journal/old_journal.c 0 → 100644
View file @ 8e5493eb
/* Copyright (C) 2016 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <assert.h>
#include "knot/common/log.h"
#include "contrib/files.h"
#include "knot/journal/old_journal.h"
#include "knot/journal/serialization.h"
#include "libknot/libknot.h"
typedef enum {
JOURNAL_NULL = 0 << 0, /*!< Invalid journal entry. */
JOURNAL_FREE = 1 << 0, /*!< Free journal entry. */
JOURNAL_VALID = 1 << 1, /*!< Valid journal entry. */
JOURNAL_DIRTY = 1 << 2 /*!< Journal entry cannot be evicted. */
} journal_flag_t;
typedef struct {
uint64_t id; /*!< Node ID. */
uint16_t flags; /*!< Node flags. */
uint16_t next; /*!< UNUSED */
uint32_t pos; /*!< Position in journal file. */
uint32_t len; /*!< Entry data length. */
} journal_node_t;
typedef struct {
int fd;
char *path; /*!< Path to journal file. */
uint16_t tmark; /*!< Transaction start mark. */
uint16_t max_nodes; /*!< Number of nodes. */
uint16_t qhead; /*!< Node queue head. */
uint16_t qtail; /*!< Node queue tail. */
uint16_t bflags; /*!< Initial flags for each written node. */
size_t fsize; /*!< Journal file size. */
size_t fslimit; /*!< File size limit. */
journal_node_t free; /*!< Free segment. */
journal_node_t *nodes; /*!< Array of nodes. */
} old_journal_t;
#define JOURNAL_NCOUNT 1024 /*!< Default node count. */
#define JOURNAL_MAGIC {'k', 'n', 'o', 't', '1', '5', '2'}
#define MAGIC_LENGTH 7
/* HEADER = magic, crc, max_entries, qhead, qtail */
#define JOURNAL_HSIZE (MAGIC_LENGTH + sizeof(uint32_t) + sizeof(uint16_t) * 3)
/*! \brief Infinite file size limit. */
#define FSLIMIT_INF (~((size_t)0))
/*! \brief Next node. */
#define jnode_next(j, i) (((i) + 1) % (j)->max_nodes)
/*! \brief Previous node. */
#define jnode_prev(j, i) (((i) == 0) ? (j)->max_nodes - 1 : (i) - 1)
/*! \bref Starting node data position. */
#define jnode_base_pos(max_nodes) (JOURNAL_HSIZE + (max_nodes + 1) * sizeof(journal_node_t))
static inline int sfread(void *dst, size_t len, int fd)
{
return read(fd, dst, len) == len;
}
/*! \brief Return 'serial_from' part of the key. */
static inline uint32_t journal_key_from(uint64_t k)
{
return (uint32_t)(k & ((uint64_t)0x00000000ffffffff));
}
/*! \brief Compare function to match entries with starting serial. */
static inline int journal_key_from_cmp(uint64_t k, uint64_t from)
{
return ((uint64_t)journal_key_from(k)) - from;
}
/*! \brief Open journal file for r/w (returns error if not exists). */
static int old_journal_open_file(old_journal_t *j)
{
assert(j != NULL);
int ret = KNOT_EOK;
j->fd = open(j->path, O_RDWR);
if (j->fd < 0) {
return knot_map_errno();
}
/* File lock. */
struct flock lock = { .l_type = F_WRLCK, .l_whence = SEEK_SET,
.l_start = 0, .l_len = 0, .l_pid = 0 };
/* Attempt to lock. */
ret = fcntl(j->fd, F_SETLKW, &lock);
if (ret < 0) {
return knot_map_errno();
}
/* Read magic bytes. */
const char magic_req[MAGIC_LENGTH] = JOURNAL_MAGIC;
char magic[MAGIC_LENGTH];
if (!sfread(magic, MAGIC_LENGTH, j->fd)) {
goto open_file_error;
}
if (memcmp(magic, magic_req, MAGIC_LENGTH) != 0) {
log_warning("old journal '%s', version too old", j->path);
close(j->fd);
j->fd = -1;
return KNOT_ENOTSUP;
}
/* Skip CRC */
if (lseek(j->fd, MAGIC_LENGTH + sizeof(uint32_t), SEEK_SET) < 0) {
goto open_file_error;
}
/* Get journal file size. */
struct stat st;
if (fstat(j->fd, &st) < 0) {
goto open_file_error;
}
/* Set file size. */
j->fsize = st.st_size;
/* Read maximum number of entries. */
if (!sfread(&j->max_nodes, sizeof(uint16_t), j->fd)) {
goto open_file_error;
}
/* Allocate nodes. */
const size_t node_len = sizeof(journal_node_t);
j->nodes = malloc(j->max_nodes * node_len);
if (j->nodes == NULL) {
goto open_file_error;
} else {
memset(j->nodes, 0, j->max_nodes * node_len);
}
/* Load node queue state. */
j->qhead = j->qtail = 0;
if (!sfread(&j->qhead, sizeof(uint16_t), j->fd)) {
goto open_file_error;
}
/* Load queue tail. */
if (!sfread(&j->qtail, sizeof(uint16_t), j->fd)) {
goto open_file_error;
}
/* Load empty segment descriptor. */
if (!sfread(&j->free, node_len, j->fd)) {
goto open_file_error;
}
/* Read journal descriptors table. */
if (!sfread(j->nodes, j->max_nodes * node_len, j->fd)) {
goto open_file_error;
}
/* Save file lock and return. */
return KNOT_EOK;
/* Unlock and close file and return error. */
open_file_error:
free(j->nodes);
j->nodes = NULL;
close(j->fd);
j->fd = -1;
return KNOT_ERROR;
}
/*! \brief Close journal file. */
static int old_journal_close_file(old_journal_t *journal)
{
/* Close file. */
if (journal->fd > 0) {
close(journal->fd);
journal->fd = -1;
}
/* Free nodes. */
free(journal->nodes);
journal->nodes = NULL;
return KNOT_EOK;
}
static int old_journal_close(old_journal_t *journal)
{
/* Check journal. */
if (journal == NULL) {
return KNOT_EINVAL;
}
/* Close file. */
old_journal_close_file(journal);
/* Free allocated resources. */
free(journal->path);
free(journal);
return KNOT_EOK;
}
static int old_journal_open(old_journal_t **journal, const char *path, size_t fslimit)
{
if (journal == NULL || path == NULL) {
return KNOT_EINVAL;
}
old_journal_t *j = malloc(sizeof(*j));
if (j == NULL) {
return KNOT_ENOMEM;
}
memset(j, 0, sizeof(*j));
j->bflags = JOURNAL_DIRTY;
j->fd = -1;
j->fslimit = fslimit;
/* Copy path. */
j->path = strdup(path);
if (j->path == NULL) {
free(j);
return KNOT_ENOMEM;
}
/* Open journal file. */
int ret = old_journal_open_file(j);
if (ret != KNOT_EOK) {
log_error("old journal '%s', failed to open (%s)", path,
knot_strerror(ret));
old_journal_close(j);
return ret;
}
*journal = j;
return KNOT_EOK;
}
typedef int (*journal_cmp_t)(uint64_t k1, uint64_t k2);
static int old_journal_fetch(old_journal_t *journal, uint64_t id,
journal_cmp_t cf, journal_node_t** dst)
{
if (journal == NULL || dst == NULL) {
return KNOT_EINVAL;
}
size_t i = jnode_prev(journal, journal->qtail);
size_t endp = jnode_prev(journal, journal->qhead);
for(; i != endp; i = jnode_prev(journal, i)) {
journal_node_t *n = journal->nodes + i;
/* Skip invalid nodes. */
if (!(n->flags & JOURNAL_VALID)) {
continue;
}
if (cf(n->id, id) == 0) {
*dst = journal->nodes + i;
return KNOT_EOK;
}
}
return KNOT_ENOENT;
}
static int old_journal_read_node(old_journal_t *journal, journal_node_t *n, char *dst)
{
/* Check valid flag. */
if (!(n->flags & JOURNAL_VALID)) {
return KNOT_EINVAL;
}
/* Seek journal node. */
int seek_ret = lseek(journal->fd, n->pos, SEEK_SET);
/* Read journal node content. */
if (seek_ret < 0 || !sfread(dst, n->len, journal->fd)) {
return KNOT_ERROR;
}
return KNOT_EOK;
}
bool old_journal_exists(const char *path)
{
if (path == NULL) {
return false;
}
struct stat st;
return stat(path, &st) == 0;
}
/*! \brief No doc here. Moved from zones.h (@mvavrusa) */
static int changesets_unpack(changeset_t *chs)
{
/* Read changeset flags. */
if (chs->data == NULL) {
return KNOT_EMALF;
}
size_t remaining = chs->size;
/* Read initial changeset RRSet - SOA. */
uint8_t *stream = chs->data + (chs->size - remaining);
knot_rrset_t rrset;
int ret = rrset_deserialize(stream, &remaining, &rrset);
if (ret != KNOT_EOK) {
return KNOT_EMALF;
}
assert(rrset.type == KNOT_RRTYPE_SOA);
chs->soa_from = knot_rrset_copy(&rrset, NULL);
knot_rrset_clear(&rrset, NULL);
if (chs->soa_from == NULL) {
return KNOT_ENOMEM;
}
/* Read remaining RRSets */
bool in_remove_section = true;
while (remaining > 0) {
/* Parse next RRSet. */
stream = chs->data + (chs->size - remaining);
knot_rrset_init_empty(&rrset);
ret = rrset_deserialize(stream, &remaining, &rrset);
if (ret != KNOT_EOK) {
return KNOT_EMALF;
}
/* Check for next SOA. */
if (rrset.type == KNOT_RRTYPE_SOA) {
/* Move to ADD section if in REMOVE. */
if (in_remove_section) {
chs->soa_to = knot_rrset_copy(&rrset, NULL);
if (chs->soa_to == NULL) {
ret = KNOT_ENOMEM;
break;
}
in_remove_section = false;
} else {
/* Final SOA, no-op. */
;
}
} else {
/* Remove RRSets. */
if (in_remove_section) {
ret = changeset_add_removal(chs, &rrset, 0);
} else {
/* Add RRSets. */
ret = changeset_add_addition(chs, &rrset, 0);
}
}
knot_rrset_clear(&rrset, NULL);
if (ret != KNOT_EOK) {