Commit 9775d0ea authored by Libor Peltan's avatar Libor Peltan
Browse files

Journal: re-implemented completely, LMDB, all zones in one db

parent 0f8d449e
......@@ -258,6 +258,10 @@ src/knot/events/handlers/refresh.c
src/knot/events/handlers/update.c
src/knot/events/replan.c
src/knot/events/replan.h
src/knot/journal/serialization.c
src/knot/journal/serialization.h
src/knot/journal/journal.c
src/knot/journal/journal.h
src/knot/modules/dnsproxy/dnsproxy.c
src/knot/modules/dnsproxy/dnsproxy.h
src/knot/modules/dnstap/dnstap.c
......@@ -312,10 +316,6 @@ src/knot/query/requestor.c
src/knot/query/requestor.h
src/knot/server/dthreads.c
src/knot/server/dthreads.h
src/knot/server/journal.c
src/knot/server/journal.h
src/knot/server/serialization.c
src/knot/server/serialization.h
src/knot/server/server.c
src/knot/server/server.h
src/knot/server/tcp-handler.c
......@@ -562,7 +562,7 @@ tests/contrib/test_wire_ctx.c
tests/dthreads.c
tests/fake_server.h
tests/fdset.c
tests/journal.c
tests/journal_lmdb.c
tests/libknot/test_control.c
tests/libknot/test_cookies-client.c
tests/libknot/test_cookies-opt.c
......
......@@ -871,9 +871,17 @@ This option has no effect with enabled
\fIDefault:\fP off
.SS max\-journal\-size
.sp
Maximum size of the zone journal file.
Maximum size of the journal DB.
.sp
\fIDefault:\fP 2^64
\fIDefault:\fP 1 GiB
.sp
\fBNOTE:\fP
.INDENT 0.0
.INDENT 3.5
Decreasing this value will lead to discarding
whole journal history of all zones.
.UNINDENT
.UNINDENT
.SS max\-zone\-size
.sp
Maximum size of the zone. The size is measured as size of the zone records
......
......@@ -1013,9 +1013,13 @@ is a master server for the zone.
max-journal-size
----------------
Maximum size of the zone journal file.
Maximum size of the journal DB.
*Default:* 2^64
*Default:* 1 GiB
.. NOTE::
Decreasing this value will lead to discarding
whole journal history of all zones.
.. _zone_max_zone_size:
......
......@@ -335,10 +335,10 @@ libknotd_la_SOURCES = \
knot/common/stats.h \
knot/server/dthreads.c \
knot/server/dthreads.h \
knot/server/journal.c \
knot/server/journal.h \
knot/server/serialization.c \
knot/server/serialization.h \
knot/journal/journal.c \
knot/journal/journal.h \
knot/journal/serialization.c \
knot/journal/serialization.h \
knot/server/server.c \
knot/server/server.h \
knot/server/tcp-handler.c \
......
......@@ -1048,22 +1048,19 @@ char* conf_zonefile_txn(
char* conf_journalfile_txn(
conf_t *conf,
knot_db_txn_t *txn,
const knot_dname_t *zone)
knot_db_txn_t *txn)
{
if (zone == NULL) {
return NULL;
}
conf_val_t val;
conf_val_t val = conf_zone_get_txn(conf, txn, C_JOURNAL, zone);
const char *journal = conf_str(&val);
val = conf_default_get_txn(conf, txn, C_STORAGE);
char *storage = conf_abs_path(&val, NULL);
// Use default journalfile name pattern if not specified.
if (journal == NULL) {
journal = "%s.db";
}
val = conf_default_get_txn(conf, txn, C_JOURNAL);
char *journaldir = conf_abs_path(&val, storage);
free(storage);
return get_filename(conf, txn, zone, journal);
return journaldir;
}
size_t conf_udp_threads_txn(
......
......@@ -584,16 +584,12 @@ static inline char* conf_zonefile(
*
* \return Absolute journal file path string pointer.
*/
char* conf_journalfile_txn(
conf_t *conf,
knot_db_txn_t *txn,
const knot_dname_t *zone
);
char* conf_journalfile_txn(conf_t *conf,
knot_db_txn_t *txn);
static inline char* conf_journalfile(
conf_t *conf,
const knot_dname_t *zone)
conf_t *conf)
{
return conf_journalfile_txn(conf, &conf->read_txn, zone);
return conf_journalfile_txn(conf, &conf->read_txn);
}
/*!
......
......@@ -225,10 +225,14 @@ static const yp_item_t desc_remote[] = {
{ NULL }
};
#define VIRT_MEM_TOP (2LLU * 1024 * 1204 * 1204)
#define VIRT_MEM_LIMIT(x) (((sizeof(void *) < 8) && ((x) > VIRT_MEM_TOP)) ? VIRT_MEM_TOP : (x))
#define ZONE_ITEMS(FLAGS) \
{ C_STORAGE, YP_TSTR, YP_VSTR = { STORAGE_DIR }, FLAGS }, \
{ C_FILE, YP_TSTR, YP_VNONE, FLAGS }, \
{ C_JOURNAL, YP_TSTR, YP_VNONE, FLAGS }, \
{ C_MASTER, YP_TREF, YP_VREF = { C_RMT }, YP_FMULTI, { check_ref } }, \
{ C_DDNS_MASTER, YP_TREF, YP_VREF = { C_RMT }, YP_FNONE, { check_ref } }, \
{ C_NOTIFY, YP_TREF, YP_VREF = { C_RMT }, YP_FMULTI, { check_ref } }, \
......@@ -237,8 +241,6 @@ static const yp_item_t desc_remote[] = {
{ C_DISABLE_ANY, YP_TBOOL, YP_VNONE }, \
{ C_ZONEFILE_SYNC, YP_TINT, YP_VINT = { -1, INT32_MAX, 0, YP_STIME } }, \
{ C_IXFR_DIFF, YP_TBOOL, YP_VNONE }, \
{ C_MAX_JOURNAL_SIZE, YP_TINT, YP_VINT = { 0, INT64_MAX, INT64_MAX, YP_SSIZE }, \
FLAGS }, \
{ C_MAX_ZONE_SIZE, YP_TINT, YP_VINT = { 0, INT64_MAX, INT64_MAX, YP_SSIZE }, \
FLAGS }, \
{ C_KASP_DB, YP_TSTR, YP_VSTR = { "keys" }, FLAGS }, \
......@@ -256,6 +258,9 @@ static const yp_item_t desc_template[] = {
{ C_TIMER_DB, YP_TSTR, YP_VSTR = { "timers" }, CONF_IO_FRLD_ZONES }, \
{ C_GLOBAL_MODULE, YP_TDATA, YP_VDATA = { 0, NULL, mod_id_to_bin, mod_id_to_txt }, \
YP_FMULTI | CONF_IO_FRLD_MOD, { check_modref } }, \
{ C_JOURNAL, YP_TSTR, YP_VSTR = { "journal.db" }, CONF_IO_FRLD_ZONES }, \
{ C_MAX_JOURNAL_SIZE, YP_TINT, YP_VINT = { 1024 * 1024, VIRT_MEM_LIMIT(100LLU * 1024 * 1024 * 1024 * 1024), \
VIRT_MEM_LIMIT(20LLU * 1024 * 1024 * 1024), YP_SSIZE } }, \
{ NULL }
};
......
......@@ -514,6 +514,24 @@ int check_template(
return KNOT_EINVAL;
}
// Check journal.
conf_val_t journal = conf_rawid_get_txn(args->conf, args->txn, C_TPL,
C_JOURNAL, args->id, args->id_len);
if (journal.code == KNOT_EOK) {
args->err_str = "journal location in non-default template";
return KNOT_EINVAL;
}
// Check max-journal-size.
conf_val_t max_journal_size = conf_rawid_get_txn(args->conf, args->txn, C_TPL,
C_MAX_JOURNAL_SIZE, args->id, args->id_len);
if (max_journal_size.code == KNOT_EOK) {
args->err_str = "journal size in non-default template";
return KNOT_EINVAL;
}
return KNOT_EOK;
}
......
......@@ -930,9 +930,9 @@ static int zone_purge(zone_t *zone, ctl_args_t *args)
free(zonefile);
// Purge the zone journal.
char *journalfile = conf_journalfile(conf(), zone->name);
(void)unlink(journalfile);
free(journalfile);
if (journal_open(zone->journal, zone->journal_db, zone->name) == KNOT_EOK) {
(void)scrape_journal(zone->journal);
}
// Purge the zone timers.
(void)remove_timer_db(args->server->timers_db, args->server->zone_db,
......
This diff is collapsed.
/* Copyright (C) 2016 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <pthread.h>
#include "libknot/libknot.h"
#include "contrib/ucw/lists.h"
#include "knot/updates/changesets.h"
#include "knot/journal/serialization.h"
/*!
* \brief j->fslimit special value to open with minimal possible mapsize
*
* ...it is equal to the actual DB file size.
* Beware of using this value for the first time initialized DB !
* It is mostly useful for read only access
*/
#define KNOT_JOURNAL_FSLIMIT_SAMEASLAST (400 * 1024)
typedef struct {
knot_db_t *db;
const knot_db_api_t *db_api;
char *path;
size_t fslimit;
pthread_mutex_t db_mutex; // please delete this once you move DB opening from journal_open to db_init
// common metadata: last_inserter_zone, last_total_occupied, journal_count
} journal_db_t;
typedef struct {
uint32_t first_serial; // serial_from of the first changeset
uint32_t last_serial; // serial_from of the last changeset
uint32_t last_serial_to; // serial_to of the last changeset
uint32_t last_flushed; // serial_from of the last flushed (or merged) chengeset
uint32_t merged_serial; // "serial_from" of merged changeset
uint32_t dirty_serial; // serial_from of an incompletely inserted changeset which shall be deleted (see DB_MAX_INSERT_TXN)
uint32_t flags; // LAST_FLUSHED_VALID, SERIAL_TO_VALID, MERGED_SERIAL_VALID
// specific metadata: occupied
} journal_metadata_t;
typedef struct {
journal_db_t *db;
knot_dname_t *zone;
//journal_metadata_t md;
} journal_t;
enum {
KNOT_JOURNAL_CHECK_SILENT = 0,
KNOT_JOURNAL_CHECK_WARN = 1,
KNOT_JOURNAL_CHECK_INFO = 2
};
/*!
* \brief Allocate a new journal structure.
*
* \retval new journal instance if successful.
* \retval NULL on error.
*/
journal_t *journal_new(void);
/*!
* \brief Free a journal structure.
*
* \param journal A journal structure to free.
*/
void journal_free(journal_t **journal);
/*!
* \brief Open journal.
*
* \param j Journal struct to use.
* \param db Shared journal database
* \param zone_name Name of the zone this journal belongs to.
*
* \retval KNOT_EOK on success.
* \return < KNOT_EOK on other errors.
*/
int journal_open(journal_t *j, journal_db_t **db, const knot_dname_t *zone_name);
/*!
* \brief Close journal.
*
* \param journal Journal to close.
*/
void journal_close(journal_t *journal);
/*!
* \brief Initialize shared journal DB file. The DB will be open on first use.
*
* \param db Database to be initialized. Must be (*db == NULL) before!
* \param lmdb_dir_path Path to the directory with DB
* \param lmdb_fslimit Maximum size of DB data file
*
* \return KNOT_E*
*/
int init_journal_db(journal_db_t **db, const char *lmdb_dir_path, size_t lmdb_fslimit);
/*!
* \brief Close shared journal DB file.
*
* \param db DB to close.
*/
void close_journal_db(journal_db_t **db);
/*!
* \brief Load changesets from journal.
*
* \param journal Journal to load from.
* \param dst Store changesets here.
* \param from Start serial.
*
* \retval KNOT_EOK on success.
* \retval KNOT_ENOENT when the lookup of the first entry fails.
* \return < KNOT_EOK on other error.
*/
int journal_load_changesets(journal_t *journal, list_t *dst, uint32_t from);
/*!
* \brief Store changesets in journal.
*
* \param journal Journal to store in.
* \param src Changesets to store.
*
* \retval KNOT_EOK on success.
* \retval KNOT_EBUSY when full, asking zone to flush itself to zonefile
* to allow cleaning up history and freeing up space
* \retval KNOT_ESPACE when full and not able to free up any space
* \return < KNOT_EOK on other errors.
*/
int journal_store_changesets(journal_t *journal, list_t *src);
/*!
* \brief Store changesets in journal.
*
* \param journal Journal to store in.
* \param change Changeset to store.
*
* \retval (same as for journal_store_changesets())
*/
int journal_store_changeset(journal_t *journal, changeset_t *change);
/*!
* \brief Check if this (zone's) journal is present in shared journal DB.
*
* \param db Shared journal DB
* \param zone_name Name of the zone of the journal in question
*
* \return true or false
*/
bool journal_exists(journal_db_t **db, knot_dname_t *zone_name);
/*! \brief Tell the journal that zone has been flushed.
*
* \param journal Journal to flush.
*
* \return KNOT_E*
*/
int journal_flush(journal_t *journal);
/*! \brief Remove completely this (zone's) journal from shared journal DB.
*
* This must be called with opened journal.
*
* \param j Journal to be deleted
*
* \return KNOT_E*
*/
int scrape_journal(journal_t *j);
/*! \brief Obtain public information from journal metadata
*
* \param[in] j Journal
* \param[out] is_empty 1 if j contains no changesets
* \param[out] serial_from [if !is_empty] starting serial of changesets history
* \param[out] serial_to [if !is_empty] ending serial of changesets history
*/
void journal_metadata_info(journal_t *j, int *is_empty, uint32_t *serial_from, uint32_t *serial_to);
/*!
* \brief List the zones contained in journal DB.
*
* \param db[in] Shared journal DB
* \param zones[out] List of strings (char *) of zone names
*
* \return KNOT_EOK ok
* \retval KNOT_ENOMEM no zones found
* \retval KNOT_EMALF different # of zones found than expected
* \retval KNOT_E* other error
*/
int journal_db_list_zones(journal_db_t **db, list_t *zones);
/*! \brief Check the journal consistency, errors to stderr.
*
* \param journal Journal to check.
* \param warn_level SILENT: no logging, just curious for return value; WARN: log journal inconsistencies; INFO: log journal state
*
* \return KNOT_E*
*/
int journal_check(journal_t *j, int warn_level);
/*! @} */
/* Copyright (C) 2014 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz>
/* Copyright (C) 2016 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
......@@ -16,8 +16,9 @@
#include <assert.h>
#include "knot/server/serialization.h"
#include "knot/journal/serialization.h"
#include "libknot/libknot.h"
#include "contrib/wire_ctx.h"
static size_t rr_binary_size(const knot_rrset_t *rrset, size_t rdata_pos)
{
......@@ -35,11 +36,11 @@ static uint64_t rrset_binary_size(const knot_rrset_t *rrset)
if (rrset == NULL || rrset->rrs.rr_count == 0) {
return 0;
}
uint64_t size = sizeof(uint64_t) + // size at the beginning
knot_dname_size(rrset->owner) + // owner data
sizeof(uint16_t) + // type
sizeof(uint16_t) + // class
sizeof(uint16_t); //RR count
uint64_t size = /* sizeof(uint64_t) + // size at the beginning */
knot_dname_size(rrset->owner) + // owner data
sizeof(uint16_t) + // type
sizeof(uint16_t) + // class
sizeof(uint16_t); //RR count
uint16_t rdata_count = rrset->rrs.rr_count;
for (uint16_t i = 0; i < rdata_count; i++) {
/* Space to store length of one RR. */
......@@ -52,7 +53,7 @@ static uint64_t rrset_binary_size(const knot_rrset_t *rrset)
}
static void serialize_rr(const knot_rrset_t *rrset, size_t rdata_pos,
uint8_t *stream)
uint8_t *stream)
{
const knot_rdata_t *rr = knot_rdataset_at(&rrset->rrs, rdata_pos);
assert(rr);
......@@ -66,7 +67,80 @@ static int deserialize_rr(knot_rrset_t *rrset, const uint8_t *stream, uint32_t r
uint32_t ttl;
memcpy(&ttl, stream, sizeof(uint32_t));
return knot_rrset_add_rdata(rrset, stream + sizeof(uint32_t),
rdata_size - sizeof(uint32_t), ttl, NULL);
rdata_size - sizeof(uint32_t), ttl, NULL);
}
static int serialize_rrset(wire_ctx_t *wire, const knot_rrset_t *rrset)
{
assert(wire);
assert(rrset);
/* Write owner. */
int size = knot_dname_to_wire(wire->position, rrset->owner,
wire_ctx_available(wire));
if (size < 0) {
return size;
}
wire_ctx_skip(wire, size);
/* Write rtype, rclass and RR count. */
wire_ctx_write_u16(wire, rrset->type);
wire_ctx_write_u16(wire, rrset->rclass);
wire_ctx_write_u16(wire, rrset->rrs.rr_count);
/* Write rdata items. */
for (uint16_t i = 0; i < rrset->rrs.rr_count; i++) {
const knot_rdata_t *rr = knot_rdataset_at(&rrset->rrs, i);
assert(rr != NULL);
wire_ctx_write_u32(wire, knot_rdata_ttl(rr));
wire_ctx_write_u32(wire, knot_rdata_rdlen(rr));
wire_ctx_write(wire, knot_rdata_data(rr), knot_rdata_rdlen(rr));
}
return wire->error;
}
static int deserialize_rrset(wire_ctx_t *wire, knot_rrset_t *rrset)
{
assert(wire);
assert(rrset);
/* Read owner. */
int size = knot_dname_size(wire->position);
if (size < 0) {
return size;
}
knot_dname_t *owner = knot_dname_copy_part(wire->position, size, NULL);
if (owner == NULL) {
return KNOT_EMALF;
}
wire_ctx_skip(wire, size);
/* Read rtype, rclass and RR count. */
uint16_t type = wire_ctx_read_u16(wire);
uint16_t rclass = wire_ctx_read_u16(wire);
uint16_t count = wire_ctx_read_u16(wire);
if (wire->error != KNOT_EOK) {
return wire->error;
}
knot_rrset_init(rrset, owner, type, rclass);
/* Read rdata items. */
for (uint16_t i = 0; i < count; i++) {
uint32_t ttl = wire_ctx_read_u32(wire);
uint32_t rdata_size = wire_ctx_read_u32(wire);
if (wire->error != KNOT_EOK ||
wire_ctx_available(wire) < rdata_size ||
knot_rrset_add_rdata(rrset, wire->position, rdata_size,
ttl, NULL) != KNOT_EOK) {
knot_rrset_clear(rrset, NULL);
return KNOT_EMALF;
}
wire_ctx_skip(wire, rdata_size);
}
return wire->error;
}
int changeset_binary_size(const changeset_t *chgset, size_t *size)
......@@ -132,7 +206,7 @@ int rrset_serialize(const knot_rrset_t *rrset, uint8_t *stream, size_t *size)
}
int rrset_deserialize(const uint8_t *stream, size_t *stream_size,
knot_rrset_t *rrset)
knot_rrset_t *rrset)
{
if (stream == NULL || stream_size == NULL ||
rrset == NULL) {
......@@ -190,3 +264,205 @@ int rrset_deserialize(const uint8_t *stream, size_t *stream_size,
return KNOT_EOK;
}
size_t changeset_serialized_size(const changeset_t *ch)
{
if (ch == NULL) {
return 0;
}
size_t soa_from_size = rrset_binary_size(ch->soa_from);
size_t soa_to_size = rrset_binary_size(ch->soa_to);
changeset_iter_t it;
changeset_iter_all(&it, ch);
size_t change_size = 0;
knot_rrset_t rrset = changeset_iter_next(&it);
while (!knot_rrset_empty(&rrset)) {
change_size += rrset_binary_size(&rrset);
rrset = changeset_iter_next(&it);
}
changeset_iter_clear(&it);
return soa_from_size + soa_to_size + change_size;
}
int serialize_rrset_chunks(wire_ctx_t *wire, const knot_rrset_t *rrset, uint8_t *dst_chunks[], size_t chunk_size, int chunks_count, size_t *chunks_real_sizes, int *cur_chunk)
{
if (wire == NULL || chunks_real_sizes == NULL || cur_chunk == NULL || *cur_chunk < 0) return KNOT_EINVAL;
while (wire_ctx_available(wire) < rrset_binary_size(rrset)) {
chunks_real_sizes[*cur_chunk] = wire_ctx_offset(wire);
if (*cur_chunk >= chunks_count - 1) {
return KNOT_ESPACE;
}
// move to next chunk
if (wire->error != KNOT_EOK) {
return wire->error;
}
(*cur_chunk)++;
*wire = wire_ctx_init(dst_chunks[*cur_chunk], chunk_size);
}
return serialize_rrset(wire, rrset);
}
/*!
* \brief Serializes given changeset into chunked area.
*
* \param ch The changeset; dst_chunks The chunks to serialize into; chunk_size Maximum size of each chunk; chunks_count Maximum number of used chunks
* \param chunks_real_sizes Output: real size of each chunk after serialization, or zeros for unused chunks
* \param chunks_real_count Output: real # of chunks after serialization. Can be wrong if error returned!
*
* \retval KNOT_E*
*/