Journal: re-implemented completely, LMDB, all zones in one db

Knot.files

@@ -258,6 +258,10 @@ src/knot/events/handlers/refresh.c
 src/knot/events/handlers/update.c
 src/knot/events/replan.c
 src/knot/events/replan.h
+src/knot/journal/serialization.c
+src/knot/journal/serialization.h
+src/knot/journal/journal.c
+src/knot/journal/journal.h
 src/knot/modules/dnsproxy/dnsproxy.c
 src/knot/modules/dnsproxy/dnsproxy.h
 src/knot/modules/dnstap/dnstap.c
@@ -312,10 +316,6 @@ src/knot/query/requestor.c
 src/knot/query/requestor.h
 src/knot/server/dthreads.c
 src/knot/server/dthreads.h
-src/knot/server/journal.c
-src/knot/server/journal.h
-src/knot/server/serialization.c
-src/knot/server/serialization.h
 src/knot/server/server.c
 src/knot/server/server.h
 src/knot/server/tcp-handler.c
@@ -562,7 +562,7 @@ tests/contrib/test_wire_ctx.c
 tests/dthreads.c
 tests/fake_server.h
 tests/fdset.c
-tests/journal.c
+tests/journal_lmdb.c
 tests/libknot/test_control.c
 tests/libknot/test_cookies-client.c
 tests/libknot/test_cookies-opt.c

doc/man/knot.conf.5in

@@ -871,9 +871,17 @@ This option has no effect with enabled
 \fIDefault:\fP off
 .SS max\-journal\-size
 .sp
-Maximum size of the zone journal file.
+Maximum size of the journal DB.
 .sp
-\fIDefault:\fP 2^64
+\fIDefault:\fP 1 GiB
+.sp
+\fBNOTE:\fP
+.INDENT 0.0
+.INDENT 3.5
+Decreasing this value will lead to discarding
+whole journal history of all zones.
+.UNINDENT
+.UNINDENT
 .SS max\-zone\-size
 .sp
 Maximum size of the zone. The size is measured as size of the zone records

doc/reference.rst

@@ -1013,9 +1013,13 @@ is a master server for the zone.
 max-journal-size
 ----------------
 
-Maximum size of the zone journal file.
+Maximum size of the journal DB.
 
-*Default:* 2^64
+*Default:* 1 GiB
+
+.. NOTE::
+   Decreasing this value will lead to discarding
+   whole journal history of all zones.
 
 .. _zone_max_zone_size:
 

src/Makefile.am

@@ -335,10 +335,10 @@ libknotd_la_SOURCES =				\
 	knot/common/stats.h			\
 	knot/server/dthreads.c			\
 	knot/server/dthreads.h			\
-	knot/server/journal.c			\
-	knot/server/journal.h			\
-	knot/server/serialization.c		\
-	knot/server/serialization.h		\
+	knot/journal/journal.c			\
+	knot/journal/journal.h			\
+	knot/journal/serialization.c		\
+	knot/journal/serialization.h		\
 	knot/server/server.c			\
 	knot/server/server.h			\
 	knot/server/tcp-handler.c		\

src/knot/conf/conf.c

@@ -1048,22 +1048,19 @@ char* conf_zonefile_txn(
 
 char* conf_journalfile_txn(
 	conf_t *conf,
-	knot_db_txn_t *txn,
-	const knot_dname_t *zone)
+	knot_db_txn_t *txn)
 {
-	if (zone == NULL) {
-		return NULL;
-	}
+	conf_val_t val;
 
-	conf_val_t val = conf_zone_get_txn(conf, txn, C_JOURNAL, zone);
-	const char *journal = conf_str(&val);
+	val = conf_default_get_txn(conf, txn, C_STORAGE);
+	char *storage = conf_abs_path(&val, NULL);
 
-	// Use default journalfile name pattern if not specified.
-	if (journal == NULL) {
-		journal = "%s.db";
-	}
+	val = conf_default_get_txn(conf, txn, C_JOURNAL);
+
+	char *journaldir = conf_abs_path(&val, storage);
+	free(storage);
 
-	return get_filename(conf, txn, zone, journal);
+	return journaldir;
 }
 
 size_t conf_udp_threads_txn(

src/knot/conf/conf.h

@@ -584,16 +584,12 @@ static inline char* conf_zonefile(
  *
  * \return Absolute journal file path string pointer.
  */
-char* conf_journalfile_txn(
-	conf_t *conf,
-	knot_db_txn_t *txn,
-	const knot_dname_t *zone
-);
+char* conf_journalfile_txn(conf_t *conf,
+	knot_db_txn_t *txn);
 static inline char* conf_journalfile(
-	conf_t *conf,
-	const knot_dname_t *zone)
+	conf_t *conf)
 {
-	return conf_journalfile_txn(conf, &conf->read_txn, zone);
+	return conf_journalfile_txn(conf, &conf->read_txn);
 }
 
 /*!

src/knot/conf/scheme.c

@@ -225,10 +225,14 @@ static const yp_item_t desc_remote[] = {
 	{ NULL }
 };
 
+
+#define VIRT_MEM_TOP (2LLU * 1024 * 1204 * 1204)
+#define VIRT_MEM_LIMIT(x) (((sizeof(void *) < 8) && ((x) > VIRT_MEM_TOP)) ? VIRT_MEM_TOP : (x))
+
+
 #define ZONE_ITEMS(FLAGS) \
 	{ C_STORAGE,             YP_TSTR,  YP_VSTR = { STORAGE_DIR }, FLAGS }, \
 	{ C_FILE,                YP_TSTR,  YP_VNONE, FLAGS }, \
-	{ C_JOURNAL,             YP_TSTR,  YP_VNONE, FLAGS }, \
 	{ C_MASTER,              YP_TREF,  YP_VREF = { C_RMT }, YP_FMULTI, { check_ref } }, \
 	{ C_DDNS_MASTER,         YP_TREF,  YP_VREF = { C_RMT }, YP_FNONE, { check_ref } }, \
 	{ C_NOTIFY,              YP_TREF,  YP_VREF = { C_RMT }, YP_FMULTI, { check_ref } }, \
@@ -237,8 +241,6 @@ static const yp_item_t desc_remote[] = {
 	{ C_DISABLE_ANY,         YP_TBOOL, YP_VNONE }, \
 	{ C_ZONEFILE_SYNC,       YP_TINT,  YP_VINT = { -1, INT32_MAX, 0, YP_STIME } }, \
 	{ C_IXFR_DIFF,           YP_TBOOL, YP_VNONE }, \
-	{ C_MAX_JOURNAL_SIZE,    YP_TINT,  YP_VINT = { 0, INT64_MAX, INT64_MAX, YP_SSIZE }, \
-	                                   FLAGS }, \
 	{ C_MAX_ZONE_SIZE,       YP_TINT,  YP_VINT = { 0, INT64_MAX, INT64_MAX, YP_SSIZE }, \
 	                                   FLAGS }, \
 	{ C_KASP_DB,             YP_TSTR,  YP_VSTR = { "keys" }, FLAGS }, \
@@ -256,6 +258,9 @@ static const yp_item_t desc_template[] = {
 	{ C_TIMER_DB,            YP_TSTR,  YP_VSTR = { "timers" }, CONF_IO_FRLD_ZONES }, \
 	{ C_GLOBAL_MODULE,       YP_TDATA, YP_VDATA = { 0, NULL, mod_id_to_bin, mod_id_to_txt }, \
 	                                   YP_FMULTI | CONF_IO_FRLD_MOD, { check_modref } }, \
+	{ C_JOURNAL,             YP_TSTR,  YP_VSTR = { "journal.db" }, CONF_IO_FRLD_ZONES }, \
+	{ C_MAX_JOURNAL_SIZE,    YP_TINT,  YP_VINT = { 1024 * 1024, VIRT_MEM_LIMIT(100LLU * 1024 * 1024 * 1024 * 1024), \
+	                                               VIRT_MEM_LIMIT(20LLU * 1024 * 1024 * 1024), YP_SSIZE } }, \
 	{ NULL }
 };
 

src/knot/conf/tools.c

@@ -514,6 +514,24 @@ int check_template(
 		return KNOT_EINVAL;
 	}
 
+	// Check journal.
+	conf_val_t journal = conf_rawid_get_txn(args->conf, args->txn, C_TPL,
+	                                        C_JOURNAL, args->id, args->id_len);
+
+	if (journal.code == KNOT_EOK) {
+		args->err_str = "journal location in non-default template";
+		return KNOT_EINVAL;
+	}
+
+	// Check max-journal-size.
+	conf_val_t max_journal_size = conf_rawid_get_txn(args->conf, args->txn, C_TPL,
+	                                                 C_MAX_JOURNAL_SIZE, args->id, args->id_len);
+
+	if (max_journal_size.code == KNOT_EOK) {
+		args->err_str = "journal size in non-default template";
+		return KNOT_EINVAL;
+	}
+
 	return KNOT_EOK;
 }
 

src/knot/ctl/commands.c

@@ -930,9 +930,9 @@ static int zone_purge(zone_t *zone, ctl_args_t *args)
 	free(zonefile);
 
 	// Purge the zone journal.
-	char *journalfile = conf_journalfile(conf(), zone->name);
-	(void)unlink(journalfile);
-	free(journalfile);
+	if (journal_open(zone->journal, zone->journal_db, zone->name) == KNOT_EOK) {
+		(void)scrape_journal(zone->journal);
+	}
 
 	// Purge the zone timers.
 	(void)remove_timer_db(args->server->timers_db, args->server->zone_db,

src/knot/journal/journal.h

+/*  Copyright (C) 2016 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <pthread.h>
+
+#include "libknot/libknot.h"
+#include "contrib/ucw/lists.h"
+#include "knot/updates/changesets.h"
+#include "knot/journal/serialization.h"
+
+/*!
+ * \brief j->fslimit special value to open with minimal possible mapsize
+ *
+ * ...it is equal to the actual DB file size.
+ * Beware of using this value for the first time initialized DB !
+ * It is mostly useful for read only access
+ */
+#define KNOT_JOURNAL_FSLIMIT_SAMEASLAST (400 * 1024)
+
+typedef struct {
+	knot_db_t *db;
+	const knot_db_api_t *db_api;
+	char *path;
+	size_t fslimit;
+	pthread_mutex_t db_mutex; // please delete this once you move DB opening from journal_open to db_init
+	// common metadata: last_inserter_zone, last_total_occupied, journal_count
+} journal_db_t;
+
+typedef struct {
+	uint32_t first_serial;		// serial_from of the first changeset
+	uint32_t last_serial;		// serial_from of the last changeset
+	uint32_t last_serial_to;	// serial_to of the last changeset
+	uint32_t last_flushed;		// serial_from of the last flushed (or merged) chengeset
+	uint32_t merged_serial;		// "serial_from" of merged changeset
+	uint32_t dirty_serial;		// serial_from of an incompletely inserted changeset which shall be deleted (see DB_MAX_INSERT_TXN)
+	uint32_t flags;			// LAST_FLUSHED_VALID, SERIAL_TO_VALID, MERGED_SERIAL_VALID
+	// specific metadata: occupied
+} journal_metadata_t;
+
+typedef struct {
+	journal_db_t *db;
+	knot_dname_t *zone;
+	//journal_metadata_t md;
+} journal_t;
+
+enum {
+	KNOT_JOURNAL_CHECK_SILENT = 0,
+	KNOT_JOURNAL_CHECK_WARN = 1,
+	KNOT_JOURNAL_CHECK_INFO = 2
+};
+
+/*!
+ * \brief Allocate a new journal structure.
+ *
+ * \retval new journal instance if successful.
+ * \retval NULL on error.
+ */
+journal_t *journal_new(void);
+
+/*!
+ * \brief Free a journal structure.
+ *
+ * \param journal  A journal structure to free.
+ */
+void journal_free(journal_t **journal);
+
+/*!
+ * \brief Open journal.
+ *
+ * \param j          Journal struct to use.
+ * \param db         Shared journal database
+ * \param zone_name  Name of the zone this journal belongs to.
+ *
+ * \retval KNOT_EOK on success.
+ * \return < KNOT_EOK on other errors.
+ */
+int journal_open(journal_t *j, journal_db_t **db, const knot_dname_t *zone_name);
+
+/*!
+ * \brief Close journal.
+ *
+ * \param journal  Journal to close.
+ */
+void journal_close(journal_t *journal);
+
+/*!
+ * \brief Initialize shared journal DB file. The DB will be open on first use.
+ *
+ * \param db            Database to be initialized. Must be (*db == NULL) before!
+ * \param lmdb_dir_path Path to the directory with DB
+ * \param lmdb_fslimit  Maximum size of DB data file
+ *
+ * \return KNOT_E*
+ */
+int init_journal_db(journal_db_t **db, const char *lmdb_dir_path, size_t lmdb_fslimit);
+
+/*!
+ * \brief Close shared journal DB file.
+ *
+ * \param db DB to close.
+ */
+void close_journal_db(journal_db_t **db);
+
+/*!
+ * \brief Load changesets from journal.
+ *
+ * \param journal  Journal to load from.
+ * \param dst      Store changesets here.
+ * \param from     Start serial.
+ *
+ * \retval KNOT_EOK on success.
+ * \retval KNOT_ENOENT when the lookup of the first entry fails.
+ * \return < KNOT_EOK on other error.
+ */
+int journal_load_changesets(journal_t *journal, list_t *dst, uint32_t from);
+
+/*!
+ * \brief Store changesets in journal.
+ *
+ * \param journal  Journal to store in.
+ * \param src      Changesets to store.
+ *
+ * \retval KNOT_EOK on success.
+ * \retval KNOT_EBUSY when full, asking zone to flush itself to zonefile
+ *                    to allow cleaning up history and freeing up space
+ * \retval KNOT_ESPACE when full and not able to free up any space
+ * \return < KNOT_EOK on other errors.
+ */
+int journal_store_changesets(journal_t *journal, list_t *src);
+
+/*!
+ * \brief Store changesets in journal.
+ *
+ * \param journal  Journal to store in.
+ * \param change   Changeset to store.
+ *
+ * \retval (same as for journal_store_changesets())
+ */
+int journal_store_changeset(journal_t *journal, changeset_t *change);
+
+/*!
+ * \brief Check if this (zone's) journal is present in shared journal DB.
+ *
+ * \param db        Shared journal DB
+ * \param zone_name Name of the zone of the journal in question
+ *
+ * \return true or false
+ */
+bool journal_exists(journal_db_t **db, knot_dname_t *zone_name);
+
+/*! \brief Tell the journal that zone has been flushed.
+ *
+ * \param journal  Journal to flush.
+ *
+ * \return KNOT_E*
+ */
+int journal_flush(journal_t *journal);
+
+/*! \brief Remove completely this (zone's) journal from shared journal DB.
+ *
+ * This must be called with opened journal.
+ *
+ * \param j Journal to be deleted
+ *
+ * \return KNOT_E*
+ */
+int scrape_journal(journal_t *j);
+
+/*! \brief Obtain public information from journal metadata
+ *
+ * \param[in]  j           Journal
+ * \param[out] is_empty    1 if j contains no changesets
+ * \param[out] serial_from [if !is_empty] starting serial of changesets history
+ * \param[out] serial_to   [if !is_empty] ending serial of changesets history
+ */
+void journal_metadata_info(journal_t *j, int *is_empty, uint32_t *serial_from, uint32_t *serial_to);
+
+/*!
+ * \brief List the zones contained in journal DB.
+ *
+ * \param db[in]        Shared journal DB
+ * \param zones[out]    List of strings (char *) of zone names
+ *
+ * \return KNOT_EOK    ok
+ * \retval KNOT_ENOMEM no zones found
+ * \retval KNOT_EMALF  different # of zones found than expected
+ * \retval KNOT_E*     other error
+ */
+int journal_db_list_zones(journal_db_t **db, list_t *zones);
+
+/*! \brief Check the journal consistency, errors to stderr.
+ *
+ * \param journal  Journal to check.
+ * \param warn_level SILENT: no logging, just curious for return value; WARN: log journal inconsistencies; INFO: log journal state
+ *
+ * \return KNOT_E*
+ */
+int journal_check(journal_t *j, int warn_level);
+
+/*! @} */

src/knot/server/serialization.c → src/knot/journal/serialization.c

-/*  Copyright (C) 2014 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz>
+/*  Copyright (C) 2016 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz>
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -16,8 +16,9 @@
 
 #include <assert.h>
 
-#include "knot/server/serialization.h"
+#include "knot/journal/serialization.h"
 #include "libknot/libknot.h"
+#include "contrib/wire_ctx.h"
 
 static size_t rr_binary_size(const knot_rrset_t *rrset, size_t rdata_pos)
 {
@@ -35,11 +36,11 @@ static uint64_t rrset_binary_size(const knot_rrset_t *rrset)
 	if (rrset == NULL || rrset->rrs.rr_count == 0) {
 		return 0;
 	}
-	uint64_t size = sizeof(uint64_t) + // size at the beginning
-	              knot_dname_size(rrset->owner) + // owner data
-	              sizeof(uint16_t) + // type
-	              sizeof(uint16_t) + // class
-	              sizeof(uint16_t);  //RR count
+	uint64_t size = /* sizeof(uint64_t) + // size at the beginning */
+		      knot_dname_size(rrset->owner) + // owner data
+		      sizeof(uint16_t) + // type
+		      sizeof(uint16_t) + // class
+		      sizeof(uint16_t);  //RR count
 	uint16_t rdata_count = rrset->rrs.rr_count;
 	for (uint16_t i = 0; i < rdata_count; i++) {
 		/* Space to store length of one RR. */
@@ -52,7 +53,7 @@ static uint64_t rrset_binary_size(const knot_rrset_t *rrset)
 }
 
 static void serialize_rr(const knot_rrset_t *rrset, size_t rdata_pos,
-                         uint8_t *stream)
+			 uint8_t *stream)
 {
 	const knot_rdata_t *rr = knot_rdataset_at(&rrset->rrs, rdata_pos);
 	assert(rr);
@@ -66,7 +67,80 @@ static int deserialize_rr(knot_rrset_t *rrset, const uint8_t *stream, uint32_t r
 	uint32_t ttl;
 	memcpy(&ttl, stream, sizeof(uint32_t));
 	return knot_rrset_add_rdata(rrset, stream + sizeof(uint32_t),
-	                         rdata_size - sizeof(uint32_t), ttl, NULL);
+				 rdata_size - sizeof(uint32_t), ttl, NULL);
+}
+
+static int serialize_rrset(wire_ctx_t *wire, const knot_rrset_t *rrset)
+{
+	assert(wire);
+	assert(rrset);
+
+	/* Write owner. */
+	int size = knot_dname_to_wire(wire->position, rrset->owner,
+				      wire_ctx_available(wire));
+	if (size < 0) {
+		return size;
+	}
+	wire_ctx_skip(wire, size);
+
+	/* Write rtype, rclass and RR count. */
+	wire_ctx_write_u16(wire, rrset->type);
+	wire_ctx_write_u16(wire, rrset->rclass);
+	wire_ctx_write_u16(wire, rrset->rrs.rr_count);
+
+	/* Write rdata items. */
+	for (uint16_t i = 0; i < rrset->rrs.rr_count; i++) {
+		const knot_rdata_t *rr = knot_rdataset_at(&rrset->rrs, i);
+		assert(rr != NULL);
+		wire_ctx_write_u32(wire, knot_rdata_ttl(rr));
+		wire_ctx_write_u32(wire, knot_rdata_rdlen(rr));
+		wire_ctx_write(wire, knot_rdata_data(rr), knot_rdata_rdlen(rr));
+	}
+
+	return wire->error;
+}
+
+static int deserialize_rrset(wire_ctx_t *wire, knot_rrset_t *rrset)
+{
+	assert(wire);
+	assert(rrset);
+
+	/* Read owner. */
+	int size = knot_dname_size(wire->position);
+	if (size < 0) {
+		return size;
+	}
+	knot_dname_t *owner = knot_dname_copy_part(wire->position, size, NULL);
+	if (owner == NULL) {
+		return KNOT_EMALF;
+	}
+	wire_ctx_skip(wire, size);
+
+	/* Read rtype, rclass and RR count. */
+	uint16_t type = wire_ctx_read_u16(wire);
+	uint16_t rclass = wire_ctx_read_u16(wire);
+	uint16_t count = wire_ctx_read_u16(wire);
+	if (wire->error != KNOT_EOK) {
+		return wire->error;
+	}
+
+	knot_rrset_init(rrset, owner, type, rclass);
+
+	/* Read rdata items. */
+	for (uint16_t i = 0; i < count; i++) {
+		uint32_t ttl = wire_ctx_read_u32(wire);
+		uint32_t rdata_size = wire_ctx_read_u32(wire);
+		if (wire->error != KNOT_EOK ||
+		    wire_ctx_available(wire) < rdata_size ||
+		    knot_rrset_add_rdata(rrset, wire->position, rdata_size,
+					 ttl, NULL) != KNOT_EOK) {
+			knot_rrset_clear(rrset, NULL);
+			return KNOT_EMALF;
+		}
+		wire_ctx_skip(wire, rdata_size);
+	}
+
+	return wire->error;
 }
 
 int changeset_binary_size(const changeset_t *chgset, size_t *size)
@@ -132,7 +206,7 @@ int rrset_serialize(const knot_rrset_t *rrset, uint8_t *stream, size_t *size)
 }
 
 int rrset_deserialize(const uint8_t *stream, size_t *stream_size,
-                      knot_rrset_t *rrset)
+		      knot_rrset_t *rrset)
 {
 	if (stream == NULL || stream_size == NULL ||
 	    rrset == NULL) {
@@ -190,3 +264,205 @@ int rrset_deserialize(const uint8_t *stream, size_t *stream_size,
 
 	return KNOT_EOK;
 }
+
+size_t changeset_serialized_size(const changeset_t *ch)
+{
+	if (ch == NULL) {
+		return 0;
+	}
+
+	size_t soa_from_size = rrset_binary_size(ch->soa_from);
+	size_t soa_to_size = rrset_binary_size(ch->soa_to);
+
+	changeset_iter_t it;
+	changeset_iter_all(&it, ch);
+
+	size_t change_size = 0;
+	knot_rrset_t rrset = changeset_iter_next(&it);
+	while (!knot_rrset_empty(&rrset)) {
+		change_size += rrset_binary_size(&rrset);
+		rrset = changeset_iter_next(&it);
+	}
+
+	changeset_iter_clear(&it);
+
+	return soa_from_size + soa_to_size + change_size;
+}
+
+int serialize_rrset_chunks(wire_ctx_t *wire, const knot_rrset_t *rrset, uint8_t *dst_chunks[], size_t chunk_size, int chunks_count, size_t *chunks_real_sizes, int *cur_chunk)
+{
+	if (wire == NULL || chunks_real_sizes == NULL || cur_chunk == NULL || *cur_chunk < 0) return KNOT_EINVAL;
+
+	while (wire_ctx_available(wire) < rrset_binary_size(rrset)) {
+		chunks_real_sizes[*cur_chunk] = wire_ctx_offset(wire);
+		if (*cur_chunk >= chunks_count - 1) {
+			return KNOT_ESPACE;
+		}
+		// move to next chunk
+		if (wire->error != KNOT_EOK) {
+			return wire->error;
+		}
+		(*cur_chunk)++;
+		*wire = wire_ctx_init(dst_chunks[*cur_chunk], chunk_size);
+	}
+
+	return serialize_rrset(wire, rrset);
+}
+
+/*!
+ * \brief Serializes given changeset into chunked area.
+ *
+ * \param ch The changeset; dst_chunks The chunks to serialize into; chunk_size Maximum size of each chunk; chunks_count Maximum number of used chunks
+ * \param chunks_real_sizes Output: real size of each chunk after serialization, or zeros for unused chunks
+ * \param chunks_real_count Output: real # of chunks after serialization. Can be wrong if error returned!
+ *
+ * \retval KNOT_E*
+ */