Merge remote-tracking branch 'origin/mem-estim'

Conflicts:
	doc/running.texi

Knot.files

@@ -281,3 +281,5 @@ src/zscanner/test/processing.h
 src/zscanner/test/tests.c
 src/zscanner/test/tests.h
 src/zscanner/test/zscanner-tool.c
+src/knot/zone/estimator.c
+src/knot/zone/estimator.h

doc/running.texi

@@ -63,6 +63,7 @@ Actions:
  zonestatus                 Show status of configured zones.
  checkconf                  Check current server configuration.
  checkzone [zone]           Check zone (all if not specified).
+ memstats [zone]            Estimate memory consumption for zone (all if not specified).
 @end example
 
 Also, the server needs to create several files in order to run properly.
@@ -104,8 +105,9 @@ $ knotd -c slave.conf
 
 To start it as a daemon, just add a @code{-d} parameter. Unlike the foreground mode,
 PID file will be created in @code{rundir} directory.
+
 @example
-$ knotd -d -c slave.conf start # start the daemon
+$ knotd -d -c slave.conf # start the daemon
 $ knotc -c slave.conf stop # stop the daemon
 @end example
 
@@ -120,6 +122,16 @@ you can use @code{knotc checkzone} action.
 $ knotc -c master.conf checkzone example.com
 @end example
 
+For an approximate estimate of server's memory consumption, you can use the @code{knotc memstats} action.
+This action prints count of resource records, percentage of signed records and finally estimation
+of memory consumption for each zone, unless specified otherwise. Please note that estimated values might
+differ from the actual consumption. Also, for slave servers with incoming transfers enabled,
+be aware that the actual memory consumption might be double or more during transfers.
+
+@example
+$ knotc -c master.conf memstats example.com
+@end example
+
 Starting and stopping the daemon is the same as with the slave server in the previous section.
 
 @node Controlling running daemon
@@ -143,7 +155,7 @@ If you encounter a bug using this feature, please send it to Knot developers (@p
 You can also choose to tear-down the server fully and restart with the @code{knotc restart} action.
 Note that some actions like start, stop and restart cannot be done remotely.
 @example
-$ knotc -c master.conf running # check if running
+$ knotc -c master.conf status # check if running
 $ knotc -c master.conf restart # fully restart
 @end example
 

man/knotc.8in

@@ -68,6 +68,9 @@ Check server configuration.
 .TP
 checkzone
 Check zones before compiling (accepts specific zones, f.e. 'knotc checkzone example1.com example2.com').
+.TP
+memstats
+Estimate memory consumption for zone files. Useful mainly for big zones.
 .SS "EXAMPLES"
 .TP
 .B Setup a keyfile for remote control

src/Makefile.am

@@ -285,6 +285,8 @@ libknotd_la_SOURCES =				\
 	knot/zone/zone-dump.c			\
 	knot/zone/zone-load.h			\
 	knot/zone/zone-load.c			\
+	knot/zone/estimator.h			\
+	knot/zone/estimator.c			\
 	knot/server/server.h
 
 zscanner_tool_SOURCES =				\

src/common/hattrie/ahtable.h

@@ -57,7 +57,7 @@ size_t     ahtable_size   (const ahtable_t*); // Number of stored keys.
 
 
 /** Find the given key in the table, inserting it if it does not exist, and
- * returning a pointer to it's key.
+ * returning a pointer to its key.
  *
  * This pointer is not guaranteed to be valid after additional calls to
  * ahtable_get, ahtable_del, ahtable_clear, or other functions that modifies the

src/common/hattrie/hat-trie.c

@@ -382,11 +382,30 @@ static void node_apply(node_ptr node, void (*f)(value_t*,void*), void* d)
     }
 }
 
+static void node_apply_ahtable(node_ptr node, void (*f)(void*,void*), void* d)
+{
+    if (*node.flag & NODE_TYPE_TRIE) {
+        size_t i;
+        for (i = 0; i < NODE_CHILDS; ++i) {
+            if (i > 0 && node.t->xs[i].t == node.t->xs[i - 1].t) continue;
+            if (node.t->xs[i].t) node_apply_ahtable(node.t->xs[i], f, d);
+        }
+    }
+    else {
+	    f(node.b, d);
+	}
+}
+
 void hattrie_apply_rev(hattrie_t* T, void (*f)(value_t*,void*), void* d)
 {
 	node_apply(T->root, f, d);
 }
 
+void hattrie_apply_rev_ahtable(hattrie_t* T, void (*f)(void*,void*), void* d)
+{
+	node_apply_ahtable(T->root, f, d);
+}
+
 int hattrie_split_mid(node_ptr node, unsigned *left_m, unsigned *right_m)
 {
     /* count the number of occourances of every leading character */

src/common/hattrie/hat-trie.h

@@ -54,6 +54,7 @@ hattrie_t* hattrie_dup (const hattrie_t*, value_t (*nval)(value_t));
 void hattrie_build_index (hattrie_t*);
 
 void hattrie_apply_rev (hattrie_t*, void (*f)(value_t*,void*), void* d);
+void hattrie_apply_rev_ahtable(hattrie_t* T, void (*f)(void*,void*), void* d);
 
 /** Find the given key in the trie, inserting it if it does not exist, and
  * returning a pointer to it's key.

src/knot/ctl/knotc_main.c

@@ -44,6 +44,7 @@
 #include "libknot/packet/query.h"
 #include "libknot/packet/response.h"
 #include "knot/zone/zone-load.h"
+#include "knot/zone/estimator.h"
 
 /*! \brief Controller flags. */
 enum knotc_flag_t {
@@ -86,6 +87,7 @@ static int cmd_status(int argc, char *argv[], unsigned flags);
 static int cmd_zonestatus(int argc, char *argv[], unsigned flags);
 static int cmd_checkconf(int argc, char *argv[], unsigned flags);
 static int cmd_checkzone(int argc, char *argv[], unsigned flags);
+static int cmd_memstats(int argc, char *argv[], unsigned flags);
 
 /*! \brief Table of remote commands. */
 knot_cmd_t knot_cmd_tbl[] = {
@@ -98,6 +100,7 @@ knot_cmd_t knot_cmd_tbl[] = {
 	{&cmd_zonestatus, 0, "zonestatus", "",       "\tShow status of configured zones."},
 	{&cmd_checkconf,  1, "checkconf",  "",       "\tCheck current server configuration."},
 	{&cmd_checkzone,  1, "checkzone",  "[zone]", "Check zone (all if not specified)."},
+	{&cmd_memstats,  1, "memstats",  "[zone]", "Estimate memory use for zone (all if not specified)."},
 	{NULL, 0, NULL, NULL, NULL}
 };
 
@@ -712,9 +715,123 @@ static int cmd_checkzone(int argc, char *argv[], unsigned flags)
 
 		knot_zone_deep_free(&z);
 		knot_zload_close(l);
-
 		log_zone_info("Zone %s OK.\n", zone->name);
 	}
 
 	return rc;
 }
+
+static int cmd_memstats(int argc, char *argv[], unsigned flags)
+{
+	UNUSED(flags);
+
+	/* Zone checking */
+	int rc = 0;
+	node *n = 0;
+	size_t total_size = 0;
+
+	/* Generate databases for all zones */
+	WALK_LIST(n, conf()->zones) {
+		/* Fetch zone */
+		conf_zone_t *zone = (conf_zone_t *) n;
+		int zone_match = 0;
+		for (unsigned i = 0; i < (unsigned)argc; ++i) {
+			size_t len = strlen(zone->name);
+
+			/* All (except root) without final dot */
+			if (len > 1) {
+				len -= 1;
+			}
+			if (strncmp(zone->name, argv[i], len) == 0) {
+				zone_match = 1;
+				break;
+			}
+		}
+
+		if (!zone_match && argc > 0) {
+			/* WALK_LIST is a for-cycle. */
+			continue;
+		}
+
+		/* Init malloc wrapper for trie size estimation. */
+		size_t malloc_size = 0;
+		mm_ctx_t mem_ctx = { .ctx = &malloc_size,
+		                     .alloc = estimator_malloc,
+		                     .free = estimator_free };
+
+		/* Init memory estimation context. */
+		zone_estim_t est = {.node_table = hattrie_create_n(TRIE_BUCKET_SIZE, &mem_ctx),
+		                    .dname_table = hattrie_create_n(TRIE_BUCKET_SIZE, &mem_ctx),
+		                    .dname_size = 0, .rrset_size = 0,
+		                    .node_size = 0, .ahtable_size = 0,
+		                    .rdata_size = 0, .record_count = 0 };
+		if (est.node_table == NULL) {
+			if (est.dname_table) {
+				hattrie_free(est.dname_table);
+			}
+			log_server_error("Not enough memory.\n");
+			continue;
+		}
+		if (est.dname_table == NULL) {
+			if (est.node_table) {
+				hattrie_free(est.node_table);
+			}
+			log_server_error("Not enough memory.\n");
+			continue;
+		}
+
+		/* Create file loader. */
+		file_loader_t *loader = file_loader_create(zone->file, zone->name,
+		                                           KNOT_CLASS_IN, 3600,
+		                                           estimator_rrset_memsize_wrap,
+		                                           process_error,
+		                                           &est);
+		if (loader == NULL) {
+			log_zone_error("Could not load zone.\n");
+			hattrie_apply_rev(est.node_table, estimator_free_trie_node, NULL);
+			hattrie_apply_rev(est.dname_table, estimator_free_trie_node, NULL);
+			hattrie_free(est.node_table);
+			hattrie_free(est.dname_table);
+			return KNOT_ERROR;
+		}
+
+		/* Do a parser run, but do not actually create the zone. */
+		int ret = file_loader_process(loader);
+		if (ret != KNOT_EOK) {
+			log_zone_error("Failed to parse zone.\n");
+			hattrie_apply_rev(est.node_table, estimator_free_trie_node, NULL);
+			hattrie_apply_rev(est.dname_table, estimator_free_trie_node, NULL);
+			hattrie_free(est.node_table);
+			hattrie_free(est.dname_table);
+			return KNOT_ERROR;
+		}
+
+		/* Only size of ahtables inside trie's nodes is missing. */
+		assert(est.ahtable_size == 0);
+		est.ahtable_size = estimator_trie_ahtable_memsize(est.node_table);
+
+		/* Cleanup */
+		hattrie_apply_rev(est.node_table, estimator_free_trie_node, NULL);
+		hattrie_apply_rev(est.dname_table, estimator_free_trie_node, NULL);
+		hattrie_free(est.node_table);
+		hattrie_free(est.dname_table);
+
+		size_t zone_size = (size_t)(((double)(est.rdata_size +
+		                   est.node_size +
+		                   est.rrset_size +
+		                   est.dname_size +
+		                   est.ahtable_size +
+		                   malloc_size) * ESTIMATE_MAGIC) / (1024.0 * 1024.0));
+
+		log_zone_info("Zone %s: %zu RRs, used memory estimation is %zuMB.\n",
+		              zone->name, est.record_count, zone_size);
+		file_loader_free(loader);
+		total_size += zone_size;
+	}
+
+	if (argc == 0) { // for all zones
+		log_zone_info("Estimated memory consumption for all zones is %zuMB.\n", total_size);
+	}
+
+	return rc;
+}

src/knot/zone/estimator.c

+/*  Copyright (C) 2011 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <config.h>
+#include <assert.h>
+
+#include "estimator.h"
+#include "dname.h"
+#include "common/lists.h"
+#include "libknot/zone/node.h"
+#include "common/hattrie/ahtable.h"
+#include "zscanner/scanner.h"
+#include "common/descriptor.h"
+
+// Constants used for tweaking, mostly malloc overhead
+enum estim_consts {
+	MALLOC_OVER = sizeof(size_t), // according to malloc.c, this is minimum
+	DNAME_MULT = 1,
+	DNAME_ADD = MALLOC_OVER * 3, // dname itself, labels, name
+	RDATA_MULT = 1,
+	RDATA_ADD = MALLOC_OVER, // just raw rdata, but allocation is there
+	RRSET_MULT = 1,
+	RRSET_ADD = MALLOC_OVER * 3, // rrset itself, rdata array, indices
+	NODE_MULT = 1,
+	NODE_ADD = MALLOC_OVER * 2, // node itself, rrset array
+	AHTABLE_ADD = MALLOC_OVER * 3, // table, slots, slot sizes
+	MALLOC_MIN = 3 * MALLOC_OVER // minimun size of malloc'd chunk, -overhead
+};
+
+typedef struct type_list_item {
+	node n;
+	uint16_t type;
+} type_list_item_t;
+
+typedef struct dummy_node {
+	// For now only contains list of RR types
+	list node_list;
+} dummy_node_t;
+
+// return: 0 not present, 1 - present
+static int find_in_list(list *node_list, uint16_t type)
+{
+	node *n = NULL;
+	WALK_LIST(n, *node_list) {
+		type_list_item_t *l_entr = (type_list_item_t *)n;
+		assert(l_entr);
+		if (l_entr->type == type) {
+			return 1;
+		}
+	}
+
+	type_list_item_t *new_entry = xmalloc(sizeof(type_list_item_t));
+	new_entry->type = type;
+
+	add_head(node_list, (node *)new_entry);
+	return 0;
+}
+
+// return: 0 not present (added), 1 - present
+static int dummy_node_add_type(dummy_node_t *n, uint16_t t)
+{
+	return find_in_list(&n->node_list, t);
+}
+
+static size_t dname_memsize(const knot_dname_t *d)
+{
+	size_t d_size = d->size;
+	size_t l_size = d->label_count;
+	if (d->size < MALLOC_MIN) {
+		d_size = MALLOC_MIN;
+	}
+	if (d->label_count < MALLOC_MIN) {
+		l_size = MALLOC_MIN;
+	}
+
+	return (sizeof(knot_dname_t) + d_size + l_size)
+	       * DNAME_MULT + DNAME_ADD;
+}
+
+// return: 0 - unique, 1 - duplicate
+static int insert_dname_into_table(hattrie_t *table, knot_dname_t *d,
+                                   dummy_node_t **n)
+{
+	value_t *val = hattrie_tryget(table, (char *)d->name, d->size);
+	if (val == NULL) {
+		// Create new dummy node to use for this dname
+		*n = xmalloc(sizeof(dummy_node_t));
+		init_list(&(*n)->node_list);
+		*hattrie_get(table, (char *)d->name, d->size) = *n;
+		return 0;
+	} else {
+		// Return previously found dummy node
+		*n = (dummy_node_t *)(*val);
+		return 1;
+	}
+}
+
+// return: RDATA memsize, minus size of dnames inside
+static size_t rdata_memsize(zone_estim_t *est, const scanner_t *scanner)
+{
+	const rdata_descriptor_t *desc = get_rdata_descriptor(scanner->r_type);
+	size_t size = 0;
+	for (int i = 0; desc->block_types[i] != KNOT_RDATA_WF_END; ++i) {
+		// DNAME - pointer in memory
+		int item = desc->block_types[i];
+		if (descriptor_item_is_dname(item)) {
+			size += sizeof(knot_dname_t *);
+			knot_dname_t *dname =
+				knot_dname_new_from_wire(scanner->r_data +
+			                                 scanner->r_data_blocks[i],
+			                                 scanner->r_data_blocks[i + 1] -
+			                                 scanner->r_data_blocks[i],
+			                                 NULL);
+			if (dname == NULL) {
+				return KNOT_ERROR;
+			}
+
+			knot_dname_to_lower(dname);
+			dummy_node_t *n = NULL;
+			if (insert_dname_into_table(est->dname_table,
+			                            dname, &n) == 0) {
+				// First time we see this dname, add size
+				est->dname_size += dname_memsize(dname);
+			}
+			knot_dname_free(&dname);
+		} else if (descriptor_item_is_fixed(item)) {
+		// Fixed length
+			size += item;
+		} else {
+		// Variable length
+			size += scanner->r_data_blocks[i + 1] -
+			        scanner->r_data_blocks[i];
+		}
+	}
+
+	return size * RDATA_MULT + RDATA_ADD;
+}
+
+static void rrset_memsize(zone_estim_t *est, const scanner_t *scanner)
+{
+	const rdata_descriptor_t *desc = get_rdata_descriptor(scanner->r_type);
+
+	// Handle RRSet's owner
+	knot_dname_t *owner = knot_dname_new_from_wire(scanner->r_owner,
+	                         scanner->r_owner_length,
+	                         NULL);
+	dummy_node_t *n;
+	if (insert_dname_into_table(est->node_table, owner, &n) == 0) {
+		// First time we see this name == new node
+		est->node_size += sizeof(knot_node_t) * NODE_MULT + NODE_ADD;
+		// Also, RRSet's owner will now contain full dname
+		est->dname_size += dname_memsize(owner);
+		// Trie's nodes handled at the end of computation
+	}
+	knot_dname_free(&owner);
+	assert(n);
+
+	// We will always add RDATA
+	size_t rdlen = rdata_memsize(est, scanner);
+	if (rdlen < MALLOC_MIN) {
+		rdlen = MALLOC_MIN;
+	}
+	// DNAME's size not included (handled inside rdata_memsize())
+	est->rdata_size += rdlen;
+
+	est->record_count++;
+
+	/*
+	 * RDATA size done, now add static part of RRSet to size.
+	 * Do not add for RRs that would be merged.
+	 * All possible duplicates will be added to total size.
+	 */
+
+	if (dummy_node_add_type(n, scanner->r_type) == 0) {
+		/*
+		 * New RR type, add actual RRSet struct's size:
+		 * MALLOC_MIN is added because of index array - usually not many RRs
+		 * are in the RRSet and values in the array are type uint32, so
+		 * 3 would be needed on 32bit system and 6 on 32bit system in order to
+		 * be larger than MALLOC_MIN, so we use it instead. Of course, if there
+		 * are more than 3/6 records in RRSet, measurement will not be precise.
+		 */
+		est->rrset_size += (sizeof(knot_rrset_t) + MALLOC_MIN)
+		                   * RRSET_MULT + RRSET_ADD;
+		// Add pointer in node's array
+		est->node_size += sizeof(knot_rrset_t *);
+	} else {
+		// Merge would happen, so just RDATA index is added
+		//est->rrset_size += sizeof(uint32_t);
+	}
+}
+
+void *estimator_malloc(void *ctx, size_t len)
+{
+	size_t *count = (size_t *)ctx;
+	*count += len + MALLOC_OVER;
+	return xmalloc(len);
+}
+
+void estimator_free(void *p)
+{
+	free(p);
+}
+
+static void get_ahtable_size(void *t, void *d)
+{
+	ahtable_t *table = (ahtable_t *)t;
+	size_t *size = (size_t *)d;
+	// info about allocated chunks starts at table->n-th index
+	for (size_t i = table->n; i < table->n * 2; ++i) {
+		// add actual slot size (= allocated for slot)
+		*size += table->slot_sizes[i];
+		// each non-empty slot means allocation overhead
+		*size += table->slot_sizes[i] ? MALLOC_OVER : 0;
+	}
+	*size += sizeof(ahtable_t);
+	// slot sizes + allocated sizes
+	*size += (table->n * 2) * sizeof(uint32_t);
+	// slots
+	*size += table->n * sizeof(void *);
+	*size += AHTABLE_ADD;
+}
+
+size_t estimator_trie_ahtable_memsize(hattrie_t *table)
+{
+	/*
+	 * Iterate through trie's node, and get stats from each ahtable.
+	 * Space taken up by the trie itself is measured using malloc wrapper.
+	 * (Even for large zones, space taken by trie itself is very small)
+	 */
+	size_t size = 0;
+	hattrie_apply_rev_ahtable(table, get_ahtable_size, &size);
+	return size;
+}
+
+void estimator_rrset_memsize_wrap(const scanner_t *scanner)
+{
+	rrset_memsize(scanner->data, scanner);
+}
+
+void estimator_free_trie_node(value_t *val, void *data)
+{
+	UNUSED(data);
+	dummy_node_t *trie_n = (dummy_node_t *)(*val);
+	WALK_LIST_FREE(trie_n->node_list);
+	free(trie_n);
+}

src/knot/zone/estimator.h

+/*  Copyright (C) 2013 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/*!
+ * \file estimator.h
+ *
+ * \author Jan Kadlec <jan.kadlec@nic.cz>
+ *
+ * \brief Memory estimation for zone files.
+ *
+ * \addtogroup config
+ * @{
+ */
+
+#ifndef _KNOT_ESTIMATOR_H_
+#define _KNOT_ESTIMATOR_H_
+
+#include "common/hattrie/hat-trie.h"
+#include "zscanner/scanner.h"
+
+// Mutiplicative constant, needed because of malloc's fragmentation
+static const double ESTIMATE_MAGIC = 1.2;
+
+/*!
+ * \brief Memory estimation context.
+ */
+typedef struct zone_estim {
+	hattrie_t *node_table; /*!< Same trie is in actual zone. */
+	hattrie_t *dname_table; /*!< RDATA section DNAMEs. */
+	size_t rdata_size; /*!< Estimated RDATA size. */
+	size_t dname_size; /*!< Estimated DNAME size. */
+	size_t node_size; /*!< Estimated node size. */
+	size_t ahtable_size; /*!< Estimated ahtable size. */
+	size_t rrset_size; /*!< Estimated RRSet size. */
+	size_t record_count; /*!< Total record count for zone. */
+} zone_estim_t;
+
+/*!
+ * \brief Size counting malloc wrapper.
+ *
+ * \param ctx Data for malloc wrapper.
+ * \param len Size to allocate.
+ *
+ * \retval Alloc'd data on succes.
+ * \retval NULL on error.
+ */
+void *estimator_malloc(void* ctx, size_t len);
+
+/*!
+ * \brief Size counting free wrapper.
+ *
+ * \param p Data to free.
+ */
+void estimator_free(void *p);
+
+/*!
+ * \brief Goes through trie's ahtables and estimates their memory requirements.
+ *
+ * \param table Trie to traverse.
+ */
+size_t estimator_trie_ahtable_memsize(hattrie_t *table);
+
+/*!
+ * \brief For use with scanner - counts memsize of RRSets.
+ *
+ * \param scanner Scanner context.
+ */
+void estimator_rrset_memsize_wrap(const scanner_t *scanner);
+
+/*!
+ * \brief Cleanup function for use with hattrie.
+ *
+ * \param p Data to free.
+ */
+void estimator_free_trie_node(value_t *val, void *data);
+
+#endif /* _KNOT_ESTIMATOR_H_ */

src/knot/zone/zone-load.c

@@ -189,7 +189,7 @@ static void process_rrsigs_in_node(parser_context_t *parser,
 	}
 }
 
-static void process_error(const scanner_t *s)
+void process_error(const scanner_t *s)
 {
 	if (s->stop == true) {
 		log_zone_error("Fatal error in zone file %s:%"PRIu64": %s "

src/knot/zone/zone-load.h

@@ -101,6 +101,8 @@ knot_zone_t *knot_zload_load(zloader_t *loader);
  */
 void knot_zload_close(zloader_t *loader);
 
+void process_error(const scanner_t *scanner);
+
 #endif /* _KNOTD_ZONELOAD_H_ */
 
 /*! @} */