Initial commit for memory estatimation.

- Works, but there are leaks
- Missing doxygens
- The estimate is not accurate - it reports about a half of the actual consumption on a 32bit system

Refs #37

Knot.files

@@ -281,3 +281,5 @@ src/zscanner/test/processing.h
 src/zscanner/test/tests.c
 src/zscanner/test/tests.h
 src/zscanner/test/zscanner-tool.c
+src/knot/zone/estimator.c
+src/knot/zone/estimator.h

src/Makefile.am

@@ -282,6 +282,8 @@ libknotd_la_SOURCES =				\
 	knot/zone/zone-dump.c			\
 	knot/zone/zone-load.h			\
 	knot/zone/zone-load.c			\
+	knot/zone/estimator.h			\
+	knot/zone/estimator.c			\
 	knot/server/server.h
 
 zscanner_tool_SOURCES =				\

src/knot/ctl/knotc_main.c

@@ -44,6 +44,7 @@
 #include "libknot/packet/query.h"
 #include "libknot/packet/response.h"
 #include "knot/zone/zone-load.h"
+#include "knot/zone/estimator.h"
 
 /*! \brief Controller constants. */
 enum knotc_constants_t {
@@ -92,6 +93,7 @@ static int cmd_status(int argc, char *argv[], unsigned flags);
 static int cmd_zonestatus(int argc, char *argv[], unsigned flags);
 static int cmd_checkconf(int argc, char *argv[], unsigned flags);
 static int cmd_checkzone(int argc, char *argv[], unsigned flags);
+static int cmd_memstats(int argc, char *argv[], unsigned flags);
 
 /*! \brief Table of remote commands. */
 knot_cmd_t knot_cmd_tbl[] = {
@@ -105,6 +107,7 @@ knot_cmd_t knot_cmd_tbl[] = {
 	{&cmd_zonestatus, 0, "zonestatus", "",       "\tShow status of configured zones."},
 	{&cmd_checkconf,  1, "checkconf",  "",       "\tCheck current server configuration."},
 	{&cmd_checkzone,  1, "checkzone",  "[zone]", "Check zone (all if not specified)."},
+	{&cmd_memstats,  1, "memstats",  "[zone]", "Estimate memory use for zone (all if not specified)."},
 	{NULL, 0, NULL, NULL, NULL}
 };
 
@@ -852,3 +855,66 @@ static int cmd_checkzone(int argc, char *argv[], unsigned flags)
 
 	return rc;
 }
+
+static int cmd_memstats(int argc, char *argv[], unsigned flags)
+{
+	UNUSED(flags);
+
+	/* Zone checking */
+	int rc = 0;
+	node *n = 0;
+
+	/* Generate databases for all zones */
+	WALK_LIST(n, conf()->zones) {
+		/* Fetch zone */
+		conf_zone_t *zone = (conf_zone_t *) n;
+		int zone_match = 0;
+		for (unsigned i = 0; i < (unsigned)argc; ++i) {
+			size_t len = strlen(zone->name);
+
+			/* All (except root) without final dot */
+			if (len > 1) {
+				len -= 1;
+			}
+			if (strncmp(zone->name, argv[i], len) == 0) {
+				zone_match = 1;
+				break;
+			}
+		}
+
+		if (!zone_match && argc > 0) {
+			/* WALK_LIST is a for-cycle. */
+			continue;
+		}
+
+		zone_estim_t est = {.table = ahtable_create(),
+		                    .size = 0, .record_count = 0,
+		                    .signed_count = 0 };
+		if (est.table == NULL) {
+			log_server_error("Not enough memory.\n");
+		}
+
+		/* Create file loader. */
+		file_loader_t *loader = file_loader_create(zone->file, zone->name,
+		                                           KNOT_CLASS_IN, 3600,
+		                                           rrset_memsize_wrap,
+		                                           process_error,
+		                                           &est);
+		if (loader == NULL) {
+			log_server_error("Could not load zone.\n");
+			ahtable_free(est.table);
+			return KNOT_ERROR;
+		}
+
+		file_loader_process(loader);
+		ahtable_free(est.table);
+
+		log_zone_info("Zone %s: %zu records, signed=%f%%, estimated size in memory=%fMB\n",
+		              zone->name, est.record_count,
+		              est.signed_count ? ((double)est.record_count / est.signed_count) * 100 : 0.0,
+		              (double)est.size / (1024 * 1024));
+		file_loader_free(loader);
+	}
+
+	return rc;
+}

src/knot/zone/estimator.c

+/*  Copyright (C) 2011 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <config.h>
+#include <assert.h>
+
+#include "estimator.h"
+#include "dname.h"
+#include "libknot/zone/node.h"
+#include "common/hattrie/ahtable.h"
+#include "zscanner/scanner.h"
+#include "common/descriptor.h"
+
+enum estim_consts {
+	DNAME_MULT = 1,
+	DNAME_ADD = 0,
+	RDATA_MULT = 1,
+	RDATA_ADD = 0,
+	RRSET_MULT = 1,
+	RRSET_ADD = 0,
+	NODE_MULT = 1,
+	NODE_ADD = 0
+};
+
+typedef struct dummy_node {
+	// Types are enough for now, later we can add more
+	uint16_t *types;
+	size_t allocated;
+	size_t used;
+} dummy_node_t;
+
+// Insert value to position - move the rest of the array
+static void insert_on_pos(dummy_node_t *n, size_t pos, uint16_t t)
+{
+	assert(pos <= n->used);
+	memmove(n->types + pos + 1, n->types + pos, n->used - pos);
+	n->types[pos] = t;
+	n->used++;
+}
+
+// Binary search in sorted array
+static int find_in_array(dummy_node_t *n, uint16_t t, size_t *pos, size_t bound)
+{
+	if (*pos < 0) {
+		*pos = 0;
+		return 0;
+	} else if (*pos > n->used) {
+		*pos = n->used;
+		return 0;
+	} else if (bound == 0) {
+		return 0;
+	} else if (n->types[*pos] == t) {
+		return 1;
+	} else if (n->types[*pos] > t) {
+		*pos -= bound / 2;
+		return find_in_array(n, t, pos, bound / 2);
+	} else {
+		*pos += bound / 2;
+		return find_in_array(n, t, pos, bound / 2);
+	}
+}
+
+// return: 0 not present, 1 - present
+static int dummy_node_add_type(dummy_node_t *n, uint16_t t)
+{
+	size_t pos = n->used / 2;
+	int found = find_in_array(n, t, &pos, n->used);
+	if (found) {
+		// Found, nothing to do
+		return 1;
+	} else {
+		// Not found, need to insert and possibly realloc
+		if (n->used + 1 >= n->allocated) {
+			n->allocated *= 2;
+			n->types = xrealloc(n->types,
+			                    n->allocated * sizeof(uint16_t));
+		}
+		insert_on_pos(n, pos, t);
+		return 0;
+	}
+}
+
+static size_t dname_memsize(const knot_dname_t *d)
+{
+	return (sizeof(knot_dname_t) + d->size + d->label_count)
+	       * DNAME_MULT + DNAME_ADD;
+}
+
+// return: 0 - unique, 1 - duplicate
+static int insert_dname_into_table(zone_estim_t *est, knot_dname_t *d,
+                                   dummy_node_t **n)
+{
+	value_t *val = ahtable_tryget(est->table, d->name, d->size);
+	if (val == NULL) {
+		// Create new dummy node to use for this dname
+		*n = xmalloc(sizeof(dummy_node_t));
+		(*n)->allocated = 16;
+		(*n)->used = 0;
+		(*n)->types = xmalloc(16 * sizeof(uint16_t));
+		memset((*n)->types, 0, 16 * sizeof(uint16_t))	;
+		*ahtable_get(est->table, d->name, d->size) = *n;
+		return 0;
+	} else {
+		// Return previously found dummy node
+		*n = (dummy_node_t *)(*val);
+		assert((*n)->allocated);
+		return 1;
+	}
+}
+
+static size_t rdata_memsize(zone_estim_t *est, const scanner_t *scanner)
+{
+	const rdata_descriptor_t *desc = get_rdata_descriptor(scanner->r_type);
+	size_t size = 0;
+	for (int i = 0; desc->block_types[i] != KNOT_RDATA_WF_END; ++i) {
+		// DNAME - pointer in memory
+		int item = desc->block_types[i];
+		if (descriptor_item_is_dname(item)) {
+			size += sizeof(knot_dname_t *);
+			knot_dname_t *dname =
+				knot_dname_new_from_wire(scanner->r_data +
+			                                 scanner->r_data_blocks[i],
+			                                 scanner->r_data_blocks[i + 1] -
+			                                 scanner->r_data_blocks[i],
+			                                 NULL);
+			if (dname == NULL) {
+				return KNOT_ERROR;
+			}
+
+			knot_dname_to_lower(dname);
+			dummy_node_t *n;
+			if (insert_dname_into_table(est, dname, &n) == 0) {
+				// First time we see this dname, add size
+				size += dname_memsize(dname);
+			}
+			knot_dname_free(&dname);
+		} else if (descriptor_item_is_fixed(item)) {
+		// Fixed length
+			size += item;
+		} else {
+		// Variable length
+			size += scanner->r_data_blocks[i + 1] -
+			        scanner->r_data_blocks[i];
+		}
+	}
+
+	return size * RDATA_MULT + RDATA_ADD;
+}
+
+static void rrset_memsize(zone_estim_t *est, const scanner_t *scanner)
+{
+	const rdata_descriptor_t *desc = get_rdata_descriptor(scanner->r_type);
+
+	// Handle RRSet's owner
+	knot_dname_t *owner = knot_dname_new_from_wire(scanner->r_owner,
+	                         scanner->r_owner_length,
+	                         NULL);
+	dummy_node_t *n;
+	if (insert_dname_into_table(est, owner, &n) == 0) {
+		// First time we see this dname
+		est->size += dname_memsize(owner);
+		// Add node constants
+		est->size += sizeof(knot_node_t) * NODE_MULT + NODE_ADD;
+	}
+	knot_dname_free(&owner);
+	assert(n);
+
+	// We will always add RDATA
+	size_t rdlen = rdata_memsize(est, scanner);
+	est->size += rdlen;
+
+	est->record_count++;
+	est->signed_count += scanner->r_type == KNOT_RRTYPE_RRSIG ? 1 : 0;
+
+	/*
+	 * RDATA size done, now add static part of RRSet to size, included
+	 * owner. Do not add for RRs that would be merged.
+	 * All possible duplicates will be added to total size.
+	 */
+
+	if (dummy_node_add_type(n, scanner->r_type) == 0) {
+		// New RR type, add actual RRSet struct's size
+		est->size += sizeof(knot_rrset_t) * RRSET_MULT + RRSET_ADD;
+		// Add pointer in node's array
+		est->size += sizeof(knot_rrset_t *);
+	} else {
+		// Merge would happen, so just RDATA index is added
+		est->size += sizeof(uint32_t);
+	}
+}
+
+void rrset_memsize_wrap(const scanner_t *scanner) {
+	rrset_memsize(scanner->data, scanner);
+}
+

src/knot/zone/estimator.h

+/*  Copyright (C) 2013 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "common/hattrie/ahtable.h"
+#include "zscanner/scanner.h"
+
+typedef struct zone_estim {
+	ahtable_t *table;
+	size_t size;
+	size_t record_count;
+	size_t signed_count;
+} zone_estim_t;
+
+void rrset_memsize_wrap(const scanner_t *scanner);

src/knot/zone/zone-load.c

@@ -189,7 +189,7 @@ static void process_rrsigs_in_node(parser_context_t *parser,
 	}
 }
 
-static void process_error(const scanner_t *s)
+void process_error(const scanner_t *s)
 {
 	if (s->stop == true) {
 		log_zone_error("Fatal error in zone file %s:%"PRIu64": %s "

src/knot/zone/zone-load.h

@@ -101,6 +101,8 @@ knot_zone_t *knot_zload_load(zloader_t *loader);
  */
 void knot_zload_close(zloader_t *loader);
 
+void process_error(const scanner_t *scanner);
+
 #endif /* _KNOTD_ZONELOAD_H_ */
 
 /*! @} */