Unit Testing for BIRD

 - Unit Testing Framework (BirdTest)
 - Integration of BirdTest into the BIRD build system
 - Tests for several BIRD modules

 Based on squashed Pavel Tvrdik's int-test branch, updated for
 current int-new branch.

Makefile.in

@@ -22,7 +22,7 @@ INSTALL_DATA=@INSTALL_DATA@
 
 client=$(addprefix $(exedir)/,@CLIENT@)
 daemon=$(exedir)/bird
-protocols  = @protocols@
+protocols=@protocols@
 
 prefix=@prefix@
 exec_prefix=@exec_prefix@
@@ -49,24 +49,26 @@ else
 endif
 
 # Meta rules
-cleangoals := clean distclean
 docgoals := docs userdocs progdocs
-.PHONY: all daemon cli $(cleangoals) $(docgoals) tags
+testgoals := check test tests tests_run
+cleangoals := clean distclean testsclean
+.PHONY: all daemon cli $(docgoals) $(testgoals) $(cleangoals) tags
 all: daemon cli
 
 daemon: $(daemon)
 cli: $(client)
 
 # Include directories
-dirs := client conf doc filter lib nest $(addprefix proto/,$(protocols)) @sysdep_dirs@
+dirs := client conf doc filter lib nest test $(addprefix proto/,$(protocols)) @sysdep_dirs@
 
 conf-y-targets := $(addprefix $(objdir)/conf/,cf-parse.y keywords.h commands.h)
 cf-local = $(conf-y-targets): $(s)config.Y
 
 src-o-files = $(patsubst %.c,$(o)%.o,$(src))
+tests-target-files = $(patsubst %.c,$(o)%,$(tests_src))
 
-all-daemon = $(exedir)/bird: $(obj)
-all-client = $(exedir)/birdc $(exedir)/birdcl: $(obj)
+all-daemon = $(daemon): $(obj)
+all-client = $(client): $(obj)
 
 s = $(dir $(lastword $(MAKEFILE_LIST)))
 ifeq ($(srcdir),.)
@@ -109,6 +111,22 @@ $(objdir)/sysdep/paths.h: Makefile
 	echo >>$@ "#define PATH_CONTROL_SOCKET \"@CONTROL_SOCKET@\""
 	if test -n "@iproutedir@" ; then echo >>$@ "#define PATH_IPROUTE_DIR \"@iproutedir@\"" ; fi
 
+# Unit tests rules
+
+tests_targets_ok = $(addsuffix .ok,$(tests_targets))
+
+$(tests_targets): %: %.o $(tests_objs)
+	$(E)echo LD $(LDFLAGS) -o $@ $^ $(LIBS)
+	$(Q)$(CC) $(LDFLAGS) -o $@ $^ $(LIBS)
+
+$(tests_targets_ok): %.ok: %
+	$(Q)$* 2>/dev/null && touch $*.ok
+
+test: testsclean check
+check: tests tests_run
+tests: $(tests_targets)
+tests_run: $(tests_targets_ok)
+
 # Finally include the computed dependencies
 
 ifneq ($(filter-out $(cleangoals),$(MAKECMDGOALS)),)
@@ -147,6 +165,9 @@ clean::
 	rm -f $(addprefix $(exedir)/,bird birdc birdcl)
 	find $(objdir) -name "*.[od]" -exec rm -f '{}' '+'
 
+testsclean:
+	rm -f $(tests_targets_ok)
+
 ifeq ($(objdir),obj)
 distclean: clean
 	rm -rf $(objdir)

conf/Makefile

@@ -3,6 +3,8 @@ obj := $(src-o-files)
 
 $(all-daemon)
 
+tests_objs := $(tests_objs) $(src-o-files)
+
 ifdef DEBUG
 BISON_DEBUG=-t
 #FLEX_DEBUG=-d

conf/conf.c

@@ -49,12 +49,10 @@
 #include "nest/route.h"
 #include "nest/protocol.h"
 #include "nest/iface.h"
-#include "lib/resource.h"
-#include "lib/string.h"
 #include "lib/event.h"
 #include "sysdep/unix/timer.h"
 #include "conf/conf.h"
-#include "filter/filter.h"
+
 
 static jmp_buf conf_jmpbuf;
 
@@ -85,7 +83,7 @@ int undo_available;			/* Undo was not requested from last reconfiguration */
  * further use. Returns a pointer to the structure.
  */
 struct config *
-config_alloc(const byte *name)
+config_alloc(const char *name)
 {
   pool *p = rp_new(&root_pool, "Config");
   linpool *l = lp_new(p, 4080);
@@ -96,6 +94,7 @@ config_alloc(const byte *name)
   char *ndup = lp_allocu(l, nlen);
   memcpy(ndup, name, nlen);
 
+  init_list(&c->tests);
   c->mrtdump_file = -1; /* Hack, this should be sysdep-specific */
   c->pool = p;
   c->mem = l;

conf/conf.h

@@ -9,6 +9,8 @@
 #ifndef _BIRD_CONF_H_
 #define _BIRD_CONF_H_
 
+#include "sysdep/config.h"
+#include "lib/ip.h"
 #include "lib/resource.h"
 #include "sysdep/unix/timer.h"
 
@@ -21,6 +23,7 @@ struct config {
   list protos;				/* Configured protocol instances (struct proto_config) */
   list tables;				/* Configured routing tables (struct rtable_config) */
   list logfiles;			/* Configured log files (sysdep) */
+  list tests;				/* Configured unit tests */
 
   int mrtdump_file;			/* Configured MRTDump file (sysdep, fd in unix) */
   char *syslog_name;			/* Name used for syslog (NULL -> no syslog) */
@@ -60,7 +63,7 @@ struct config {
 extern struct config *config;		/* Currently active configuration */
 extern struct config *new_config;	/* Configuration being parsed */
 
-struct config *config_alloc(const byte *name);
+struct config *config_alloc(const char *name);
 int config_parse(struct config *);
 int cli_parse(struct config *);
 void config_free(struct config *);
@@ -161,6 +164,7 @@ static inline int cf_symbol_is_constant(struct symbol *sym)
 
 /* Parser */
 
+extern char *cf_text;
 int cf_parse(void);
 
 /* Sysdep hooks */

configure.in

@@ -56,7 +56,7 @@ if test "$ac_test_CFLAGS" != set ; then
 	bird_cflags_default=yes
 fi
 
-AC_PROG_CC
+AC_PROG_CC_C99
 if test -z "$GCC" ; then
 	AC_MSG_ERROR([This program requires the GNU C Compiler.])
 fi
@@ -220,6 +220,9 @@ BIRD_CHECK_STRUCT_IP_MREQN
 
 if test "$enable_debug" = yes ; then
 	AC_DEFINE(DEBUGGING)
+	AC_CHECK_HEADER(execinfo.h, [AC_SEARCH_LIBS([backtrace, backtrace_symbols], [c execinfo], [AC_DEFINE(HAVE_EXECINFO_H)])])
+	LDFLAGS="$LDFLAGS -rdynamic"
+	CFLAGS="$CFLAGS -O0 -ggdb -g3 -gdwarf-4"
 	if test "$enable_memcheck" = yes ; then
 		AC_CHECK_LIB(dmalloc, dmalloc_debug)
 		if test $ac_cv_lib_dmalloc_dmalloc_debug != yes ; then

doc/bird.sgml

@@ -1253,7 +1253,7 @@ foot).
 	<cf/!&tilde;/ membership operators) can be used to modify or test
 	eclists, with ECs instead of pairs as arguments.
 
-	<tag/lclist/
+	<tag><label id="type-lclist">lclist/</tag>
 	Lclist is a data type used for BGP large community lists. Like eclists,
 	lclists are very similar to clists, but they are sets of LCs instead of
 	pairs. The same operations (like <cf/add/, <cf/delete/ or <cf/&tilde;/

filter/Makefile

@@ -2,3 +2,7 @@ src := filter.c f-util.c tree.c trie.c
 obj := $(src-o-files)
 $(all-daemon)
 $(cf-local)
+
+tests_src := tree_test.c filter_test.c trie_test.c
+tests_targets := $(tests_targets) $(tests-target-files)
+tests_objs := $(tests_objs) $(src-o-files)

filter/config.Y

@@ -323,7 +323,71 @@ f_generate_lc(struct f_inst *t1, struct f_inst *t2, struct f_inst *t3)
   return rv;
 }
 
+/*
+ * Remove all new lines and doubled whitespaces
+ * and convert all tabulators to spaces
+ * and return a copy of string
+ */
+char *
+assert_copy_expr(const char *start, size_t len)
+{
+  /* XXX: Allocates maybe a little more memory than we really finally need */
+  char *str = cfg_alloc(len + 1);
+
+  char *dst = str;
+  const char *src = start - 1;
+  const char *end = start + len;
+  while (++src < end)
+  {
+    if (*src == '\n')
+      continue;
+
+    /* Skip doubled whitespaces */
+    if (src != start)
+    {
+      const char *prev = src - 1;
+      if ((*src == ' ' || *src == '\t') && (*prev == ' ' || *prev == '\t'))
+	continue;
+    }
+
+    if (*src == '\t')
+      *dst = ' ';
+    else
+      *dst = *src;
+
+    dst++;
+  }
+  *dst = '\0';
+
+  return str;
+}
+
+/*
+ * assert_done - create f_instruction of bt_assert
+ * @expr: expression in bt_assert()
+ * @start: pointer to first char of test expression
+ * @end: pointer to the last char of test expression
+ */
+static struct f_inst *
+assert_done(struct f_inst *expr, const char *start, const char *end)
+{
+  struct f_inst *i;
+  i = f_new_inst();
+  i->code = P('a','s');
+  i->a1.p = expr;
+
+  if (end >= start)
+  {
+    i->a2.p = assert_copy_expr(start, end - start + 1);
+  }
+  else
+  {
+    /* this is a break of lexer buffer */
+    i->a2.p = "???";
+  }
 
+  return i;
+}
 
 CF_DECLS
 
@@ -341,12 +405,13 @@ CF_KEYWORDS(FUNCTION, PRINT, PRINTN, UNSET, RETURN,
 	ADD, DELETE, CONTAINS, RESET,
 	PREPEND, FIRST, LAST, LAST_NONAGGREGATED, MATCH,
 	EMPTY,
-	FILTER, WHERE, EVAL)
+	FILTER, WHERE, EVAL,
+	BT_ASSERT, BT_TEST_SUITE)
 
 %nonassoc THEN
 %nonassoc ELSE
 
-%type <x> term block cmds cmds_int cmd function_body constant constructor print_one print_list var_list var_listn dynamic_attr static_attr function_call symbol bgp_path_expr
+%type <x> term block cmds cmds_int cmd function_body constant constructor print_one print_list var_list var_listn dynamic_attr static_attr function_call symbol bgp_path_expr bt_assert
 %type <f> filter filter_body where_filter
 %type <i> type break_command ec_kind
 %type <i32> cnum
@@ -356,6 +421,7 @@ CF_KEYWORDS(FUNCTION, PRINT, PRINTN, UNSET, RETURN,
 %type <px> fprefix
 %type <s> decls declsn one_decl function_params
 %type <h> bgp_path bgp_path_tail1 bgp_path_tail2
+%type <t> get_cf_position
 
 CF_GRAMMAR
 
@@ -375,6 +441,21 @@ filter_eval:
    EVAL term { f_eval_int($2); }
  ;
 
+CF_ADDTO(conf, bt_test_suite)
+bt_test_suite:
+ BT_TEST_SUITE '(' SYM ',' text ')' {
+  if (!($3->class & SYM_FUNCTION))
+    cf_error("Function expected");
+
+  struct f_bt_test_suite *t = cfg_alloc(sizeof(struct f_bt_test_suite));
+  t->fn = $3->def;
+  t->fn_name = $3->name;
+  t->dsc = $5;
+
+  add_tail(&new_config->tests, &t->n);
+ }
+ ;
+
 type:
    INT { $$ = T_INT; }
  | BOOL { $$ = T_BOOL; }
@@ -835,6 +916,8 @@ term:
  | ROA_CHECK '(' rtable ')' { $$ = f_generate_roa_check($3, NULL, NULL); }
  | ROA_CHECK '(' rtable ',' term ',' term ')' { $$ = f_generate_roa_check($3, $5, $7); }
 
+ | bt_assert { $$ = $1; }
+
 /* | term '.' LEN { $$->code = P('P','l'); } */
 
 /* function_call is inlined here */
@@ -966,6 +1049,7 @@ cmd:
       $$->a1.p = $2;
       $$->a2.p = build_tree( $4 );
    }
+ | bt_assert ';' { $$ = $1; }
 
 
  | rtadot dynamic_attr '.' EMPTY ';' { $$ = f_generate_empty($2); }
@@ -975,4 +1059,14 @@ cmd:
  | rtadot dynamic_attr '.' FILTER '(' term ')' ';'    { $$ = f_generate_complex( P('C','a'), 'f', $2, $6 ); }
  ;
 
+ bt_assert:
+   BT_ASSERT '(' get_cf_position term get_cf_position ')' { $$ = assert_done($4, $3 + 1, $5 - 1); }
+ ;
+
+get_cf_position:
+{
+  $$ = cf_text;
+};
+
+
 CF_END

filter/filter.c

@@ -52,6 +52,8 @@
 
 #define CMP_ERROR 999
 
+void (*bt_assert_hook)(int result, struct f_inst *assert);
+
 static struct adata *
 adata_empty(struct linpool *pool, int l)
 {
@@ -563,8 +565,8 @@ f_rta_cow(void)
 
 static struct tbf rl_runtime_err = TBF_DEFAULT_LOG_LIMITS;
 
-#define runtime(x) do { \
-    log_rl(&rl_runtime_err, L_ERR "filters, line %d: %s", what->lineno, x); \
+#define runtime(fmt, ...) do { \
+    log_rl(&rl_runtime_err, L_ERR "filters, line %d: " fmt, what->lineno, ##__VA_ARGS__); \
     res.type = T_RETURN; \
     res.val.i = F_ERROR; \
     return res; \
@@ -1475,6 +1477,17 @@ interpret(struct f_inst *what)
 
     break;
 
+  case P('a', 's'):	/* Birdtest Assert */
+    ONEARG;
+
+    if (v1.type != T_BOOL)
+      runtime("Should be boolean value");
+
+    res.type = v1.type;
+    res.val = v1.val;
+
+    CALL(bt_assert_hook, res.val.i, what);
+    break;
 
   default:
     bug( "Unknown instruction %d (%c)", what->code, what->code & 0xff);

filter/filter.h

@@ -16,16 +16,16 @@
 
 struct f_inst {		/* Instruction */
   struct f_inst *next;	/* Structure is 16 bytes, anyway */
-  u16 code;
-  u16 aux;
+  u16 code;		/* Instruction code, see the interpret() function and P() macro */
+  u16 aux;		/* Extension to instruction code, T_*, EA_*, EAF_*  */
   union {
     int i;
     void *p;
-  } a1;
+  } a1;			/* The first argument */
   union {
     int i;
     void *p;
-  } a2;
+  } a2;			/* The second argument */
   int lineno;
 };
 
@@ -55,7 +55,7 @@ struct f_prefix {
 };
 
 struct f_val {
-  int type;
+  int type;		/* T_*  */
   union {
     uint i;
     u64 ec;
@@ -205,4 +205,15 @@ struct f_trie
 
 #define FF_FORCE_TMPATTR 1		/* Force all attributes to be temporary */
 
+/* Bird Tests */
+struct f_bt_test_suite {
+  node n;			/* Node in config->tests */
+  struct f_inst *fn;		/* Root of function */
+  const char *fn_name;		/* Name of test */
+  const char *dsc;		/* Description */
+};
+
+/* Hook for call bt_assert() function in configuration */
+extern void (*bt_assert_hook)(int result, struct f_inst *assert);
+
 #endif

filter/filter_test.c

+/*
+ *	Filters: Tests
+ *
+ *	(c) 2015 CZ.NIC z.s.p.o.
+ *
+ *	Can be freely distributed and used under the terms of the GNU GPL.
+ */
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <string.h>
+#include <stdlib.h>
+
+#include "test/birdtest.h"
+#include "test/bt-utils.h"
+
+#include "filter/filter.h"
+#include "conf/conf.h"
+
+#define BT_CONFIG_FILE "filter/test.conf"
+
+
+static struct config *
+parse_config_file(const void *filename_void)
+{
+  bt_bird_init();
+
+  size_t fn_size = strlen((const char *) filename_void) + 1;
+  char *filename = alloca(fn_size);
+  strncpy(filename, filename_void, fn_size);
+
+  struct config *c = bt_config_file_parse(filename);
+  bt_bird_cleanup();
+
+  return c;
+}
+
+static int
+run_function(const void *parsed_fn_def)
+{
+  /* XXX: const -> non-const */
+  struct f_inst *f = (struct f_inst *) parsed_fn_def;
+
+  linpool *tmp = lp_new(&root_pool, 4096);
+  struct f_val res = f_eval(f, tmp);
+  rfree(tmp);
+
+  if (res.type == T_RETURN && res.val.i >= F_REJECT)
+    return BT_FAILURE;
+
+  return BT_SUCCESS;
+}
+
+static void
+bt_assert_filter(int result, struct f_inst *assert)
+{
+  int bt_suit_case_result = BT_SUCCESS;
+  if (!result)
+  {
+    bt_result = BT_FAILURE;
+    bt_suite_result = BT_FAILURE;
+    bt_suit_case_result = BT_FAILURE;
+  }
+
+  bt_log_suite_case_result(bt_suit_case_result, "Assertion at line %d (%s)", assert->lineno, (char *) assert->a2.p);
+}
+
+int
+main(int argc, char *argv[])
+{
+  bt_init(argc, argv);
+
+  struct config *c = parse_config_file(BT_CONFIG_FILE);
+
+  if (c)
+  {
+    bt_assert_hook = bt_assert_filter;
+
+    struct f_bt_test_suite *t;
+    WALK_LIST(t, c->tests)
+      bt_test_suite_base(run_function, t->fn_name, t->fn, BT_FORKING, BT_TIMEOUT, "%s", t->dsc);
+  }
+
+  return bt_exit_value();
+}

filter/test.conf.inc

@@ -2,4 +2,5 @@
 print "Entering include";
 print "Should be 2:  ", 1+1;
 print "Leaving include";
+i = 42;
 

filter/test.conf2

@@ -18,6 +18,7 @@ protocol direct {
 
 protocol kernel {
 	disabled;
+	ipv4;			# Must be specified at least one channel
 #	learn;			# Learn all routes from the kernel
 #	scan time 10;		# Scan kernel tables every 10 seconds
 }
@@ -25,51 +26,58 @@ protocol kernel {
 protocol static {
 #	disabled;
 
-	import filter { print "ahoj"; 
-	print source;
-	if source = RTS_STATIC then {
-		print "It is from static";
-	}
-	print from;
-	from = 1.2.3.4;
-	print from;
-	print scope;
-	scope = SCOPE_HOST;
-	print scope;
-	if !(scope ~ [ SCOPE_HOST, SCOPE_SITE ]) then {
-		print "Failed in test";
-		quitbird;
-	}
-		
-	preference = 15;
-	print preference;
-	preference = 29;
-	print preference;
-	rip_metric = 1;
-	print rip_metric;
-	rip_metric = rip_metric + 5;
-	print rip_metric;
-	bgp_community = - empty - ; 
-	print "nazdar";
-	bgp_community = add(bgp_community, (1,2));
-	print "cau";
-	bgp_community = add(bgp_community, (2,3));
-	bgp_community.add((4,5));
-	print "community = ", bgp_community;
-	bgp_community.delete((2,3));
-	print "community = ", bgp_community;
-	bgp_community.empty;
-	print "community = ", bgp_community;
-	print "done";
-	};
+	ipv4 {
+		export all;
+
+		import filter {
+			print "ahoj";
+			print source;
+			if source = RTS_STATIC then {
+				print "It is from static";
+			}
+			print from;
+			from = 1.2.3.4;
+			print from;
+			print scope;
+			scope = SCOPE_HOST;
+			print scope;
+			if !(scope ~ [ SCOPE_HOST, SCOPE_SITE ]) then {
+				print "Failed in test";
+				quitbird;
+			}
 
+			preference = 15;
+			print preference;
+			preference = 29;
+			print preference;
+			rip_metric = 1;
+			print rip_metric;
+			rip_metric = rip_metric + 5;
+			print rip_metric;
+
+#
+# TODO: uncomment this part after finishing BGP integration version
+#
+#			bgp_community = -empty-;
+#			print "hi";
+#			bgp_community = add(bgp_community, (1,2));
+#			print "hello";
+#			bgp_community = add(bgp_community, (2,3));
+#			bgp_community.add((4,5));
+#			print "community = ", bgp_community;
+#			bgp_community.delete((2,3));
+#			print "community = ", bgp_community;
+#			bgp_community.empty;
+#			print "community = ", bgp_community;
+#			print "done";
+		};
+	};
 	route 0.0.0.0/0 via 195.113.31.113;
 	route 62.168.0.0/25 reject;
 	route 1.2.3.4/32 via 195.113.31.124;
-#	route 10.0.0.0/8 reject;
-#	route 10.1.1.0:255.255.255.0 via 62.168.0.3;
-#	route 10.1.2.0:255.255.255.0 via 62.168.0.3;
-#	route 10.1.3.0:255.255.255.0 via 62.168.0.4;
-#	route 10.2.0.0/24 via "arc0";
-	export all;
+	route 10.0.0.0/8 reject;
+	route 10.1.1.0:255.255.255.0 via 62.168.0.3;
+	route 10.1.2.0:255.255.255.0 via 62.168.0.3;
+	route 10.1.3.0:255.255.255.0 via 62.168.0.4;
+	route 10.2.0.0/24 via "arc0";
 }

filter/test6.conf

@@ -9,6 +9,8 @@ router id 62.168.0.1;
 
 define xyzzy = (120+10);
 
+protocol device {}
+
 function callme(int arg1; int arg2)
 int local1;
 int local2;
@@ -30,7 +32,7 @@ function fifteen()
 	return 15;
 }
 
-function paths()
+function _paths()
 bgpmask pm1;
 bgpmask pm2;
 bgppath p2;
@@ -97,7 +99,7 @@ ip p;
 pair pp;
 int set is;
 prefix set pxs;
-string s;
+string str;
 { 
 	print "Testing filter language:";
 	i = four; 
@@ -129,8 +131,8 @@ string s;
 	print "Testing pairs: (1,2) = ", (1,2), " = ", pp;
 	print "Testing enums: ", RTS_DUMMY, " ", RTS_STATIC;
 
-	s = "Hello";
-	print "Testing string: ", s, " true: ", s ~ "Hell*", " false: ", s ~ "ell*";
+	str = "Hello";
+	print "Testing string: ", str, " true: ", str ~ "Hell*", " false: ", str ~ "ell*";
 	
 	b = true;
 	print "Testing bool: ", b, ", ", !b;
@@ -156,11 +158,12 @@ string s;
 	i = fifteen();
 	print "Testing function calls: 15 = ", i;