modules/http: initial commit

doc/modules.rst

@@ -13,6 +13,7 @@ Knot DNS Resolver modules
 .. include:: ../modules/policy/README.rst
 .. include:: ../modules/view/README.rst
 .. include:: ../modules/predict/README.rst
+.. include:: ../modules/http/README.rst
 .. include:: ../modules/graphite/README.rst
 .. include:: ../modules/kmemcached/README.rst
 .. include:: ../modules/redis/README.rst

modules/http/README.rst

+.. _mod-http:
+
+HTTP interface
+--------------
+
+This module provides both DNS/HTTP(s) and web interface that cooperates with the internal
+scheduler. It preloads all static assets, so nothing is read from disk after startup and
+provides basic foundation for other services wishing to export services over HTTP endpoints.
+The module supports HTTP/2, server push and all other shiny things thanks to lua-http.
+
+Example configuration
+^^^^^^^^^^^^^^^^^^^^^
+
+By default, the web interface starts at port 8053 if HTTP or 4453 if running on TLS.
+
+.. code-block:: lua
+
+	-- Load modules
+	modules = {
+		http = {
+      host = 'localhost',
+      port = 8080,
+		}
+	}
+
+Dependencies
+^^^^^^^^^^^^
+
+* `lua-http <https://github.com/daurnimator/lua-http>`_ available in LuaRocks
+
+    ``$ luarocks install --server=http://luarocks.org/dev http``
\ No newline at end of file

modules/http/http.lua

+local cqueues = require('cqueues')
+local ce = require('cqueues.errno')
+local server = require('http.server')
+local headers = require('http.headers')
+local websocket = require('http.websocket')
+local kres = require('kres')
+
+-- Module declaration
+local cq = cqueues.new()
+local M = {
+	servers = {},
+}
+
+-- Load dependent modules
+if not stats then modules.load('stats') end
+local function stream_stats(h, ws)
+	local ok, err = true, nil
+	while ok do
+		-- Receive PINGs
+		ok, err = ws:receive()
+		if not ok then
+			ws.socket:clearerr()
+		end
+		-- Stream 
+		ok, err = ws:send(tojson(stats.list()))
+	end
+	ws:close()
+end
+
+-- Preload static contents, nothing on runtime will touch the disk
+local function pgload(relpath)
+	local fp, err = io.open(moduledir..'/http/'..relpath, 'r')
+	if not fp then error(err) end
+	local data = fp:read('*all')
+	fp:close()
+	return data
+end
+local pages = {
+	root = pgload('main.tpl'):gsub('{{.Title}}', 'kresd @ '..hostname()),
+	rootjs = pgload('tinyweb.js'),
+	datamaps = pgload('datamaps.world.min.js'),
+	topojson = pgload('topojson.js'),
+	jquery = pgload('jquery.js'),
+	epochcss = pgload('epoch.css'),
+	epoch = pgload('epoch.js'),
+	favicon = pgload('favicon.ico'),
+	d3 = pgload('d3.js'),
+}
+
+-- Export HTTP service endpoints
+M.endpoints = {
+	['/']                      = {'text/html', pages.root},
+	['/tinyweb.js']            = {'application/json', pages.rootjs},
+	['/datamaps.world.min.js'] = {'application/json', pages.datamaps},
+	['/topojson.js']           = {'application/json', pages.topojson},
+	['/jquery.js']             = {'application/json', pages.jquery},
+	['/epoch.js']              = {'application/json', pages.epoch},
+	['/epoch.css']             = {'text/css', pages.epochcss},
+	['/favicon.ico']           = {'text/html', pages.favicon},
+	['/d3.js']                 = {'text/html', pages.d3},
+	['/stats']                 = {'application/json', stats.list, stream_stats},
+	['/feed']                  = {'application/json', stats.frequent},
+
+}
+
+-- Serve GET requests, we only support a fixed
+-- number of endpoints that are actually preloaded
+-- in memory or constructed on request
+local function serve_get(h, stream)
+	local hsend = headers.new()
+	local path = h:get(':path')
+	local ctype, data = M.endpoints[path]
+	-- Unpack ctype
+	if ctype then
+		ctype, data = unpack(ctype)
+	end
+	-- Get string data out of service endpoint
+	if type(data) == 'function' then data = data(h) end
+	if type(data) == 'table' then data = tojson(data) end
+	if not ctype or type(data) ~= 'string' then
+		hsend:append(':status', '404')
+		hsend:append('connection', 'close')
+		assert(stream:write_headers(hsend, true))
+	else
+		-- Serve content type appropriately
+		hsend:append(':status', '200')
+		hsend:append('content/type', ctype)
+		hsend:append('connection', 'close')
+		assert(stream:write_headers(hsend, false))
+		assert(stream:write_chunk(data, true))
+	end
+end
+
+-- Web server service closure
+function M.route(endpoints)
+	return function (stream)
+		local h = assert(stream:get_headers())
+		local m = h:get(':method')
+		local path = h:get(':path')
+		-- Upgrade connection to WebSocket
+		local ws = websocket.new_from_stream(h, stream)
+		if ws then
+			assert(ws:accept { protocols = {'json'} })
+			-- Continue streaming results to client
+			local ep = endpoints[path]
+			local cb = ep[3]
+			if cb then
+				cb(h, ws)
+			end
+			return
+		-- Handle HTTP method appropriately
+		elseif m == 'GET' then
+			serve_get(h, stream)
+		else
+			-- Method is not supported
+			local hsend = headers.new()
+			hsend:append(':status', '500')
+			hsend:append('connection', 'close')
+			assert(stream:write_headers(hsend, true))
+		end
+		stream:shutdown()
+		stream.connection:shutdown()
+	end
+end
+
+-- @function Listen on given HTTP(s) host
+function M.listen(m, host, port, cb, cert)
+	local s, err = server.listen {
+		host = host,
+		port = port,
+	}
+	if not s then
+		error(string.format('failed to listen on %s#%d: %s', host, port, err))
+	end
+	-- Compose server handler
+	cq:wrap(function ()
+		assert(s:run(cb))
+		s:close()
+	end)
+	table.insert(M.servers, s)
+end
+
+-- @function 
+
+-- @function Cleanup module
+function M.deinit()
+	if M.ev then event.cancel(M.ev) end
+	M.servers = {}
+end
+
+-- @function Configure module
+function M.config(conf)
+		assert(type(conf) == 'table', 'config { host = "...", port = 443, cert = "..." }')
+		-- Configure web interface for resolver
+		if not conf.port then conf.port = conf.cert and 80 or 443 end
+		if not conf.host then conf.host = 'localhost' end 
+		M:listen(conf.host, conf.port, M.route(M.endpoints))
+		-- TODO: configure DNS/HTTP(s) interface
+		-- M:listen(conf.dns.host, conf.dns/port, serve_web)
+		if M.ev then return end
+		M.ev = event.socket(cq:pollfd(), function (ev, status, events)
+			local ok, err, _, co = cq:step(0)
+			if not ok then print('[http] '..err, debug.traceback(co)) end
+		end)
+end
+
+return M

modules/http/http.mk

+http_SOURCES := http.lua
+http_INSTALL := $(wildcard modules/http/static/*)
+$(call make_lua_module,http)

modules/http/static/LICENSE

+jQuery is provided under MIT license <https://jquery.org/license/>
+D3 under BSD license <https://github.com/mbostock/d3/blob/master/LICENSE>
+Epoch under MIT license <https://github.com/epochjs/epoch/blob/master/LICENSE>
+TopoJSON under BSD license <https://github.com/mbostock/topojson/blob/master/LICENSE>
+DataMaps under MIT license <https://github.com/markmarkoh/datamaps/blob/master/LICENSE>

modules/http/static/main.tpl

+<!DOCTYPE html>
+<title>{{.Title}}</title>
+<style>
+	body { font-family: 'Gill Sans', 'Gill Sans MT', Verdana, sans-serif; color: #555; }
+	h1, h2, h3 { line-height: 2em; color: #000; text-align: center; border-bottom: 1px solid #ccc; }
+	h1, h2, h3 { font-weight: 300; }
+	th { text-align: left; font-weight: normal; margin-bottom: 0.5em; }
+	#page { font-weight: 300; }
+	#page { width: 900px; margin: 0 auto; }
+	#stats { height: 300px; }
+	#stats .layer-cached , .l-cached  { fill: #2CA02C; color: #2CA02C; }
+	#stats .layer-10ms   , .l-10ms    { fill: #165683; color: #165683; }
+	#stats .layer-100ms  , .l-100ms   { fill: #258FDA; color: #258FDA; }
+	#stats .layer-1000ms , .l-1000ms  { fill: #51A5E1; color: #51A5E1; }
+	#stats .layer-slow   , .l-slow    { fill: #E1AC51; color: #E1AC51; }
+	#feed { width: 100%; }
+	#feed .secure { color: #74c476; }
+	.stats-legend { text-align: center; }
+	.stats-legend li { display: inline; list-style-type: none; padding-right: 20px; }
+	.map-legend { font-size: 10px; }
+</style>
+<script type="text/javascript" src="jquery.js"></script>
+<script type="text/javascript" src="d3.js"></script>
+<script type="text/javascript" src="epoch.js"></script>
+<script type="text/javascript" src="topojson.js"></script>
+<script type="text/javascript" src="datamaps.world.min.js"></script>
+<script type="text/javascript" src="tinyweb.js"></script>
+<link rel="icon" type="image/ico" href="favicon.ico">
+<link rel="stylesheet" type="text/css" href="epoch.css">
+<div id="page">
+	<h1>{{.Title}}</h1>
+	<div class="epoch" id="stats"></div>
+	<ul class="stats-legend"></ul>
+	<h2>Frequent queries</h2>
+	<table id="feed"></table>
+</div>

modules/http/static/tinyweb.js

+// Unit conversion
+function tounit(d) {
+	d = parseInt(d);
+	if (d < 1000) return d.toFixed(0);
+	else if (d < 1000000) return (d / 1000.0).toFixed(1) + 'K';
+	else return (d / 1000000.0).toFixed(1) + 'M';
+}
+// Set up UI and pollers
+window.onload = function() {
+	var statsLabels = ['cached', '10ms', '100ms', '1000ms', 'slow'];
+	var statsHistory = [];
+	var now = Date.now();
+	for (i = 0; i < statsLabels.length; ++i) {
+		statsHistory.push({ label: 'Layer ' + statsLabels[i], values: [{time: now, y:0}] });
+		$('.stats-legend').append('<li class="l-' + statsLabels[i] + '">' + statsLabels[i]);
+	}
+	var statsChart = $('#stats').epoch({
+		type: 'time.bar',
+		axes: ['right', 'bottom'],
+		ticks: { right: 2 },
+		margins: { right: 60 },
+		tickFormats: {
+			right: function(d) { return tounit(d) + ' pps'; },
+			bottom: function(d) { return new Date(d).toTimeString().split(' ')[0]; },
+		},
+		data: statsHistory
+	});
+	var statsPrev = null;
+
+	/*
+	 * Realtime updates over WebSockets
+	 */
+	function pushMetrics(resp) {
+		var now = Date.now();
+		var next = [];
+		for (i = 0; i < statsLabels.length; ++i) {
+			next.push(resp['answer.' + statsLabels[i]]);
+		}
+		if (statsPrev) {
+			var delta = [];
+			for (i = 0; i < statsLabels.length; ++i) {
+				delta.push({time: now, y: next[i]-statsPrev[i]});
+			}
+			statsChart.push(delta);
+		}
+		statsPrev = next;
+	}
+
+	/* WebSocket endpoints */
+	var wsStats = 'ws://' + location.host + '/stats';
+    var Socket = "MozWebSocket" in window ? MozWebSocket : WebSocket;
+    var ws = new Socket(wsStats);
+    ws.onmessage = function(evt) {
+      var data = $.parseJSON(evt.data);
+      pushMetrics(data);
+    };
+    setInterval(function() {
+    	ws.send('ping')
+    }, 500);
+}

modules/http/static/topojson.js

+!function(){function t(n,t){function r(t){var r,e=n.arcs[0>t?~t:t],o=e[0];return n.transform?(r=[0,0],e.forEach(function(n){r[0]+=n[0],r[1]+=n[1]})):r=e[e.length-1],0>t?[r,o]:[o,r]}function e(n,t){for(var r in n){var e=n[r];delete t[e.start],delete e.start,delete e.end,e.forEach(function(n){o[0>n?~n:n]=1}),f.push(e)}}var o={},i={},u={},f=[],c=-1;return t.forEach(function(r,e){var o,i=n.arcs[0>r?~r:r];i.length<3&&!i[1][0]&&!i[1][1]&&(o=t[++c],t[c]=r,t[e]=o)}),t.forEach(function(n){var t,e,o=r(n),f=o[0],c=o[1];if(t=u[f])if(delete u[t.end],t.push(n),t.end=c,e=i[c]){delete i[e.start];var a=e===t?t:t.concat(e);i[a.start=t.start]=u[a.end=e.end]=a}else i[t.start]=u[t.end]=t;else if(t=i[c])if(delete i[t.start],t.unshift(n),t.start=f,e=u[f]){delete u[e.end];var s=e===t?t:e.concat(t);i[s.start=e.start]=u[s.end=t.end]=s}else i[t.start]=u[t.end]=t;else t=[n],i[t.start=f]=u[t.end=c]=t}),e(u,i),e(i,u),t.forEach(function(n){o[0>n?~n:n]||f.push([n])}),f}function r(n,r,e){function o(n){var t=0>n?~n:n;(s[t]||(s[t]=[])).push({i:n,g:a})}function i(n){n.forEach(o)}function u(n){n.forEach(i)}function f(n){"GeometryCollection"===n.type?n.geometries.forEach(f):n.type in l&&(a=n,l[n.type](n.arcs))}var c=[];if(arguments.length>1){var a,s=[],l={LineString:i,MultiLineString:u,Polygon:u,MultiPolygon:function(n){n.forEach(u)}};f(r),s.forEach(arguments.length<3?function(n){c.push(n[0].i)}:function(n){e(n[0].g,n[n.length-1].g)&&c.push(n[0].i)})}else for(var h=0,p=n.arcs.length;p>h;++h)c.push(h);return{type:"MultiLineString",arcs:t(n,c)}}function e(r,e){function o(n){n.forEach(function(t){t.forEach(function(t){(f[t=0>t?~t:t]||(f[t]=[])).push(n)})}),c.push(n)}function i(n){return l(u(r,{type:"Polygon",arcs:[n]}).coordinates[0])>0}var f={},c=[],a=[];return e.forEach(function(n){"Polygon"===n.type?o(n.arcs):"MultiPolygon"===n.type&&n.arcs.forEach(o)}),c.forEach(function(n){if(!n._){var t=[],r=[n];for(n._=1,a.push(t);n=r.pop();)t.push(n),n.forEach(function(n){n.forEach(function(n){f[0>n?~n:n].forEach(function(n){n._||(n._=1,r.push(n))})})})}}),c.forEach(function(n){delete n._}),{type:"MultiPolygon",arcs:a.map(function(e){var o=[];if(e.forEach(function(n){n.forEach(function(n){n.forEach(function(n){f[0>n?~n:n].length<2&&o.push(n)})})}),o=t(r,o),(n=o.length)>1)for(var u,c=i(e[0][0]),a=0;n>a;++a)if(c===i(o[a])){u=o[0],o[0]=o[a],o[a]=u;break}return o})}}function o(n,t){return"GeometryCollection"===t.type?{type:"FeatureCollection",features:t.geometries.map(function(t){return i(n,t)})}:i(n,t)}function i(n,t){var r={type:"Feature",id:t.id,properties:t.properties||{},geometry:u(n,t)};return null==t.id&&delete r.id,r}function u(n,t){function r(n,t){t.length&&t.pop();for(var r,e=s[0>n?~n:n],o=0,i=e.length;i>o;++o)t.push(r=e[o].slice()),a(r,o);0>n&&f(t,i)}function e(n){return n=n.slice(),a(n,0),n}function o(n){for(var t=[],e=0,o=n.length;o>e;++e)r(n[e],t);return t.length<2&&t.push(t[0].slice()),t}function i(n){for(var t=o(n);t.length<4;)t.push(t[0].slice());return t}function u(n){return n.map(i)}function c(n){var t=n.type;return"GeometryCollection"===t?{type:t,geometries:n.geometries.map(c)}:t in l?{type:t,coordinates:l[t](n)}:null}var a=g(n.transform),s=n.arcs,l={Point:function(n){return e(n.coordinates)},MultiPoint:function(n){return n.coordinates.map(e)},LineString:function(n){return o(n.arcs)},MultiLineString:function(n){return n.arcs.map(o)},Polygon:function(n){return u(n.arcs)},MultiPolygon:function(n){return n.arcs.map(u)}};return c(t)}function f(n,t){for(var r,e=n.length,o=e-t;o<--e;)r=n[o],n[o++]=n[e],n[e]=r}function c(n,t){for(var r=0,e=n.length;e>r;){var o=r+e>>>1;n[o]<t?r=o+1:e=o}return r}function a(n){function t(n,t){n.forEach(function(n){0>n&&(n=~n);var r=o[n];r?r.push(t):o[n]=[t]})}function r(n,r){n.forEach(function(n){t(n,r)})}function e(n,t){"GeometryCollection"===n.type?n.geometries.forEach(function(n){e(n,t)}):n.type in u&&u[n.type](n.arcs,t)}var o={},i=n.map(function(){return[]}),u={LineString:t,MultiLineString:r,Polygon:r,MultiPolygon:function(n,t){n.forEach(function(n){r(n,t)})}};n.forEach(e);for(var f in o)for(var a=o[f],s=a.length,l=0;s>l;++l)for(var h=l+1;s>h;++h){var p,v=a[l],g=a[h];(p=i[v])[f=c(p,g)]!==g&&p.splice(f,0,g),(p=i[g])[f=c(p,v)]!==v&&p.splice(f,0,v)}return i}function s(n,t){function r(n){u.remove(n),n[1][2]=t(n),u.push(n)}var e,o=g(n.transform),i=m(n.transform),u=v(),f=0;for(t||(t=h),n.arcs.forEach(function(n){var r=[];n.forEach(o);for(var i=1,f=n.length-1;f>i;++i)e=n.slice(i-1,i+2),e[1][2]=t(e),r.push(e),u.push(e);n[0][2]=n[f][2]=1/0;for(var i=0,f=r.length;f>i;++i)e=r[i],e.previous=r[i-1],e.next=r[i+1]});e=u.pop();){var c=e.previous,a=e.next;e[1][2]<f?e[1][2]=f:f=e[1][2],c&&(c.next=a,c[2]=e[2],r(c)),a&&(a.previous=c,a[0]=e[0],r(a))}return n.arcs.forEach(function(n){n.forEach(i)}),n}function l(n){for(var t,r=-1,e=n.length,o=n[e-1],i=0;++r<e;)t=o,o=n[r],i+=t[0]*o[1]-t[1]*o[0];return.5*i}function h(n){var t=n[0],r=n[1],e=n[2];return Math.abs((t[0]-e[0])*(r[1]-t[1])-(t[0]-r[0])*(e[1]-t[1]))}function p(n,t){return n[1][2]-t[1][2]}function v(){function n(n,t){for(;t>0;){var r=(t+1>>1)-1,o=e[r];if(p(n,o)>=0)break;e[o._=t]=o,e[n._=t=r]=n}}function t(n,t){for(;;){var r=t+1<<1,i=r-1,u=t,f=e[u];if(o>i&&p(e[i],f)<0&&(f=e[u=i]),o>r&&p(e[r],f)<0&&(f=e[u=r]),u===t)break;e[f._=t]=f,e[n._=t=u]=n}}var r={},e=[],o=0;return r.push=function(t){return n(e[t._=o]=t,o++),o},r.pop=function(){if(!(0>=o)){var n,r=e[0];return--o>0&&(n=e[o],t(e[n._=0]=n,0)),r}},r.remove=function(r){var i,u=r._;if(e[u]===r)return u!==--o&&(i=e[o],(p(i,r)<0?n:t)(e[i._=u]=i,u)),u},r}function g(n){if(!n)return y;var t,r,e=n.scale[0],o=n.scale[1],i=n.translate[0],u=n.translate[1];return function(n,f){f||(t=r=0),n[0]=(t+=n[0])*e+i,n[1]=(r+=n[1])*o+u}}function m(n){if(!n)return y;var t,r,e=n.scale[0],o=n.scale[1],i=n.translate[0],u=n.translate[1];return function(n,f){f||(t=r=0);var c=(n[0]-i)/e|0,a=(n[1]-u)/o|0;n[0]=c-t,n[1]=a-r,t=c,r=a}}function y(){}var d={version:"1.6.9",mesh:function(n){return u(n,r.apply(this,arguments))},meshArcs:r,merge:function(n){return u(n,e.apply(this,arguments))},mergeArcs:e,feature:o,neighbors:a,presimplify:s};"function"==typeof define&&define.amd?define(d):"object"==typeof module&&module.exports?module.exports=d:this.topojson=d}();
\ No newline at end of file

modules/modules.mk

@@ -19,7 +19,8 @@ modules_TARGETS += ketcd \
                    view \
                    predict \
                    dns64 \
-                   renumber
+                   renumber \
+                   http
 endif
 
 # List of Golang modules
@@ -44,9 +45,17 @@ endef
 define lua_target
 $(1) := $$(addprefix $(2)/,$$($(1)_SOURCES))
 $(1)-clean:
-$(1)-install: $$(addprefix $(2)/,$$($(1)_SOURCES)) $(DESTDIR)$(MODULEDIR)
+ifeq ($$(strip $$($(1)_INSTALL)),)
+$(1)-dist:
+	$(INSTALL) -d $(DESTDIR)$(MODULEDIR)
+else
+$(1)-dist: $$($(1)_INSTALL)
+	$(INSTALL) -d $(DESTDIR)$(MODULEDIR)/$(1)
+	$(INSTALL) -m 0644 $$^ $(DESTDIR)$(MODULEDIR)/$(1)
+endif
+$(1)-install: $$(addprefix $(2)/,$$($(1)_SOURCES)) $(DESTDIR)$(MODULEDIR) $(1)-dist
 	$(INSTALL) -m 0644 $$(addprefix $(2)/,$$($(1)_SOURCES)) $(DESTDIR)$(MODULEDIR)
-.PHONY: $(1) $(1)-install $(1)-clean
+.PHONY: $(1) $(1)-install $(1)-clean $(1)-dist
 endef
 
 # Make Go module

modules/stats/stats.c

@@ -23,6 +23,7 @@
  */
 
 #include <libknot/packet/pkt.h>
+#include <libknot/descriptor.h>
 #include <ccan/json/json.h>
 #include <contrib/cleanup.h>
 
@@ -276,7 +277,7 @@ static char* dump_list(void *env, struct kr_module *module, const char *args, na
 		return NULL;
 	}
 	uint16_t key_type = 0;
-	char key_name[KNOT_DNAME_MAXLEN];
+	char key_name[KNOT_DNAME_MAXLEN], type_str[16];
 	JsonNode *root = json_mkarray();
 	for (unsigned i = 0; i < table->size; ++i) {
 		struct lru_slot *slot = lru_slot_at((struct lru_hash_base *)table, i);
@@ -284,12 +285,13 @@ static char* dump_list(void *env, struct kr_module *module, const char *args, na
 			/* Extract query name, type and counter */
 			memcpy(&key_type, slot->key, sizeof(key_type));
 			knot_dname_to_str(key_name, (uint8_t *)slot->key + sizeof(key_type), sizeof(key_name));
+			knot_rrtype_to_string(key_type, type_str, sizeof(type_str));
 			unsigned *slot_val = lru_slot_val(slot, lru_slot_offset(table));
 			/* Convert to JSON object */
 			JsonNode *json_val = json_mkobject();
 			json_append_member(json_val, "count", json_mknumber(*slot_val));
 			json_append_member(json_val, "name",  json_mkstring(key_name));
-			json_append_member(json_val, "type",  json_mknumber(key_type));
+			json_append_member(json_val, "type",  json_mkstring(type_str));
 			json_append_element(root, json_val);
 		}
 	}

modules/tinyweb/README.rst

@@ -9,6 +9,8 @@ including a feed of recent iterative queries. It also includes bindings_ to `Max
 The *stats* module is required for plotting query rate.
 By default, it listens on ``localhost:8053``.
 
+.. warning:: This is a proof of concept module for embedding Go, which has several drawbacks - it runs in separate threads, is relatively heavy-weight due to the nature of Go, and is opaque for other modules. Look at :ref:`http module <mod-http>` if you want to expose services over HTTP from other modules.
+
 Examples
 ^^^^^^^^