Commit ac8f55eb authored by Marek Vavruša's avatar Marek Vavruša
Browse files

modules/tinyweb: Go-based module with an embedded web interface

parent 6c684140
......@@ -26,6 +26,7 @@ $(eval $(call find_lib,libmemcached,1.0))
$(eval $(call find_lib,hiredis))
$(eval $(call find_lib,socket_wrapper))
$(eval $(call find_lib,libdnssec))
$(eval $(call find_gopkg,geoip,github.com/abh/geoip))
# Find Go compiler version
E :=
GO_VERSION := $(subst $(E) $(E),,$(subst go,,$(wordlist 1,3,$(subst ., ,$(word 3,$(shell $(GO) version))))))
......
ifeq ($(HAS_doxygen)|$(HAS_sphinx-build), yes|yes)
doc-doxygen:
@cd doc && $(doxygen_BIN)
@cd doc && $(doxygen_BIN)
doc-html: doc-doxygen
@cd doc && $(sphinx-build_BIN) -b html . html
else
doc-html:
$(error doxygen and sphinx must be installed)
endif
doc-clean:
rm -rf doc/doxyxml doc/*.db doc/html
......
......@@ -17,4 +17,5 @@ Knot DNS Resolver modules
.. include:: ../modules/kmemcached/README.rst
.. include:: ../modules/redis/README.rst
.. include:: ../modules/ketcd/README.rst
.. include:: ../modules/cachectl/README.rst
\ No newline at end of file
.. include:: ../modules/cachectl/README.rst
.. include:: ../modules/tinyweb/README.rst
\ No newline at end of file
......@@ -19,6 +19,7 @@ info:
$(info --------)
$(info [$(HAS_doxygen)] doxygen (doc))
$(info [$(HAS_go)] Go (modules/go))
$(info [$(HAS_geoip)] github.com/abh/geoip (modules/tinyweb))
$(info [$(HAS_libmemcached)] libmemcached (modules/memcached))
$(info [$(HAS_hiredis)] hiredis (modules/redis))
$(info [$(HAS_cmocka)] cmocka (tests/unit))
......
......@@ -49,7 +49,7 @@ libkres_HEADERS := \
# Dependencies
libkres_DEPEND :=
libkres_LIBS := $(libknot_LIBS) $(libdnssec_LIBS)
libkres_TARGET := -Llib -lkres
libkres_TARGET := -L$(abspath lib) -lkres
# Make library
$(eval $(call make_static,libkres,lib))
......
......@@ -190,12 +190,12 @@ The Go modules use CGO_ to interface C resolver library, there are no native bin
import "C"
import "unsafe"
/* Mandatory functions */
//export mymodule_api
func mymodule_api() C.uint32_t {
return C.KR_MODULE_API
}
// Mandatory function
func main() {}
.. warning:: Do not forget to prefix function declarations with ``//export symbol_name``, as only these will be exported in module.
......@@ -244,6 +244,14 @@ Now we can add the implementations for the ``finish`` layer and finalize the mod
See the CGO_ for more information about type conversions and interoperability between the C/Go.
Gotchas
-------
* ``main()`` function is mandatory in each module, otherwise it won't compile.
* Module layer function implementation must be done in C during ``import "C"``, as Go doesn't support pointers to functions.
* The library doesn't have a Go-ified bindings yet, so interacting with it requires CGO shims, namely structure traversal and type conversions (strings, numbers).
* Other modules can be called through C call ``C.kr_module_call(kr_context, module_name, module_propery, input)``
Configuring modules
===================
......
package main
/*
#include "lib/layer.h"
#include "lib/module.h"
int begin(knot_layer_t *, void *);
int finish(knot_layer_t *);
static inline const knot_layer_api_t *_layer(void)
{
static const knot_layer_api_t api = {
.begin = &begin,
.finish = &finish
};
return &api;
}
*/
import "C"
import "unsafe"
import "fmt"
//export gostats_api
func gostats_api() C.uint32_t {
return C.KR_MODULE_API
}
//export gostats_init
func gostats_init(module *C.struct_kr_module) int {
return 0
}
//export gostats_deinit
func gostats_deinit(module *C.struct_kr_module) int {
return 0
}
//export begin
func begin(ctx *C.knot_layer_t, param unsafe.Pointer) C.int {
ctx.data = param
return 0
}
//export finish
func finish(ctx *C.knot_layer_t) C.int {
var param *C.struct_kr_request = (*C.struct_kr_request)(ctx.data)
fmt.Printf("[gostats] resolved %d queries\n", C.list_size(&param.rplan.resolved))
return 0
}
//export gostats_layer
func gostats_layer(module *C.struct_kr_module) *C.knot_layer_api_t {
return C._layer()
}
func main() {}
\ No newline at end of file
gostats_SOURCES := modules/gostats/gostats.go
gostats_DEPEND := $(libkres)
gostats_LIBS := $(libkres_TARGET) $(libkres_LIBS)
$(call make_go_module,gostats)
\ No newline at end of file
......@@ -23,7 +23,9 @@ endif
# List of Golang modules
ifeq ($(HAS_go),yes)
modules_TARGETS += gostats
ifeq ($(HAS_geoip),yes)
modules_TARGETS += tinyweb
endif
endif
# Make C module
......@@ -58,10 +60,17 @@ $(2)/$(1)$(LIBEXT): $$($(1)_SOURCES) $$($(1)_DEPEND)
@echo " GO $(2)"; CGO_CFLAGS="$(BUILD_CFLAGS)" CGO_LDFLAGS="$$($(1)_LIBS)" $(GO) build -buildmode=c-shared -o $$@ $$($(1)_SOURCES)
$(1)-clean:
$(RM) -r $(2)/$(1).h $(2)/$(1)$(LIBEXT)
$(1)-install: $(2)/$(1)$(LIBEXT)
ifeq ($$(strip $$($(1)_INSTALL)),)
$(1)-dist:
$(INSTALL) -d $(PREFIX)/$(MODULEDIR)
$(INSTALL) $$^ $(PREFIX)/$(MODULEDIR)
.PHONY: $(1)-clean $(1)-install
else
$(1)-dist: $$($(1)_INSTALL)
$(INSTALL) -d $(PREFIX)/$(MODULEDIR)/$(1)
$(INSTALL) $$^ $(PREFIX)/$(MODULEDIR)/$(1)
endif
$(1)-install: $(2)/$(1)$(LIBEXT) $(1)-dist
$(INSTALL) $(2)/$(1)$(LIBEXT) $(PREFIX)/$(MODULEDIR)
.PHONY: $(1)-clean $(1)-install $(1)-dist
endef
# Include modules
......
package main
/*
#include "lib/layer.h"
#include "lib/module.h"
#include "lib/utils.h"
#include <libknot/descriptor.h>
int consume(knot_layer_t *, knot_pkt_t *);
static inline const char *module_path(void)
{ return PREFIX MODULEDIR; }
static inline const knot_layer_api_t *_layer(void)
{ static const knot_layer_api_t api = { .consume = &consume, }; return &api; }
*/
import "C"
import (
"os"
"sync"
"unsafe"
"fmt"
"net"
"net/http"
"html"
"html/template"
"encoding/json"
"github.com/abh/geoip"
)
type Sample struct {
qname string
qtype int
addr net.IP
secure bool
}
type QueryInfo struct {
Qname string
Qtype string
Addr string
Secure bool
}
// Global context
var resolver *C.struct_kr_context
// Synchronisation
var wg sync.WaitGroup
// Global channel for metrics
var ch_metrics chan Sample
// FIFO of last-seen metrics
var fifo_metrics [10] QueryInfo
var fifo_metrics_i = 0
// Geo frequency table
var geo_freq map[string] int
var geo_db *geoip.GeoIP
var geo_db6 *geoip.GeoIP
/*
* Callbacks for serving static content.
*/
func resource_path(filename string) string {
return C.GoString(C.module_path()) + "/tinyweb" + filename;
}
func serve_page(w http.ResponseWriter, r *http.Request) {
t, err := template.ParseFiles(resource_path("/tinyweb.tpl"))
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
host, err := os.Hostname()
t.Execute(w, struct {
Title string
}{
Title: "kresd @ " + host,
})
w.Header().Set("Content-Type", "text/html; charset=utf-8")
}
func serve_file(w http.ResponseWriter, r *http.Request) {
http.ServeFile(w, r, resource_path(html.EscapeString(r.URL.Path)))
}
/*
* Serving dynamic contents.
*/
func serve_json(w http.ResponseWriter, r *http.Request, v interface{}) {
js, err := json.Marshal(v)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
w.Write(js)
}
func serve_geo(w http.ResponseWriter, r *http.Request) {
serve_json(w, r, geo_freq)
}
func serve_feed(w http.ResponseWriter, r *http.Request) {
// Walk back FIFO to preserve ordering
const nsamples = len(fifo_metrics)
var samples [nsamples] QueryInfo
for i := 0; i < nsamples; i++ {
samples[i] = fifo_metrics[(nsamples + (fifo_metrics_i - i - 1)) % nsamples]
}
serve_json(w, r, samples)
}
func serve_stats(w http.ResponseWriter, r *http.Request) {
mod_name := C.CString("stats")
defer C.free(unsafe.Pointer(mod_name))
prop_name := C.CString("list")
defer C.free(unsafe.Pointer(prop_name))
out := C.kr_module_call(resolver, mod_name, prop_name, nil)
defer C.free(unsafe.Pointer(out))
if out != nil {
fmt.Fprintf(w, C.GoString(out))
} else {
http.Error(w, "No stats module", http.StatusInternalServerError)
}
}
/*
* Module implementation.
*/
//export tinyweb_init
func tinyweb_init(module *C.struct_kr_module) int {
resolver = (*C.struct_kr_context)(module.data)
ch_metrics = make(chan Sample, 10)
geo_freq = make(map[string]int)
// Start sample collector goroutine
wg.Add(1)
go func() {
defer wg.Done()
for msg := range ch_metrics {
var qtype_str [16] byte
C.knot_rrtype_to_string(C.uint16_t(msg.qtype), (*C.char)(unsafe.Pointer(&qtype_str[0])), C.size_t(16))
fifo_metrics[fifo_metrics_i] = QueryInfo{msg.qname, string(qtype_str[:]), msg.addr.String(), msg.secure}
fifo_metrics_i = (fifo_metrics_i + 1) % len(fifo_metrics)
// Sample NS country code
var cc string
switch len(msg.addr) {
case 4: if (geo_db != nil) { cc, _ = geo_db.GetCountry(msg.addr.String()) }
case 16: if (geo_db6 != nil) { cc, _ = geo_db6.GetCountry_v6(msg.addr.String()) }
default: continue
}
// Count occurences
if freq, exists := geo_freq[cc]; exists {
geo_freq[cc] = freq + 1
} else {
geo_freq[cc] = 1
}
}
}()
return 0
}
//export tinyweb_config
func tinyweb_config(module *C.struct_kr_module, conf *C.char) int {
var err error
var config map[string] interface{}
addr := "localhost:8053"
if err = json.Unmarshal([]byte(C.GoString(conf)), &config); err != nil {
fmt.Printf("[tinyweb] %s\n", err)
} else {
if v, ok := config["addr"]; ok {
addr = v.(string)
}
if v, ok := config["geoip"]; ok {
geoip.SetCustomDirectory(v.(string))
}
}
geo_db, err = geoip.OpenTypeFlag(geoip.GEOIP_COUNTRY_EDITION, geoip.GEOIP_MEMORY_CACHE)
if err != nil {
fmt.Printf("[tinyweb] couldn't open GeoIP IPv4 Country Edition\n");
}
geo_db6, err = geoip.OpenTypeFlag(geoip.GEOIP_COUNTRY_EDITION_V6, geoip.GEOIP_MEMORY_CACHE)
if err != nil {
fmt.Printf("[tinyweb] couldn't open GeoIP IPv6 Country Edition\n");
}
// Start web interface
http.HandleFunc("/feed", serve_feed)
http.HandleFunc("/stats", serve_stats)
http.HandleFunc("/geo", serve_geo)
http.HandleFunc("/tinyweb.js", serve_file)
http.HandleFunc("/datamaps.world.min.js", serve_file)
http.HandleFunc("/topojson.js", serve_file)
http.HandleFunc("/jquery.js", serve_file)
http.HandleFunc("/epoch.css", serve_file)
http.HandleFunc("/epoch.js", serve_file)
http.HandleFunc("/d3.js", serve_file)
http.HandleFunc("/", serve_page)
// @todo Not sure how to cancel this routine yet
// wg.Add(1)
go http.ListenAndServe(addr, nil)
return 0
}
//export tinyweb_deinit
func tinyweb_deinit(module *C.struct_kr_module) int {
close(ch_metrics)
wg.Wait()
return 0
}
//export consume
func consume(ctx *C.knot_layer_t, pkt *C.knot_pkt_t) C.int {
req := (*C.struct_kr_request)(ctx.data)
qry := req.current_query
state := (C.int)(ctx.state)
if qry.flags & C.QUERY_CACHED != 0 {
return state
}
// Parse answer source address
sa := (*C.struct_sockaddr)(unsafe.Pointer(&qry.ns.addr[0]))
var ip net.IP
if sa.sa_family == C.AF_INET {
sa_v4 := (*C.struct_sockaddr_in)(unsafe.Pointer(sa))
ip = net.IP(C.GoBytes(unsafe.Pointer(&sa_v4.sin_addr), 4))
} else if sa.sa_family == C.AF_INET6 {
sa_v6 := (*C.struct_sockaddr_in6)(unsafe.Pointer(sa))
ip = net.IP(C.GoBytes(unsafe.Pointer(&sa_v6.sin6_addr), 16))
}
// Parse metadata
qname := C.knot_dname_to_str_alloc(C.knot_pkt_qname(pkt))
defer C.free(unsafe.Pointer(qname))
qtype := C.knot_pkt_qtype(pkt)
secure := (bool)(C.knot_pkt_has_dnssec(pkt))
// Process metric
ch_metrics <- Sample{C.GoString(qname), (int)(qtype), ip, secure}
return state
}
//export tinyweb_layer
func tinyweb_layer(module *C.struct_kr_module) *C.knot_layer_api_t {
return C._layer()
}
//export tinyweb_api
func tinyweb_api() C.uint32_t {
return C.KR_MODULE_API
}
func main() {}
tinyweb_SOURCES := modules/tinyweb/tinyweb.go
tinyweb_INSTALL := $(wildcard modules/tinyweb/tinyweb/*)
tinyweb_DEPEND := $(libkres)
tinyweb_LIBS := $(libkres_TARGET) $(libkres_LIBS)
$(call make_go_module,tinyweb)
jQuery is provided under MIT license <https://jquery.org/license/>
D3 under BSD license <https://github.com/mbostock/d3/blob/master/LICENSE>
Epoch under MIT license <https://github.com/epochjs/epoch/blob/master/LICENSE>
TopoJSON under BSD license <https://github.com/mbostock/topojson/blob/master/LICENSE>
DataMaps under MIT license <https://github.com/markmarkoh/datamaps/blob/master/LICENSE>
.. _mod-tinyweb:
Web interface
-------------
This module provides an embedded web interface for resolver. It plots current performance in real-time,
including a feed of recent iterative queries. It also includes bindings_ to `MaxMind GeoIP`_, and presents a world map coloured by frequency of queries, so you can see where do your queries go.
By default, it listens on ``localhost:8053``.
Examples
^^^^^^^^
.. code-block:: lua
-- Load web interface
modules = { 'tinyweb' }
-- Listen on specific address/port
modules = {
tinyweb = {
addr = 'localhost:8080', -- Custom address
geoip = '/usr/local/var/GeoIP' -- Different path to GeoIP DB
}
}
Dependencies
^^^^^^^^^^^^
It depends on Go 1.5+, `github.com/abh/geoip <bindings>`_ package.
.. code-block:: bash
$ <install> libgeoip
$ go get github.com/abh/geoip
.. _`MaxMind GeoIP`: https://www.maxmind.com/en/home
.. _bindings: https://github.com/abh/geoip
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
// Country code conversion
var iso2_to_iso3 = {
"AF": "AFG", "AL": "ALB", "DZ": "DZA", "AS": "ASM", "AD": "AND", "AO": "AGO", "AI": "AIA", "AQ": "ATA", "AG": "ATG", "AR": "ARG", "AM": "ARM", "AW": "ABW", "AU": "AUS", "AT": "AUT", "AZ": "AZE", "BS": "BHS", "BH": "BHR", "BD": "BGD", "BB": "BRB", "BY": "BLR", "BE": "BEL", "BZ": "BLZ", "BJ": "BEN", "BM": "BMU", "BT": "BTN", "BO": "BOL", "BA": "BIH", "BW": "BWA", "BV": "BVT", "BR": "BRA", "IO": "IOT", "VG": "VGB", "BN": "BRN", "BG": "BGR", "BF": "BFA", "BI": "BDI", "KH": "KHM", "CM": "CMR", "CA": "CAN", "CV": "CPV", "KY": "CYM", "CF": "CAF", "TD": "TCD", "CL": "CHL", "CN": "CHN", "CX": "CXR", "CC": "CCK", "CO": "COL", "KM": "COM", "CD": "COD", "CG": "COG", "CK": "COK", "CR": "CRI", "CI": "CIV", "CU": "CUB", "CY": "CYP", "CZ": "CZE", "DK": "DNK", "DJ": "DJI", "DM": "DMA", "DO": "DOM", "EC": "ECU", "EG": "EGY", "SV": "SLV", "GQ": "GNQ", "ER": "ERI", "EE": "EST", "ET": "ETH", "FO": "FRO", "FK": "FLK", "FJ": "FJI", "FI": "FIN", "FR": "FRA", "GF": "GUF", "PF": "PYF", "TF": "ATF", "GA": "GAB", "GM": "GMB", "GE": "GEO", "DE": "DEU", "GH": "GHA", "GI": "GIB", "GR": "GRC", "GL": "GRL", "GD": "GRD", "GP": "GLP", "GU": "GUM", "GT": "GTM", "GN": "GIN", "GW": "GNB", "GY": "GUY", "HT": "HTI", "HM": "HMD", "VA": "VAT", "HN": "HND", "HK": "HKG", "HR": "HRV", "HU": "HUN", "IS": "ISL", "IN": "IND", "ID": "IDN", "IR": "IRN", "IQ": "IRQ", "IE": "IRL", "IL": "ISR", "IT": "ITA", "JM": "JAM", "JP": "JPN", "JO": "JOR", "KZ": "KAZ", "KE": "KEN", "KI": "KIR", "KP": "PRK", "KR": "KOR", "KW": "KWT", "KG": "KGZ", "LA": "LAO", "LV": "LVA", "LB": "LBN", "LS": "LSO", "LR": "LBR", "LY": "LBY", "LI": "LIE", "LT": "LTU", "LU": "LUX", "MO": "MAC", "MK": "MKD", "MG": "MDG", "MW": "MWI", "MY": "MYS", "MV": "MDV", "ML": "MLI", "MT": "MLT", "MH": "MHL", "MQ": "MTQ", "MR": "MRT", "MU": "MUS", "YT": "MYT", "MX": "MEX", "FM": "FSM", "MD": "MDA", "MC": "MCO", "MN": "MNG", "MS": "MSR", "MA": "MAR", "MZ": "MOZ", "MM": "MMR", "NA": "NAM", "NR": "NRU", "NP": "NPL", "AN": "ANT", "NL": "NLD", "NC": "NCL", "NZ": "NZL", "NI": "NIC", "NE": "NER", "NG": "NGA", "NU": "NIU", "NF": "NFK", "MP": "MNP", "NO": "NOR", "OM": "OMN", "PK": "PAK", "PW": "PLW", "PS": "PSE", "PA": "PAN", "PG": "PNG", "PY": "PRY", "PE": "PER", "PH": "PHL", "PN": "PCN", "PL": "POL", "PT": "PRT", "PR": "PRI", "QA": "QAT", "RE": "REU", "RO": "ROU", "RU": "RUS", "RW": "RWA", "SH": "SHN", "KN": "KNA", "LC": "LCA", "PM": "SPM", "VC": "VCT", "WS": "WSM", "SM": "SMR", "ST": "STP", "SA": "SAU", "SN": "SEN", "CS": "SCG", "SC": "SYC", "SL": "SLE", "SG": "SGP", "SK": "SVK", "SI": "SVN", "SB": "SLB", "SO": "SOM", "ZA": "ZAF", "GS": "SGS", "ES": "ESP", "LK": "LKA", "SD": "SDN", "SR": "SUR", "SJ": "SJM", "SZ": "SWZ", "SE": "SWE", "CH": "CHE", "SY": "SYR", "TW": "TWN", "TJ": "TJK", "TZ": "TZA", "TH": "THA", "TL": "TLS", "TG": "TGO", "TK": "TKL", "TO": "TON", "TT": "TTO", "TN": "TUN", "TR": "TUR", "TM": "TKM", "TC": "TCA", "TV": "TUV", "VI": "VIR", "UG": "UGA", "UA": "UKR", "AE": "ARE", "GB": "GBR", "UM": "UMI", "US": "USA", "UY": "URY", "UZ": "UZB", "VU": "VUT", "VE": "VEN", "VN": "VNM", "WF": "WLF", "EH": "ESH", "YE": "YEM", "ZM": "ZMB", "ZW": "ZWE",
};
// Set up UI and pollers
window.onload = function() {
var statsLabels = ['cached', '10ms', '100ms', '1000ms', 'slow'];
var statsHistory = [];
for (i = 0; i < statsLabels.length; ++i) {
statsHistory.push({ label: 'Layer ' + statsLabels[i], values: [] });
$('.legend').append('<li class="l-' + statsLabels[i] + '">' + statsLabels[i]);
}
var statsChart = $('#stats').epoch({
type: 'time.area',
axes: ['left', 'right'],
data: statsHistory
});
var statsPrev = null;
var map = new Datamap({
element: document.getElementById('map'),
fills: { defaultFill: '#F5F5F5' },
});
/* Realtime updates */
function poller(feed, interval, cb) {
setInterval(function () {
$.ajax({
url: feed,
type: 'get',
dataType: 'json',
success: cb
});
}, interval);
}
poller('stats', 1000, function(resp) {
var now = Date.now();
var next = [];
for (i = 0; i < statsLabels.length; ++i) {
next.push(resp['answer.' + statsLabels[i]]);
}
if (statsPrev) {
var delta = [];
for (i = 0; i < statsLabels.length; ++i) {
delta.push({time: now, y: next[i]-statsPrev[i]});
}
statsChart.push(delta);
}
statsPrev = next;
});
poller('feed', 2000, function(resp) {
var feed = $('#feed')
feed.children().remove();
feed.append('<tr><th>Type</th><th>Query</th><th>Nameserver</th><th>DNSSEC</th></tr>')
for (i = 0; i < resp.length; ++i) {
if (resp[i].Qname != "") {
var row = $('<tr />');
row.append('<td>' + resp[i].Qtype + '</td>');
row.append('<td>' + resp[i].Qname + '</td>');
row.append('<td>' + resp[i].Addr + '</td>');
if (resp[i].Secure) {
row.append('<td class="secure">SECURE</td>');
} else {
row.append('<td></td>');
}
feed.append(row);
}
}
});
poller('geo', 2000, function(resp) {
var update = {};
var max = 0.0;
/* Convert country code, calculate maximum. */
for (var key in resp) {
if (resp.hasOwnProperty(key)) {
max = Math.max(max, resp[key]);
var iso3_key = iso2_to_iso3[key];
if (iso3_key) {
update[iso3_key] = resp[key];
}
}
}
/* Normalize, convert to HSL. */
for (var key in update) {
var ratio = 1.0 - update[key]/max;
update[key] = 'hsl(205,70%,' + Math.floor(20.0 + 70.0 * ratio) + '%)'
}
map.updateChoropleth(update);
});
}
\ No newline at end of file
<!DOCTYPE html>
<title>{{.Title}}</title>
<style>
body { font-family: 'Gill Sans', 'Gill Sans MT', Verdana, sans-serif; color: #555; }
h1, h2, h3 { line-height: 2em; color: #000; text-align: center; border-bottom: 1px solid #ccc; }
h1, h2, h3 { font-weight: 300; }
th { text-align: left; font-weight: normal; margin-bottom: 0.5em; }