Experimental version using WebWorkers and WordCloud

Displaying and processing all data using WebWorker, support basics like
pagination, sorting, aggregation. All data are used to create a WordCloud that
can be used to provide some kind of overview.

What is currently missing is error handling and some more styling.

src/__init__.py

@@ -41,10 +41,6 @@ class PakonPluginPage(ConfigPageMixin, PakonPluginConfigHandler):
         args['PLUGIN_STYLES'] = PakonPlugin.PLUGIN_STYLES
         args['PLUGIN_STATIC_SCRIPTS'] = PakonPlugin.PLUGIN_STATIC_SCRIPTS
         args['PLUGIN_DYNAMIC_SCRIPTS'] = PakonPlugin.PLUGIN_DYNAMIC_SCRIPTS
-        args['query'] = json.dumps({})
-        data = current_state.backend.perform(
-            "pakon", "perform_query", {"query_data": args['query']})
-        args['results'] = self._prepare_data(data)
 
     def render(self, **kwargs):
         self._prepare_render_args(kwargs)
@@ -55,20 +51,14 @@ class PakonPluginPage(ConfigPageMixin, PakonPluginConfigHandler):
 
     def call_ajax_action(self, action):
         if action == "perform_query":
-            if bottle.request.method != 'POST':
-                raise bottle.HTTPError(404, "Wrong http method (only POST is allowed.")
+            if bottle.request.method != 'GET':
+                raise bottle.HTTPError(404, "Wrong http method (only GET is allowed.")
 
-            query_data = bottle.request.POST.get('query', {})
+            query_data = bottle.request.GET.get('query', {})
             data = current_state.backend.perform(
                 "pakon", "perform_query", {"query_data": query_data})
-            if "text/html" in bottle.request.headers["Accept"]:
-                return bottle.template(
-                    "pakon/_results",
-                    results=self._prepare_data(data),
-                )
-            else:
-                bottle.response.content_type = "application/json"
-                return data["response_data"]
+            bottle.response.content_type = "application/json"
+            return data["response_data"]
 
         raise ValueError("Unknown AJAX action.")
 
@@ -81,6 +71,7 @@ class PakonPlugin(ForisPlugin):
         "css/pakon.css"
     ]
     PLUGIN_STATIC_SCRIPTS = [
+        "js/jQWCloudv3.1.js"
     ]
     PLUGIN_DYNAMIC_SCRIPTS = [
         "pakon.js"

src/static/css/pakon.css

+@keyframes spinner {
+    from {transform:rotate(0deg);}
+    to {transform: rotate(360deg);}
+}
+    
+.spinner:after {
+    content: '';
+    box-sizing: border-box;
+    position: fixed;
+    top: 50%;
+    left: 50%;
+    width: 150px;
+    height: 150px;
+    margin-top: -75px;
+    margin-left: -75px;
+    border-radius: 50%;
+    border: 5px solid #ccc;
+    border-top-color: #00a2e2;
+    animation: spinner .6s linear infinite;
+}
+
+i {
+    padding-left: 5px;
+    padding-right: 5px;
+}
+#pakon-pager-page li {
+    display: inline;
+    margin: 5px;
+    text-decoration: underline;
+}
+#pakon-pager-page li.current-page {
+    text-decoration: none;
+    font-weight: bold;
+}
+
+.extra_row {
+    display: none;
+    font-style: italic;
+}
+
+#date-from, #date-to, #time-from, #time-to, #apply-changes {
+    float: right;
+    margin-left: 10px;
+    margin-right: 0px;
+}
+
+#page-pakon-plugin {
+    display: inline-block;
+}
+
+#filter-form {
+    display: inline-block;
+    clear: both;
+}
+
+#client-filter, #hostname-filter {
+    width: 100%;
+}
+
+label {
+    padding-top: 10px;
+    display: block;
+    width: 100%;
+}
+
+#no-data {
+    text-align: center;
+}
+
+#pakon-results {
+    white-space: nowrap;
+}
+
+#pakon-results td, th {
+    padding: 5px;
+    padding-left: 10px;
+    padding-right: 10px;
+}
+
+#pakon-results th {
+    font-weight: bold;
+}
+
+tr.odd, tr.even, tr.extra_row.even, tr.extra_row.odd {
+    border-style: none none dashed none;
+    border-width: 2px;
+    border-color: rgba(255,255,255,0);
+}
+
+tr.odd:hover, tr.even:hover, tr.extra_row.even:hover, tr.extra_row.odd:hover {
+    border-color: rgba(0,0,0,1);
+}
+
+tr.odd {
+    background: #EEE;
+}
+tr.even {
+    background: #FFF;
+}
+tr.extra_row.even {
+    background: #DDD;
+}
+tr.extra_row.odd {
+    background: #EEE;
+}
+
+td:nth-child(7), td:nth-child(8), th:nth-child(7), th:nth-child(8) {
+    text-align: right;
+}
+
+#pakon-pager {
+    margin-top: 10px;
+    text-align: right;
+}
+
+#pakon-pager-page {
+    margin: 10px;
+    display: inline-block;
+}
+
+#pakon-pager-pagesize {
+    display: inline-block;
+    width: 10em;
+}

src/static/js/data_manager.js

+var full_data = [];
+var sort_by = [0, -1];
+var page = 0;
+var page_size = 200;
+var limit = 20;
+var filtered_data = [];
+var aggregated_data = [];
+var filter_hosts = [];
+var filter_clients = [];
+var aggregate_by = 'hostname';
+var word_list = {};
+var word_list_txt = [];
+
+function format_data(data) {
+    const BASE = 1024;
+    const RIDGE = '\u00A0'; // NO-BREAK SPACE
+    const SIZES = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB']; // @todo : add abbr title for units
+    const getMeasurementUnit = Math.floor(Math.log(Number(data)) / Math.log(BASE));
+    if(this === 0) {
+        return '0' + RIDGE + BYTES[lang];
+    }
+    return parseFloat((Number(data) / Math.pow(BASE, getMeasurementUnit)).toFixed(2)) + RIDGE + SIZES[getMeasurementUnit];
+}
+
+function zero_pad(data) {
+    if(data < 10) {
+        return '0' + data;
+    }
+    return data;
+}
+
+function time_interval(data) {
+    let h = Math.floor(data / 3600);
+    let m = Math.floor((data - h * 3600) / 60);
+    let s = Math.floor(data - h * 3600 - m * 60);
+    return zero_pad(h) + ':' + zero_pad(m) + ':' + zero_pad(s);
+}
+
+function shorten(data) {
+    if(data.length > 30) {
+        return '...' + data.substring(data.length - 27);
+    }
+    return data;
+}
+
+function render_entry(data) {
+    let table = '';
+    for(y in data) {
+        switch (y) {
+        case '1':
+            table += '<td data-sort-value="' + data[y] + '">' + time_interval(data[y]) + '</td>';
+            break;
+        case '3':
+            table += '<td data-sort-value="' + data[y] + '" title="' + data[y] + '">' + shorten(data[y]) + '</td>';
+            break;
+        case '5':
+            break;
+        case '6':
+        case '7':
+            table += '<td data-sort-value="' + data[y] + '">' + format_data(data[y]) + '</td>';
+            break;
+        case '8':
+            break;
+        default:
+            table += '<td data-sort-value="' + data[y] + '">' + data[y] + '</td>';
+            break;
+        }
+    }
+    return table;
+
+}
+
+function render_results() {
+    console.log('Rendering data');
+    if(filtered_data == null) {
+        return {
+            'table': '<tr><td></td><td colspan=7 id="no-data">No data</td></tr>',
+            'word_list': '',
+            'pager': '',
+            'page_size': page_size,
+            'sort_by': sort_by
+        }
+    }
+    let table = '';
+    let x = 0;
+    let wordcloud = {};
+    for(x = page * page_size;
+        (x < filtered_data.length) && ((page_size == 0) || (x < (page + 1) * page_size)); x++) {
+        table += '<tr id="line_' + x + '" class="' + (x % 2 ? 'odd' : 'even') + '">';
+        if(filtered_data[x][8]) {
+            table += '<td onClick="toogle_lines(' + x + ')"><i class="fas fa-plus"></i></td>';
+        } else {
+            table += '<td></td>';
+        }
+        table += render_entry(filtered_data[x]);
+        table += '</tr>\n';
+        if(filtered_data[x][8]) {
+            let y = 0;
+            for(y = 0; y < filtered_data[x][8].length; y++) {
+                table += '<tr class="extra_row extra_' + x + ' ' + (y % 2 ? 'odd' : 'even') + '"><td></td>' + render_entry(filtered_data[x][8][y]) + '</tr>\n';
+            }
+        }
+    }
+    var pager = '<ul>'
+    if(page_size > 0) {
+        for(x = 0; x < filtered_data.length / page_size; x++) {
+            if(x == page) {
+                pager += '<li class="current-page" onClick="goto_page(' + x + ')">' + x + '</li>';
+            } else {
+                pager += '<li onClick="goto_page(' + x + ')"><a href="#">' + x + '</a></li>';
+            }
+        }
+    }
+    pager += '</ul>';
+    return {
+        'table': table,
+        'pager': pager,
+        'page_size': page_size,
+        'sort_by': sort_by,
+        'word_list': word_list_txt
+    }
+}
+
+function print_date(date) {
+    return '' + date.getFullYear() + '-' + zero_pad(date.getMonth() + 1) + '-' + zero_pad(date.getDate()) + ' ' + zero_pad(date.getHours()) + ':' + zero_pad(date.getMinutes()) + ':' + zero_pad(date.getSeconds());
+}
+
+function get_word_list() {
+    word_list = {};
+    if(filtered_data == null)
+        return;
+    let word_index = 3;
+    let word_value_index = 1;
+    for(x in filtered_data) {
+        word_list[filtered_data[x][3]] = (word_list[filtered_data[x][3]] ? word_list[filtered_data[x][3]] : 0) + filtered_data[x][word_value_index];
+    }
+    word_list_txt = [];
+    for(k in word_list) {
+        word_list_txt.push({
+            word: k,
+            weight: word_list[k]
+        });
+    }
+}
+
+function aggregate_data() {
+    console.log('Aggregating data');
+    if((aggregate_by == 'none') || (filtered_data == null)) {
+        aggregated_data = filtered_data;
+        return
+    }
+    let index = 3;
+    let other_index = 2;
+    if(aggregate_by == 'hostname') {
+        index = 3;
+        other_index = 2;
+    }
+    if(aggregate_by == 'client') {
+        index = 2;
+        other_index = 3;
+    }
+    let tmp = filtered_data.sort(function (a, b) {
+        if(a[index] == b[index]) {
+            return a[other_index] < b[other_index]
+        } else {
+            return a[index] < b[index];
+        }
+    });
+    console.log('Aggregating data (' + tmp.length + ')');
+    aggregated_data = [];
+    let cur_entry = ['', '', '', '', '', '', ''];
+    let a_entries = [];
+    let last_entry = cur_entry;
+    let st_date;
+    let nd_date;
+    let tmp_date;
+    let send;
+    let recv;
+    let i;
+    let proto = ""
+    aggregated_data = [];
+    for(i = 1; i < tmp.length; i++) {
+        cur_entry = tmp[i];
+        if(cur_entry[2] == last_entry[2] && cur_entry[3] == last_entry[3]) {
+            if(proto != cur_entry[4])
+                proto = "";
+            send += cur_entry[6];
+            recv += cur_entry[7];
+            tmp_date = new Date(Date.parse(cur_entry[0]));
+            if(tmp_date < st_date) st_date = tmp_date;
+            tmp_date = new Date(tmp_date.getTime() + cur_entry[1] * 1000);
+            if(tmp_date > nd_date) nd_date = tmp_date;
+            a_entries.push(cur_entry);
+        } else {
+            if(a_entries.length < 2) {
+                if(a_entries.length == 1)
+                    aggregated_data.push(a_entries[0]);
+            } else {
+                aggregated_data.push([print_date(st_date), (nd_date.getTime() - st_date.getTime()) / 1000, last_entry[2], last_entry[3], proto, '', send, recv, a_entries]);
+            }
+            send = cur_entry[6];
+            recv = cur_entry[7];
+            proto = cur_entry[4];
+            a_entries = [cur_entry];
+            st_date = new Date(Date.parse(cur_entry[0]));
+            nd_date = new Date(st_date.getTime() + cur_entry[1] * 1000);
+        }
+        last_entry = cur_entry;
+    }
+    console.log('Aggregated ' + tmp.length + ' -> ' + aggregated_data.length);
+    sort_data();
+}
+
+function sort_data() {
+    console.log(`Sorting data by ${sort_by[0]} direction ${sort_by[1]}`);
+    if(aggregated_data == null)
+        return;
+    if(sort_by[0] > 4) sort_by[0]++;
+    filtered_data = aggregated_data.sort(function (a, b) {
+        return sort_by[1] < 0 ? a[sort_by[0]] < b[sort_by[0]] : a[sort_by[0]] > b[sort_by[0]]
+    });
+    if(sort_by[0] > 4) sort_by[0]--;
+}
+
+function filter_data() {
+    console.log('Filtering data');
+    if(full_data == null) {
+        filtered_data = null;
+    } else {
+        filtered_data = full_data.filter(function (e) {
+            let ret = 0;
+            if(filter_hosts.length > 0) {
+                for(i in filter_hosts) {
+                    if(filter_hosts[i].test(e[3])) {
+                        ret |= 0x1;
+                    }
+                }
+            } else {
+                ret |= 0x1;
+            }
+            if(filter_clients.length > 0) {
+                for(i in filter_clients) {
+                    if(filter_clients[i].test(e[2])) {
+                        ret |= 0x2;
+                    }
+                }
+            } else {
+                ret |= 0x2;
+            }
+            if(ret == 0x3) {
+                return true;
+            }
+            return false;
+        });
+        console.log('Filtering done: ' + full_data.length + ' -> ' + filtered_data.length);
+    }
+    aggregate_data();
+    get_word_list();
+}
+
+onmessage = function (e) {
+    console.log('Message "' + e.data.command + '" received from main script');
+    let render = false;
+    switch (e.data.command) {
+    case "change":
+        page = 0;
+        if(e.data.filter_hosts) {
+            console.log("Updating host filters");
+            filter_hosts = e.data.filter_hosts;
+            console.log(filter_hosts);
+        }
+        if(e.data.filter_clients) {
+            console.log("Updating client filters");
+            filter_clients = e.data.filter_clients;
+            console.log(filter_clients);
+        }
+        if(e.data.aggregate) {
+            aggregate_by = 'hostname';
+        } else {
+            aggregate_by = 'none';
+        }
+        if(e.data.date_from || e.data.date_to) {
+            full_data = [];
+            console.log('Updating interval to ' + e.data.date_from + ' - ' + e.data.date_to);
+            let req = new XMLHttpRequest();
+            let query = '{"aggregate": false, "start":' + e.data.date_from + (e.data.date_to ? ',",end":' + e.data.date_to + '}' : '}');
+            console.log('Getting response to ' + query);
+            req.open('GET', e.data.ajax_url + '?action=perform_query&query=' + query, true);
+            req.responseType = 'json';
+            req.setRequestHeader('Content-type', 'application/json')
+            req.onreadystatechange = function () {
+                if(this.readyState == 4 && this.status == 200) {
+                    full_data = this.response;
+                    filter_data();
+                    postMessage(render_results());
+                }
+            };
+            req.send();
+        } else {
+            filter_data();
+            sort_data();
+            postMessage(render_results());
+        }
+        break;
+    case "page":
+        page = e.data.page;
+        page_size = e.data.page_size;
+        postMessage(render_results());
+        break;
+    case "sort":
+        page = 0;
+        sort_by = e.data.sort_by;
+        sort_data();
+        postMessage(render_results());
+        break;
+    }
+}

src/static/js/jQWCloudv3.1.js

+var LB=1,  //Left Bottom
+	LT=2,  //Left Top
+	RT=3,  //Right Top
+	RB=4,  //Right Bottom
+	HR=1, 	//Horizontal		
+	VR=2,    //Vertical
+	WordObjType='span',
+	DIV='div',
+	Word_Default_font_Family='Impact',
+	distance_Counter=1,
+	word_counter=1;
+
+function Util(){}
+//To Generate Random Colors For Words
+Util.getRandomColor = function(){
+	var letters = '0123456789ABCDEF'.split('');
+    var color = '#';
+    for (var i = 0; i < 6; i++ ) {
+        color += letters[Math.round(Math.random() * 15)];
+    }
+    return color;
+};
+
+function space(spaceType,width,height,x,y){
+	this.spaceType=spaceType;
+	this.width=width;
+	this.height=height;
+	this.x=x;
+	this.y=y;
+}
+
+function Word(wordConfig){
+	this.word=wordConfig.word;
+	this.weight=wordConfig.weight;
+	
+	this.fontFactor=wordConfig.fontFactor;
+	this.fontOffset=wordConfig.fontOffset;
+	this.minWeight=wordConfig.minWeight;
+	this.padding_left=wordConfig.padding_left;
+	
+	this.font_family=wordConfig.font_family;
+	this.font=null;
+	this.color=wordConfig.color;
+	this.span=null;
+	this.width=null;
+    this.height=null;
+    this.word_class=wordConfig.word_class;
+    
+    this._init();
+}
+Word.prototype = {
+	_init: function(){
+		this._setFont();
+		this._setSpan_Size();
+	},
+	_setFont: function(){
+		this.font=Math.floor(((this.weight-this.minWeight) *this.fontFactor ) + this.fontOffset);
+	},
+	_setSpan_Size: function(){
+		var span = document.createElement(WordObjType);
+		span.setAttribute('id', "Word_"+(word_counter++)+"_"+this.weight);
+		document.body.appendChild(span);
+		$(span).css({
+			   position: 'absolute',
+			   display: 'block',
+			   left: -999990,
+			   top: 0
+		});
+		$(span).css("font-size",this.font+"px");
+		
+		if(this.font_family!=null && this.font_family!='')
+			$(span).css("font-family",this.font_family);
+		else
+			$(span).css("font-family",Word_Default_font_Family);
+		
+		
+		if(this.word_class!=null && this.word_class!='')
+			$(span).addClass(this.word_class);
+		
+		if(this.color!=null && this.color!='')
+			$(span).css("color",this.color);
+		else
+			$(span).css("color",Util.getRandomColor());
+		
+		$(span).css("-webkit-user-select","none").css("-moz-user-select","none").css("-ms-user-select","none");
+		$(span).css("user-select","none").css("-o-user-select","none");
+		$(span).css("line-height",this.font+"px");
+		if(this.padding_left==null)
+			this.padding_left=0;
+			
+		$(span).css("padding-left",this.padding_left+"px");
+		$(span).html(this.word);
+		
+		this.width=$(span).outerWidth()+(this.padding_left*2);
+	    this.height=$(span).outerHeight();
+	    
+	    $(span).remove();
+	    this.span=span;	
+}	
+};
+
+
+function WordCloud() {
+	this.defaultOptions={
+			title: 'JQ WOrd Cloud',
+			words: [],
+			minFont: 10,
+			maxFont: 50,
+			fontOffset