Unverified Commit f641ba59 authored by Michal Hrusecky's avatar Michal Hrusecky 🐉

Experimental version using WebWorkers and WordCloud

Displaying and processing all data using WebWorker, support basics like
pagination, sorting, aggregation. All data are used to create a WordCloud that
can be used to provide some kind of overview.

What is currently missing is error handling and some more styling.
parent f88b8747
......@@ -41,10 +41,6 @@ class PakonPluginPage(ConfigPageMixin, PakonPluginConfigHandler):
args['PLUGIN_STYLES'] = PakonPlugin.PLUGIN_STYLES
args['PLUGIN_STATIC_SCRIPTS'] = PakonPlugin.PLUGIN_STATIC_SCRIPTS
args['PLUGIN_DYNAMIC_SCRIPTS'] = PakonPlugin.PLUGIN_DYNAMIC_SCRIPTS
args['query'] = json.dumps({})
data = current_state.backend.perform(
"pakon", "perform_query", {"query_data": args['query']})
args['results'] = self._prepare_data(data)
def render(self, **kwargs):
self._prepare_render_args(kwargs)
......@@ -55,20 +51,14 @@ class PakonPluginPage(ConfigPageMixin, PakonPluginConfigHandler):
def call_ajax_action(self, action):
if action == "perform_query":
if bottle.request.method != 'POST':
raise bottle.HTTPError(404, "Wrong http method (only POST is allowed.")
if bottle.request.method != 'GET':
raise bottle.HTTPError(404, "Wrong http method (only GET is allowed.")
query_data = bottle.request.POST.get('query', {})
query_data = bottle.request.GET.get('query', {})
data = current_state.backend.perform(
"pakon", "perform_query", {"query_data": query_data})
if "text/html" in bottle.request.headers["Accept"]:
return bottle.template(
"pakon/_results",
results=self._prepare_data(data),
)
else:
bottle.response.content_type = "application/json"
return data["response_data"]
bottle.response.content_type = "application/json"
return data["response_data"]
raise ValueError("Unknown AJAX action.")
......@@ -81,6 +71,7 @@ class PakonPlugin(ForisPlugin):
"css/pakon.css"
]
PLUGIN_STATIC_SCRIPTS = [
"js/jQWCloudv3.1.js"
]
PLUGIN_DYNAMIC_SCRIPTS = [
"pakon.js"
......
@keyframes spinner {
from {transform:rotate(0deg);}
to {transform: rotate(360deg);}
}
.spinner:after {
content: '';
box-sizing: border-box;
position: fixed;
top: 50%;
left: 50%;
width: 150px;
height: 150px;
margin-top: -75px;
margin-left: -75px;
border-radius: 50%;
border: 5px solid #ccc;
border-top-color: #00a2e2;
animation: spinner .6s linear infinite;
}
i {
padding-left: 5px;
padding-right: 5px;
}
#pakon-pager-page li {
display: inline;
margin: 5px;
text-decoration: underline;
}
#pakon-pager-page li.current-page {
text-decoration: none;
font-weight: bold;
}
.extra_row {
display: none;
font-style: italic;
}
#date-from, #date-to, #time-from, #time-to, #apply-changes {
float: right;
margin-left: 10px;
margin-right: 0px;
}
#page-pakon-plugin {
display: inline-block;
}
#filter-form {
display: inline-block;
clear: both;
}
#client-filter, #hostname-filter {
width: 100%;
}
label {
padding-top: 10px;
display: block;
width: 100%;
}
#no-data {
text-align: center;
}
#pakon-results {
white-space: nowrap;
}
#pakon-results td, th {
padding: 5px;
padding-left: 10px;
padding-right: 10px;
}
#pakon-results th {
font-weight: bold;
}
tr.odd, tr.even, tr.extra_row.even, tr.extra_row.odd {
border-style: none none dashed none;
border-width: 2px;
border-color: rgba(255,255,255,0);
}
tr.odd:hover, tr.even:hover, tr.extra_row.even:hover, tr.extra_row.odd:hover {
border-color: rgba(0,0,0,1);
}
tr.odd {
background: #EEE;
}
tr.even {
background: #FFF;
}
tr.extra_row.even {
background: #DDD;
}
tr.extra_row.odd {
background: #EEE;
}
td:nth-child(7), td:nth-child(8), th:nth-child(7), th:nth-child(8) {
text-align: right;
}
#pakon-pager {
margin-top: 10px;
text-align: right;
}
#pakon-pager-page {
margin: 10px;
display: inline-block;
}
#pakon-pager-pagesize {
display: inline-block;
width: 10em;
}
var full_data = [];
var sort_by = [0, -1];
var page = 0;
var page_size = 200;
var limit = 20;
var filtered_data = [];
var aggregated_data = [];
var filter_hosts = [];
var filter_clients = [];
var aggregate_by = 'hostname';
var word_list = {};
var word_list_txt = [];
function format_data(data) {
const BASE = 1024;
const RIDGE = '\u00A0'; // NO-BREAK SPACE
const SIZES = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB']; // @todo : add abbr title for units
const getMeasurementUnit = Math.floor(Math.log(Number(data)) / Math.log(BASE));
if(this === 0) {
return '0' + RIDGE + BYTES[lang];
}
return parseFloat((Number(data) / Math.pow(BASE, getMeasurementUnit)).toFixed(2)) + RIDGE + SIZES[getMeasurementUnit];
}
function zero_pad(data) {
if(data < 10) {
return '0' + data;
}
return data;
}
function time_interval(data) {
let h = Math.floor(data / 3600);
let m = Math.floor((data - h * 3600) / 60);
let s = Math.floor(data - h * 3600 - m * 60);
return zero_pad(h) + ':' + zero_pad(m) + ':' + zero_pad(s);
}
function shorten(data) {
if(data.length > 30) {
return '...' + data.substring(data.length - 27);
}
return data;
}
function render_entry(data) {
let table = '';
for(y in data) {
switch (y) {
case '1':
table += '<td data-sort-value="' + data[y] + '">' + time_interval(data[y]) + '</td>';
break;
case '3':
table += '<td data-sort-value="' + data[y] + '" title="' + data[y] + '">' + shorten(data[y]) + '</td>';
break;
case '5':
break;
case '6':
case '7':
table += '<td data-sort-value="' + data[y] + '">' + format_data(data[y]) + '</td>';
break;
case '8':
break;
default:
table += '<td data-sort-value="' + data[y] + '">' + data[y] + '</td>';
break;
}
}
return table;
}
function render_results() {
console.log('Rendering data');
if(filtered_data == null) {
return {
'table': '<tr><td></td><td colspan=7 id="no-data">No data</td></tr>',
'word_list': '',
'pager': '',
'page_size': page_size,
'sort_by': sort_by
}
}
let table = '';
let x = 0;
let wordcloud = {};
for(x = page * page_size;
(x < filtered_data.length) && ((page_size == 0) || (x < (page + 1) * page_size)); x++) {
table += '<tr id="line_' + x + '" class="' + (x % 2 ? 'odd' : 'even') + '">';
if(filtered_data[x][8]) {
table += '<td onClick="toogle_lines(' + x + ')"><i class="fas fa-plus"></i></td>';
} else {
table += '<td></td>';
}
table += render_entry(filtered_data[x]);
table += '</tr>\n';
if(filtered_data[x][8]) {
let y = 0;
for(y = 0; y < filtered_data[x][8].length; y++) {
table += '<tr class="extra_row extra_' + x + ' ' + (y % 2 ? 'odd' : 'even') + '"><td></td>' + render_entry(filtered_data[x][8][y]) + '</tr>\n';
}
}
}
var pager = '<ul>'
if(page_size > 0) {
for(x = 0; x < filtered_data.length / page_size; x++) {
if(x == page) {
pager += '<li class="current-page" onClick="goto_page(' + x + ')">' + x + '</li>';
} else {
pager += '<li onClick="goto_page(' + x + ')"><a href="#">' + x + '</a></li>';
}
}
}
pager += '</ul>';
return {
'table': table,
'pager': pager,
'page_size': page_size,
'sort_by': sort_by,
'word_list': word_list_txt
}
}
function print_date(date) {
return '' + date.getFullYear() + '-' + zero_pad(date.getMonth() + 1) + '-' + zero_pad(date.getDate()) + ' ' + zero_pad(date.getHours()) + ':' + zero_pad(date.getMinutes()) + ':' + zero_pad(date.getSeconds());
}
function get_word_list() {
word_list = {};
if(filtered_data == null)
return;
let word_index = 3;
let word_value_index = 1;
for(x in filtered_data) {
word_list[filtered_data[x][3]] = (word_list[filtered_data[x][3]] ? word_list[filtered_data[x][3]] : 0) + filtered_data[x][word_value_index];
}
word_list_txt = [];
for(k in word_list) {
word_list_txt.push({
word: k,
weight: word_list[k]
});
}
}
function aggregate_data() {
console.log('Aggregating data');
if((aggregate_by == 'none') || (filtered_data == null)) {
aggregated_data = filtered_data;
return
}
let index = 3;
let other_index = 2;
if(aggregate_by == 'hostname') {
index = 3;
other_index = 2;
}
if(aggregate_by == 'client') {
index = 2;
other_index = 3;
}
let tmp = filtered_data.sort(function (a, b) {
if(a[index] == b[index]) {
return a[other_index] < b[other_index]
} else {
return a[index] < b[index];
}
});
console.log('Aggregating data (' + tmp.length + ')');
aggregated_data = [];
let cur_entry = ['', '', '', '', '', '', ''];
let a_entries = [];
let last_entry = cur_entry;
let st_date;
let nd_date;
let tmp_date;
let send;
let recv;
let i;
let proto = ""
aggregated_data = [];
for(i = 1; i < tmp.length; i++) {
cur_entry = tmp[i];
if(cur_entry[2] == last_entry[2] && cur_entry[3] == last_entry[3]) {
if(proto != cur_entry[4])
proto = "";
send += cur_entry[6];
recv += cur_entry[7];
tmp_date = new Date(Date.parse(cur_entry[0]));
if(tmp_date < st_date) st_date = tmp_date;
tmp_date = new Date(tmp_date.getTime() + cur_entry[1] * 1000);
if(tmp_date > nd_date) nd_date = tmp_date;
a_entries.push(cur_entry);
} else {
if(a_entries.length < 2) {
if(a_entries.length == 1)
aggregated_data.push(a_entries[0]);
} else {
aggregated_data.push([print_date(st_date), (nd_date.getTime() - st_date.getTime()) / 1000, last_entry[2], last_entry[3], proto, '', send, recv, a_entries]);
}
send = cur_entry[6];
recv = cur_entry[7];
proto = cur_entry[4];
a_entries = [cur_entry];
st_date = new Date(Date.parse(cur_entry[0]));
nd_date = new Date(st_date.getTime() + cur_entry[1] * 1000);
}
last_entry = cur_entry;
}
console.log('Aggregated ' + tmp.length + ' -> ' + aggregated_data.length);
sort_data();
}
function sort_data() {
console.log(`Sorting data by ${sort_by[0]} direction ${sort_by[1]}`);
if(aggregated_data == null)
return;
if(sort_by[0] > 4) sort_by[0]++;
filtered_data = aggregated_data.sort(function (a, b) {
return sort_by[1] < 0 ? a[sort_by[0]] < b[sort_by[0]] : a[sort_by[0]] > b[sort_by[0]]
});
if(sort_by[0] > 4) sort_by[0]--;
}
function filter_data() {
console.log('Filtering data');
if(full_data == null) {
filtered_data = null;
} else {
filtered_data = full_data.filter(function (e) {
let ret = 0;
if(filter_hosts.length > 0) {
for(i in filter_hosts) {
if(filter_hosts[i].test(e[3])) {
ret |= 0x1;
}
}
} else {
ret |= 0x1;
}
if(filter_clients.length > 0) {
for(i in filter_clients) {
if(filter_clients[i].test(e[2])) {
ret |= 0x2;
}
}
} else {
ret |= 0x2;
}
if(ret == 0x3) {
return true;
}
return false;
});
console.log('Filtering done: ' + full_data.length + ' -> ' + filtered_data.length);
}
aggregate_data();
get_word_list();
}
onmessage = function (e) {
console.log('Message "' + e.data.command + '" received from main script');
let render = false;
switch (e.data.command) {
case "change":
page = 0;
if(e.data.filter_hosts) {
console.log("Updating host filters");
filter_hosts = e.data.filter_hosts;
console.log(filter_hosts);
}
if(e.data.filter_clients) {
console.log("Updating client filters");
filter_clients = e.data.filter_clients;
console.log(filter_clients);
}
if(e.data.aggregate) {
aggregate_by = 'hostname';
} else {
aggregate_by = 'none';
}
if(e.data.date_from || e.data.date_to) {
full_data = [];
console.log('Updating interval to ' + e.data.date_from + ' - ' + e.data.date_to);
let req = new XMLHttpRequest();
let query = '{"aggregate": false, "start":' + e.data.date_from + (e.data.date_to ? ',",end":' + e.data.date_to + '}' : '}');
console.log('Getting response to ' + query);
req.open('GET', e.data.ajax_url + '?action=perform_query&query=' + query, true);
req.responseType = 'json';
req.setRequestHeader('Content-type', 'application/json')
req.onreadystatechange = function () {
if(this.readyState == 4 && this.status == 200) {
full_data = this.response;
filter_data();
postMessage(render_results());
}
};
req.send();
} else {
filter_data();
sort_data();
postMessage(render_results());
}
break;
case "page":
page = e.data.page;
page_size = e.data.page_size;
postMessage(render_results());
break;
case "sort":
page = 0;
sort_by = e.data.sort_by;
sort_data();
postMessage(render_results());
break;
}
}
var LB=1, //Left Bottom
LT=2, //Left Top
RT=3, //Right Top
RB=4, //Right Bottom
HR=1, //Horizontal
VR=2, //Vertical
WordObjType='span',
DIV='div',
Word_Default_font_Family='Impact',
distance_Counter=1,
word_counter=1;
function Util(){}
//To Generate Random Colors For Words
Util.getRandomColor = function(){
var letters = '0123456789ABCDEF'.split('');
var color = '#';
for (var i = 0; i < 6; i++ ) {
color += letters[Math.round(Math.random() * 15)];
}
return color;
};
function space(spaceType,width,height,x,y){
this.spaceType=spaceType;
this.width=width;
this.height=height;
this.x=x;
this.y=y;
}
function Word(wordConfig){
this.word=wordConfig.word;
this.weight=wordConfig.weight;
this.fontFactor=wordConfig.fontFactor;
this.fontOffset=wordConfig.fontOffset;
this.minWeight=wordConfig.minWeight;
this.padding_left=wordConfig.padding_left;
this.font_family=wordConfig.font_family;
this.font=null;
this.color=wordConfig.color;
this.span=null;
this.width=null;
this.height=null;
this.word_class=wordConfig.word_class;
this._init();
}
Word.prototype = {
_init: function(){
this._setFont();
this._setSpan_Size();
},
_setFont: function(){
this.font=Math.floor(((this.weight-this.minWeight) *this.fontFactor ) + this.fontOffset);
},
_setSpan_Size: function(){
var span = document.createElement(WordObjType);
span.setAttribute('id', "Word_"+(word_counter++)+"_"+this.weight);
document.body.appendChild(span);
$(span).css({
position: 'absolute',
display: 'block',
left: -999990,
top: 0
});
$(span).css("font-size",this.font+"px");
if(this.font_family!=null && this.font_family!='')
$(span).css("font-family",this.font_family);
else
$(span).css("font-family",Word_Default_font_Family);
if(this.word_class!=null && this.word_class!='')
$(span).addClass(this.word_class);
if(this.color!=null && this.color!='')
$(span).css("color",this.color);
else
$(span).css("color",Util.getRandomColor());
$(span).css("-webkit-user-select","none").css("-moz-user-select","none").css("-ms-user-select","none");
$(span).css("user-select","none").css("-o-user-select","none");
$(span).css("line-height",this.font+"px");
if(this.padding_left==null)
this.padding_left=0;
$(span).css("padding-left",this.padding_left+"px");
$(span).html(this.word);
this.width=$(span).outerWidth()+(this.padding_left*2);
this.height=$(span).outerHeight();
$(span).remove();
this.span=span;
}
};
function WordCloud() {
this.defaultOptions={
title: 'JQ WOrd Cloud',
words: [],
minFont: 10,
maxFont: 50,
fontOffset: 0,
showSpaceDIV: false,
verticalEnabled: true,
cloud_color: null,
cloud_font_family: null,
spaceDIVColor: 'white',
padding_left: null,
word_common_classes: null,
word_click : function(){},
word_mouseOver : function(){},
word_mouseEnter : function(){},
word_mouseOut : function(){},
beforeCloudRender: function(){},
afterCloudRender: function(){}
};
this.minWeight=null;
this.maxWeight=null;
this.spaceDataObject=null;
this.spaceIdArray=null;
this.words=null;
this.fontFactor=1,
this.methods = {
destroy : this._destroy<