Commit 3f0a81b4 authored by Edvard Rejthar's avatar Edvard Rejthar 💬
Browse files

UI scans and cache working

parent f3914864
import json
import logging
import subprocess
from urllib.parse import urlparse
from peewee import IntegrityError
from ...templates.crawl_view import CrawlView
from ..domains import Domains
from .scan_controller import ScanController
from ..config import Config
from ..model.dbp import Turris, Whitelist
from ..parser.traffic_log_parser import TrafficLogParser
from ...templates.crawl_view import CrawlView
class Api:
......@@ -20,20 +22,31 @@ class Api:
:type path: dict from URL request. /api/analyze=cache/http://example.com → {"api": True, "analyze": cache, "page": "http://example.com"}
"""
crawl = None
if "analyze" in request:
crawl = ScanController().launch(request["page"], {"cached": 1, "weekcache":7, "oldcache": True, True: None}[request["analyze"]])
if request["api"] == "json":
return CrawlView.output_json(crawl)
map_ = {"fresh": None, "cached": 1, "weekcache": 7, "oldcache": True, True: None}
if request["analyze"] in map_:
days = map_[request["analyze"]]
else:
return CrawlView.output_html(crawl)
days = int(request["analyze"])
crawl = ScanController().launch(request["page"], days)
elif "decide" in request: # XX deprecated?
return self.get_undecided()
elif "nicify" in request:
return TrafficLogParser.getStylesheet() + TrafficLogParser.nicifyFile(request["page"])
elif "vote" in request: # /api/vote/block/example.org/10.0.0.1
elif "vote" in request: # /api/vote/block/example.org/10.0.0.1
logging.debug("vote cmd")
return Turris.vote(request["vote"], request["page"])
elif "whitelist" in request: # XXX not implemented yet
elif "scans" in request:
if "url" in request: # /api/scans/url/http://example.com
domain = Domains.domain2dir(request["page"])
if not domain:
return "Wrong domain"
return ScanController().get_domain_snapdirs(domain, full_dirs=False)
else:
return "Not implemented"
elif "whitelist" in request: # XXX not implemented yet
"""url = path.split("/", 3)
if len(url) > 3:
self._setWebsite(url[2]) # osetrit self.website, ze je URL, a nikoli shell
......@@ -47,6 +60,14 @@ class Api:
elif "reset" in request:
self.reset()
return "reset"
else:
return "Unknown method."
if crawl:
if request["api"] == "json":
return CrawlView.output_json(crawl)
else:
return CrawlView.output_html(crawl)
@staticmethod
def reset():
......
......@@ -28,30 +28,31 @@ class ScanController:
# cacheDir = None
# logDir = None
def get_domain_snapdirs(self, domain, full_dirs=True):
dir = Config.CACHE_DIR + domain + "/"
if os.path.isdir(dir):
return [str(dir + subdir) if full_dirs else str(subdir) for subdir in os.listdir(dir) # adresare vsech moznych snapshotu
if os.path.isdir(str(dir + subdir)) and os.path.isfile(dir + subdir + "/" + ScanController.CRAWL_FILE)]
##
# @param cached Pokud chceme zobrazit cachovanou verzi analyzy, dejme True. Pokud dame int, je to maximalni stari (ve dnech). Kdyz se nenalezne, zanazyzuje se znovu.
def launch(self, url, cached=None):
if cached:
# """ Pokud je k dispozici analyza, vratit ji """
dir = Config.CACHE_DIR + Domains.domain2dir(url) + "/"
if os.path.isdir(dir):
snapdirs = [str(dir + subdir) for subdir in os.listdir(dir) # adresare vsech moznych snapshotu
if os.path.isdir(str(dir + subdir)) and os.path.isfile(dir + subdir + "/" + ScanController.CRAWL_FILE)]
if snapdirs:
cache_dir = max(snapdirs, key=os.path.getmtime) + "/" # nejnovejsi dir analyzy
if type(cached) != int or os.path.getmtime(
cache_dir) > time.time() - 3600 * 24 * cached: # maximalni stari analyzy
try:
logging.debug(f"Returning a previous crawl from: {cache_dir + ScanController.CRAWL_FILE}")
crawl = Crawl.load_from_file(cache_dir + ScanController.CRAWL_FILE)
return crawl
except ValueError:
pass
logging.debug("({-1}) Cachovana analyza nenalezena")
# provest novou analyzu
if self.queue(): # /api/analyze/web - zaradi web do fronty
snapdirs = self.get_domain_snapdirs(Domains.domain2dir(url))
if snapdirs:
# get the most recent snapdir and check if it's not too old
cache_dir = max(snapdirs, key=os.path.getmtime) + "/"
if cached is True or os.path.getmtime(cache_dir) > time.time() - (3600 * 24 * cached):
try:
logging.debug(f"Returning a previous crawl from: {cache_dir + ScanController.CRAWL_FILE}")
crawl = Crawl.load_from_file(cache_dir + ScanController.CRAWL_FILE)
return crawl
except ValueError:
pass
logging.debug("({-1}) Convenient cached analysis not found")
# perform fresh analysis
if self.queue(): # /api/analyze/web - queue current analysis
print("({}) start crawl".format(self.profile))
self.url = Domains.assureUrl(url)
try:
......
import json
import logging
import mimetypes
import os
......@@ -42,7 +43,7 @@ class Server(SimpleHTTPRequestHandler):
bytearray([7, 8, 9, 10, 12, 13, 27]) + bytearray(
range(0x20, 0x100)))) else 'r'
with open(url, type_) as f:
self.output(f.read(), content_type=mimetypes.guess_type(url))
self.output(f.read(), content_type=mimetypes.guess_type(url)[0])
def do_GET(self):
"""
......@@ -90,9 +91,16 @@ class Server(SimpleHTTPRequestHandler):
if "api" in request: # /api/analyze/web
output = Api().run(request)
if request["api"] == "json":
if type(output) is dict:
output = json.dumps(output)
elif type(output) in [str, list]:
output = json.dumps(output)
if "destination" in request:
# send everything up, we are in an iframe
self.render_template("_message.html", contents=output, cmd=request, url=self.path,
self.render_template("_message.html", contents=output, url=self.path,
destination=f"https://{request['destination']}/")
else:
self.output(output)
......
......@@ -35,7 +35,12 @@ class Domains:
'(((((http|https|ftp)://)?[\w\-_]+(?:(?:\.[\w\-_]+)+)))([\w\-\.,@?^=%&:/~\+#]*[\w\-\@?^=%&/~\+#])?)',
url).group(0)
def domain2dir(url): # friendly nazev adresare z domeny v url
def domain2dir(url) -> str:
"""
Friendly dir name from a domain contained in an url
XXmay be replaced by urlparse(url).netloc?
"""
url = url.lower()
url = re.sub('^(http://|https://|ftp://)', '', url) # odstrihnout protokol
url = re.sub('(/.*)', '', url) # nestojim o cestu, jen o domene
......
auxiliary.org-netbeans-modules-css-prep.less_2e_compiler_2e_options=
auxiliary.org-netbeans-modules-css-prep.less_2e_configured=true
auxiliary.org-netbeans-modules-css-prep.less_2e_enabled=true
auxiliary.org-netbeans-modules-css-prep.less_2e_mappings=/static:/static
auxiliary.org-netbeans-modules-css-prep.sass_2e_compiler_2e_options=
auxiliary.org-netbeans-modules-css-prep.sass_2e_enabled=false
auxiliary.org-netbeans-modules-css-prep.sass_2e_mappings=/scss:/css
file.reference.mdmaug-installer-mdmaug=.
files.encoding=UTF-8
site.root.folder=${file.reference.mdmaug-installer-mdmaug}
source.folder=${file.reference.mdmaug-installer-mdmaug}
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://www.netbeans.org/ns/project/1">
<type>org.netbeans.modules.web.clientproject</type>
<configuration>
<data xmlns="http://www.netbeans.org/ns/clientside-project/1">
<name>mdmaug</name>
</data>
</configuration>
</project>
/**
* Cache radio buttons responsivity
*/
$("input[name=cache]").change(function () {
let $days_input = $("input[name=days]");
if ($(this).val() === "days") {
$days_input = $("input[name=days]");
$days_input.show().siblings("span").hide();
setTimeout(() => {
$days_input.select();
}, 100);
} else {
$days_input.hide().siblings("span").show();
}
});
/**
* Initialization
*/
$("input[type=url]").val(new URL(window.location.href).searchParams.get("url")); // fetch the default URL from the address bar
/**
* list scans button
$("#list-scans").click(() => {
let urls = get_urls();
if (urls) {
let url = urls[0];
launch_request("/scans/url/{0}".format(url), "json");
}
return false;
});*/
/**
* Send request, analyze button
*/
var $requests = $("#requests-panel");
$("form").submit(() => {
let cache = $(".active > input[name=cache]").val();
if (cache === "days") {
cache = $(".active > input[type=number]").val();
}
let method = "html";
for (let url of get_urls()) {
let rest_url = "/analyze={0}/{1}".format(cache, url);
launch_request(rest_url, method, url);
}
//.map(); // let's create a closure to preserve counter etc
return false;
});
/**
* Toggle single and multiple URLs
*/
$("#request-url-toggler").change(function () {
if ($(this).prop("checked")) {
$("#request-url").hide();
let $el = $("#request-urls").show();
if (!$el.val().trim()) {
$el.val($("#request-url").val());
}
} else {
let $el = $("#request-url").show();
$("#request-urls").hide();
if (!$el.val().trim()) {
$el.val($("#request-urls").val().split("\n")[0]);
}
}
});
/**
* Launches a new request to the api
* @param {type} header Ex: Scanned domain
* @param {string} method html or json
* @param {type} rest_url Part of the URI of the API.
* @returns {undefined}
*/
function launch_request(rest_url, method = "json", header = "Result", callback=null) {
let counter = 0;
//let uri = "/api={0}/analyze={1}/{2}".format(method, cache, url);
let uri = "/api={0}{1}".format(method, rest_url);
let $el = $(".template >", $requests).clone();
$(".uri", $el).text(uri);
$("> .placeholder", $requests).remove();
$("> b", $requests).after($el);
// elapsed time counter
let increase_elapsed = setInterval(() => {
$(".timer", $el).html("{0} s".format(++counter));
if (counter > 10) {
bootstrap_change_bg($el, "bg-warning");
$el.addClass();
}
}, 1000);
// start the ajax
$.ajax({
dataType: method,
url: uri
}).done((data) => {
bootstrap_change_bg($el, "bg-success").find(".timer").prop("disabled", true).prepend("");
let s;
if (callback) {
s = callback(data);
} else {
if (method === "json") {
s = "<pre>{0}</pre>".format(syntaxHighlight(JSON.stringify(data, undefined, 4)));
} else {
s = data;
}
$("<div/>", {"class": "analysis", "html": "<h3>{0}</h3>".format(header) + s}).appendTo($("#analysis-result-panel"));
}
}).fail(() => {
bootstrap_change_bg($el, "bg-danger").find(".timer").prepend("× ");
}).always(() => {
clearInterval(increase_elapsed);
});
}
/**
* Prints out a JSON in a nice manner.
* @param {JSON} json
* @returns {HTML}
*/
function syntaxHighlight(json) {
json = json.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
return json.replace(/("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+\-]?\d+)?)/g, function (match) {
var cls = 'number';
if (/^"/.test(match)) {
if (/:$/.test(match)) {
cls = 'key';
} else {
cls = 'string';
}
} else if (/true|false/.test(match)) {
cls = 'boolean';
} else if (/null/.test(match)) {
cls = 'null';
}
return '<span class="' + cls + '">' + match + '</span>';
});
}
// String formatting function usage "string {0}".format("1") => "string 1"
if (!String.prototype.format) {
String.prototype.format = function () {
var args = arguments;
return this.replace(/{(\d+)}/g, function (match, number) {
return typeof args[number] !== 'undefined'
? args[number]
: match
;
});
};
}
/**
* Removes all the bg- classes.
* @param {type} $el
* @param {type} new_class Optionally append new class.
* @returns {$el} for chaining
*/
function bootstrap_change_bg($el, new_class = false) {
$el.removeClass(function (index, className) {
return (className.match(/\bbg-\S+/g) || []).join(' ');
});
if (new_class) {
$el.addClass(new_class);
}
return $el;
}
/**
* get list of urls that we want to analyze
*/
function get_urls() {
let urls = $("#request-url-toggler").prop("checked") ? $("#request-urls").val() : $("#request-url").val();
return urls.split("\n");
}
\ No newline at end of file
......@@ -108,7 +108,8 @@ if (document.location.href.indexOf("https://mdm.nic.cz/detail/") === 0) {
* @returns {undefined}
*/
safebrowsingIcon = function (webEl, url) {
a = $(webEl).attr("data-safebrowsing-suspicious");
let a = $(webEl).attr("data-safebrowsing-suspicious");
let str;
if (a === "1") {// je suspicious
str = "!";
} else if (a === "0") {// neni suspicious
......@@ -123,10 +124,6 @@ if (document.location.href.indexOf("https://mdm.nic.cz/detail/") === 0) {
window.addEventListener("message", function (e) {
//prevzit vysledky | e.data ve formatu: <div class=web><div class=domain>jedinecna domena</div></div>
console.log("event", e);
if (e.data["analyze=cached"]) {
e.data["analyze"] = e.data["analyze=cached"];
}
if (e.data.analyze) {//jedna se o zpravu pro me
// dat vedet uzivateli
$("#analysis-info").text("");
......@@ -135,12 +132,12 @@ if (document.location.href.indexOf("https://mdm.nic.cz/detail/") === 0) {
//ulozit do sessionStorage (pri prostem refreshi nedojde k narocne reanalyze)
if (e.data.url !== "refresh") {//neukladame do storage, protoze tohle je load ze storage, nikoli fresh analyza
analysisHost = $(".analysis", $analysisTemp).attr("data-host");
let analysisHost = $(".analysis", $analysisTemp).attr("data-host");
console.log("HOST", analysisHost);
if (typeof analysisHost == "undefined") {//mdmaug skoncil chybou, zkusime host dosta z URL iframu
// "/api/analyze=cached/http://www.klein-automotive.cz/aktuality/page/2/" -> http://...
if (typeof analysisHost === "undefined") {//mdmaug skoncil chybou, zkusime host dostat z URL iframu
// "/destination=mdm.nic.cz/api/analyze=cached/http://www.klein-automotive.cz/aktuality/page/2/" -> http://...
var path = e.data.url.split("/");
for (var i = 0; i < 3; i++) {
for (var i = 0; i < 4; i++) {
path.shift();
}
analysisHost = path.join("/");
......@@ -150,17 +147,18 @@ if (document.location.href.indexOf("https://mdm.nic.cz/detail/") === 0) {
}
$(".analysis", $analysisTemp).each(function () {// probehla jedina analyza, ale radsi to prozenu eachem
// add the safebrowsing icon to the main page header
$("body h1:eq(0)").append(safebrowsingIcon(this, $(this).attr("data-host")));
//help text
// help text
$("#advice").remove();
$advice = $("<p/>", {id: "advice", "text": "Když je to ! nebo wp a stránka není sundaná a ještě nemáme korespondenci, dát poslat."}).insertAfter($("body h1:eq(0)"));
$(".web", this).each(function () {//primerguje weby nalezene v nove analyze do stare analyzy.
host = $.trim($(".domain", this).text());
let host = $.trim($(".domain", this).text());
safebrowsingIcon(this, host);
contained = false;
let contained = false;
$(".web > .domain", $analysis).each(function () {
if ($(this).text() == host) {
console.log($(this).text());
......@@ -177,7 +175,7 @@ if (document.location.href.indexOf("https://mdm.nic.cz/detail/") === 0) {
} else {
//voting, zda je web malicious
voting = $("<span class='voting'/>").insertBefore($(".domain", this));
let voting = $("<span class='voting'/>").insertBefore($(".domain", this));
$.each(["block", "log", "n/a", "allow"], function (i, val) {//vlozit 4 hlasovaci tlacitka
$("<input type=radio name='" + host + "' value='" + val + "'/>").appendTo(voting).change(function () {
console.log("VOTING CHANGE");
......@@ -186,7 +184,9 @@ if (document.location.href.indexOf("https://mdm.nic.cz/detail/") === 0) {
});
//localStorage -> default voting value
if ((vote = $(this).attr("data-vote")) == "") {//vote pro web je v jeho atributu, protoze k domene webu nepatri zadne ip
if ((vote = $(".address:eq(0)", this).attr("data-vote")) == "") {//vote pro web se bere z votu pro jeho prvni ip XXTakze kdyz se budou voty pro IP z nejakeho duvodu lisit, admin na to neprijde a vote premaze.
if ((vote = $(".address:eq(0)", this).attr("data-vote")) == "") {
//vote pro web se bere z votu pro jeho prvni ip
//XXTakze kdyz se budou voty pro IP z nejakeho duvodu lisit, admin na to neprijde a vote premaze.
vote = "n/a"; //localStorage.getItem("domain_" + host)
}
}
......@@ -198,7 +198,6 @@ if (document.location.href.indexOf("https://mdm.nic.cz/detail/") === 0) {
} else {
$(analysis).prepend(this);
}*/
}
});
......@@ -210,7 +209,7 @@ if (document.location.href.indexOf("https://mdm.nic.cz/detail/") === 0) {
//refreshOutput();
//rovnou zaciname hlasovat, analyza kompletni
$(".voting:eq(0) input:eq(0)",sections.$undecided).focus();//focusujeme pouze z bloku undecided; nema smysl focusovat allowed web, jeste nekomu ujede ruka
$(".voting:eq(0) input:eq(0)", sections.$undecided).focus();//focusujeme pouze z bloku undecided; nema smysl focusovat allowed web, jeste nekomu ujede ruka
} else if (e.data.voted) {//ve vedlejsim tabu se hlasovalo, zmen hodnotu i zde
changeVote($("[type=radio][value='" + e.data.voted + "']", $(".web .domain:contains('" + e.data.domain + "'):eq(0)").siblings(".voting")).prop("checked", true), false);//hlasovat
}
......
#page0 {
margin:20px;
}
#page1 {
background:#888;
margin:0;
padding:1%;
}
#container {
overflow:hidden;
}
#menu {
background-color:#eed;
float:left;
margin:0 0 -100000px;
padding:1% 1% 100000px;
width:20%;
}
#menu ul {
border:none;
list-style:none;
margin:0 auto;
padding:0;
width:100%;
}
#menu ul li {
background-color:#ccb;
margin:2px 0 0;
padding:5px;
text-align:left;
}
#menu ul li.indent {
margin-left:25px;
}
#menu ul li a {
display:block;
font-weight:700;
width:auto;
}
#menu ul li:hover {
background-color:#edb;
}
#menu > div {
background-color:#ccb;
display:block;
padding:5px;
}
#menu img {
border:0;
}
#content {
background-color:#9bc;
float:left;
margin:0 0 0 1%;
min-height:650px;
overflow:auto;
padding:1%;
width:75%;
}
#content h1 {
margin-top:0;
}
.roundedcorners {
-khtml-border-radius:5px;
-moz-border-radius:5px;
-webkit-border-radius:5px;
border-radius:5px;
}
.fail {
background-color:#fdd;
color:#800;
display:block;
text-align:center;
}
.info {
background-color:#dfd;
color:#800;
display:block;
text-align:center;
}
.loading {
text-decoration:blink;
}
label.upper {
display:block;
margin-top:.33em;
}
#ticket {
margin-top:5px;
padding:5px;
}
#ticket>div {
background-color:#def;
border:2px solid #888;
margin-top:5px;
padding:10px;
}
#ticket>div.odd {
background-color:#cde;
}
#ticket>div>div.caption {
float:left;
}