Skip to content

GitLab

Commit f0ec24e8 authored by Edvard Rejthar's avatar Edvard Rejthar
Browse files
Options

#25 dir structure, "rest" -> "api"

parent 3a4dbd60
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 230 additions and 810 deletions
.mozilla/firefox/0/prefs.js
View file @ f0ec24e8
......@@ -30,6 +30,7 @@ user_pref("browser.newtabpage.storageVersion", 1);
user_pref("browser.pagethumbnails.storage_version", 3);
user_pref("browser.places.smartBookmarksVersion", 8);
user_pref("browser.preferences.advanced.selectedTabIndex", 2);
user_pref("browser.privatebrowsing.autostart", true);
user_pref("browser.safebrowsing.provider.google.lastupdatetime", "1486578575144");
user_pref("browser.safebrowsing.provider.google.nextupdatetime", "1486580358144");
user_pref("browser.safebrowsing.provider.mozilla.lastupdatetime", "1486578502288");
......@@ -92,8 +93,11 @@ user_pref("pdfjs.previousHandler.alwaysAskBeforeHandling", true);
user_pref("pdfjs.previousHandler.preferredAction", 4);
user_pref("places.history.expiration.transient_current_max_pages", 122334);
user_pref("plugin.disable_full_page_plugin_for_types", "application/pdf");
user_pref("privacy.clearOnShutdown.offlineApps", true);
user_pref("privacy.clearOnShutdown.*", true);
user_pref("privacy.clearOnShutdown.openWindows", true);
user_pref("privacy.clearOnShutdown.cookies", true);
user_pref("privacy.clearOnShutdown.sessions", true);
user_pref("privacy.clearOnShutdown.siteSettings", true);
user_pref("privacy.clearOnShutdown.passwords", true);
user_pref("privacy.clearOnShutdown.siteSettings", true);
user_pref("privacy.sanitize.didShutdownSanitize", true);
......
INSTALL
View file @ f0ec24e8
......@@ -5,7 +5,6 @@
DESTINATION=/opt/mdmaug
PROFILE_COUNT=21
# sometimes, there is no repository with pip on the machine, trying to add some sources
apt install software-properties-common
add-apt-repository "deb http://archive.ubuntu.com/ubuntu $(lsb_release -sc) main universe restricted multiverse"
......
mdmaug/lib/analysis/__init__.py deleted 100644 → 0
View file @ 3a4dbd60
__author__ = "edvard"
__date__ = "$Apr 27, 2015 5:40:11 AM$"
\ No newline at end of file
mdmaug/lib/analysis/parser/example.log deleted 100644 → 0
View file @ 3a4dbd60
DEBUG:root:This message should go to the log file
INFO:root:So should this
WARNING:root:And this, too
WARNING:root:warning1
DEBUG:root:debug1
mdmaug/lib/config.py
View file @ f0ec24e8
import threading
import os
import logging
from glob import glob
from peewee import MySQLDatabase
......@@ -10,7 +11,8 @@ class Config:
APP_PORT = 8000
APP_DOMAIN = 'https://217.31.202.41:' + str(APP_PORT) #csirt.csirt.office.nic.cz
LOG_DIR = "/opt/mdmaug/.cache/mdmaug-scans/_tmp/" # X /tmp/mdm/
CACHE_DIR = "/opt/mdmaug/.cache/mdmaug-scans/"
CACHE_DIR = "/opt/mdmaug/.cache/mdmaug-scans/"
DIR = os.path.dirname(os.path.realpath(__file__)) + "/../"
myDB = ""
lock = threading.RLock() # doufam, ze kdyz je lock tady, ze je funknci. Closure...? XX nejak otestovat
THUMBNAIL_SIZE = 640, 640
......@@ -21,7 +23,7 @@ class Config:
def connect():
# XX resim problem peewee.OperationalError: (2006, "MySQL server has gone away (BrokenPipeError(32, 'Broken pipe'))") po 7 hodinach timeoutu
# XX kupodivu pripojeni nemuze byt v dbp DBModel.connect. Prestoze type je pak spravne (MySQLDatabase), nic udelat nejde a pokusy o select konci NoneType.
print("Connecting to DB.")
logging.debug("Connecting to DB.")
Config.myDB = MySQLDatabase("mdmaug", host='localhost', port=3306, user="mdmaug", passwd="fidFDSs676") # XX dal jsem pryc: , threadlocals=False
Config.myDB.register_fields({'primary_key': 'BIGINT AUTOINCREMENT'})
......
mdmaug/lib/rest.py → mdmaug/lib/controller/api.py
View file @ f0ec24e8
#import os
#from pprint import pprint
#import queue
#import re
#import threading
#import dill
#import pickle
#import pdb
import json
import subprocess
import logging
logging.basicConfig(level=logging.WARNING, format="%(message)s")
from lib.config import Config
from lib.analysis.scan_controller import ScanController
from lib.dbp import Status, Export, Turris, Whitelist
from lib.controller.scan_controller import ScanController
from lib.model.dbp import Status, Export, Turris, Whitelist
from lib.analysis.parser.traffic_log_parser import TrafficLogParser
class Rest:
allow_reuse_address = True # XX tohle nevim, co dela
class Api:
website = "" # http://site.cz
websiteDomain = "" # site.cz
#spojit s mysql
def __init__(self, path):
self.path = path
##
# Spusti prikaz
def run(self, cmd):
if cmd == "analyze": #Xqueue se do db pripojuje sama (je mozne, ze vsechny FF jsou obsazene, pak se db zbytecne vytezuje)
""" Accept command """
if cmd == "analyze":
return ScanController().launch(self.path)
if cmd == "analyze=cached":
return ScanController().launch(self.path, cached = 1)
......@@ -37,45 +23,39 @@ class Rest:
return ScanController().launch(self.path, cached = 7)
if cmd == "analyze=oldcache":
return ScanController().launch(self.path, cached = True)
elif cmd == "export=view": # XX asi deprecated
elif cmd == "export=view": # XX deprecated?
return Export.exportView()
elif cmd == "export=confirm": # XX asi deprecated
elif cmd == "export=confirm": # XX deprecated?
return Export.exportConfirm()
elif cmd == "decide": # XX asi deprecated
elif cmd == "decide": # XX deprecated?
return self.getUndecided()
elif cmd == "nicify":
url = self.path.split("/", 3)
return TrafficLogParser.getStylesheet() + TrafficLogParser.nicifyFile(url[3])
elif cmd == "vote": # /rest/vote/block/example.org/10.0.0.1
elif cmd == "vote": # /api/vote/block/example.org/10.0.0.1
logging.debug("vote cmd")
url = self.path.split("/", 4)
logging.debug(url[3])
return Turris.vote(url[3], url[4])
elif cmd == "whitelist":
elif cmd == "whitelist": # XXX not implemented yet
url = self.path.split("/", 3)
if len(url) > 3:
self._setWebsite(url[2]) #osetrit self.website, ze je URL, a nikoli shell
print("XXX nejsem si jist, zda url je spravne na url[2]") # XXX
print(url) # XXX
self._setWebsite(url[2]) # osetrit self.website, ze je URL, a nikoli shell
logging.debug("XXX nejsem si jist, zda url je spravne na url[2]") # XXX
logging.debug(url) # XXX
quit() # XXX
logging.debug(self.website)
logging.debug(self.websiteDomain)
return self.whitelist()
elif cmd == "reset":
Server.reset()
return "reset"
return "reset"
def reset():
logging.debug("resetting running browsers")
#smazat frontu
with open(Config.configFile, 'w') as f:
json.dump({}, f)
#zabit zamrzle prohlizece
subprocess.call(["pkill", Config.browser])
logging.debug("resetting running browsers")
with open(Config.configFile, 'w') as f: # clear the queue
json.dump({}, f)
subprocess.call(["pkill", Config.browser]) # kill frozen browsers
......@@ -84,7 +64,7 @@ class Rest:
def whitelist(self):
logging.debug("whitelistuju")
#Db.cur = Db.connection.cursor()
#self._print(Db.cur.execute("""REPLACE INTO whitelist set domain = %s""", (self.websiteDomain, )))
#self._logging.debug(Db.cur.execute("""REPLACE INTO whitelist set domain = %s""", (self.websiteDomain, )))
#Db.connection.commit()
#Db.cur.close()
try:Whitelist.insert(domain=self.websiteDomain).execute()
......@@ -94,7 +74,4 @@ class Rest:
def getUndecided(self):
logging.debug("XXX jeste jsem neudelal - ma vylezt tabulka vsech nerozhodlych domen od posledniho exportu")
pass
pass
\ No newline at end of file
mdmaug/lib/analysis/scan_controller.py → mdmaug/lib/controller/scan_controller.py
View file @ f0ec24e8
# Sprava dat scannovani
from pprint import pprint
import json
import datetime
import time
import os
from glob import glob
from random import randint
import subprocess
import traceback
import logging
logging.basicConfig(level=logging.WARNING, format="%(message)s")
from glob import glob
from random import randint
from lib.config import Config
from lib.domains import Domains
from lib.analysis.crawl import Crawl
from lib.model.crawl import Crawl
from templates.crawl_view import CrawlView
from lib.analysis.parser.traffic_log_parser import TrafficLogParser
from lib.analysis.parser.nspr_log_parser import NsprLogParser
from lib.analysis.parser.metadata_parser import MetadataParser
from lib.analysis.parser.screenshot_parser import ScreenshotParser
from lib.analysis.parser.spy_parser import SpyParser
import pdb
from lib.parser.traffic_log_parser import TrafficLogParser
from lib.parser.nspr_log_parser import NsprLogParser
from lib.parser.metadata_parser import MetadataParser
from lib.parser.screenshot_parser import ScreenshotParser
from lib.parser.spy_parser import SpyParser
class ScanController:
......@@ -53,25 +47,25 @@ class ScanController:
cacheDir = max(snapdirs, key = os.path.getmtime)+ "/" # nejnovejsi dir analyzy
if type(cached) != int or os.path.getmtime(cacheDir) > time.time()-3600*24*cached: # maximalni stari analyzy
try:
print("returning")
logging.debug("returning")
return CrawlView.outputHtml(Crawl.loadFromFile(cacheDir + ScanController.CRAWL_FILE)) #"crawlSave.tmp"
except ValueError:
pass
print("({-1}) Cachovana analyza nenalezena")
logging.debug("({-1}) Cachovana analyza nenalezena")
# provest novou analyzu
if self.queue(): # /rest/analyze/web - zaradi web do fronty
if self.queue(): # /api/analyze/web - zaradi web do fronty
print ("({}) start crawl".format(self.profile))
self.url = Domains.assureUrl(url)
try:
crawl = self.analyze()
except Exception as e:
print("PROFILE EXCEPTION {}".format(self.profile))
print(traceback.format_exc())
logging.debug("PROFILE EXCEPTION {}".format(self.profile))
logging.debug(traceback.format_exc())
# XX Pokud je potiz, ze JS zabiji FF, mozno experimentovat s ulimit -Sv 500000;
return ("PROFILE EXCEPTION ({}) {} See logs, i.e. mdmaug/nohup.out. ".format(self.profile, e))
print("SAVE")
logging.debug("SAVE")
crawl.saveToFile(crawl.cacheDir + ScanController.CRAWL_FILE) # ulozit vysledky hledani
return CrawlView.outputHtml(crawl)
......@@ -81,11 +75,6 @@ class ScanController:
return "<div id='analysis-results'>{}</div>".format(result)
def _print(self,str):
#logging.debug("({}) {}".format(self.profile ,str))
#print("({}) {}".format(self.profile ,str))
pass
def analyze(self):
# spustit firefox pod profilem
print ("({}) browser launch".format(self.profile))
......@@ -96,23 +85,23 @@ class ScanController:
#max_time = 3 # XXX
#,nsSocketTransport:5,nsStreamPump:5,nsHostResolver:5
print("({}) FF -P {} -no-remote {}".format(self.profile,self.profile,self.url))
logging.debug("({}) FF -P {} -no-remote {}".format(self.profile,self.profile,self.url))
command = "export NSPR_LOG_MODULES=timestamp,nsHttp:5 ; export NSPR_LOG_FILE={} ; {} -P {} -no-remote '{}'".format(logfile, Config.browser, self.profile , "http://localhost/redirect/" + self.url) # http://localhost/redirect/ gets stripped by the extension
# terminate Config.browser if hes not able to (everything has to be in single command because there is no heritance of $! amongst subprocesses)
command += " & echo $!;ii=0; while [ -n \"`ps -p $! | grep {}`\" ];do echo \"({}) running\" ;ii=$((ii+1)); if [ $ii -gt {} ]; then echo '({}) kill';kill $!; break;fi; sleep 1; done".format(Config.browser, self.profile , Config.MAX_BROWSER_RUN_TIME,self.profile) #(pokud bezi proces $! (posledni backgroudovany process), spi 1 s)
#> /dev/null
print(command)
logging.debug(command)
subprocess.call([command], shell=True)
print("({}) stopped!".format(self.profile))
logging.debug("({}) stopped!".format(self.profile))
# shromazdit informace z analyz
crawl = Crawl(host = self.url, logDir = logDir, cacheDir = cacheDir)
expiration = 0
while os.path.isfile(logfile) == False: # i po zavreni FF nekdy trva, nez se soubor zapise
expiration += 1
print("({}) waiting to close...".format(self.profile))
logging.debug("({}) waiting to close...".format(self.profile))
if expiration > Config.MAX_BROWSER_EXPIRATION:
print("({}) time is run!".format(self.profile))
logging.debug("({}) time is run!".format(self.profile))
raise FileNotFoundError("time is run - browser expired")
time.sleep(1)
......@@ -179,23 +168,23 @@ class ScanController:
def dump():
with open(Config.configFile, 'w') as f:
json.dump(self.queueFF, f)
#print("UNKBOOK")
#logging.debug("UNKBOOK")
try:
self.queueFF.pop(self.profile)
dump()
except KeyError:
print("Unbook failed")
print(self.queueFF)
logging.debug("Unbook failed")
logging.debug(self.queueFF)
raise
except OSError:
print("({}) OS Error - interferuje s pustenym FF, ktere zere prilis pameti. Zkusime pockat.".format(self.profile))
logging.debug("({}) OS Error - interferuje s pustenym FF, ktere zere prilis pameti. Zkusime pockat.".format(self.profile))
time.sleep(10) # XX jestli funkcionalitu zachovat, dat sem pocitadlo, at je na konzoli videt akce
try:
dump()
except OSError:
print("({}) System se nezotavil.".format(self.profile))
logging.debug("({}) System se nezotavil.".format(self.profile))
return "Memory may be exhausted. See mdmaug-server/scan_controller.py for details." # FF sezral vsechnu pamet asi. Stranka je problematicka. UrlQuery podle me taky selze.
#print("UNKBOOKED")
#logging.debug("UNKBOOKED")
def queue(self):
......@@ -209,13 +198,13 @@ class ScanController:
self.bookProfile()
break
if self.profile == -1:
print("(-1) PLNO, cekame par vterin")
logging.debug("(-1) PLNO, cekame par vterin")
time.sleep(randint(5, 10)) #pockame par vterin
else:
break #volny slot jsme nasli, muzeme dal
#logging.debug(" profile " + str(self.profile ) + " queueFF:")
print(self.queueFF)
logging.debug(self.queueFF)
#povedlo se zabookovat profil FF?
return self.profile > -1
\ No newline at end of file
mdmaug/lib/server.py → mdmaug/lib/controller/server.py
View file @ f0ec24e8
from http.server import SimpleHTTPRequestHandler
from jinja2 import Environment
from jinja2 import FileSystemLoader
from lib.config import Config
from lib.controller.api import Api
from lib.model.dbp import DbModel
from lib.model.dbp import Export
import logging
import mimetypes
import os
from lib.config import Config
#from lib.dbp import Db
from lib.rest import Rest
import time
from lib.dbp import Export, DbModel
env = Environment()
env.loader = FileSystemLoader(".")
env.loader = FileSystemLoader(Config.DIR + "templates/")
class Server (SimpleHTTPRequestHandler):
class Server(SimpleHTTPRequestHandler):
def favicon(self):
with open('favicon.ico', 'rb') as f:
self.output(f.read(), "image/x-icon")
def render_template(self, filename, ** kwargs):
#pars = parse_qs(urlparse(self.path).query)
#if 'view' in pars:
#if pars["view"] == "message": # jsme nacteni pres iframe
self.output(env.get_template(filename).render(kwargs))
#with open(filename,'r') as f:
#self.output(Environment().from_string(f.read()).render(kwargs))
def output(self, contents, contentType="text/html"):
#http response
self.send_response(200)
self.send_header("Content-type", contentType)
self.end_headers()
try:
self.wfile.write(contents)
except:
self.wfile.write(contents.encode("UTF-8"))
self.send_response(200)
self.send_header("Content-type", contentType)
self.end_headers()
try:
self.wfile.write(contents)
except:
self.wfile.write(contents.encode("UTF-8"))
def homepage(self):
self.render_template("templates/homepage.html")
#self.output(render_template('_layout.html'))
self.render_template("homepage.html")
def static_file(self, url):
is_binary_string = lambda bytes: bool(bytes.translate(None, bytearray([7, 8, 9, 10, 12, 13, 27]) + bytearray(range(0x20, 0x100))))
......@@ -48,24 +41,21 @@ class Server (SimpleHTTPRequestHandler):
with open(url, type) as f:
self.output(f.read(), contentType=mimetypes.guess_type(url))
def do_GET(self):
#XX Config.myDB.get_conn() # vytvorit spojeni, kdyby po 9 hodinach vypadlo. (Funguje?)
#from lib.dbp import Whitelist
DbModel.assureConnection()
def do_GET(self):
path = self.path.split("/")
print("processing", path[1])
logging.debug("Request: {}".format(path[1]))
if path[1] == "":
self.homepage()
elif os.path.isfile("static/" + path[1]): #faviconka, nebo jiny existujici soubor
self.static_file("static/" + path[1])
elif path[1] == "rest": # /rest/analyze/web
cmd = path[2]
#url = path[2] if len(path) > 3 else ""
return self.homepage()
elif os.path.isfile(Config.DIR + "static/" + path[1]): #faviconka, nebo jiny existujici soubor
return self.static_file(Config.DIR + "static/" + path[1])
rest = Rest(self.path)
#poslat vsechna data vys, pokud jsme vlozeni v iframe (https-mdm nemuze nacitat z http stroje - mixed content)
self.render_template("templates/_message.html", contents = rest.run(cmd), cmd=cmd, url=self.path, destination="https://mdm.nic.cz/")
DbModel.assureConnection()
if path[1] == "api": # /api/analyze/web
cmd = path[2]
api = Api(self.path)
# send everything up, we are in an iframe
self.render_template("_message.html", contents=api.run(cmd), cmd=cmd, url=self.path, destination="https://mdm.nic.cz/")
elif path[1] == "export": # /export/{days} - csv za poslednich 7 dni
url = self.path.split("/", 2)
self.output(Export.exportView(days = url[2]))
\ No newline at end of file
self.output(Export.exportView(days=url[2]))
\ No newline at end of file
mdmaug/lib/domains.py
View file @ f0ec24e8
......@@ -21,8 +21,8 @@ class Domains:
try:
return re.search('(([\w\-_]+(?:(?:\.[\w\-_]+)+)))', url).group(0)
except TypeError:
print("Domains/url2domain type error")
print(url)
logging.debug("Domains/url2domain type error")
logging.debug(url)
raise #return ""
......
mdmaug/lib/analysis/crawl.py → mdmaug/lib/model/crawl.py
View file @ f0ec24e8
......@@ -47,7 +47,7 @@ class Crawl(defaultdict):
return state
def __setstate__(self,state):
#print("fdsfsfds",self.__dict__)
#logging.debug("fdsfsfds",self.__dict__)
for tup in state["keys"]:
key, val = tup
self[key].__setstate__(val)
......@@ -151,13 +151,13 @@ c["seznam.cz"].addresses["8.8.8.8"].country = "preague"
e = Crawl()
e.__setstate__( c.__getstate__() )
#e = dill.loads(dill.dumps(c))
print(str(c) == str(e))
logging.debug(str(c) == str(e))
print(c)
logging.debug(c)
output = dump(c.__getstate__(), Dumper=Dumper)
e = Crawl()
e.__setstate__(load(output, Loader=Loader))
print(e)
print(str(c) == str(e))
logging.debug(e)
logging.debug(str(c) == str(e))
"""
\ No newline at end of file
mdmaug/lib/dbp.py → mdmaug/lib/model/dbp.py
View file @ f0ec24e8
......@@ -20,34 +20,34 @@ if(1): # Do not print all queries to stderr.
class DbModel(Model):
def assureConnection():
print ("Assure start.")
logging.debug("Assure connection.")
try: # zkousim spustit nejaky prikaz
#print("1")
#logging.debug("1")
Whitelist.select().count()
#print("2")
#logging.debug("2")
except:# OperationalError:
#print("3")
#logging.debug("3")
Config.connect()
#print("4")
#logging.debug("4")
try: # zkousim spustit nejaky prikaz
#print("5")
#logging.debug("5")
Whitelist.select().count()
#print("6")
#logging.debug("6")
except:
print("7 - failed")
logging.debug("7 - failed")
raise
print("Connection to DB assured.")
logging.debug("Connection to DB assured.")
"""A base model that will use our MySQL database"""
def connect():
logging.debug("connecting db....")
#DbModel.Meta.myDb = Config.myDB
# Config.myDB.connect() # XX kupodivu toto neni potreba
#print(Whitelist.select().count())
#print("Connecting to DB.")
#print("Start test.")
#print(Whitelist.matches("www.mozilla.org"))
#print("End test.")
#logging.debug(Whitelist.select().count())
#logging.debug("Connecting to DB.")
#logging.debug("Start test.")
#logging.debug(Whitelist.matches("www.mozilla.org"))
#logging.debug("End test.")
def disconnect():
logging.debug("... db ends.")
......@@ -57,9 +57,9 @@ class DbModel(Model):
database = Config.myDB
pass
#print("DVA")
#logging.debug("DVA")
#DbModel.connect()
#print(Config.myDB)
#logging.debug(Config.myDB)
class Status(DbModel):
id = PrimaryKeyField()
......@@ -107,7 +107,7 @@ class Export(DbModel):
q += "NOW() - INTERVAL {} DAY ".format(int(days))
else:
q += "(select case when MAX(timestamp IS NULL)=0 THEN max(timestamp) ELSE 0 END from export)"
print(q)
logging.debug(q)
rq = RawQuery(Turris, q).execute()
print (rq)
for r in rq: #for r in rows:
......@@ -163,15 +163,14 @@ class Turris(DbModel):
rows = Turris.select(Turris.ip).where(Turris.remoteHost == host)
except:
logging.error("domain should have been inserted in the database, but it hasnt been")
print("vote error")
logging.debug("vote error")
raise
ipList = list(set([o.ip for o in rows if o.ip != None]))
count = 0
if ipList:
count += Turris.update(status=str(status)).where(Turris.ip << ipList).execute()
count += Turris.update(status=str(status)).where(Turris.remoteHost == host).execute() # ovlivnit i remoteHost s IP = NULL
logging.debug("vote:" + vote + " host:" + host + " count:" + str(count))
print("vote:" + vote + " host:" + host + " count:" + str(count))
logging.debug("vote:" + vote + " host:" + host + " count:" + str(count))
return str(count) + " updated"
except:
return "no update, didnt find ip"
......@@ -186,8 +185,8 @@ class Whitelist(DbModel):
try:
url = Domains.url2domain(url)
s = url.split(".")
#print(urlparse(url))
#print(s)
#logging.debug(urlparse(url))
#logging.debug(s)
except AttributeError:
logging.error("ATTRIBUTE ERROR! Whitelist.matches({})".format(url))
return False
......
mdmaug/lib/old_db.py deleted 100644 → 0
View file @ 3a4dbd60
# Stary soubor - pouzivame dbp
#import pymysql
import MySQLdb
class Db:
connection = "" #Xvytvorime connection pozdej, az kdyz budeme mit volny slot
def select(sql,vars):
cursor = Db.connection.cursor()
cursor.execute(sql, vars)
cursor.close()
def commit(sql,vars):
cursor = Db.connection.cursor()
cur.execute(sql, vars)
Db.connection.commit()
cursor.close()
def connect():
Db.connection = pymysql.connect(host='172.20.7.10', user='root', passwd='lopuch', db='mdmaug', charset='utf8') #vytvorime connection, az kdyz mame slot
def disconnect():
Db.connection.commit()