unikátní záznamy ip X port v exportu

kompletní migrace na nový stroj

README.md

@@ -5,7 +5,7 @@ Scans a website for a sign of a parasite hosts or commands.
 ## Installation
 
 1. ```git clone git@gitlab.labs.nic.cz:csirt/mdmaug.git /tmp/mdmaug```
-2. edit config.py
+2. edit mdmaug/lib/config.py
 3. ```/tmp/mdmaug/INSTALL```
 
 ### Notes
@@ -17,12 +17,12 @@ Scans a website for a sign of a parasite hosts or commands.
 
 
 ## What is done to Firefox profiles?
-We want no block nor safebrowsing warning. If you created the profiles manually, you'd use ```firefox -P```, the profiles names being: 0,1...   
-For about:config changes, see prefs.js. IE:   
+We want no block nor safebrowsing warning. If you created the profiles manually, you'd use ```firefox -P```, the profiles names being: 0,1...
+For about:config changes, see prefs.js. IE:
     * toolkit.startup.max_resumed_crashes = -1 (protoze i kdyz prohlizec nekdy killnu, nesmi me pri spusteni otravovat gui popupem)
     * network.http.accept-encoding = "" # ukladame streamy, ale neumim je rozzipovat
     * extensions.autoDisableScopes = "0" # moznost instalovat ze vsech umisteni
     * browser.selfsupport.url = "" # tato moznost normalne v about:config neni, ale omezuje to nejake zbytecnou telemetrii, viz Mozilla Heartbeat
     * # nepamatovat si historii (Preferences / Privacy / Firefox will use custom settings for history / Clear history when closes / Setting / All)
     * # nejsem si jist, nakolik to funguje, zrejme dost
-    * ...
\ No newline at end of file
+    * ...

mdmaug/bin/start

+# XX shouldnt I delete this file?
+
 su - mdmaug -c 'cd /home/mdmaug/mdmaug/ ; python3 mdmaug.py'
 pkill python3 #pri Ctrl+C v prikazu su se uzavre jen terminal, ale ne uz python skript. Takhle to zas zabije veskery Python, ale sandboxovy-zavirovany uzivatel mdmaug stejne ma byt jen na spousteni tohohle skriptu.
\ No newline at end of file

mdmaug/lib/controller/api.py

@@ -6,23 +6,23 @@ from lib.controller.scan_controller import ScanController
 from lib.model.dbp import Status, Export, Turris, Whitelist
 from lib.analysis.parser.traffic_log_parser import TrafficLogParser
 
-class Api:    
+class Api:
     website = "" # http://site.cz
     websiteDomain = "" # site.cz
 
     def __init__(self, path):
         self.path = path
-    
+
     def run(self, cmd):
         """ Accept command """
         if cmd == "analyze":
             return ScanController().launch(self.path)
         if cmd == "analyze=cached":
             return ScanController().launch(self.path, cached = 1)
-        if cmd == "analyze=weekcache": 
+        if cmd == "analyze=weekcache":
             return ScanController().launch(self.path, cached = 7)
         if cmd == "analyze=oldcache":
-            return ScanController().launch(self.path, cached = True)        
+            return ScanController().launch(self.path, cached = True)
         elif cmd == "export=view": # XX deprecated?
             return Export.exportView()
         elif cmd == "export=confirm": # XX deprecated?
@@ -49,16 +49,16 @@ class Api:
             return self.whitelist()
         elif cmd == "reset":
             Server.reset()
-            return "reset" 
+            return "reset"
 
     def reset():
-        logging.debug("resetting running browsers")                
+        logging.debug("resetting running browsers")
         with open(Config.configFile, 'w') as f: # clear the queue
-            json.dump({}, f)        
+            json.dump({}, f)
         subprocess.call(["pkill", Config.browser]) # kill frozen browsers
 
 
-        
+
 
     #prida 2ld domenu mezi whitelistovane
     def whitelist(self):

mdmaug/lib/controller/scan_controller.py

@@ -39,7 +39,7 @@ class ScanController:
 
         if cached:
             # """ Pokud je k dispozici analyza, vratit ji """
-            dir = Config.CACHE_DIR + Domains.domain2dir(url) + "/"            
+            dir = Config.CACHE_DIR + Domains.domain2dir(url) + "/"
             if os.path.isdir(dir):
                 snapdirs = [str(dir + subdir) for subdir in os.listdir(dir) # adresare vsech moznych snapshotu
                             if os.path.isdir(str(dir + subdir)) and os.path.isfile(dir+subdir + "/"+ScanController.CRAWL_FILE)]
@@ -54,7 +54,7 @@ class ScanController:
             logging.debug("({-1}) Cachovana analyza nenalezena")
 
         # provest novou analyzu
-        if self.queue(): # /api/analyze/web - zaradi web do fronty            
+        if self.queue(): # /api/analyze/web - zaradi web do fronty
             print ("({}) start crawl".format(self.profile))
             self.url = Domains.assureUrl(url)
             try:
@@ -104,7 +104,7 @@ class ScanController:
                 logging.debug("({}) time is run!".format(self.profile))
                 raise FileNotFoundError("time is run - browser expired")
             time.sleep(1)
-            
+
 
         NsprLogParser(logfile, crawl)
         self.unbookProfile() # uvolnit browser profil
@@ -117,7 +117,7 @@ class ScanController:
 
 
 
-    def _getCacheDirStamp():        
+    def _getCacheDirStamp():
         # pro archiv logu pouzit timestamp:  #return "current"
         return datetime.datetime.fromtimestamp(time.time()).strftime('%y%m%d%H%M%S')
 
@@ -136,7 +136,7 @@ class ScanController:
         """
         logDir = ScanController._assureDir(Config.LOG_DIR  + str(self.profile) + "-log/")
         cacheDir = ScanController._assureDir(Config.CACHE_DIR + Domains.domain2dir(self.url) + "/" + ScanController._getCacheDirStamp() + "/")
-        
+
 
         # info pro FF
         with open(logDir + ScanController.FF_INFO_FILE,"w") as f: # v logDiru mu dame odkaz do cacheDiru
@@ -145,9 +145,9 @@ class ScanController:
         return logDir, cacheDir
 
 
-            
 
-    
+
+
     def _loadProfileQueue(self):
         #load queue from config file
         try:
@@ -167,7 +167,7 @@ class ScanController:
     def unbookProfile(self):
         def dump():
          with open(Config.configFile, 'w') as f:
-            json.dump(self.queueFF, f)            
+            json.dump(self.queueFF, f)
         #logging.debug("UNKBOOK")
         try:
             self.queueFF.pop(self.profile)
@@ -176,7 +176,7 @@ class ScanController:
             logging.debug("Unbook failed")
             logging.debug(self.queueFF)
             raise
-        except OSError:            
+        except OSError:
             logging.debug("({}) OS Error - interferuje s pustenym FF, ktere zere prilis pameti. Zkusime pockat.".format(self.profile))
             time.sleep(10) # XX jestli funkcionalitu zachovat, dat sem pocitadlo, at je na konzoli videt akce
             try:
@@ -185,25 +185,24 @@ class ScanController:
                 logging.debug("({}) System se nezotavil.".format(self.profile))
                 return "Memory may be exhausted. See mdmaug-server/scan_controller.py for details." # FF sezral vsechnu pamet asi. Stranka je problematicka. UrlQuery podle me taky selze.
         #logging.debug("UNKBOOKED")
-            
+
 
     def queue(self):
-        """ Ze souboru queue.cache nacte, ktery profil je volny a zabookuje ho"""
+        """ Reads from queue.cache what profile is available and books it """
         self._loadProfileQueue()
         self.profile  = -1
-        for i2 in range(4): #na volny slot zkusime nekolikrat pockat
+        for _ in range(4): #na volny slot zkusime nekolikrat pockat
             for i in range(Config.profileCount):             #i = 10 if i ==10:
                 if self.queueFF.get(str(i)) == None:
-                    self.profile  = i                    
+                    self.profile  = i
                     self.bookProfile()
                     break
             if self.profile  == -1:
-                logging.debug("(-1) PLNO, cekame par vterin")
+                logging.debug("(-1) FULL, let's wait few secs")
                 time.sleep(randint(5, 10)) #pockame par vterin
             else:
-                break #volny slot jsme nasli, muzeme dal
+                break # we found a free slot, let's proceed
 
-        #logging.debug(" profile " + str(self.profile ) + " queueFF:")
         logging.debug(self.queueFF)
 
         #povedlo se zabookovat profil FF?

mdmaug/lib/controller/server.py

@@ -8,16 +8,14 @@ from lib.model.dbp import Export
 import logging
 import mimetypes
 import os
-import time
-
 
 env = Environment()
 env.loader = FileSystemLoader(Config.DIR + "templates/")
 
-class Server(SimpleHTTPRequestHandler):    
-  
+class Server(SimpleHTTPRequestHandler):
+
     def favicon(self):
-        with open('favicon.ico', 'rb') as f:                
+        with open('favicon.ico', 'rb') as f:
             self.output(f.read(), "image/x-icon")
 
     def render_template(self, filename, ** kwargs):
@@ -41,9 +39,9 @@ class Server(SimpleHTTPRequestHandler):
         with open(url, type) as f:
             self.output(f.read(), contentType=mimetypes.guess_type(url))
 
-    def do_GET(self):                                
+    def do_GET(self):
         path = self.path.split("/")
-        
+
         logging.debug("Request: {}".format(path[1]))
         if path[1] == "":
             return self.homepage()
@@ -56,6 +54,6 @@ class Server(SimpleHTTPRequestHandler):
             api = Api(self.path)
             # send everything up, we are in an iframe
             self.render_template("_message.html", contents=api.run(cmd), cmd=cmd, url=self.path, destination="https://mdm.nic.cz/")
-        elif path[1] == "export": # /export/{days}  - csv za poslednich 7 dni            
+        elif path[1] == "export": # /export/{days}  - csv za poslednich 7 dni
             url = self.path.split("/", 2)
             self.output(Export.exportView(days=url[2]))
\ No newline at end of file

mdmaug/lib/model/dbp.py

@@ -17,7 +17,7 @@ if(1): # Do not print all queries to stderr.
     logger.setLevel(logging.WARNING)
 
 
-class DbModel(Model):    
+class DbModel(Model):
 
     def assureConnection():
         logging.debug("Assure connection.")
@@ -40,7 +40,7 @@ class DbModel(Model):
 
     """A base model that will use our MySQL database"""
     def connect():
-        logging.debug("connecting db....")        
+        logging.debug("connecting db....")
         #DbModel.Meta.myDb = Config.myDB
         # Config.myDB.connect() # XX kupodivu toto neni potreba
         #logging.debug(Whitelist.select().count())
@@ -107,6 +107,8 @@ class Export(DbModel):
                 q += "NOW() - INTERVAL {} DAY ".format(int(days))
             else:
                 q += "(select case when MAX(timestamp IS NULL)=0 THEN max(timestamp) ELSE 0 END from export)"
+            q += " GROUP BY concat(`ip`,`port`) " # group by concat may be a performance issue
+            q += " ORDER BY `timestamp` DESC"
             logging.debug(q)
             rq = RawQuery(Turris, q).execute()
             print (rq)
@@ -164,17 +166,17 @@ class Turris(DbModel):
             except:
                 logging.error("domain should have been inserted in the database, but it hasnt been")
                 logging.debug("vote error")
-                raise            
-            ipList = list(set([o.ip for o in rows if o.ip != None]))            
+                raise
+            ipList = list(set([o.ip for o in rows if o.ip != None]))
             count = 0
             if ipList:
                 count += Turris.update(status=str(status)).where(Turris.ip << ipList).execute()
             count += Turris.update(status=str(status)).where(Turris.remoteHost == host).execute() # ovlivnit i remoteHost s IP = NULL
-            logging.debug("vote:" + vote + " host:" + host + " count:" + str(count))            
+            logging.debug("vote:" + vote + " host:" + host + " count:" + str(count))
             return str(count) + " updated"
         except:
             return "no update, didnt find ip"
-    
+
 class Whitelist(DbModel):
     id = PrimaryKeyField()
     timestamp = DateTimeField(datetime.datetime.now())

mdmaug/mdmaug-launch

@@ -4,5 +4,5 @@ echo "mdmaug-launch start" >> ~/log.log
 pkill -f mdmaug.py
 pkill -f Xvfb
 export PYTHONPATH=$PYTHONPATH:/opt/mdmaug/mdmaug/
-cd /opt/mdmaug/mdmaug && ./mdmaug.py 2>&1 | /usr/bin/logger -t yourtag
+cd /opt/mdmaug/mdmaug && ./mdmaug.py 2>&1 | /usr/bin/logger -t mdmaugtag
 whoami >> ~/log.log
\ No newline at end of file

mdmaug/mdmaug.py

@@ -27,7 +27,7 @@ vdisplay = Xvfb()
 vdisplay.start()
 try:
     print('Listening at https://0.0.0.0:{}'.format(Config.APP_PORT))
-    for _ in range(2): # XX Config.profileCount
+    for _ in range(Config.profileCount):
         threading.Thread(target=httpd.serve_forever).start()
 except (KeyboardInterrupt, SystemExit):
     vdisplay.stop()
@@ -39,7 +39,7 @@ How to debug mysql:
 conn = pymysql.connect(host='localhost', user='root', passwd='lopuch', db='mdmaug', charset='utf8')
 cur = conn.cursor()
 cur.execute("""SELECT name from turris JOIN status ON status.id = turris.status WHERE ip = %s""", (ip,))
-vote = self.cur.fetchone()[0]            
+vote = self.cur.fetchone()[0]
 logging.debug(vote)
 exit()
 

nbproject/project.properties

+file.reference.opt-mdmaug-installer=.
 java.lib.path=
 platform.active=Python_3.5.1
 python.lib.path=