custom fields definable in config.ini!

firefox-mdmaug-extension/README.md

@@ -9,7 +9,6 @@ Test it:
 Sign it:
 
 1. Change version in `manifest.json`
-2. `web-ext sign --api-key <from addons.mozilla.org> --api-secret <the same>`
-3. Newly generated file rename to `mdmaug@csirt.cz.xpi`
-4. Move it to `.mozilla/extensions/{ec8030f7-c20a-464f-9b0e-13a3a9e97384}/`
+2. `web-ext sign --api-key <from addons.mozilla.org> --api-secret <the same> --artifacts-dir ../.mozilla/extensions/{ec8030f7-c20a-464f-9b0e-13a3a9e97384}/`
+3. Newly generated file in rename to `mdmaug@csirt.cz.xpi`
 

firefox-mdmaug-extension/manifest.json

@@ -3,7 +3,7 @@
     "description": "MDMaug scanner",
     "manifest_version": 2,
     "name": "MDMaug",
-    "version": "1.0.0rc1",
+    "version": "1.0.0rc2",
     "homepage_url": "https://gitlab.labs.nic.cz/csirt/mdmaug/",
     "icons": {
         "48": "icons/cznic.png"

mdmaug/bin/firefox_mdmaug_writer.py

@@ -7,12 +7,13 @@ import sys
 
 # Read a message from stdin and decode it.
 profile = os.environ["PROFILE"] if "PROFILE" in os.environ else "unknown-profile"
-log_dir = "/tmp/mdmaug/.cache/mdmaug-scans/_tmp/"  # /tmp/ is small (200 MB) and takes precious RAM
+#log_dir = "/tmp/mdmaug/.cache/mdmaug-scans/_tmp/"  # /tmp/ is small (200 MB) and takes precious RAM
+cache_dir = os.environ["CACHE_DIR"] if "CACHE_DIR" in os.environ else "/tmp/"
 files_encountered = set()
 #cache_dir = log_dir # // default dir to store the analysis is log_dir if we fail to identify a better storage point
 
-with open(log_dir+"cache.dir", "r") as f:
-    cache_dir = f.read().strip()
+#with open(log_dir+"cache.dir", "r") as f:
+#    cache_dir = f.read().strip()
 
 def get_message():
     raw_length = sys.stdin.buffer.read(4)
@@ -33,5 +34,5 @@ while True:
     with open(file, method) as f:
         f.write(message["text"])
 
-    #with open("/tmp/ram/5/log.txt", "a") as f:
-    #    f.write(f"{file} {method}\n")
+    with open("/tmp/ram/zde.txt", "a") as f:
+        f.write(f"{file} {method}\n")

mdmaug/lib/controller/scan_controller.py

@@ -82,8 +82,8 @@ class ScanController:
         # max_time = 3 # XXX
         # ,nsSocketTransport:5,nsStreamPump:5,nsHostResolver:5
         logging.debug("({}) FF -P {} -no-remote {}".format(self.profile, self.profile, self.url))
-        command = "export NSPR_LOG_MODULES=timestamp,nsHttp:5 ; export NSPR_LOG_FILE={} ; export PROFILE={};{} -P {} -no-remote '{}'".format(
-            logfile, self.profile, Config.browser, self.profile,
+        command = "export NSPR_LOG_MODULES=timestamp,nsHttp:5 ; export NSPR_LOG_FILE={} ; export CACHE_DIR={}; export PROFILE={};{} -P {} -no-remote '{}'".format(
+            logfile, cacheDir, self.profile, Config.browser, self.profile,
             "http://localhost/redirect/" + self.url)  # http://localhost/redirect/ gets stripped by the extension
         # terminate Config.browser if hes not able to (everything has to be in single command because there is no heritance of $! amongst subprocesses)
         command += " & echo $!;ii=0; while [ -n \"`ps -p $! | grep {}`\" ];do echo \"({}) running\" ;ii=$((ii+1)); if [ $ii -gt {} ]; then echo '({}) kill';kill $!; break;fi; sleep 1; done".format(
@@ -94,6 +94,7 @@ class ScanController:
         subprocess.call([command], shell=True)
         logging.debug("({}) stopped!".format(self.profile))
         # shromazdit informace z analyz
+
         crawl = Crawl(host=self.url, log_dir=logDir, cache_dir=cacheDir)
 
         expiration = 0

mdmaug/lib/controller/server.py

@@ -77,7 +77,11 @@ class Server(SimpleHTTPRequestHandler):
                     page = True
                 else:
                     request[c] = d[0] if len(d) else True
-                    continue
+                    if c == "nicify":
+                        l = l[6:]
+                        page = True
+                    else:
+                        continue
             request["page"] += l + "/"
         if request["page"]:  # strip last slash
             request["page"] = request["page"][:-1]

mdmaug/lib/parser/traffic_log_parser.py

 # Zpracuje log z firefoxu. Ten v tmp necha .tmp fily s html a js a screenshot.
-import cgi
+from html import escape
 import io
 import logging
 from contextlib import redirect_stdout
@@ -57,24 +57,26 @@ class TrafficLogParser:
             with open(nicefile, "r") as f:
                 return f.read()
         else:
+            import ipdb; ipdb.set_trace()
             with open(sourcefile, 'r') as f:
-                mime = ""
-                url = f.readline().rstrip().split(" ", 1)  # prvni radek obsahuje url a mime-type dat
+                type_ = ""
+                # first row is URL a type of stream (since Webextensions no more mime type)
+                url = f.readline().rstrip().split(" ", 1)
                 if (len(url) == 2):
-                    url, mime = url
+                    url, type_ = url
 
-                contents = f.read()  # zbytek souboru obsahuje zdrojova data
+                contents = f.read()  # the rest of the file contains source data
 
                 buf = io.StringIO()
                 with redirect_stdout(buf):  # print -> promenna
                     logging.debug("<h3>{}</h3>".format(url))
 
-                    if "html" in mime:
+                    if "main_frame" in type_ or "sub_frame" in type_: # X"html"
                         TrafficLogParser.HtmlParse(contents)
-                    elif "javascript" in mime:
+                    elif "script" in type_: # X"javascript"
                         TrafficLogParser.JsParse(contents)
-                    else:  # vypise neznamy soubor v plain-textu
-                        logging.debug(cgi.escape(contents))
+                    else:  # output file of an unknown type in plain-text at least
+                        print(escape(contents))
 
                 data = buf.getvalue()
                 with open(nicefile, "w")  as f2:  # zapsat hezke formatovani do souboru