Občas nefungovalo voting.

Zjistil jsem, že to bylo, protože se použával (INNER) JOIN místo LEFT JOINu.
Ale opravdu, opravdu jsem nepochopil, proč INNER JOIN nefungoval:


SELECT status FROM `turris` AS t1
left outer  JOIN `status` AS t2 ON (`t2`.`id` = `t1`.`status`)
WHERE (`t1`.`remoteHost` = 'api.w.org') LIMIT 1 # vraci status = 1
SELECT status FROM `turris` AS t1
outer JOIN `status` AS t2 ON (`t2`.`id` = `t1`.`status`)
WHERE (`t1`.`remoteHost` = 'api.w.org') LIMIT 1 # vraci status = 0. A to prestoze t1.status samo o sobe je rovno 1 (v tabulce status je pod id = 1 -> name = allow, místo name = n/a, které mi to házelo do prezentace.)

.mozilla/extensions/{ec8030f7-c20a-464f-9b0e-13a3a9e97384}/mdmaug@jetpack/resources/mdmaug/lib/funkcni_mail.js

@@ -101,12 +101,12 @@ originalListener: null,
         //if (1){ //XXX XXX XXX
         //ulozit nactenou stranku
 
-        console.error("current last", request.originalURI.spec, lastRequest);//XX debug
+        //console.error("current last", request.originalURI.spec, lastRequest);//XX debug
         if (request.originalURI.spec != lastRequest) {//novy request, vytisknout do souboru hlavicku
         lastRequest = request.originalURI.spec;
-                console.error("request", request.originalURI.spec);
+                //console.error("request", request.originalURI.spec);
                 header = request.originalURI.spec + " " + request.getResponseHeader("Content-Type") + "\n";
-                console.error("("+profileName+") " + header);//xx debug
+                //console.error("("+profileName+") " + header);//xx debug
         } else {
         header = ""; //preskakujeme hlavicku
         }

.mozilla/extensions/{ec8030f7-c20a-464f-9b0e-13a3a9e97384}/mdmaug@jetpack/resources/mdmaug/lib/main.js

@@ -144,7 +144,7 @@ TracingListener.prototype = {
                 lastRequest = request.originalURI.spec;
                 //console.error("request", request.originalURI.spec);
                 header = request.originalURI.spec + " " + request.getResponseHeader("Content-Type") + "\n";
-                console.error("(" + profileName + ") " + header);//xx debug
+                //console.error("(" + profileName + ") " + header);//xx debug
             } else {
                 header = ""; //preskakujeme hlavicku
             }

mdmaug/lib/analysis/parser/metadata_parser.py

@@ -126,7 +126,9 @@ class MetadataParser:
             print("("+str(threadNumber),"EDVARD 5")
             crawlDomainIp.country, crawlDomainIp.city = Domains.ip2countryAndCity(ip)
             # kontaktovat PDNS
-            pdns.update(Domains.ip2pdnsDomains(ip))
+            items = Domains.ip2pdnsDomains(ip)
+            if items:
+                pdns.update(items)
             print("("+str(threadNumber),"EDVARD 6")
             return None
         else: #zadna ip neni k dispozici, domena je asi propadla, hlas patri jmenu domeny

mdmaug/lib/analysis/scan_controller.py

@@ -70,7 +70,8 @@ class ScanController:
                 print(traceback.format_exc())
                 # XX Pokud je potiz, ze JS zabiji FF, mozno experimentovat s ulimit -Sv 500000;
                 return ("PROFILE EXCEPTION ({}) {} See logs, i.e. mdmaug/nohup.out. ".format(self.profile, e))
-            
+
+            print("SAVE")
             crawl.saveToFile(crawl.cacheDir + ScanController.CRAWL_FILE) # ulozit vysledky hledani
             return CrawlView.outputHtml(crawl)
 

mdmaug/lib/dbp.py

-from peewee import *
-from peewee import RawQuery
 import datetime
-import logging
 from lib.config import Config
-from urllib.parse import urlparse
+from lib.domains import Domains
 import logging
+from peewee import *
+from peewee import RawQuery
 import time
-#logging.basicConfig(level=logging.WARNING, format="%(message)s")
-
-from lib.domains import Domains
+from urllib.parse import urlparse
 
 #myDB = MySQLDatabase("mdmaug", host='172.20.7.10', port=3306, user="root", passwd="lopuch", threadlocals=False)
 #myDB.register_fields({'primary_key': 'BIGINT AUTOINCREMENT'})
 #myDB.connect()
 
 
+if(0): # Do not print all queries to stderr.
+    logger = logging.getLogger('peewee')
+    logger.setLevel(logging.WARNING)
+
+
 class DbModel(Model):    
 
     def assureConnection():
@@ -83,15 +85,14 @@ class Status(DbModel):
         if status == 3:
             return "block"
 
-
 class Export(DbModel):
-    id = IntegerField(primary_key = True)
+    id = IntegerField(primary_key=True)
     timestamp = DateTimeField(datetime.datetime.now())
 
     ##
     # Exportuje csv
     # Kdyz je urcen parametr days, bere export za tolik dni. Jinak bere export od posledniho exportu v tabulce Export. (Od tebe doby, co naposled byla volana funkce exportConfirm.)
-    def exportView(days = None):
+    def exportView(days=None):
         #nacte vsechna ip, ktera loggujeme (status=2) ci blokujeme (status=3), jejichz timestamp je vetsi nez posledni export
         xstr = lambda s:'' if s is None else str(s)
         delimiter = ","
@@ -104,7 +105,7 @@ class Export(DbModel):
             q = "SELECT ip, port, url, `remoteHost`, `otherDetails`,status from turris where status > 1 and `ip` IS NOT NULL and timestamp > "
             if days:
                 q += "NOW() - INTERVAL {} DAY ".format(int(days))
-            else :
+            else:
                 q += "(select case when MAX(timestamp IS NULL)=0 THEN max(timestamp) ELSE 0 END from export)"
             print(q)
             rq = RawQuery(Turris, q).execute()
@@ -134,21 +135,21 @@ class Turris(DbModel):
     id = PrimaryKeyField()
     timestamp = DateTimeField(datetime.datetime.now())
     status = IntegerField()
-    date	 = IntegerField(default = 0)
+    date	 = IntegerField(default=0)
     ip  = CharField(45, default=None)
-    port	 = IntegerField(default = 0)
+    port	 = IntegerField(default=0)
     url	 = CharField(255, default="")
-    block	 = IntegerField(default = 0)
+    block	 = IntegerField(default=0)
     remoteHost	 = CharField(255, default="")
     otherDetails = CharField(255, default="")
 
-    def getVote(host = None, ip = None):
+    def getVote(host=None, ip=None):
         status = None
         try:
             if host:
-                status = Turris.select().join(Status, on=(Status.id == Turris.status)).where(Turris.remoteHost == host).limit(1).get().status
+                status = Turris.select().join(Status,JOIN.LEFT_OUTER, on=(Status.id == Turris.status)).where(Turris.remoteHost == host).limit(1).get().status
             if ip:
-                status = Turris.select().join(Status, on=(Status.id == Turris.status)).where(Turris.ip == ip).limit(1).get().status
+                status = Turris.select().join(Status, JOIN.LEFT_OUTER, on=(Status.id == Turris.status)).where(Turris.ip == ip).limit(1).get().status
         except:
             status = 0
         return Status.int2word(status)

mdmaug/templates/crawl_view.py

@@ -16,7 +16,7 @@ class CrawlView:
                 r += "<div class='screenshot'><img class='thumbnail' src='data:image/png;base64,{}' /></div>".format(f.read())
         for domain in crawl.keys():            
             if (Whitelist.matches(domain)):
-                continue
+                continue            
 
             r += "<div class=web data-vote='{}' data-safebrowsing-suspicious='{}'>".format(Turris.getVote(host=domain), Domains.isSuspicious(domain, 'attr')) # crawl[domain].vote
             r += "<span class=domain>{}</span>".format(domain)