Commit a2c7179f authored by Edvard Rejthar's avatar Edvard Rejthar 💬
Browse files

Občas nefungovalo voting.

Zjistil jsem, že to bylo, protože se použával (INNER) JOIN místo LEFT JOINu.
Ale opravdu, opravdu jsem nepochopil, proč INNER JOIN nefungoval:


SELECT status FROM `turris` AS t1
left outer  JOIN `status` AS t2 ON (`t2`.`id` = `t1`.`status`)
WHERE (`t1`.`remoteHost` = 'api.w.org') LIMIT 1 # vraci status = 1
SELECT status FROM `turris` AS t1
outer JOIN `status` AS t2 ON (`t2`.`id` = `t1`.`status`)
WHERE (`t1`.`remoteHost` = 'api.w.org') LIMIT 1 # vraci status = 0. A to prestoze t1.status samo o sobe je rovno 1 (v tabulce status je pod id = 1 -> name = allow, místo name = n/a, které mi to házelo do prezentace.)
parent 197e6772
......@@ -101,12 +101,12 @@ originalListener: null,
//if (1){ //XXX XXX XXX
//ulozit nactenou stranku
console.error("current last", request.originalURI.spec, lastRequest);//XX debug
//console.error("current last", request.originalURI.spec, lastRequest);//XX debug
if (request.originalURI.spec != lastRequest) {//novy request, vytisknout do souboru hlavicku
lastRequest = request.originalURI.spec;
console.error("request", request.originalURI.spec);
//console.error("request", request.originalURI.spec);
header = request.originalURI.spec + " " + request.getResponseHeader("Content-Type") + "\n";
console.error("("+profileName+") " + header);//xx debug
//console.error("("+profileName+") " + header);//xx debug
} else {
header = ""; //preskakujeme hlavicku
}
......
......@@ -144,7 +144,7 @@ TracingListener.prototype = {
lastRequest = request.originalURI.spec;
//console.error("request", request.originalURI.spec);
header = request.originalURI.spec + " " + request.getResponseHeader("Content-Type") + "\n";
console.error("(" + profileName + ") " + header);//xx debug
//console.error("(" + profileName + ") " + header);//xx debug
} else {
header = ""; //preskakujeme hlavicku
}
......
......@@ -126,7 +126,9 @@ class MetadataParser:
print("("+str(threadNumber),"EDVARD 5")
crawlDomainIp.country, crawlDomainIp.city = Domains.ip2countryAndCity(ip)
# kontaktovat PDNS
pdns.update(Domains.ip2pdnsDomains(ip))
items = Domains.ip2pdnsDomains(ip)
if items:
pdns.update(items)
print("("+str(threadNumber),"EDVARD 6")
return None
else: #zadna ip neni k dispozici, domena je asi propadla, hlas patri jmenu domeny
......
......@@ -70,7 +70,8 @@ class ScanController:
print(traceback.format_exc())
# XX Pokud je potiz, ze JS zabiji FF, mozno experimentovat s ulimit -Sv 500000;
return ("PROFILE EXCEPTION ({}) {} See logs, i.e. mdmaug/nohup.out. ".format(self.profile, e))
print("SAVE")
crawl.saveToFile(crawl.cacheDir + ScanController.CRAWL_FILE) # ulozit vysledky hledani
return CrawlView.outputHtml(crawl)
......
from peewee import *
from peewee import RawQuery
import datetime
import logging
from lib.config import Config
from urllib.parse import urlparse
from lib.domains import Domains
import logging
from peewee import *
from peewee import RawQuery
import time
#logging.basicConfig(level=logging.WARNING, format="%(message)s")
from lib.domains import Domains
from urllib.parse import urlparse
#myDB = MySQLDatabase("mdmaug", host='172.20.7.10', port=3306, user="root", passwd="lopuch", threadlocals=False)
#myDB.register_fields({'primary_key': 'BIGINT AUTOINCREMENT'})
#myDB.connect()
if(0): # Do not print all queries to stderr.
logger = logging.getLogger('peewee')
logger.setLevel(logging.WARNING)
class DbModel(Model):
def assureConnection():
......@@ -83,15 +85,14 @@ class Status(DbModel):
if status == 3:
return "block"
class Export(DbModel):
id = IntegerField(primary_key = True)
id = IntegerField(primary_key=True)
timestamp = DateTimeField(datetime.datetime.now())
##
# Exportuje csv
# Kdyz je urcen parametr days, bere export za tolik dni. Jinak bere export od posledniho exportu v tabulce Export. (Od tebe doby, co naposled byla volana funkce exportConfirm.)
def exportView(days = None):
def exportView(days=None):
#nacte vsechna ip, ktera loggujeme (status=2) ci blokujeme (status=3), jejichz timestamp je vetsi nez posledni export
xstr = lambda s:'' if s is None else str(s)
delimiter = ","
......@@ -104,7 +105,7 @@ class Export(DbModel):
q = "SELECT ip, port, url, `remoteHost`, `otherDetails`,status from turris where status > 1 and `ip` IS NOT NULL and timestamp > "
if days:
q += "NOW() - INTERVAL {} DAY ".format(int(days))
else :
else:
q += "(select case when MAX(timestamp IS NULL)=0 THEN max(timestamp) ELSE 0 END from export)"
print(q)
rq = RawQuery(Turris, q).execute()
......@@ -134,21 +135,21 @@ class Turris(DbModel):
id = PrimaryKeyField()
timestamp = DateTimeField(datetime.datetime.now())
status = IntegerField()
date = IntegerField(default = 0)
date = IntegerField(default=0)
ip = CharField(45, default=None)
port = IntegerField(default = 0)
port = IntegerField(default=0)
url = CharField(255, default="")
block = IntegerField(default = 0)
block = IntegerField(default=0)
remoteHost = CharField(255, default="")
otherDetails = CharField(255, default="")
def getVote(host = None, ip = None):
def getVote(host=None, ip=None):
status = None
try:
if host:
status = Turris.select().join(Status, on=(Status.id == Turris.status)).where(Turris.remoteHost == host).limit(1).get().status
status = Turris.select().join(Status,JOIN.LEFT_OUTER, on=(Status.id == Turris.status)).where(Turris.remoteHost == host).limit(1).get().status
if ip:
status = Turris.select().join(Status, on=(Status.id == Turris.status)).where(Turris.ip == ip).limit(1).get().status
status = Turris.select().join(Status, JOIN.LEFT_OUTER, on=(Status.id == Turris.status)).where(Turris.ip == ip).limit(1).get().status
except:
status = 0
return Status.int2word(status)
......
This diff is collapsed.
......@@ -16,7 +16,7 @@ class CrawlView:
r += "<div class='screenshot'><img class='thumbnail' src='data:image/png;base64,{}' /></div>".format(f.read())
for domain in crawl.keys():
if (Whitelist.matches(domain)):
continue
continue
r += "<div class=web data-vote='{}' data-safebrowsing-suspicious='{}'>".format(Turris.getVote(host=domain), Domains.isSuspicious(domain, 'attr')) # crawl[domain].vote
r += "<span class=domain>{}</span>".format(domain)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment