Commit a2c7179f authored by Edvard Rejthar's avatar Edvard Rejthar
Browse files

Občas nefungovalo voting.

Zjistil jsem, že to bylo, protože se použával (INNER) JOIN místo LEFT JOINu.
Ale opravdu, opravdu jsem nepochopil, proč INNER JOIN nefungoval:


SELECT status FROM `turris` AS t1
left outer  JOIN `status` AS t2 ON (`t2`.`id` = `t1`.`status`)
WHERE (`t1`.`remoteHost` = 'api.w.org') LIMIT 1 # vraci status = 1
SELECT status FROM `turris` AS t1
outer JOIN `status` AS t2 ON (`t2`.`id` = `t1`.`status`)
WHERE (`t1`.`remoteHost` = 'api.w.org') LIMIT 1 # vraci status = 0. A to prestoze t1.status samo o sobe je rovno 1 (v tabulce status je pod id = 1 -> name = allow, místo name = n/a, které mi to házelo do prezentace.)
parent 197e6772
...@@ -101,12 +101,12 @@ originalListener: null, ...@@ -101,12 +101,12 @@ originalListener: null,
//if (1){ //XXX XXX XXX //if (1){ //XXX XXX XXX
//ulozit nactenou stranku //ulozit nactenou stranku
console.error("current last", request.originalURI.spec, lastRequest);//XX debug //console.error("current last", request.originalURI.spec, lastRequest);//XX debug
if (request.originalURI.spec != lastRequest) {//novy request, vytisknout do souboru hlavicku if (request.originalURI.spec != lastRequest) {//novy request, vytisknout do souboru hlavicku
lastRequest = request.originalURI.spec; lastRequest = request.originalURI.spec;
console.error("request", request.originalURI.spec); //console.error("request", request.originalURI.spec);
header = request.originalURI.spec + " " + request.getResponseHeader("Content-Type") + "\n"; header = request.originalURI.spec + " " + request.getResponseHeader("Content-Type") + "\n";
console.error("("+profileName+") " + header);//xx debug //console.error("("+profileName+") " + header);//xx debug
} else { } else {
header = ""; //preskakujeme hlavicku header = ""; //preskakujeme hlavicku
} }
......
...@@ -144,7 +144,7 @@ TracingListener.prototype = { ...@@ -144,7 +144,7 @@ TracingListener.prototype = {
lastRequest = request.originalURI.spec; lastRequest = request.originalURI.spec;
//console.error("request", request.originalURI.spec); //console.error("request", request.originalURI.spec);
header = request.originalURI.spec + " " + request.getResponseHeader("Content-Type") + "\n"; header = request.originalURI.spec + " " + request.getResponseHeader("Content-Type") + "\n";
console.error("(" + profileName + ") " + header);//xx debug //console.error("(" + profileName + ") " + header);//xx debug
} else { } else {
header = ""; //preskakujeme hlavicku header = ""; //preskakujeme hlavicku
} }
......
...@@ -126,7 +126,9 @@ class MetadataParser: ...@@ -126,7 +126,9 @@ class MetadataParser:
print("("+str(threadNumber),"EDVARD 5") print("("+str(threadNumber),"EDVARD 5")
crawlDomainIp.country, crawlDomainIp.city = Domains.ip2countryAndCity(ip) crawlDomainIp.country, crawlDomainIp.city = Domains.ip2countryAndCity(ip)
# kontaktovat PDNS # kontaktovat PDNS
pdns.update(Domains.ip2pdnsDomains(ip)) items = Domains.ip2pdnsDomains(ip)
if items:
pdns.update(items)
print("("+str(threadNumber),"EDVARD 6") print("("+str(threadNumber),"EDVARD 6")
return None return None
else: #zadna ip neni k dispozici, domena je asi propadla, hlas patri jmenu domeny else: #zadna ip neni k dispozici, domena je asi propadla, hlas patri jmenu domeny
......
...@@ -70,7 +70,8 @@ class ScanController: ...@@ -70,7 +70,8 @@ class ScanController:
print(traceback.format_exc()) print(traceback.format_exc())
# XX Pokud je potiz, ze JS zabiji FF, mozno experimentovat s ulimit -Sv 500000; # XX Pokud je potiz, ze JS zabiji FF, mozno experimentovat s ulimit -Sv 500000;
return ("PROFILE EXCEPTION ({}) {} See logs, i.e. mdmaug/nohup.out. ".format(self.profile, e)) return ("PROFILE EXCEPTION ({}) {} See logs, i.e. mdmaug/nohup.out. ".format(self.profile, e))
print("SAVE")
crawl.saveToFile(crawl.cacheDir + ScanController.CRAWL_FILE) # ulozit vysledky hledani crawl.saveToFile(crawl.cacheDir + ScanController.CRAWL_FILE) # ulozit vysledky hledani
return CrawlView.outputHtml(crawl) return CrawlView.outputHtml(crawl)
......
from peewee import *
from peewee import RawQuery
import datetime import datetime
import logging
from lib.config import Config from lib.config import Config
from urllib.parse import urlparse from lib.domains import Domains
import logging import logging
from peewee import *
from peewee import RawQuery
import time import time
#logging.basicConfig(level=logging.WARNING, format="%(message)s") from urllib.parse import urlparse
from lib.domains import Domains
#myDB = MySQLDatabase("mdmaug", host='172.20.7.10', port=3306, user="root", passwd="lopuch", threadlocals=False) #myDB = MySQLDatabase("mdmaug", host='172.20.7.10', port=3306, user="root", passwd="lopuch", threadlocals=False)
#myDB.register_fields({'primary_key': 'BIGINT AUTOINCREMENT'}) #myDB.register_fields({'primary_key': 'BIGINT AUTOINCREMENT'})
#myDB.connect() #myDB.connect()
if(0): # Do not print all queries to stderr.
logger = logging.getLogger('peewee')
logger.setLevel(logging.WARNING)
class DbModel(Model): class DbModel(Model):
def assureConnection(): def assureConnection():
...@@ -83,15 +85,14 @@ class Status(DbModel): ...@@ -83,15 +85,14 @@ class Status(DbModel):
if status == 3: if status == 3:
return "block" return "block"
class Export(DbModel): class Export(DbModel):
id = IntegerField(primary_key = True) id = IntegerField(primary_key=True)
timestamp = DateTimeField(datetime.datetime.now()) timestamp = DateTimeField(datetime.datetime.now())
## ##
# Exportuje csv # Exportuje csv
# Kdyz je urcen parametr days, bere export za tolik dni. Jinak bere export od posledniho exportu v tabulce Export. (Od tebe doby, co naposled byla volana funkce exportConfirm.) # Kdyz je urcen parametr days, bere export za tolik dni. Jinak bere export od posledniho exportu v tabulce Export. (Od tebe doby, co naposled byla volana funkce exportConfirm.)
def exportView(days = None): def exportView(days=None):
#nacte vsechna ip, ktera loggujeme (status=2) ci blokujeme (status=3), jejichz timestamp je vetsi nez posledni export #nacte vsechna ip, ktera loggujeme (status=2) ci blokujeme (status=3), jejichz timestamp je vetsi nez posledni export
xstr = lambda s:'' if s is None else str(s) xstr = lambda s:'' if s is None else str(s)
delimiter = "," delimiter = ","
...@@ -104,7 +105,7 @@ class Export(DbModel): ...@@ -104,7 +105,7 @@ class Export(DbModel):
q = "SELECT ip, port, url, `remoteHost`, `otherDetails`,status from turris where status > 1 and `ip` IS NOT NULL and timestamp > " q = "SELECT ip, port, url, `remoteHost`, `otherDetails`,status from turris where status > 1 and `ip` IS NOT NULL and timestamp > "
if days: if days:
q += "NOW() - INTERVAL {} DAY ".format(int(days)) q += "NOW() - INTERVAL {} DAY ".format(int(days))
else : else:
q += "(select case when MAX(timestamp IS NULL)=0 THEN max(timestamp) ELSE 0 END from export)" q += "(select case when MAX(timestamp IS NULL)=0 THEN max(timestamp) ELSE 0 END from export)"
print(q) print(q)
rq = RawQuery(Turris, q).execute() rq = RawQuery(Turris, q).execute()
...@@ -134,21 +135,21 @@ class Turris(DbModel): ...@@ -134,21 +135,21 @@ class Turris(DbModel):
id = PrimaryKeyField() id = PrimaryKeyField()
timestamp = DateTimeField(datetime.datetime.now()) timestamp = DateTimeField(datetime.datetime.now())
status = IntegerField() status = IntegerField()
date = IntegerField(default = 0) date = IntegerField(default=0)
ip = CharField(45, default=None) ip = CharField(45, default=None)
port = IntegerField(default = 0) port = IntegerField(default=0)
url = CharField(255, default="") url = CharField(255, default="")
block = IntegerField(default = 0) block = IntegerField(default=0)
remoteHost = CharField(255, default="") remoteHost = CharField(255, default="")
otherDetails = CharField(255, default="") otherDetails = CharField(255, default="")
def getVote(host = None, ip = None): def getVote(host=None, ip=None):
status = None status = None
try: try:
if host: if host:
status = Turris.select().join(Status, on=(Status.id == Turris.status)).where(Turris.remoteHost == host).limit(1).get().status status = Turris.select().join(Status,JOIN.LEFT_OUTER, on=(Status.id == Turris.status)).where(Turris.remoteHost == host).limit(1).get().status
if ip: if ip:
status = Turris.select().join(Status, on=(Status.id == Turris.status)).where(Turris.ip == ip).limit(1).get().status status = Turris.select().join(Status, JOIN.LEFT_OUTER, on=(Status.id == Turris.status)).where(Turris.ip == ip).limit(1).get().status
except: except:
status = 0 status = 0
return Status.int2word(status) return Status.int2word(status)
......
This diff is collapsed.
...@@ -16,7 +16,7 @@ class CrawlView: ...@@ -16,7 +16,7 @@ class CrawlView:
r += "<div class='screenshot'><img class='thumbnail' src='data:image/png;base64,{}' /></div>".format(f.read()) r += "<div class='screenshot'><img class='thumbnail' src='data:image/png;base64,{}' /></div>".format(f.read())
for domain in crawl.keys(): for domain in crawl.keys():
if (Whitelist.matches(domain)): if (Whitelist.matches(domain)):
continue continue
r += "<div class=web data-vote='{}' data-safebrowsing-suspicious='{}'>".format(Turris.getVote(host=domain), Domains.isSuspicious(domain, 'attr')) # crawl[domain].vote r += "<div class=web data-vote='{}' data-safebrowsing-suspicious='{}'>".format(Turris.getVote(host=domain), Domains.isSuspicious(domain, 'attr')) # crawl[domain].vote
r += "<span class=domain>{}</span>".format(domain) r += "<span class=domain>{}</span>".format(domain)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment