funkcni mdmaug server

0d75e703 · edvard · 0d75e703 · 0d75e703 · 0d75e703 · 0d75e703
Commit 0d75e703 authored Jun 11, 2015 by edvard
39 changed files
--- a/.gitignore
+++ b/.gitignore
+/__pycache__/
+/lib/__pycache__/
+/lib/analysis/__pycache__/
+/lib/analysis/parser/__pycache__/
+/nbproject/
+/templates/__pycache__/
\ No newline at end of file
--- a/bin/profile_copy.py
+++ b/bin/profile_copy.py
+# mdmaug@jetpack.xpi je treba nakopirovat do ~/.mozilla/firefox/[1-20]/extensions/mdmaug@jetpack.xpi. To same pro prefs.js
+# Pouzita funkce copyfile stary xpi-soubor nahradi.
+import os
+from shutil import copyfile
+
+#xpiSourceFilepath = "/home/edvard/edvard/skola/diplomka/mdmaug-addon/mdmaug.xpi" # zdrojova cesta k souboru vcetne jmena
+#xpiFilename = "mdmaug@jetpack.xpi" # jak ma vypadat jmeno souboru
+#profilesDir = "/mnt/mdmaug/home/mdmaug/.mozilla/firefox/" # cesta k rozcestniku profilu v .mozilla
+
+
+sourceFilepath = "/mnt/mdmaug/home/mdmaug/.mozilla/firefox/0/prefs.js" # zdrojova cesta k souboru vcetne jmena
+filenameNew = "prefs.js" # jak ma vypadat jmeno souboru
+profilesDir = "/mnt/mdmaug/home/mdmaug/.mozilla/firefox/" # cesta k rozcestniku profilu v .mozilla
+
+
+#quit("Edit before running")
+
+
+
+for profileNum in range(1,21):
+    dir = profilesDir + str(profileNum) + "/"
+    if not os.path.exists(dir): os.makedirs(dir)
+    #dir += "extensions/"
+    #if not os.path.exists(dir): os.makedirs(dir)
+    print("From {} to {}".format(sourceFilepath, dir + filenameNew))
+    copyfile(sourceFilepath, dir + filenameNew)
\ No newline at end of file
--- a/bin/start
+++ b/bin/start
+su - mdmaug -c 'cd /home/mdmaug/mdmaug/ ; python3 mdmaug.py'
+killall python3 #pri Ctrl+C v prikazu su se uzavre jen terminal, ale ne uz python skript. Takhle to zas zabije veskery Python, ale sandboxovy-zavirovany uzivatel mdmaug stejne ma byt jen na spousteni tohohle skriptu.
\ No newline at end of file
--- a/crawlSave.tmp
+++ b/crawlSave.tmp
+cacheDir: /home/mdmaug/.cache/mdmaug-scans/tjetri.s3websiteuswest2.amazonaws.com/current/
+keys:
+- - www.telize.com
+  - addresses:
+    - - 185.24.220.241
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 2a02:2770:9:0:21a:4aff:feed:aa95
+      - {city: ' (Private Address)', country: ' (Private Address) (XX)', vote: n/a}
+    urls:
+    - - /geoip/
+      - {sourcefile: null, spyfile: null}
+    vote: null
+- - j.maxmind.com
+  - addresses:
+    - - 127.0.0.1
+      - {city: ' (Private Address)', country: ' (Private Address) (XX)', vote: n/a}
+    urls:
+    - - /app/geoip.js
+      - {sourcefile: /home/mdmaug/.cache/mdmaug-scans/tjetri.s3websiteuswest2.amazonaws.com/current/httpjmaxmindcomappgeoipjs.tmp,
+        spyfile: null}
+    vote: null
+- - whos.amung.us
+  - addresses:
+    - - 67.202.94.94
+      - {city: ' Plano, TX', country: ' UNITED STATES (US)', vote: n/a}
+    - - 67.202.94.86
+      - {city: ' Plano, TX', country: ' UNITED STATES (US)', vote: n/a}
+    - - 67.202.94.93
+      - {city: ' Plano, TX', country: ' UNITED STATES (US)', vote: n/a}
+    urls:
+    - - /pingjs/?k=ditiadea
+      - {sourcefile: /home/mdmaug/.cache/mdmaug-scans/tjetri.s3websiteuswest2.amazonaws.com/current/httpwhosamunguspingjskditiadea.tmp,
+        spyfile: null}
+    vote: null
+- - cracks4free.info
+  - addresses:
+    - - 216.55.138.103
+      - {city: ' (Unknown city)', country: ' UNITED STATES (US)', vote: n/a}
+    urls:
+    - - /soarb/player.swf
+      - {sourcefile: null, spyfile: null}
+    vote: null
+- - s27.postimg.org
+  - addresses:
+    - - 141.101.112.62
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 190.93.243.61
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 141.101.113.62
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 141.101.123.62
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 190.93.240.62
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    urls:
+    - - /fp28nvyar/arrow.png
+      - {sourcefile: null, spyfile: null}
+    vote: null
+- - code.jquery.com
+  - addresses:
+    - - 94.31.29.230
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 94.31.29.53
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    urls:
+    - - /jquery-1.9.1.min.js
+      - {sourcefile: /home/mdmaug/.cache/mdmaug-scans/tjetri.s3websiteuswest2.amazonaws.com/current/httpcodejquerycomjquery191minjs.tmp,
+        spyfile: null}
+    vote: null
+- - ssl.gstatic.com
+  - addresses:
+    - - 173.194.116.248
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.239
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.247
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.255
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 2a00:1450:4014:80a::101f
+      - {city: ' (Private Address)', country: ' (Private Address) (XX)', vote: n/a}
+    urls:
+    - - /
+      - {sourcefile: null, spyfile: null}
+    - - /s2/oz/images/logo/1x/googleplus_color_33-d28e37b6be3328c1aab7a37fa108901f.png
+      - {sourcefile: null, spyfile: null}
+    vote: null
+- - www.html-manager.com
+  - addresses:
+    - - 54.245.242.253
+      - {city: ' (Unknown city)', country: ' UNITED STATES (US)', vote: n/a}
+    urls:
+    - - /scripts/AMIdownload.js
+      - {sourcefile: /home/mdmaug/.cache/mdmaug-scans/tjetri.s3websiteuswest2.amazonaws.com/current/httpwwwhtmlmanagercomscriptsamidownloadjs.tmp,
+        spyfile: null}
+    vote: null
+- - s.ytimg.com
+  - addresses:
+    - - 173.194.116.224
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.226
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 2a00:1450:4014:80a::1003
+      - {city: ' (Private Address)', country: ' (Private Address) (XX)', vote: n/a}
+    - - 173.194.116.229
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.228
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.231
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.225
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.233
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.227
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.232
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.238
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.230
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    urls:
+    - - /yts/cssbin/www-player-webp-vflDCq0qG.css
+      - {sourcefile: null, spyfile: null}
+    - - /yts/img/favicon_32-vflWoMFGx.png
+      - {sourcefile: null, spyfile: null}
+    - - /yts/cssbin/www-core-webp-vfl5WLlKM.css
+      - {sourcefile: null, spyfile: null}
+    - - /yts/cssbin/www-guide-webp-vflf71p2P.css
+      - {sourcefile: null, spyfile: null}
+    - - /yts/imgbin/www-hitchhiker-vflBSZ2Yi.webp
+      - {sourcefile: null, spyfile: null}
+    - - /yts/img/favicon-vfldLzJxy.ico
+      - {sourcefile: null, spyfile: null}
+    - - /yts/img/pixel-vfl3z5WfW.gif
+      - {sourcefile: null, spyfile: null}
+    - - /yts/img/meh7-vflGevej7.png
+      - {sourcefile: null, spyfile: null}
+    vote: null
+- - yt3.ggpht.com
+  - addresses:
+    - - 2a00:1450:4014:80a::100a
+      - {city: ' (Private Address)', country: ' (Private Address) (XX)', vote: n/a}
+    - - 173.194.116.234
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.236
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.235
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    urls:
+    - - /
+      - {sourcefile: null, spyfile: null}
+    - - /-RZPcUyz4cSo/AAAAAAAAAAI/AAAAAAAAAAA/V-UhQqK_QhA/s48-c-k-no/photo.jpg
+      - {sourcefile: null, spyfile: null}
+    vote: null
+- - www.google.com
+  - addresses:
+    - - 216.58.209.164
+      - {city: ' Bloomington, CA', country: ' UNITED STATES (US)', vote: n/a}
+    - - 2a00:1450:400d:806::2004
+      - {city: ' (Private Address)', country: ' (Private Address) (XX)', vote: n/a}
+    urls:
+    - - /support/accounts/bin/answer.py?hl=en&answer=151657
+      - {sourcefile: null, spyfile: null}
+    vote: null
+- - www.google-analytics.com
+  - addresses:
+    - - 173.194.116.224
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.226
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.231
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.229
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.228
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 2a00:1450:4014:80a::1009
+      - {city: ' (Private Address)', country: ' (Private Address) (XX)', vote: n/a}
+    - - 173.194.116.225
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.233
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.227
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.232
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.238
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.230
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    urls:
+    - - /ga.js
+      - {sourcefile: /home/mdmaug/.cache/mdmaug-scans/tjetri.s3websiteuswest2.amazonaws.com/current/httpwwwgoogleanalyticscomgajs.tmp,
+        spyfile: null}
+    - - /r/__utm.gif?utmwv=5.6.4&utms=1&utmn=1440526304&utmhn=tjetri.s3-website-us-west-2.amazonaws.com&utmcs=UTF-8&utmsr=800x680&utmvp=784x485&utmsc=24-bit&utmul=cs&utmje=0&utmfl=10.1%20r999.%3Cbr%3EGnash%200.8.11dev%2C%20the%20GNU%20SWF%20Player.%20%20%20Copyright%20
+      - {sourcefile: null, spyfile: null}
+    vote: null
+- - plus.google.com
+  - addresses:
+    - - 173.194.116.224
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.226
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.231
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.229
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.228
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.225
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 2a00:1450:4014:80a::1002
+      - {city: ' (Private Address)', country: ' (Private Address) (XX)', vote: n/a}
+    - - 173.194.116.233
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.227
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.232
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.238
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.230
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    urls:
+    - - /
+      - {sourcefile: null, spyfile: null}
+    - - /_/notifications/frame?origin=http%3A%2F%2Fwww.youtube.com&source=yt&hl=tr-TR&jsh=m%3B%2F_%2Fscs%2Fabc-static%2F_%2Fjs%2Fk%3Dgapi.gapi.en.5GL3tg_C6Nc.O%2Fm%3D__features__%2Frt%3Dj%2Fd%3D1%2Frs%3DAItRSTPfsfz-ifAZeiHMk7mKfiGQJuhVFw#pid=36&rpctoken=605743366&_methods=setNotificationWidgetSize%2CsetNotificationWidgetHeight%2CsetNotificationText%2ChideNotificationWidget%2CopenSharebox%2ConError%2C_ready%2C_close%2C_open%2C_resizeMe%2C_renderstart&id=I1_1388598845054&parent=http%3A%2F%2Fwww.youtube.com&pfname=
+      - {sourcefile: /home/mdmaug/.cache/mdmaug-scans/tjetri.s3websiteuswest2.amazonaws.com/current/httpsplusgooglecomnotificationsframeoriginhttp3a2f.tmp,
+        spyfile: null}
+    - - /_/notifications/frame?origin=http%3A%2F%2Fwww.youtube.com&source=yt&hl=tr-TR&jsh=m%3B%2F_%2Fscs%2Fabc-static%2F_%2Fjs%2Fk%3Dgapi.gapi.en.5GL3tg_C6Nc.O%2Fm%3D__features__%2Frt%3Dj%2Fd%3D1%2Frs%3DAItRSTPfsfz-ifAZeiHMk7mKfiGQJuhVFw
+      - {sourcefile: null, spyfile: null}
+    vote: null
+- - www.youtube.com
+  - addresses:
+    - - 173.194.116.224
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.226
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 2a00:1450:4014:80a::1005
+      - {city: ' (Private Address)', country: ' (Private Address) (XX)', vote: n/a}
+    - - 173.194.116.231
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.229
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.228
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.225
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.233
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.227
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.232
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.238
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.230
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    urls:
+    - - ''
+      - {sourcefile: null, spyfile: null}
+    vote: null
+- - clients1.google.com
+  - addresses:
+    - - 173.194.116.224
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.226
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.231
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.229
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.228
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.225
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.233
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.227
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.232
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.238
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.230
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 2a00:1450:4014:80a::1001
+      - {city: ' (Private Address)', country: ' (Private Address) (XX)', vote: n/a}
+    urls:
+    - - /ocsp
+      - {sourcefile: null, spyfile: null}
+    vote: null
+- - tjetri.s3-website-us-west-2.amazonaws.com
+  - addresses: []
+    urls:
+    - - ''
+      - {sourcefile: null, spyfile: null}
+    - - /
+      - {sourcefile: null, spyfile: null}
+    - - /red.html spy
+      - {sourcefile: null, spyfile: /home/mdmaug/.cache/mdmaug-scans/tjetri.s3websiteuswest2.amazonaws.com/current/httptjetris3websiteuswest2amazonawscomredhtml.spy}
+    - - /BKi_Gm.png
+      - {sourcefile: null, spyfile: null}
+    - - /red.html
+      - {sourcefile: /home/mdmaug/.cache/mdmaug-scans/tjetri.s3websiteuswest2.amazonaws.com/current/httptjetris3websiteuswest2amazonawscomredhtml.tmp,
+        spyfile: null}
+    - - /mf40.html spy
+      - {sourcefile: null, spyfile: /home/mdmaug/.cache/mdmaug-scans/tjetri.s3websiteuswest2.amazonaws.com/current/httptjetris3websiteuswest2amazonawscommf40html.spy}
+    - - /favicon.ico
+      - {sourcefile: /home/mdmaug/.cache/mdmaug-scans/tjetri.s3websiteuswest2.amazonaws.com/current/httptjetris3websiteuswest2amazonawscomfaviconico.tmp,
+        spyfile: null}
+    - - /mf40.html
+      - {sourcefile: /home/mdmaug/.cache/mdmaug-scans/tjetri.s3websiteuswest2.amazonaws.com/current/httptjetris3websiteuswest2amazonawscommf40html.tmp,
+        spyfile: null}
+    - - /swfobject.js
+      - {sourcefile: /home/mdmaug/.cache/mdmaug-scans/tjetri.s3websiteuswest2.amazonaws.com/current/httptjetris3websiteuswest2amazonawscomswfobjectjs.tmp,
+        spyfile: null}
+    vote: null
+- - apis.google.com
+  - addresses:
+    - - 173.194.116.224
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.226
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.231
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.229
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.228
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 2a00:1450:4014:80a::1004
+      - {city: ' (Private Address)', country: ' (Private Address) (XX)', vote: n/a}
+    - - 173.194.116.225
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.233
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.227
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.232
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.238
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    - - 173.194.116.230
+      - {city: ' (Unknown City?)', country: ' (Unknown Country?) (XX)', vote: n/a}
+    urls:
+    - - /_/scs/abc-static/_/js/k=gapi.gapi.en.5GL3tg_C6Nc.O/m=iframes,googleapis_client/rt=j/d=1/rs=AItRSTPfsfz-ifAZeiHMk7mKfiGQJuhVFw
+      - {sourcefile: /home/mdmaug/.cache/mdmaug-scans/tjetri.s3websiteuswest2.amazonaws.com/current/httpsapisgooglecomscsabcstaticjskgapigapien5gl3tgc.tmp,
+        spyfile: null}
+    - - /
+      - {sourcefile: null, spyfile: null}
+    vote: null
+logDir: /tmp/mdm/0-log/
+screenfile: /home/mdmaug/.cache/mdmaug-scans/tjetri.s3websiteuswest2.amazonaws.com/current/screenshot_base64.txt
--- a/drump1.tmp
+++ b/drump1.tmp
+Výsledky vyhledávání - navštíveno 14 domén
+* http://tb.symcd.com 1 adres a 1 url
+ /
+ 23.63.139.27 K adrese je připojena geolokace  (Unknown Country?) (XX)  (Unknown City?), vote n/a
+* http://tj.symcd.com 1 adres a 1 url
+ /
+ 23.63.139.27 K adrese je připojena geolokace  (Unknown Country?) (XX)  (Unknown City?), vote n/a
+* https://media.mixer.cz 2 adres a 1 url
+ /icon/1505/a55cb19c30/51568f34d69a3/13cafb78ad8f76da6cf301/normal.png
+ 77.75.76.88 K adrese je připojena geolokace  (Unknown Country?) (XX)  (Unknown City?), vote n/a
+ 2a02:598:2::88 K adrese je připojena geolokace  (Private Address) (XX)  (Private Address), vote n/a
+* https://www.seznam.cz 2 adres a 50 url
+ /st/img/2011/arrow-stream.png
+ /st/img/2011/ico-top-articles.png
+ /st/3.34.46/css/large.css?3.34.46
+ /st/3.34.46/css/suggest.css?3.34.46
+ /favicons/194/536bc4e726e4fafde1b20fb95ccb13dd.jpg?r=Jak%20vydr%C5%BEet%3F%20M%C3%A1lokdo%20m%C3%A1%20na%20takov%C3%BDto%20vztah%20n%C3%A1turu
+ /st/img/2011/ikonka-aplikace.png
+ /st/3.34.46/css/0-homepage.css?3.34.46
+ /st/3.34.46/css/suggest5.css?3.34.46
+ /st/img/2011/inputbck.gif
+ /st/img/favicon.ico?3.34.46
+ /favicons/181/949c90a6db2c04b09f9d99b1184ab5a9.jpg
+ /st/img/2011/horoskopy/12.png
+ /st/img/2011/arrow-down.gif
+ /favicons/71510/b80d03d572463b8e000c8078bcb9c22b.jpg
+ /favicons/130541/41b070f96a8b0d81cb2581e476821504.jpg
+ /st/3.34.46/css/suggest5-words.css?3.34.46
+ /favicons/130541/25906be5c4d4b55bf19a769c53a1988e.jpg
+ /favicons/002/26-nAcEdb.png
+ /st/3.34.46/js/checker/checker.js
+ /favicons/194/913692dd01384c72feeb73b79fff37dc.jpg?r=Vyrobte%20si%20deodorant%20bez%20chemie.%20Vyjde%20na%2027%20korun
+ /favicons/182/41ec953f57e46a8588af40c2ce3a4541.png
+ /st/img/2011/ico-download.png
+ /favicons/019/194-DBrJCJ.png
+ /favicons/130541/9c3728c15f22dbbdfd15de79ea5f557a.jpg
+ /st/img/2011/ico-beta-is-comming.png
+ /favicons/130541/6f3931d4b2536e2b5c45938d3c130144.jpg
+ /favicons/018/181-DkKGMv.png
+ /st/3.34.46/css/small.css?3.34.46
+ /favicons/1051/e8d724a4695f4fd3cfbd360c8090668a.jpg
+ /favicons/130541/cd0a5ecceda48f4005b8480e900ffdc0.jpg
+ /st/img/2011/music-button-hover.png
+ /st/3.34.46/css/medium.css?3.34.46
+ /favicons/151/71510-qdjavj.png
+ /favicons/130541/b4a347e3a824c91dbe65d6e650894cc5.jpg
+ /favicons/194/d4d3cd7595710f09fc5376eccfdc2bab.jpg?r=Po%C5%99%C3%A1d%20je%20na%20co%20koukat.%20Skv%C4%9Bl%C3%BD%20film%20v%C3%A1s%20nadchne%20i%20p
+ /st/img/2011/ico-set-content.png
+ /favicons/130541/1e2adacd0f0b1a259852bae95f420b1c.jpg
+ /favicons/130541/0a15ffe368b068aa4d88b6d7e8b83860.jpg
+ /st/img/2011/ico-homepage.png
+ /st/img/bar_bck.gif
+ /favicons/130541/051abe304ebaf055a4c57ed09d9e4a56.jpg
+ /st/img/weather/w2.png
+ /favicons/105/1051-HdtbaH.png
+ /st/img/nic.png
+ /st/img/new.gif
+ /
+ /st/3.34.46/js/homepage-all.js?3.34.46
+ /favicons/26/0f1a089ee540503a5cce64d69b873ed8.jpg
+ /st/img/weather/w13.png
+ /st/img/2011//logo.png?3.34.46
+ 77.75.76.3 K adrese je připojena geolokace  (Unknown Country?) (XX)  (Unknown City?), vote n/a
+ 2a02:598:2::3 K adrese je připojena geolokace  (Private Address) (XX)  (Private Address), vote n/a
+* seznam.cz 0 adres a 1 url
+ /
+* https://gacz.hit.gemius.pl 3 adres a 1 url
+ /xgemius.js
+ 81.0.212.199 K adrese je připojena geolokace  CZECH REPUBLIC (CZ)  (Unknown city), vote n/a
+ 81.0.212.201 K adrese je připojena geolokace  CZECH REPUBLIC (CZ)  (Unknown city), vote n/a
+ 81.0.212.200 K adrese je připojena geolokace  CZECH REPUBLIC (CZ)  (Unknown city), vote n/a
+* http://tk.symcd.com 1 adres a 1 url
+ /
+ 23.63.139.27 K adrese je připojena geolokace  (Unknown Country?) (XX)  (Unknown City?), vote n/a
+* http://seznam.cz 0 adres a 2 url
+ 
+ /
+* http://tg.symcd.com 1 adres a 1 url
+ /
+ 23.63.139.27 K adrese je připojena geolokace  (Unknown Country?) (XX)  (Unknown City?), vote n/a
+* https://h.imedia.cz 1 adres a 1 url
+ /js/dot-small.js
+ 77.75.77.9 K adrese je připojena geolokace  CZECH REPUBLIC (CZ)  Prague, vote n/a
+* http://id.seznam.cz 2 adres a 1 url
+ /yadis
+ 77.75.77.35 K adrese je připojena geolokace  CZECH REPUBLIC (CZ)  Prague, vote n/a
+ 2a02:598:2::1035 K adrese je připojena geolokace  (Private Address) (XX)  (Private Address), vote n/a
+* https://seznam.cz 0 adres a 1 url
+ /
+* https://i.imedia.cz 1 adres a 2 url
+ /js/im3.js
+ /js/sid.js
+ 77.75.78.72 K adrese je připojena geolokace  (Unknown Country?) (XX)  (Unknown City?), vote n/a
+* http://t2.symcb.com 1 adres a 1 url
+ /
+ 23.63.139.27 K adrese je připojena geolokace  (Unknown Country?) (XX)  (Unknown City?), vote n/a
\ No newline at end of file
--- a/drump2.tmp
+++ b/drump2.tmp
+Výsledky vyhledávání - navštíveno 14 domén
+* http://tb.symcd.com 1 adres a 1 url
+ /
+ 23.63.139.27 K adrese je připojena geolokace  (Unknown Country?) (XX)  (Unknown City?), vote n/a
+* http://tj.symcd.com 1 adres a 1 url
+ /
+ 23.63.139.27 K adrese je připojena geolokace  (Unknown Country?) (XX)  (Unknown City?), vote n/a
+* https://media.mixer.cz 2 adres a 1 url
+ /icon/1505/a55cb19c30/51568f34d69a3/13cafb78ad8f76da6cf301/normal.png
+ 77.75.76.88 K adrese je připojena geolokace  (Unknown Country?) (XX)  (Unknown City?), vote n/a
+ 2a02:598:2::88 K adrese je připojena geolokace  (Private Address) (XX)  (Private Address), vote n/a
+* http://tg.symcd.com 1 adres a 1 url
+ /
+ 23.63.139.27 K adrese je připojena geolokace  (Unknown Country?) (XX)  (Unknown City?), vote n/a
+* https://gacz.hit.gemius.pl 3 adres a 1 url
+ /xgemius.js
+ 81.0.212.199 K adrese je připojena geolokace  CZECH REPUBLIC (CZ)  (Unknown city), vote n/a
+ 81.0.212.201 K adrese je připojena geolokace  CZECH REPUBLIC (CZ)  (Unknown city), vote n/a
+ 81.0.212.200 K adrese je připojena geolokace  CZECH REPUBLIC (CZ)  (Unknown city), vote n/a
+* http://tk.symcd.com 1 adres a 1 url
+ /
+ 23.63.139.27 K adrese je připojena geolokace  (Unknown Country?) (XX)  (Unknown City?), vote n/a
+* http://seznam.cz 0 adres a 2 url
+ 
+ /
+* seznam.cz 0 adres a 1 url
+ /
+* https://h.imedia.cz 1 adres a 1 url
+ /js/dot-small.js
+ 77.75.77.9 K adrese je připojena geolokace  CZECH REPUBLIC (CZ)  Prague, vote n/a
+* http://id.seznam.cz 2 adres a 1 url
+ /yadis
+ 77.75.77.35 K adrese je připojena geolokace  CZECH REPUBLIC (CZ)  Prague, vote n/a
+ 2a02:598:2::1035 K adrese je připojena geolokace  (Private Address) (XX)  (Private Address), vote n/a
+* https://seznam.cz 0 adres a 1 url
+ /
+* https://i.imedia.cz 1 adres a 2 url
+ /js/im3.js
+ /js/sid.js
+ 77.75.78.72 K adrese je připojena geolokace  (Unknown Country?) (XX)  (Unknown City?), vote n/a
+* https://www.seznam.cz 2 adres a 50 url
+ /st/img/2011/arrow-stream.png
+ /st/img/2011/ico-top-articles.png
+ /st/3.34.46/css/large.css?3.34.46
+ /st/3.34.46/css/suggest.css?3.34.46
+ /st/img/2011/ikonka-aplikace.png
+ /favicons/194/536bc4e726e4fafde1b20fb95ccb13dd.jpg?r=Jak%20vydr%C5%BEet%3F%20M%C3%A1lokdo%20m%C3%A1%20na%20takov%C3%BDto%20vztah%20n%C3%A1turu
+ /st/3.34.46/css/suggest5.css?3.34.46
+ /st/img/2011/inputbck.gif
+ /st/3.34.46/css/0-homepage.css?3.34.46
+ /st/img/favicon.ico?3.34.46
+ /st/img/2011/horoskopy/12.png
+ /st/img/2011/arrow-down.gif
+ /favicons/181/949c90a6db2c04b09f9d99b1184ab5a9.jpg
+ /favicons/71510/b80d03d572463b8e000c8078bcb9c22b.jpg
+ /favicons/130541/41b070f96a8b0d81cb2581e476821504.jpg
+ /st/3.34.46/css/suggest5-words.css?3.34.46
+ /favicons/130541/25906be5c4d4b55bf19a769c53a1988e.jpg
+ /favicons/002/26-nAcEdb.png
+ /st/3.34.46/js/checker/checker.js
+ /favicons/194/913692dd01384c72feeb73b79fff37dc.jpg?r=Vyrobte%20si%20deodorant%20bez%20chemie.%20Vyjde%20na%2027%20korun
+ /favicons/182/41ec953f57e46a8588af40c2ce3a4541.png
+ /st/img/2011/ico-download.png
+ /favicons/019/194-DBrJCJ.png
+ /favicons/130541/9c3728c15f22dbbdfd15de79ea5f557a.jpg
+ /st/img/2011/ico-beta-is-comming.png
+ /favicons/130541/6f3931d4b2536e2b5c45938d3c130144.jpg
+ /favicons/018/181-DkKGMv.png
+ /favicons/130541/cd0a5ecceda48f4005b8480e900ffdc0.jpg
+ /favicons/1051/e8d724a4695f4fd3cfbd360c8090668a.jpg
+ /st/img/2011/music-button-hover.png
+ /st/3.34.46/css/small.css?3.34.46
+ /favicons/130541/b4a347e3a824c91dbe65d6e650894cc5.jpg
+ /favicons/151/71510-qdjavj.png
+ /st/3.34.46/css/medium.css?3.34.46
+ /favicons/194/d4d3cd7595710f09fc5376eccfdc2bab.jpg?r=Po%C5%99%C3%A1d%20je%20na%20co%20koukat.%20Skv%C4%9Bl%C3%BD%20film%20v%C3%A1s%20nadchne%20i%20p
+ /st/img/2011/ico-set-content.png
+ /favicons/130541/1e2adacd0f0b1a259852bae95f420b1c.jpg
+ /st/img/2011/ico-homepage.png
+ /favicons/130541/0a15ffe368b068aa4d88b6d7e8b83860.jpg
+ /st/img/bar_bck.gif
+ /favicons/130541/051abe304ebaf055a4c57ed09d9e4a56.jpg
+ /st/img/weather/w2.png
+ /favicons/105/1051-HdtbaH.png
+ /st/img/nic.png
+ /st/img/new.gif
+ /
+ /st/3.34.46/js/homepage-all.js?3.34.46
+ /favicons/26/0f1a089ee540503a5cce64d69b873ed8.jpg
+ /st/img/weather/w13.png
+ /st/img/2011//logo.png?3.34.46
+ 77.75.76.3 K adrese je připojena geolokace  (Unknown Country?) (XX)  (Unknown City?), vote n/a
+ 2a02:598:2::3 K adrese je připojena geolokace  (Private Address) (XX)  (Private Address), vote n/a
+* http://t2.symcb.com 1 adres a 1 url
+ /
+ 23.63.139.27 K adrese je připojena geolokace  (Unknown Country?) (XX)  (Unknown City?), vote n/a
\ No newline at end of file
--- a/drump3.tmp
+++ b/drump3.tmp
+teeeeeeeeeeeeeeext
\ No newline at end of file
--- a/example.log
+++ b/example.log
+HOVVVVVVVVVVVVV....
--- a/ff_extension
+++ b/ff_extension
+/mnt/mdmaug/home/mdmaug/.mozilla/extensions/{ec8030f7-c20a-464f-9b0e-13a3a9e97384}/mdmaug@jetpack/resources/mdmaug/lib
\ No newline at end of file
--- a/lib/analysis/__init__.py
+++ b/lib/analysis/__init__.py
+__author__ = "edvard"
+__date__ = "$Apr 27, 2015 5:40:11 AM$"
\ No newline at end of file
--- a/lib/analysis/crawl.py
+++ b/lib/analysis/crawl.py
+from yaml import load, dump
+try:
+    from yaml import CLoader as Loader, CDumper as Dumper
+except ImportError:
+    from yaml import Loader, Dumper
+from collections import defaultdict
+from lib.config import Config
+
+class Crawl(defaultdict):
+    """  Objekt Crawl udružuje výsledky vyhledávání """
+    
+    def __str__(self):
+        r = "Výsledky vyhledávání - navštíveno {} domén".format(len(self))
+        for key in self.keys():
+            r += "\n* " + key + " " + str(self[key])
+        return r
+
+
+    def saveToFile(self,filename):
+        with open(filename, "w") as f:
+            f.write(dump(self.__getstate__(), Dumper=Dumper))
+
+    def loadFromFile(filename):
+        with open(filename, 'r') as f:
+            return Crawl(state = load(f.read(), Loader=Loader))
+
+
+
+    def __init__(self, state = None, logDir =  None, cacheDir = None):
+        """ State muze obsahovat vystup __getstate__() (serializace YAMLem) """
+        self.default_factory = _Domain
+        self.screenfile = None # HTML output XXX
+
+        if logDir:
+            self.logDir = logDir
+        if cacheDir:
+            self.cacheDir = cacheDir
+        if state:
+            self.__setstate__(state)
+        pass
+
+    def __getstate__(self):
+        state = self.__dict__.copy()
+        state["keys"] = [[x for x in (key, self[key].__getstate__())] for key in self.keys()]
+        return state
+
+    def __setstate__(self,state):
+        #print("fdsfsfds",self.__dict__)
+        for tup in state["keys"]:
+            key, val = tup
+            self[key].__setstate__(val)
+        del state["keys"]
+        self.__dict__ = state    
+
+class _Domain(defaultdict):
+    """ Navstivena domena behem crawlu """
+    def __str__(self):
+        r = "{} adres a {} url".format(len(self.addresses), len(self.urls))
+        for key in self.urls.keys():
+            r += "\n " + key + " " + str(self.urls[key])
+        for key in self.addresses.keys():
+            r += "\n " + key + " " + str(self.addresses[key])
+        return r
+
+    def __init__(self):
+        #self.vote = None