netmetr: Netmetr python script added.

This script communicates with netmetr control server
and starts rmbt client with received parametres.

netmetr/netmetr

+#!/usr/bin/env python2
+# coding: utf-8
+import urllib2
+import json
+import calendar
+import time
+import locale
+import subprocess
+import shlex
+import os
+
+FLOWS_FILE = "/tmp/flows.json"
+CONFIG_FILE = "/tmp/rmbt.cfg"
+RMBT_BIN = "rmbt"
+DEBUG = True   # Debug printouts
+
+
+class Settings:
+    def __init__(self):
+        self.language = locale.getdefaultlocale()[0]
+        self.timezone = subprocess.check_output(["date", "+%Z"])[:-1]
+        if os.path.isfile("/etc/turris-version"):
+            with open("/etc/turris-version", 'r') as turris_version:
+                self.os_version = turris_version.read().split('\n')[0]
+            self.hw_version = subprocess.check_output([
+                "atsha204cmd",
+                "hw-rev"
+            ])[:-1]
+            self.model = "Turris"
+        else:
+            self.os_version = "unknown"
+            self.hw_version = "unknown"
+            self.model = "unknown"
+
+    def get_time(cls):
+        return str(int(round(calendar.timegm(time.gmtime())*1000)))
+
+
+def print_debug(msg):
+    if DEBUG:
+        print('\033[93m' + msg + '\033[0m')
+    return DEBUG
+
+
+def request_uuid(sets):
+    """Creates a http request and ask the control server for correct uuid"""
+    print('\033[93m'+"Requesting test config from control server..."+'\033[0m')
+    # Create json to request uuid
+    req_json = {
+        "language": sets.language,
+        "timezone": sets.timezone,
+        "name": "RMBT",
+        "terms_and_conditions_accepted": "true",
+        "type": "DESKTOP",
+        "version_code": "1",
+        "version_name": "1.0",
+    }
+
+    # Load uuid saved in config file via uci
+    if os.path.isfile("/sbin/uci"):
+        process = subprocess.Popen(
+                ["uci", "get", "netmetr.@settings[0].uuid"],
+                stdout=subprocess.PIPE
+        )
+        if process.wait() == 0:
+            req_json['uuid'] = process.stdout.read()[:-1]
+        else:
+            print('uuid not found, requesting new one.')
+            req_json['uuid'] = 0
+    else:
+        print('uuid not found, requesting new one.')
+        req_json['uuid'] = 0
+
+    # Creating GET request to obtain / check uuid
+    req = urllib2.Request(
+            'https://netmetr-control.labs.nic.cz/RMBTControlServer/settings'
+    )
+    req.add_header('Accept', 'application/json, text/javascript, */*; q=0.01')
+    req.add_header('Content-Type', 'application/json')
+    if print_debug("Test settings request:"):
+        print(json.dumps(req_json, indent=2))
+    # Send the request
+    resp = urllib2.urlopen(req, json.dumps(req_json))
+    resp_json = json.loads(resp.read())
+    uuid_new = resp_json["settings"][0].get("uuid", '')
+    if uuid_new:  # New uuid was received
+        sets.uuid = uuid_new
+        if os.path.isfile("/sbin/uci"):
+            process = subprocess.Popen([
+                "uci", "set",
+                "netmetr.@settings[0].uuid="+sets.uuid
+            ])
+            subprocess.call(["uci", "commit"])
+    else:
+        sets.uuid = req_json['uuid']
+    if print_debug("Test settings response:"):
+        print(json.dumps(resp_json, indent=2))
+
+
+def request_settings(sets):
+    """Creates a http request to get test token, number of threads, number
+    of pings, server address and port and so on.
+    """
+    # Create request to start a test
+    req = urllib2.Request(
+            'https://netmetr-control.labs.nic.cz/RMBTControlServer/testRequest'
+    )
+    # Add headers
+    req.add_header('Accept', 'application/json, text/javascript, */*; q=0.01')
+    req.add_header('Content-Type', 'application/json')
+    # Create the json to send
+    req_json = {
+        "client": "RMBT",
+        "language": sets.language,
+        "time": sets.get_time(),
+        "timezone": sets.timezone,
+        "type": "DESKTOP",
+        "uuid": sets.uuid,
+        "version": "0.1",
+        "version_code": "1"
+    }
+    if print_debug("Test testRequest request"):
+        print(json.dumps(req_json, indent=2))
+
+    # Send the request
+    resp = urllib2.urlopen(req, json.dumps(req_json))
+    # Read the content
+    resp_json = json.loads(resp.read())
+
+    if print_debug("Test testRequest response:"):
+        print(json.dumps(resp_json, indent=2))
+
+    sets.test_server_address = resp_json["test_server_address"]
+    sets.test_server_port = resp_json["test_server_port"]
+    sets.test_token = resp_json["test_token"]
+    sets.test_uuid = resp_json["test_uuid"]
+    sets.test_numthreads = resp_json["test_numthreads"]
+    sets.test_numpings = resp_json["test_numpings"]
+    sets.test_server_encryption = resp_json["test_server_encryption"]
+    sets.test_duration = resp_json["test_duration"]
+
+
+def measure_pings(sets):
+    """Run serie of pings to the test server and computes & saves
+     the lowest one
+    """
+
+    print('\033[93m'+"Starting ping test..."+'\033[0m')
+    ping_result_lines = subprocess.check_output([
+        "ping", sets.test_server_address,
+        "-c", sets.test_numpings
+    ]).split('\n')
+    ping_values = list()
+    for i in range(1, int(sets.test_numpings)+1):
+        try:
+            start = ping_result_lines[i].index("time=") + len("time=")
+            end = ping_result_lines[i].index(" ms")
+            ping = int(float(ping_result_lines[i][start:end])*1000000)
+            ping_values.append(ping)
+        except:
+            print("Problem decoding pings.")
+            return ''
+    return min(int(s) for s in ping_values)
+
+
+def measure_speed(sets):
+    """Start RMBT client with saved arguments to measure the speed
+    """
+    # Create config file needed by rmbt-client
+    if os.path.isfile(CONFIG_FILE):
+        try:
+            os.remove(CONFIG_FILE)
+        except Exception, e:
+            print(e)
+            return ''
+
+    try:
+        with open(CONFIG_FILE, "w") as config_file:
+            config_file.write('{"cnf_file_flows": "'+FLOWS_FILE+'.xz"}')
+    except Exception, e:
+        print("Error creating config file")
+        print(e)
+        return ''
+
+    encryption = {True: "-e"}
+    print('\033[93m'+"Starting speed test..."+'\033[0m')
+    test_result = subprocess.check_output([
+            RMBT_BIN, encryption.get(sets.test_server_encryption, ""), "-h",
+            sets.test_server_address, "-p", str(sets.test_server_port), "-t",
+            sets.test_token, "-f", sets.test_numthreads, "-d",
+            sets.test_duration, "-u", sets.test_duration, "-c", CONFIG_FILE])
+    if print_debug("Speed test result:"):
+        print(test_result)
+    return json.loads(test_result.split("}")[1] + "}")
+
+
+def import_speed_flows():
+    """The speedtest flow is saved to a file during the test. This function
+    imports it so it could be sent to the control server.
+    """
+    if os.path.isfile(FLOWS_FILE):
+        try:
+            os.remove(FLOWS_FILE)
+        except Exception, e:
+            print(e)
+            return
+
+    directions = {
+        "dl": "download",
+        "ul": "upload"
+    }
+    try:
+        subprocess.call(shlex.split("unxz "+FLOWS_FILE+".xz"))
+        with open(FLOWS_FILE, 'r') as json_data:
+            flows_json = json.load(json_data)
+    except Exception, e:
+        print('Problem reading/decoding flows data.')
+        print(e)
+        return
+
+    speed_array = list()
+    for d_short, d_long in directions.iteritems():
+        thread = 0
+        # Each direction has multiple threads
+        for flow in flows_json["res_details"][d_short]:
+            last_time = 0
+            # Each thread has plenty of samples
+            # We want to use a small amount of them
+            for sample in flow["time_series"]:
+                if (sample.get("t") - last_time) > 30000000:
+                    last_time = sample["t"]
+                    speed_array.append({
+                        "direction": d_long,
+                        "thread": thread,
+                        "time": sample["t"],
+                        "bytes": sample["b"]
+                     })
+            thread += 1
+
+    # Remove generated files
+    try:
+        os.remove(FLOWS_FILE)
+    except Exception, e:
+        print(e)
+    try:
+        os.remove(CONFIG_FILE)
+    except Exception, e:
+        print(e)
+    return speed_array
+
+
+def upload_result(sets, pres, test_result_json, speed_array):
+    """Uploads the tests result to the control server.
+    """
+    req_json = {
+        "client_language": sets.language,
+        "client_name": "RMBT",
+        "client_uuid": sets.uuid,
+        "client_version": "0.1",
+        "client_software_version": "0.3",
+        "geoLocations": [],
+        "model": sets.model,
+        "network_type": 98,
+        "platform": "RMBT",
+        "product": "os: "+sets.os_version+" hw: "+sets.hw_version,
+        "test_bytes_download": test_result_json.get("res_total_bytes_dl", {}),
+        "test_bytes_upload": test_result_json.get("res_total_bytes_ul", {}),
+        "test_nsec_download": test_result_json.get("res_dl_time_ns", {}),
+        "test_nsec_upload": test_result_json.get("res_ul_time_ns", {}),
+        "test_num_threads": test_result_json.get("res_dl_num_flows", {}),
+        "test_ping_shortest": pres,
+        "num_threads_ul": test_result_json.get("res_ul_num_flows", {}),
+        "test_speed_download": test_result_json.get(
+            "res_dl_throughput_kbps", {}
+        ),
+        "test_speed_upload": test_result_json.get(
+            "res_ul_throughput_kbps", {}
+        ),
+        "test_token": sets.test_token,
+        "test_uuid": sets.test_uuid,
+        "timezone": sets.timezone,
+        "type": "DESKTOP",
+        "version_code": "1",
+        "developer_code": 0
+    }
+    if print_debug("Save result request (without speed array and pings)"):
+        print(json.dumps(req_json, indent=2))
+
+    req_json["speed_detail"] = speed_array
+    req_json["pings"] = []
+
+    # Create GET request
+    req = urllib2.Request(
+            'https://netmetr-control.labs.nic.cz/RMBTControlServer/result'
+    )
+    # Add headers
+    req.add_header('Accept', 'application/json, text/javascript, */*; q=0.01')
+    req.add_header('Content-Type', 'application/json')
+    # Send the request
+    resp = urllib2.urlopen(req, json.dumps(req_json))
+    resp_json = json.loads(resp.read())
+    if print_debug("Save result response:"):
+        print(json.dumps(resp_json, indent=2))
+
+
+settings = Settings()
+request_uuid(settings)
+request_settings(settings)
+
+shortest_ping = measure_pings(settings)
+speed_result = measure_speed(settings)
+if speed_result == '':
+    quit()
+
+speed_flows = import_speed_flows()
+
+upload_result(settings, shortest_ping, speed_result, speed_flows)