diff --git a/netmetr/netmetr b/netmetr/netmetr index 7aa6ae65b658bce6a3ed55c421cac14a7d16cfb7..5495af369607baa7c4992a2f4733c359377c61f6 100755 --- a/netmetr/netmetr +++ b/netmetr/netmetr @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python # coding: utf-8 import urllib2 import json @@ -8,17 +8,49 @@ import locale import subprocess import shlex import os +from random import randint +import argparse +import datetime +import re +import tempfile + -FLOWS_FILE = "/tmp/flows.json" -CONFIG_FILE = "/tmp/rmbt.cfg" RMBT_BIN = "rmbt" -DEBUG = False # Debug printouts +HIST_FILE = "/tmp/netmetr-history.json" +# FALLBACK_CTRL_SRV = "netmetr-control.labs.nic.cz" +FALLBACK_CTRL_SRV = "control.netmetr.cz" class Settings: def __init__(self): self.language = locale.getdefaultlocale()[0] self.timezone = subprocess.check_output(["date", "+%Z"])[:-1] + if os.path.isfile("/sbin/uci"): + process = subprocess.Popen( + ["uci", "-q", "get", "netmetr.settings.control_server"], + stdout=subprocess.PIPE + ) + if process.wait() == 0: + self.control_server = process.stdout.read()[:-1] + else: + print_info( + 'Control server not found, falling to: ' + + FALLBACK_CTRL_SRV + ) + self.control_server = FALLBACK_CTRL_SRV + subprocess.call([ + "uci", + "set", + "netmetr.settings.control_server=" + + self.control_server + ]) + subprocess.call(["uci", "commit"]) + else: + print_info( + 'Control server not found (uci missing), falling to: ' + + FALLBACK_CTRL_SRV + ) + self.control_server = FALLBACK_CTRL_SRV if os.path.isfile("/etc/turris-version"): with open("/etc/turris-version", 'r') as turris_version: self.os_version = turris_version.read().split('\n')[0] @@ -38,15 +70,66 @@ class Settings: def print_debug(msg): if DEBUG: - print('\033[93m' + msg + '\033[0m') + if COLORED_OUTPUT: + print('\033[93m' + msg + '\033[0m') + else: + print(msg) return DEBUG -def request_uuid(sets): +def print_info(msg): + if COLORED_OUTPUT: + print('\033[91m' + msg + '\033[0m') + else: + print(msg) + + +def print_progress(msg): + if COLORED_OUTPUT: + print('\033[93m' + msg + '\033[0m') + else: + print(msg) + + +def print_error(msg, error_code): + if COLORED_OUTPUT: + print('\033[41m' + msg + '\033[0m') + else: + print(msg) + exit(error_code) + + +def load_uuid(sets): + """Checks the uci config for uuid and loads it to the + script. If no uuid is found a https request is send to the control server + to download it. + """ + # Load uuid saved in config file via uci + if os.path.isfile("/sbin/uci"): + process = subprocess.Popen( + ["uci", "-q", "get", "netmetr.settings.uuid"], + stdout=subprocess.PIPE + ) + if process.wait() == 0: + sets.uuid = process.stdout.read()[:-1] + else: + print_info('Uuid not found, requesting new one.') + sets.uuid = 0 + else: + print_info('Uuid not found (uci missing), requesting new one.') + sets.uuid = 0 + + # the download request must be sent all the time - either to raquest new + # uuid or to check the existing one + download_uuid(sets) + + +def download_uuid(sets): """Creates a http request and ask the control server for correct uuid""" - print('\033[93m'+"Requesting test config from control server..."+'\033[0m') + print_progress("Checking uuid on the control server...") # Create json to request uuid req_json = { + "uuid": sets.uuid, "language": sets.language, "timezone": sets.timezone, "name": "RMBT", @@ -56,24 +139,9 @@ def request_uuid(sets): "version_name": "1.0", } - # Load uuid saved in config file via uci - if os.path.isfile("/sbin/uci"): - process = subprocess.Popen( - ["uci", "get", "netmetr.@settings[0].uuid"], - stdout=subprocess.PIPE - ) - if process.wait() == 0: - req_json['uuid'] = process.stdout.read()[:-1] - else: - print('uuid not found, requesting new one.') - req_json['uuid'] = 0 - else: - print('uuid not found, requesting new one.') - req_json['uuid'] = 0 - # Creating GET request to obtain / check uuid req = urllib2.Request( - 'https://netmetr-control.labs.nic.cz/RMBTControlServer/settings' + 'https://' + sets.control_server + '/RMBTControlServer/settings' ) req.add_header('Accept', 'application/json, text/javascript, */*; q=0.01') req.add_header('Content-Type', 'application/json') @@ -86,9 +154,13 @@ def request_uuid(sets): if uuid_new: # New uuid was received sets.uuid = uuid_new if os.path.isfile("/sbin/uci"): - process = subprocess.Popen([ + subprocess.call([ "uci", "set", - "netmetr.@settings[0].uuid="+sets.uuid + "netmetr.settings.uuid="+sets.uuid + ]) + subprocess.call([ + "uci", "-q", "delete", + "netmetr.settings.sync_code" ]) subprocess.call(["uci", "commit"]) else: @@ -101,9 +173,10 @@ def request_settings(sets): """Creates a http request to get test token, number of threads, number of pings, server address and port and so on. """ + print_progress("Requesting test config from the control server...") # Create request to start a test req = urllib2.Request( - 'https://netmetr-control.labs.nic.cz/RMBTControlServer/testRequest' + 'https://' + sets.control_server + '/RMBTControlServer/testRequest' ) # Add headers req.add_header('Accept', 'application/json, text/javascript, */*; q=0.01') @@ -145,73 +218,70 @@ def measure_pings(sets): the lowest one """ - print('\033[93m'+"Starting ping test..."+'\033[0m') - ping_result_lines = subprocess.check_output([ - "ping", "-c", sets.test_numpings, - sets.test_server_address - ]).split('\n') + print_progress("Starting ping test...") ping_values = list() for i in range(1, int(sets.test_numpings)+1): - try: - start = ping_result_lines[i].index("time=") + len("time=") - end = ping_result_lines[i].index(" ms") - ping = int(float(ping_result_lines[i][start:end])*1000000) - ping_values.append(ping) - except: - print("Problem decoding pings.") - return '' - return min(int(s) for s in ping_values) + process = subprocess.Popen([ + "ping", "-c1", + sets.test_server_address + ], stdout=subprocess.PIPE) + if (process.wait() == 0): + try: + ping_result = process.stdout.read() + start = ping_result.index("time=") + len("time=") + end = ping_result.index(" ms") + ping = float(ping_result[start:end]) + print("ping_"+str(i)+"_msec = "+format(ping, '.2f')) + ping = int(ping * 1000000) + ping_values.append(ping) + except: + print("Problem decoding pings.") + return '' + time.sleep(0.5) + try: + return min(int(s) for s in ping_values) + except: + return '' def measure_speed(sets): """Start RMBT client with saved arguments to measure the speed """ # Create config file needed by rmbt-client - if os.path.isfile(CONFIG_FILE): - try: - os.remove(CONFIG_FILE) - except Exception, e: - print(e) - return '' - + _, sets.config_file = tempfile.mkstemp() + _, sets.flows_file = tempfile.mkstemp() try: - with open(CONFIG_FILE, "w") as config_file: - config_file.write('{"cnf_file_flows": "'+FLOWS_FILE+'.xz"}') + with open(sets.config_file, "w") as config_file: + config_file.write('{"cnf_file_flows": "'+sets.flows_file+'.xz"}') except Exception, e: print("Error creating config file") print(e) return '' encryption = {True: "-e"} - print('\033[93m'+"Starting speed test..."+'\033[0m') + print_progress("Starting speed test...") test_result = subprocess.check_output([ RMBT_BIN, encryption.get(sets.test_server_encryption, ""), "-h", sets.test_server_address, "-p", str(sets.test_server_port), "-t", sets.test_token, "-f", sets.test_numthreads, "-d", - sets.test_duration, "-u", sets.test_duration, "-c", CONFIG_FILE]) + sets.test_duration, "-u", sets.test_duration, "-c", + sets.config_file]) if print_debug("Speed test result:"): print(test_result) return json.loads(test_result.split("}")[1] + "}") -def import_speed_flows(): +def import_speed_flows(sets): """The speedtest flow is saved to a file during the test. This function imports it so it could be sent to the control server. """ - if os.path.isfile(FLOWS_FILE): - try: - os.remove(FLOWS_FILE) - except Exception, e: - print(e) - return - directions = { "dl": "download", "ul": "upload" } try: - subprocess.call(shlex.split("unxz "+FLOWS_FILE+".xz")) - with open(FLOWS_FILE, 'r') as json_data: + subprocess.call(shlex.split("unxz -f "+sets.flows_file+".xz")) + with open(sets.flows_file, 'r') as json_data: flows_json = json.load(json_data) except Exception, e: print('Problem reading/decoding flows data.') @@ -239,11 +309,11 @@ def import_speed_flows(): # Remove generated files try: - os.remove(FLOWS_FILE) + os.remove(sets.flows_file) except Exception, e: print(e) try: - os.remove(CONFIG_FILE) + os.remove(sets.config_file) except Exception, e: print(e) return speed_array @@ -291,7 +361,7 @@ def upload_result(sets, pres, test_result_json, speed_array): # Create GET request req = urllib2.Request( - 'https://netmetr-control.labs.nic.cz/RMBTControlServer/result' + 'https://' + sets.control_server + '/RMBTControlServer/result' ) # Add headers req.add_header('Accept', 'application/json, text/javascript, */*; q=0.01') @@ -303,15 +373,198 @@ def upload_result(sets, pres, test_result_json, speed_array): print(json.dumps(resp_json, indent=2)) +def purge(dir, pattern): + """Lists a directory and removes every file matching the pattern + """ + for f in os.listdir(dir): + if re.search(pattern, f): + try: + os.remove(os.path.join(dir, f)) + except OSError: + pass + + +def download_history(sets): + """Creates a http request and ask the control server for a measurement + history. + """ + # Create json to request history + req_json = { + "language": sets.language, + "timezone": sets.timezone, + "uuid": sets.uuid, + } + # Creating POST request to get history + req = urllib2.Request( + 'https://' + sets.control_server + '/RMBTControlServer/history' + ) + req.add_header('Accept', 'application/json, text/javascript, */*; q=0.01') + req.add_header('Content-Type', 'application/json') + + if print_debug("Downloading measurement history from the control server."): + print(json.dumps(req_json, indent=2)) + # Send the request + resp = urllib2.urlopen(req, json.dumps(req_json)) + resp_json = json.loads(resp.read()) + if print_debug("Measurement history response:"): + print(json.dumps(resp_json, indent=2)) + _, sets.hist_file = tempfile.mkstemp() + try: + with open(sets.hist_file, "w") as hist_file: + hist_file.write(json.dumps(resp_json, indent=2)) + os.rename(sets.hist_file, HIST_FILE) + except Exception, e: + print("Error saving measurement history.") + print(e) + + +def download_sync_code(sets): + """Creates a http request and ask the control server for a synchronization + code that can be used to view saved measurements from different devices. + The new code is saved via uci. + """ + # Create json to request synchronization code + req_json = { + "language": sets.language, + "timezone": sets.timezone, + "uuid": sets.uuid, + } + # Creating POST request to get the sync code + req = urllib2.Request( + 'https://' + sets.control_server + '/RMBTControlServer/sync' + ) + req.add_header('Accept', 'application/json, text/javascript, */*; q=0.01') + req.add_header('Content-Type', 'application/json') + + if print_debug( + "Downloading synchronization code from the control server." + ): + print(json.dumps(req_json, indent=2)) + # Send the request + resp = urllib2.urlopen(req, json.dumps(req_json)) + resp_json = json.loads(resp.read()) + if print_debug("Synchronization token response:"): + print(json.dumps(resp_json, indent=2)) + + if not resp_json["error"]: + sets.sync_code = resp_json["sync"][0].get("sync_code", '') + # If we have uci and VALID sync code: + if (os.path.isfile("/sbin/uci") and sets.sync_code): + subprocess.call([ + "uci", "set", + "netmetr.settings.sync_code="+sets.sync_code + ]) + subprocess.call(["uci", "commit"]) + else: + sets.sync_code = '' + print("Error downloading synchronization code.") + + +def load_sync_code(sets): + """Checks the uci config for sychronization code and loads it to the\ + script. If no synchronization code is found a https request is send to\ + download it. + """ + # Load synchronization code saved in config file via uci + if os.path.isfile("/sbin/uci"): + process = subprocess.Popen( + ["uci", "-q", "get", "netmetr.settings.sync_code"], + stdout=subprocess.PIPE + ) + if process.wait() == 0: + sets.sync_code = process.stdout.read()[:-1] + else: + print_info('Sync code not found, requesting new one.') + download_sync_code(sets) + else: + print_info('Sync code not found (uci missing), requesting new one.') + download_sync_code(sets) + + +# Prepare argument parsing +parser = argparse.ArgumentParser(description='NetMetr - client application for\ + download and upload speed measurement.') +parser.add_argument('--rwait', nargs=1, type=int, default=[0], help='delay for\ + a random amount of time up to RWAIT seconds before the test starts') +parser.add_argument('--autostart', action='store_true', help='use this\ + option only when running as an automated service - to check whether it is\ + right time to run the test') +parser.add_argument( + '--dwlhist', + action='store_true', + help='download measurement history from the control server and save it to \ + /tmp/' + HIST_FILE +) +parser.add_argument('--debug', action='store_true', help='enables debug \ + printouts') +parser.add_argument('--no-color', action='store_true', help='disables colored \ + text output') +parser.add_argument('--no-run', action='store_true', help='this option\ + prevents from running the test. It could be used only to obtain sync code\ + or (with --dwlhist) to download measurement history') +args = parser.parse_args() + +DEBUG = args.debug +COLORED_OUTPUT = not args.no_color + +# When autostarted - check whether autostart is enabled and +# if it is right time to run the test. +# In uci config, we expect hours of a day separated by commas (,) - these hours +# are the time the test should be run. So whenever the script is started, it +# looks to it's config and if it finds the current hour of day in it, +# it will start the test +if (args.autostart and os.path.isfile("/sbin/uci")): + process = subprocess.Popen( + ["uci", "-q", "get", "netmetr.settings.autostart_enabled"], + stdout=subprocess.PIPE + ) + if (process.wait() == 0): + autostart_enabled = process.stdout.read()[:-1] + else: + print("Failed to load autostart uci settings.") + exit() + process = subprocess.Popen( + ["uci", "-q", "get", "netmetr.settings.hours_to_run"], + stdout=subprocess.PIPE + ) + if (process.wait() == 0): + hours = process.stdout.read()[:-1].split(' ') + hours = map(int, hours) + else: + print("Failed to load autostart time uci settings.") + exit() + if (autostart_enabled != '1' or datetime.datetime.now().hour not in hours): + exit() + +# Wait appropriate amount of time +time.sleep(randint(0, args.rwait[0])) + settings = Settings() -request_uuid(settings) -request_settings(settings) +# Request uuid from the control server +load_uuid(settings) + +if (not args.no_run): + # Request test settings from the control server + request_settings(settings) + + # Get the ping measurement result + shortest_ping = measure_pings(settings) + + # Get the speed measurement result + speed_result = measure_speed(settings) + if speed_result == '': + quit() + + # Get detailed test statistics + speed_flows = import_speed_flows(settings) -shortest_ping = measure_pings(settings) -speed_result = measure_speed(settings) -if speed_result == '': - quit() + # Upload result to the control server + upload_result(settings, shortest_ping, speed_result, speed_flows) -speed_flows = import_speed_flows() +# Optionally download measurement history from the control server +if (args.dwlhist): + download_history(settings) -upload_result(settings, shortest_ping, speed_result, speed_flows) +load_sync_code(settings) +if (settings.sync_code): + print_info("Your Sync code is: " + settings.sync_code)