Verified Commit c385d63a authored by Jan Miksik's avatar Jan Miksik Committed by Karel Koci
Browse files

tests: Added basic throughput tests

Added basic throughput tests for basic testing.
Test is run for 60 seconds and checks if the speed is at least
400 Mbps.
It also checks min/max of 10s chunks.
And checks if some of chunks is under the limit - raises warning.
Prints the speeds to output log.
parent 05676899
......@@ -10,6 +10,9 @@ wait4network
# Add openssh client to access router shell
apk add openssh-client
# Additional applications installation
apk add iperf3
# Configure LAN1 interface for static local network
cat >> /etc/network/interfaces <<EOF
auto lan
......
......@@ -27,10 +27,8 @@ rc-update add named
chown :named /etc/bind
chmod a+r /etc/bind/named.conf
## Additonal service
# iperf3 for benchmarking tests
apk add iperf3 iperf3-openrc
rc-update add iperf3 default
## Additonal programs
apk add iperf3
## Configure WAN interface of router
cat >> /etc/network/interfaces <<EOF
......
......@@ -24,6 +24,8 @@ class Board(abc.ABC):
self._pexpect = fdpexpect.fdspawn(self._fdlogging.socket)
# Set board to some known state
self.reset(True) # Hold in reset state
# Set default baord constants for testing
self.min_eth_throughput = 400 # Mbps
@property
def pexpect(self):
......
......@@ -107,9 +107,14 @@ class Container:
return pexp
def get_ip(self,
interfaces: typing.Union[list, tuple] = None,
versions: typing.Union[list, tuple] = frozenset([4, 6])) -> list:
interfaces: typing.Optional[typing.Container] = None,
versions: typing.Container = frozenset([4, 6])) -> list:
"""returns list of ipaddress.IP#Interface filtered according to parameters.
interfaces: Container containing string names of interfaces from which ip addresses will be obtained
versions: Container containing integer values of IP versions to be obtained
Returns a list of ip addresses as IPv4Address or IPv6Address classes
"""
ips = []
ifs_dict = self.network.addresses
......
......@@ -71,7 +71,7 @@ def pytest_runtest_setup(item):
########################################################################################################################
# Resources shared among all tests #####################################################################################
@pytest.fixture(scope="session", name="board")
@pytest.fixture(name="board", scope="session")
def fixture_board(request):
"""Brings board on. Nothing else.
This is top most fixture for board. It returns board handle.
......@@ -82,14 +82,14 @@ def fixture_board(request):
return brd
@pytest.fixture(scope="session", name="lxd")
@pytest.fixture(name="lxd", scope="session")
def fixture_lxd():
"""Provides access to nsfarm.lxd.LXDConnection instance.
"""
return nsfarm.lxd.LXDConnection()
@pytest.fixture(scope="session", name="device_map")
@pytest.fixture(name="device_map", scope="session")
def fixture_device_map(request):
"""Provides easier access to device map generated by target.
"""
......
"""These tests are doing benchmark of troughput between various ports of router.
The idea is to do minimal benchmark to load connectection. This should discover problems such is when connection is
established but troughput because of timing or stability is minimal. Another proble it discovers is instability under
load.
We do not expect full speed of line. We expect at least 60% of speed here as rule of hand.
"""
import pytest
import abc
import warnings
import nsfarm
import json
import datetime
# TODO Add some exclusive locking for these tests between NSFarm instances to ensure that we won't fail these because we
# are running too much instances in paralel of this.
BITS_IN_MBIT = 10**6 # bits in megabits - for conversion purposes
TEST_TIME = 60 # seconds of time to be tested
TEST_INTERVAL = 10 # seconds of measurement intervals
def get_test_data(shell, type):
"""type is either 'sender' or 'receiver'
"""
data = json.loads(shell.output)
speed_data = [round(val['sum']['bits_per_second']/BITS_IN_MBIT, 2) for val in data['intervals']]
client_speed = round(data['end']['streams'][0][type]['bits_per_second']/BITS_IN_MBIT, 2)
return speed_data, client_speed
class ThroughputTest(abc.ABC):
'''Throughput test
'''
@pytest.fixture(scope="class", autouse=True)
def iperf_client(self, client_board):
"""client is alwaysrouter.
"""
client_board.run('opkg update && opkg install iperf3')
yield client_board
client_board.run('opkg remove iperf3')
def test_TCP(self, iperf_server, iperf_client, board_wan, board):
"""Basic TCP throughput test using iperfgit
"""
iperf_server, iperf_server_ip = iperf_server
# check if there are more than one ip - this should not be possible
if len(iperf_server_ip) != 1:
warnings.warn(f'iperf server is having not exactly one ip address. List of ips: {iperf_server_ip}')
# setting up Daemon server, with JSON output and for only 1 session.
iperf_server.command(f'iperf3 -1sJ -i {TEST_INTERVAL}')
# setting up client, adding additional timeout for pexpect
iperf_client.run(f'iperf3 -J -c {iperf_server_ip[0].ip} -i {TEST_INTERVAL}' +
f' -t {TEST_TIME}', timeout=TEST_TIME*1.2)
iperf_server.prompt()
data_client_speed, client_speed = get_test_data(iperf_client, 'sender')
data_server_speed, server_speed = get_test_data(iperf_server, 'receiver')
# Check if some value is under the required one
if any(value < board.min_eth_throughput for value in (data_client_speed + data_server_speed)):
warnings.warn("Speed did not reach the limit in some part of test.")
maximum = max(data_client_speed + data_server_speed)
minimum = min(data_client_speed + data_server_speed)
speed = (client_speed + server_speed)/2
print(f"\n{self.__class__.__name__} measured data [Mbps]:\n"
f"Speed reached (min/max) : {speed}({minimum}/{maximum})\n"
f"Server speeds : {data_client_speed}\n"
f"Client speeds : {data_server_speed}\n")
assert speed > board.min_eth_throughput
class TestWAN(ThroughputTest):
'''Test of WAN interface only
'''
@pytest.fixture(scope="class", autouse=True)
def iperf_server(self, isp_container):
"""server for test
"""
shell = nsfarm.cli.Shell(isp_container.pexpect())
return shell, isp_container.get_ip(['wan'], versions=[4])
class TestLAN(ThroughputTest):
'''Test of LAN interface only
'''
@pytest.fixture(scope="class", autouse=True)
def iperf_server(self, lan1_client):
"""server for test
"""
shell = nsfarm.cli.Shell(lan1_client.pexpect())
return shell, lan1_client.get_ip(['lan'], versions=[4])
@pytest.mark.skip
class TestRouting(ThroughputTest):
'''Test of routing from LAN to WAN
'''
@pytest.fixture(scope="class", autouse=True)
def iperf_client(self, isp_container):
"""client is alwaysrouter.
"""
shell = nsfarm.cli.Shell(isp_container.pexpect())
return shell, isp_container.get_ip(['wan'], versions=[4])
@pytest.fixture(scope="class", autouse=True)
def iperf_server(self, lan1_client):
"""server for test
"""
shell = nsfarm.cli.Shell(lan1_client.pexpect())
return shell, lan1_client.get_ip(['lan'], versions=[4])
# TODO: This test needs dynamic lan interface assignment.
@pytest.mark.skip
class TestSwitching(ThroughputTest):
'''Test of switching in between LAN ports
'''
@pytest.fixture(scope="class", autouse=True)
def iperf_client(self, test_client):
"""client is lan device.
"""
shell = nsfarm.cli.Shell(test_client.pexpect())
return shell, test_client.get_ip(['lan'], versions=[4])
@pytest.fixture(scope="class", autouse=True)
def iperf_server(self, lan1_client):
"""server for test
"""
shell = nsfarm.cli.Shell(lan1_client.pexpect())
return shell, lan1_client.get_ip(['lan'], versions=[4])
"""These tests are doing benchmark of troughput between various ports of router.
The idea is to do minimal benchmark to load connectection. This should discover problems such is when connection is
established but troughput because of timing or stability is minimal. Another proble it discovers is instability under
load.
We do not expect full speed of line. We expect at least 60% of speed here as rule of hand.
"""
import pytest
# TODO Add some exclusive locking for these tests between NSFarm instances to ensure that we won't fail these because we
# are running too much instances in paralel of this.
@pytest.mark.skip
def test_iperf():
"""Do quick benchmark using iperf3.
"""
# TODO use parametrize to switch between -R/_, -u/_
raise NotImplementedError
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment