tests: Added basic throughput tests

Added basic throughput tests for basic testing.
Test is run for 60 seconds and checks if the speed is at least
400 Mbps.
It also checks min/max of 10s chunks.
And checks if some of chunks is under the limit - raises warning.
Prints the speeds to output log.

imgs/client.sh

@@ -10,6 +10,9 @@ wait4network
 # Add openssh client to access router shell
 apk add openssh-client
 
+# Additional applications installation
+apk add iperf3
+
 # Configure LAN1 interface for static local network
 cat >> /etc/network/interfaces <<EOF
 auto lan

imgs/isp-common.sh

@@ -27,10 +27,8 @@ rc-update add named
 chown :named /etc/bind
 chmod a+r /etc/bind/named.conf
 
-## Additonal service
-# iperf3 for benchmarking tests
-apk add iperf3 iperf3-openrc
-rc-update add iperf3 default
+## Additonal programs
+apk add iperf3
 
 ## Configure WAN interface of router
 cat >> /etc/network/interfaces <<EOF

nsfarm/board/_board.py

@@ -24,6 +24,8 @@ class Board(abc.ABC):
         self._pexpect = fdpexpect.fdspawn(self._fdlogging.socket)
         # Set board to some known state
         self.reset(True)  # Hold in reset state
+        # Set default baord constants for testing
+        self.min_eth_throughput = 400  # Mbps
 
     @property
     def pexpect(self):

nsfarm/lxd/container.py

@@ -107,9 +107,14 @@ class Container:
         return pexp
 
     def get_ip(self,
-               interfaces: typing.Union[list, tuple] = None,
-               versions: typing.Union[list, tuple] = frozenset([4, 6])) -> list:
+               interfaces: typing.Optional[typing.Container] = None,
+               versions: typing.Container = frozenset([4, 6])) -> list:
         """returns list of ipaddress.IP#Interface filtered according to parameters.
+
+        interfaces: Container containing string names of interfaces from which ip addresses will be obtained
+        versions: Container containing integer values of IP versions to be obtained
+
+        Returns a list of ip addresses as IPv4Address or IPv6Address classes
         """
         ips = []
         ifs_dict = self.network.addresses

tests/conftest.py

@@ -71,7 +71,7 @@ def pytest_runtest_setup(item):
 ########################################################################################################################
 # Resources shared among all tests #####################################################################################
 
-@pytest.fixture(scope="session", name="board")
+@pytest.fixture(name="board", scope="session")
 def fixture_board(request):
     """Brings board on. Nothing else.
     This is top most fixture for board. It returns board handle.
@@ -82,14 +82,14 @@ def fixture_board(request):
     return brd
 
 
-@pytest.fixture(scope="session", name="lxd")
+@pytest.fixture(name="lxd", scope="session")
 def fixture_lxd():
     """Provides access to nsfarm.lxd.LXDConnection instance.
     """
     return nsfarm.lxd.LXDConnection()
 
 
-@pytest.fixture(scope="session", name="device_map")
+@pytest.fixture(name="device_map", scope="session")
 def fixture_device_map(request):
     """Provides easier access to device map generated by target.
     """

tests/network/test_throughput.py

+"""These tests are doing benchmark of troughput between various ports of router.
+
+The idea is to do minimal benchmark to load connectection. This should discover problems such is when connection is
+established but troughput because of timing or stability is minimal. Another proble it discovers is instability under
+load.
+We do not expect full speed of line. We expect at least 60% of speed here as rule of hand.
+"""
+import pytest
+import abc
+import warnings
+import nsfarm
+import json
+import datetime
+
+# TODO Add some exclusive locking for these tests between NSFarm instances to ensure that we won't fail these because we
+# are running too much instances in paralel of this.
+
+BITS_IN_MBIT = 10**6  # bits in megabits - for conversion purposes
+TEST_TIME = 60  # seconds of time to be tested
+TEST_INTERVAL = 10  # seconds of measurement intervals
+
+
+def get_test_data(shell, type):
+    """type is either 'sender' or 'receiver'
+    """
+    data = json.loads(shell.output)
+    speed_data = [round(val['sum']['bits_per_second']/BITS_IN_MBIT, 2) for val in data['intervals']]
+    client_speed = round(data['end']['streams'][0][type]['bits_per_second']/BITS_IN_MBIT, 2)
+
+    return speed_data, client_speed
+
+
+class ThroughputTest(abc.ABC):
+    '''Throughput test
+    '''
+    @pytest.fixture(scope="class", autouse=True)
+    def iperf_client(self, client_board):
+        """client is alwaysrouter.
+        """
+        client_board.run('opkg update && opkg install iperf3')
+        yield client_board
+        client_board.run('opkg remove iperf3')
+
+    def test_TCP(self, iperf_server, iperf_client, board_wan, board):
+        """Basic TCP throughput test using iperfgit
+        """
+        iperf_server, iperf_server_ip = iperf_server
+        # check if there are more than one ip - this should not be possible
+        if len(iperf_server_ip) != 1:
+            warnings.warn(f'iperf server is having not exactly one ip address. List of ips: {iperf_server_ip}')
+        # setting up Daemon server, with JSON output and for only 1 session.
+        iperf_server.command(f'iperf3 -1sJ -i {TEST_INTERVAL}')
+        # setting up client, adding additional timeout for pexpect
+        iperf_client.run(f'iperf3 -J -c {iperf_server_ip[0].ip} -i {TEST_INTERVAL}' +
+                         f' -t {TEST_TIME}', timeout=TEST_TIME*1.2)
+        iperf_server.prompt()
+        data_client_speed, client_speed = get_test_data(iperf_client, 'sender')
+        data_server_speed, server_speed = get_test_data(iperf_server, 'receiver')
+        # Check if some value is under the required one
+        if any(value < board.min_eth_throughput for value in (data_client_speed + data_server_speed)):
+            warnings.warn("Speed did not reach the limit in some part of test.")
+        maximum = max(data_client_speed + data_server_speed)
+        minimum = min(data_client_speed + data_server_speed)
+        speed = (client_speed + server_speed)/2
+
+        print(f"\n{self.__class__.__name__} measured data [Mbps]:\n"
+              f"Speed reached (min/max) : {speed}({minimum}/{maximum})\n"
+              f"Server speeds : {data_client_speed}\n"
+              f"Client speeds : {data_server_speed}\n")
+
+        assert speed > board.min_eth_throughput
+
+
+class TestWAN(ThroughputTest):
+    '''Test of WAN interface only
+    '''
+    @pytest.fixture(scope="class", autouse=True)
+    def iperf_server(self, isp_container):
+        """server for test
+        """
+        shell = nsfarm.cli.Shell(isp_container.pexpect())
+        return shell, isp_container.get_ip(['wan'], versions=[4])
+
+
+class TestLAN(ThroughputTest):
+    '''Test of LAN interface only
+    '''
+    @pytest.fixture(scope="class", autouse=True)
+    def iperf_server(self, lan1_client):
+        """server for test
+        """
+        shell = nsfarm.cli.Shell(lan1_client.pexpect())
+        return shell, lan1_client.get_ip(['lan'], versions=[4])
+
+
+@pytest.mark.skip
+class TestRouting(ThroughputTest):
+    '''Test of routing from LAN to WAN
+    '''
+    @pytest.fixture(scope="class", autouse=True)
+    def iperf_client(self, isp_container):
+        """client is alwaysrouter.
+        """
+        shell = nsfarm.cli.Shell(isp_container.pexpect())
+        return shell, isp_container.get_ip(['wan'], versions=[4])
+
+    @pytest.fixture(scope="class", autouse=True)
+    def iperf_server(self, lan1_client):
+        """server for test
+        """
+        shell = nsfarm.cli.Shell(lan1_client.pexpect())
+        return shell, lan1_client.get_ip(['lan'], versions=[4])
+
+
+# TODO: This test needs dynamic lan interface assignment.
+@pytest.mark.skip
+class TestSwitching(ThroughputTest):
+    '''Test of switching in between LAN ports
+    '''
+    @pytest.fixture(scope="class", autouse=True)
+    def iperf_client(self, test_client):
+        """client is lan device.
+        """
+        shell = nsfarm.cli.Shell(test_client.pexpect())
+        return shell, test_client.get_ip(['lan'], versions=[4])
+
+    @pytest.fixture(scope="class", autouse=True)
+    def iperf_server(self, lan1_client):
+        """server for test
+        """
+        shell = nsfarm.cli.Shell(lan1_client.pexpect())
+        return shell, lan1_client.get_ip(['lan'], versions=[4])

tests/network/test_troughput.py

-"""These tests are doing benchmark of troughput between various ports of router.
-
-The idea is to do minimal benchmark to load connectection. This should discover problems such is when connection is
-established but troughput because of timing or stability is minimal. Another proble it discovers is instability under
-load.
-We do not expect full speed of line. We expect at least 60% of speed here as rule of hand.
-"""
-import pytest
-# TODO Add some exclusive locking for these tests between NSFarm instances to ensure that we won't fail these because we
-# are running too much instances in paralel of this.
-
-
-@pytest.mark.skip
-def test_iperf():
-    """Do quick benchmark using iperf3.
-    """
-    # TODO use parametrize to switch between -R/_, -u/_
-    raise NotImplementedError