nsfarm: removed nsfarm.connection

Removed LXDConnection and refactored.
Connection replaced wit pylxd.Client()
and variables and logging moved into
lxd.__init__

nsfarm/board/_board.py

@@ -4,11 +4,12 @@ import sys
 import abc
 import time
 import logging
+import pylxd
 import serial
 import serial.tools.miniterm
 from pexpect import fdpexpect
 from .. import cli
-from ..lxd import LXDConnection, Container
+from ..lxd import Container
 from ..target.target import Target
 
 
@@ -68,10 +69,9 @@ class Board(abc.ABC):
         self._pexpect.sendline("")
         return cli.Uboot(self._pexpect)
 
-    def bootup(self, lxd_connection: LXDConnection, os_branch: str) -> cli.Shell:
+    def bootup(self, lxd_client, os_branch: str) -> cli.Shell:
         """Boot board using TFTP boot. This ensures that board is booted up and ready to accept commands.
 
-        lxd_connection: instance of nsfarm.lxd.LXDConnection
         os_branch: Turris OS branch to download medkit from.
 
         Returns instance of cli.Shell
@@ -79,7 +79,7 @@ class Board(abc.ABC):
         # First get U-Boot prompt
         uboot = self.uboot()
         # Now load image from TFTP
-        with Container(lxd_connection, "boot", self.config.device_map()) as cont:
+        with Container(lxd_client, "boot", self.config.device_map()) as cont:
             ccli = cli.Shell(cont.pexpect())
             ccli.run(f"prepare_turris_image '{os_branch}'", timeout=120)
             uboot.run('setenv ipaddr 192.168.1.142')

nsfarm/lxd/__init__.py

-from .connection import LXDConnection
+import logging
 from .image import Image
 from .container import Container
 from . import exceptions
+
+
+IMAGE_REPO = "https://images.linuxcontainers.org"
+
+PROFILE_ROOT = "nsfarm-root"
+PROFILE_INTERNET = "nsfarm-internet"
+
+# Supppress logging of components pylxd uses (we are not interested that much in them)
+logging.getLogger('ws4py').setLevel(logging.ERROR)
+logging.getLogger('urllib3').setLevel(logging.ERROR)

nsfarm/lxd/__main__.py

 import sys
 import argparse
+import pylxd
 import subprocess
 import dateutil.relativedelta
 from . import utils
-from . import Container, LXDConnection
+from . import Container
 
 
 def parser(parser):
@@ -148,8 +149,8 @@ def op_inspect(args, parser):
             device, resource = device_spec.split('=', maxsplit=1)
             device_map[device] = resource
 
-    connection = LXDConnection()
-    with Container(connection, args.IMAGE, device_map=device_map, strict=False, **kwargs) as cont:
+    lxd_client = pylxd.Client()
+    with Container(lxd_client, args.IMAGE, device_map=device_map, strict=False, **kwargs) as cont:
         if args.proxy:
             for proxy in args.proxy:
                 el = proxy.split(':', maxsplit=2)
@@ -164,7 +165,6 @@ def op_inspect(args, parser):
         sys.exit(subprocess.call(['lxc', 'exec', cont.name, '/bin/sh']))
 
 
-
 def handle_args(args, parser_ret):
     handles = {
         'clean': op_clean,

nsfarm/lxd/connection.py

-"""LXD connection for NSFarm.
-"""
-import logging
-import pylxd
-
-
-class LXDConnection:
-    """This is generic connection handler for LXD handling both connections to local and images server.
-    """
-    IMAGES_SOURCE = "https://images.linuxcontainers.org"  # this is needed for remote access via simplestream
-
-    ROOT_PROFILE = "nsfarm-root"
-    INTERNET_PROFILE = "nsfarm-internet"
-
-    def __init__(self):
-        # Suppress logging of pylxd components
-        logging.getLogger('ws4py').setLevel(logging.ERROR)
-        logging.getLogger('urllib3').setLevel(logging.ERROR)
-
-        self.local = pylxd.Client()

nsfarm/lxd/container.py

@@ -6,11 +6,11 @@ import logging
 import os
 import typing
 import warnings
+import pylxd
 
 import pexpect
 
-from .. import cli
-from .connection import LXDConnection
+from .. import cli, lxd
 from .device import Device
 from .exceptions import LXDDeviceError
 from .image import Image
@@ -26,21 +26,21 @@ class Container:
 
     def __init__(
         self,
-        lxd_connection: LXDConnection,
+        lxd_client: pylxd.Client,
         image: typing.Union[str, Image],
         device_map: dict = None,
         internet: typing.Optional[bool] = None,
         strict: bool = True,
         name: typing.Optional[str] = None,
     ):
-        self._lxd = lxd_connection
+        self._lxd = lxd_client
         self._device_map = device_map
         self._override_wants_internet = internet
         self._strict = strict
         self._devices = dict()
         self._network = None
 
-        self._image = image if isinstance(image, Image) else Image(lxd_connection, image)
+        self._image = image if isinstance(image, Image) else Image(self._lxd, image)
         self._logger = logging.getLogger(f"{__package__}[{self._image.name if name is None else name}]")
 
         self.lxd_container = None
@@ -51,9 +51,9 @@ class Container:
             return
 
         # Collect profiles to be assigned to the container
-        profiles = [self._lxd.ROOT_PROFILE]
+        profiles = [lxd.PROFILE_ROOT]
         if (self._override_wants_internet is None and self._image.wants_internet) or self._override_wants_internet:
-            profiles.append(self._lxd.INTERNET_PROFILE)
+            profiles.append(lxd.PROFILE_INTERNET)
         # Collect devices to be attached to the container
         for name, device in self._image.devices().items():
             dev = device.acquire(self._device_map)
@@ -67,7 +67,7 @@ class Container:
         self._image.prepare()
 
         # Create and start container
-        self.lxd_container = self._lxd.local.containers.create(
+        self.lxd_container = self._lxd.containers.create(
             {
                 "name": self._container_name(),
                 "ephemeral": True,
@@ -90,7 +90,7 @@ class Container:
         # cleanup algorithm). Make sure that you update them when you do changes in this code.
         name = f"{prefix}-{self._image.name}-{os.getpid()}"
         i = 1
-        while self._lxd.local.containers.exists(f"{name}x{i}"):
+        while self._lxd.containers.exists(f"{name}x{i}"):
             i += 1
         name = f"{name}x{i}"
         return name

nsfarm/lxd/image.py

@@ -9,9 +9,9 @@ import hashlib
 import functools
 import platform
 import pylxd
-from .connection import LXDConnection
 from .exceptions import LXDImageUndefinedError, LXDImageParentError, LXDImageParameterError
 from .device import Device, NetInterface, CharDevice
+from .. import lxd
 
 logger = logging.getLogger(__package__)
 
@@ -22,9 +22,9 @@ class Image:
     IMAGE_INIT_PATH = "/nsfarm-init.sh"  # Where we deploy initialization script for image
     IMGS_DIR = pathlib.Path(__file__).parents[2] / "imgs"
 
-    def __init__(self, lxd_connection: LXDConnection, img_name: str):
+    def __init__(self, lxd_client: pylxd.Client, img_name: str):
         self.name = img_name
-        self._lxd = lxd_connection
+        self._lxd = lxd_client
         self._dir_path = self.IMGS_DIR / img_name
         self._file_path = self._dir_path.with_suffix(self._dir_path.suffix + ".sh")
 
@@ -43,10 +43,10 @@ class Image:
 
         image_type, image_alias = parent.split(':', maxsplit=1)
         if image_type == "nsfarm":
-            self._parent = Image(lxd_connection, image_alias)
+            self._parent = Image(self._lxd, image_alias)
         elif image_type == "images":
-            self._parent = self._lxd.local.images.create_from_simplestreams(
-                self._lxd.IMAGES_SOURCE, image_alias + "/" + self.architecture(), auto_update=True)
+            self._parent = self._lxd.images.create_from_simplestreams(
+                lxd.IMAGE_REPO, image_alias + "/" + self.architecture(), auto_update=True)
         else:
             raise LXDImageParentError(self.name, parent)
 
@@ -143,7 +143,7 @@ class Image:
 
         img_hash: specific hash (not latest one) to be checked.
         """
-        return self._lxd.local.images.exists(self.alias(img_hash), alias=True)
+        return self._lxd.images.exists(self.alias(img_hash), alias=True)
 
     def prepare(self):
         """Prepare image. It creates it if necessary and populates lxd_image attribute.
@@ -151,7 +151,7 @@ class Image:
         if self.lxd_image is not None:
             return
         if self.is_prepared(self.hash()):
-            self.lxd_image = self._lxd.local.images.get_by_alias(self.alias())
+            self.lxd_image = self._lxd.images.get_by_alias(self.alias())
             return
 
         logger.debug("Want to bootstrap image: %s", self.alias())
@@ -170,7 +170,7 @@ class Image:
         logger.warning("Bootstrapping image '%s': %s", self.alias(), container_name)
 
         try:
-            container = self._lxd.local.containers.create({
+            container = self._lxd.containers.create({
                 'name': container_name,
                 'profiles': ['nsfarm-root', 'nsfarm-internet'],
                 'source': image_source
@@ -180,7 +180,7 @@ class Image:
                 raise
             logger.warning("Other instance is already bootsrapping image probably. "
                            "Waiting for following container to go away: %s", container_name)
-            while self._lxd.local.containers.exists(container_name):
+            while self._lxd.containers.exists(container_name):
                 time.sleep(1)
             self.prepare()  # possibly get created image or try again
             return

nsfarm/lxd/utils.py

@@ -4,7 +4,7 @@ import os
 import logging
 from datetime import datetime
 import dateutil.parser
-from .connection import LXDConnection
+import pylxd
 from .container import Container
 from .image import Image
 
@@ -21,11 +21,11 @@ def clean_images(delta: dateutil.relativedelta.relativedelta, dry_run: bool = Fa
 
     Returns list of (to be) removed images.
     """
-    connection = LXDConnection()
+    lxd_client = pylxd.Client()
     since = datetime.today() - delta
 
     removed = []
-    for img in connection.local.images.all():
+    for img in lxd_client.images.all():
         # We remove our images and images without alias as those are pulled images
         if img.aliases and not any(alias["name"].startswith("nsfarm/") for alias in img.aliases):
             continue
@@ -57,11 +57,11 @@ def clean_containers(dry_run=False):
 
     Returns list of (to be) removed containers.
     """
-    connection = LXDConnection()
+    lxd_client = pylxd.Client()
     since = datetime.today() - BOOTSTRAP_LIMIT
 
     removed = []
-    for cont in connection.local.instances.all():
+    for cont in lxd_client.instances.all():
         if not cont.name.startswith("nsfarm-"):
             continue
         if cont.name.startswith("nsfarm-bootstrap-"):
@@ -95,7 +95,7 @@ def all_images():
     return (imgf[:-3] for imgf in os.listdir(Image.IMGS_DIR) if imgf.endswith(".sh"))
 
 
-def bootstrap(imgs=None):
+def bootstrap(lxd_client, imgs=None):
     """Bootstrap all defined images.
 
     imgs: list of images to bootstrap
@@ -103,8 +103,7 @@ def bootstrap(imgs=None):
     Returns True if all images were bootstrapped correctly.
     """
     success = True
-    connection = LXDConnection()
     for img in all_images() if imgs is None else imgs:
         logger.info("Trying to bootstrap: %s", img)
-        Image(connection, img).prepare()
+        Image(lxd_client, img).prepare()
     return success

nsfarm/web/container.py

@@ -3,11 +3,11 @@
 import contextlib
 import subprocess
 import typing
+import pylxd
 
 import selenium.webdriver
 
 from ..lxd import Container as LXDContainer
-from ..lxd import LXDConnection
 
 IMAGE = "selenium"
 
@@ -27,12 +27,12 @@ class Container(LXDContainer):
 
     def __init__(
         self,
-        lxd_connection: LXDConnection,
+        lxd_client: pylxd.Client,
         device_map: dict = None,
         internet: typing.Optional[bool] = None,
         strict: bool = True,
     ):
-        super().__init__(lxd_connection, IMAGE, device_map, internet, strict)
+        super().__init__(lxd_client, IMAGE, device_map, internet, strict)
         self._viewer = None
         self._viewer_port = None
 

selftests/cli/test_shell.py

@@ -65,8 +65,8 @@ class TestOpenWrt(Common):
     """These are tests in OpenWrt (ash) shell."""
 
     @pytest.fixture(scope="class", autouse=True)
-    def container(self, lxd_connection):
-        with Container(lxd_connection, "openwrt") as container:
+    def container(self, lxd_client):
+        with Container(lxd_client, "openwrt") as container:
             yield container
 
 
@@ -74,6 +74,6 @@ class TestAlpine(Common):
     """These are tests in Alpine Linux (ash) shell."""
 
     @pytest.fixture(scope="class", autouse=True)
-    def container(self, lxd_connection):
-        with Container(lxd_connection, "base-alpine") as container:
+    def container(self, lxd_client):
+        with Container(lxd_client, "base-alpine") as container:
             yield container

selftests/conftest.py

 import pytest
-from nsfarm import lxd
+import pylxd
 
 
-@pytest.fixture(name="lxd_connection", scope="package")
-def fixture_lxd_connection():
-    return lxd.LXDConnection()
+@pytest.fixture(name="lxd_client", scope="package")
+def fixture_lxd_client():
+    return pylxd.Client()

selftests/lxd/test_container.py

@@ -4,10 +4,10 @@ from nsfarm.lxd import Container, Image
 from .test_image import BASE_IMG
 
 
-def test_new_container(lxd_connection):
+def test_new_container(lxd_client):
     """Try to create container for BASE_IMG.
     """
-    container = Container(lxd_connection, BASE_IMG)
+    container = Container(lxd_client, BASE_IMG)
     assert isinstance(container.image, Image)
     assert container.image.name == BASE_IMG
     assert not container.image.wants_internet  # Base image should not have Internet enabled as a baseline
@@ -15,49 +15,49 @@ def test_new_container(lxd_connection):
     assert container.devices == dict()  # Base image has no devices assigned
 
 
-def test_new_container_image(lxd_connection):
+def test_new_container_image(lxd_client):
     """Try to create container for BASE_IMG using Image instance.
     """
-    image = Image(lxd_connection, BASE_IMG)
-    container = Container(lxd_connection, image)
+    image = Image(lxd_client, BASE_IMG)
+    container = Container(lxd_client, image)
     assert isinstance(container.image, Image)
     assert container.image.name == BASE_IMG
     assert container.image is image  # This is intentionally 'is' as it should be the same instance
 
 
-def test_start_stop(lxd_connection):
+def test_start_stop(lxd_client):
     """Prepare and cleanup container and check that it really creates and removes container.
     """
-    container = Container(lxd_connection, BASE_IMG)
+    container = Container(lxd_client, BASE_IMG)
     assert container.name is None
     container.prepare()
 
     assert container.name is not None
-    assert lxd_connection.local.containers.exists(container.name)
+    assert lxd_client.containers.exists(container.name)
 
     container.cleanup()
 
     # It takes some time before it disappears but it should go away
     for _ in range(10):
-        if not lxd_connection.local.containers.exists(container.name):
+        if not lxd_client.containers.exists(container.name):
             return
         time.sleep(1)
-    assert not lxd_connection.local.containers.exists(container.name)
+    assert not lxd_client.containers.exists(container.name)
 
 
-def test_context(lxd_connection):
+def test_context(lxd_client):
     """Check if we can correctly work with context.
     """
-    with Container(lxd_connection, BASE_IMG) as container:
+    with Container(lxd_client, BASE_IMG) as container:
         container.prepare()
-        assert lxd_connection.local.containers.exists(container.name)
+        assert lxd_client.containers.exists(container.name)
 
     # It takes some time before it disappears but it should go away
     for _ in range(10):
-        if not lxd_connection.local.containers.exists(container.name):
+        if not lxd_client.containers.exists(container.name):
             return
         time.sleep(1)
-    assert not lxd_connection.local.containers.exists(container.name)
+    assert not lxd_client.containers.exists(container.name)
 
 
 # TODO add tests for enabled and disabled internet and for devices

selftests/lxd/test_image.py

@@ -5,25 +5,25 @@ BASE_IMG = "base-alpine"
 NOEX_IMG = "no-such-image"
 
 
-def test_new_image(lxd_connection):
+def test_new_image(lxd_client):
     """Test public attributes set by Image.__init__
     """
-    img = Image(lxd_connection, BASE_IMG)
+    img = Image(lxd_client, BASE_IMG)
     assert img.name == BASE_IMG
     assert img.lxd_image is None
 
 
-def test_image_prepare(lxd_connection):
+def test_image_prepare(lxd_client):
     """Test image preparation and lxd_image attribute it should set.
     """
-    img = Image(lxd_connection, BASE_IMG)
+    img = Image(lxd_client, BASE_IMG)
     img.prepare()
     assert img.lxd_image is not None
     assert img.is_prepared()
 
 
-def test_nonexisting_image(lxd_connection):
+def test_nonexisting_image(lxd_client):
     """Try to initialize Image for undefined image name.
     """
     with pytest.raises(exceptions.LXDImageUndefinedError):
-        Image(lxd_connection, NOEX_IMG)
+        Image(lxd_client, NOEX_IMG)

selftests/lxd/test_internet.py

@@ -6,11 +6,11 @@ from nsfarm.cli import Shell
 from .test_image import BASE_IMG
 
 
-@pytest.fixture(scope="module")
-def container(lxd_connection):
+@pytest.fixture(name="container", scope="module")
+def fixture_container(lxd_client):
     """Base container to be used for testing.
     """
-    with Container(lxd_connection, BASE_IMG, internet=True) as cont:
+    with Container(lxd_client, BASE_IMG, internet=True) as cont:
         shell = Shell(cont.pexpect())
         shell.run("wait4network")
         yield shell

selftests/lxd/test_setup.py

 import pytest
+import pylxd
 from nsfarm import lxd
 
 
 @pytest.mark.parametrize("profile", [
-    lxd.LXDConnection.ROOT_PROFILE,
-    lxd.LXDConnection.INTERNET_PROFILE,
+    lxd.PROFILE_ROOT,
+    lxd.PROFILE_INTERNET,
 ])
-def test_profiles_exists(lxd_connection, profile):
+def test_profiles_exists(lxd_client, profile):
     """Check that all profiles we need are configured in LXD.
     """
-    assert lxd_connection.local.profiles.exists(profile)
+    assert lxd_client.profiles.exists(profile)
 
 
-def test_profile_root(lxd_connection):
+def test_profile_root(lxd_client):
     """Minimal sanity check of root profile.
     """
-    profile = lxd_connection.local.profiles.get(lxd.LXDConnection.ROOT_PROFILE)
+    profile = lxd_client.profiles.get(lxd.PROFILE_ROOT)
     assert any(dev for dev in profile.devices.values() if dev["type"] == "disk")
 
 
-def test_profile_internet(lxd_connection):
+def test_profile_internet(lxd_client):
     """Minimal sanity check of internet profile.
     """
-    profile = lxd_connection.local.profiles.get(lxd.LXDConnection.INTERNET_PROFILE)
+    profile = lxd_client.profiles.get(lxd.PROFILE_INTERNET)
     assert any(dev for dev in profile.devices.values() if dev["type"] == "nic" and dev["name"] == "internet")

selftests/web/test_webdrivers.py

@@ -6,10 +6,10 @@ from nsfarm.web import Container, BROWSERS
 
 
 @pytest.fixture(name="webcontainer", scope="module")
-def fixture_webcontainer(lxd_connection):
+def fixture_webcontainer(lxd_client):
     """Provides nsfarm.web.Container instance.
     """
-    with Container(lxd_connection, internet=True, strict=False) as container:
+    with Container(lxd_client, internet=True, strict=False) as container:
         yield container
 
 

tests/conftest.py

@@ -2,7 +2,7 @@ import time
 import random
 import string
 import pytest
-import warnings
+import pylxd
 import nsfarm.board
 import nsfarm.cli
 import nsfarm.lxd
@@ -81,11 +81,11 @@ def fixture_board(request):
     brd.power(False)
 
 
-@pytest.fixture(name="lxd", scope="session")
-def fixture_lxd():
-    """Provides access to nsfarm.lxd.LXDConnection instance.
+@pytest.fixture(name="lxd_client", scope="session")
+def fixture_lxd_client():
+    """Provides access to pylxd.Client() instance.
     """
-    return nsfarm.lxd.LXDConnection()
+    return pylxd.Client()
 
 
 @pytest.fixture(name="device_map", scope="session")
@@ -99,12 +99,12 @@ def fixture_device_map(request):
 # Boot and setup fixtures ##############################################################################################
 
 @pytest.fixture(name="board_serial", scope="package")
-def fixture_board_serial(request, lxd, board):
+def fixture_board_serial(lxd_client, request, board):
     """Boot board to Shell.
     Provides instance of nsfarm.cli.Shell()
     """
     request.addfinalizer(lambda: board.reset(True))
-    return board.bootup(lxd, request.config.target_branch)
+    return board.bootup(lxd_client, request.config.target_branch)
 
 
 @pytest.fixture(name="board_root_password", scope="package")
@@ -146,28 +146,28 @@ def fixture_client_board(board, board_serial, board_root_password, lan1_client):
 # Common containers ####################################################################################################
 
 @pytest.fixture(name="isp_container", scope="package")
-def fixture_isp_container(lxd, device_map):
+def fixture_isp_container(lxd_client, device_map):
     """Minimal ISP container used to provide the Internet access for the most of the tests.
     """
-    with nsfarm.lxd.Container(lxd, 'isp-common', device_map) as container:
+    with nsfarm.lxd.Container(lxd_client, 'isp-common', device_map) as container:
         container.shell.run('wait4network')
         yield container
 
 
 @pytest.fixture(name="lan1_client", scope="package")
-def fixture_lan1_client(lxd, device_map):
+def fixture_lan1_client(lxd_client, device_map):
     """Starts client container with static IP address 192.168.1.10/24 on LAN1 and provides it.
     """
-    with nsfarm.lxd.Container(lxd, 'client', {"net:lan": device_map["net:lan1"]}) as container:
+    with nsfarm.lxd.Container(lxd_client, 'client', {"net:lan": device_map["net:lan1"]}) as container:
         container.shell.run('wait4boot')
         yield container
 
 
 @pytest.fixture(name="lan1_webclient", scope="package")
-def fixture_lan1_webclient(lxd, device_map):
+def fixture_lan1_webclient(lxd_client, device_map):
     """Starts web-client container on LAN1 and provides it.
     """
-    with nsfarm.web.Container(lxd, {"net:lan": device_map["net:lan1"]}) as container:
+    with nsfarm.web.Container(lxd_client, {"net:lan": device_map["net:lan1"]}) as container:
         container.shell.run('wait4boot')
         yield container
 

tests/network/test_dhcp.py

@@ -44,13 +44,14 @@ class DHCPv4Common(abc.ABC):
         return [iface.ip for iface in sum([client.get_ip(['lan'], [4]) for client in dhcp_clients], [])]