Verified Commit 4f971f15 authored by Karel Koci's avatar Karel Koci 🤘 Committed by Karel Koci
Browse files

nsfarm/lxd: remove exclusive devices and define all devices in image

This removes concept of exclusive device. The only use of it was to pass
exclusive access to network interface but that is not essentially
required as it is even more versatile to use macvlan as thus we can
easily spawn multiple containers to simulate network.
The only known use for physical device pass trough and thus exclusive is
Wi-Fi. It won't be possible to use macvlan for it. At the same time this
is not an issue as it is not expected that we are going to be reusing
this interface in single tests run multiple times over and over. In the
end there is no need to automatically suspend containers to steal
devices as it has been implemented (and in reality not finished).

The introduced device management now required all devices to be defined
in image as attributes. This gives image definition control over name of
this device in container. It is up to container user to assign
appropriate real device for it. This is done using device map that is
simply pair of attribute and real device specifier. This concept can be
in future expanded to even encode additional configuration if have need
for it.
parent b36a51fc
import pytest
import pkg_resources
import nsfarm.target
def pytest_addoption(parser):
parser.addoption(
"-C", "--targets-config",
help="Path to configuration file with additional targets.",
metavar="PATH",
)
@pytest.hookimpl(tryfirst=True)
def pytest_configure(config):
html_plugin = config.pluginmanager.getplugin("html")
if html_plugin is not None and \
pkg_resources.parse_version(html_plugin.__version__) >= pkg_resources.parse_version("2.1.0"):
config.pluginmanager.register(HTMLReport())
# Parse target cgnfiguration
targets = nsfarm.target.Targets(config.getoption("-C") or (), rootdir=config.rootdir)
setattr(config, "targets", targets)
class HTMLReport:
......
......@@ -48,18 +48,23 @@ execution/preparation.
## Image attributes
Every image can also specify additional attributes that would be used when
container is spawned. The attributes have in general format `TYPE:VALUE` and are
separated by spaces.
container is spawned. The attributes have in general format `TYPE:VALUE` or just
plain `TYPE`. Attributes are separated by spaces.
The following types are defined:
* `internet`: specifies that container should have access to the Internet. Note
that during image preparation the Internet is always available. No argument is
expected.
* `net`: this specifies that there is going to be network interface assigned to
container. The `VALUE` is name of it in the container. The network interface
passed to container is macvlan. The master/parent interface has to be specified
in runtime using map.
* `char`: this specifies that given Unix character device should be accessible in
container. The value is path to required device.
Attributes are inherited from base image. At the moment there is no way to negate
that. At the same time specifying the same attribute again is going to create
duplicate. Depending on an attribute this can be either wrong or good thing but in
most cases wrong.
All attributes are inherited from base image. To remove/mask some attribute you
can prepend it by `!`. As an example to disable the Internet access use
`!internet`.
## Image preparation
......
#!/bin/bash
# nsfarm:base-alpine
# nsfarm:base-alpine internet net:wan
##################################################################################
# This is image used to boot medkit. It provides TFTP server with prepared image
# for u-boot.
......
#!/bin/bash
# nsfarm:base-alpine
# nsfarm:base-alpine net:lan
##################################################################################
# Common image for clients on LAN
##################################################################################
......
#!/bin/bash
# nsfarm:base-alpine
# nsfarm:base-alpine internet net:wan
##################################################################################
# This is base for various ISP like containers.
##################################################################################
......
......@@ -8,7 +8,7 @@ import serial
import serial.tools.miniterm
from pexpect import fdpexpect
from .. import cli
from ..lxd import LXDConnection, Container, NetInterface
from ..lxd import LXDConnection, Container
from ..target.target import Target
......@@ -66,11 +66,10 @@ class Board(abc.ABC):
self._pexpect.sendline("")
return cli.Uboot(self._pexpect)
def bootup(self, lxd_connection: LXDConnection, device_wan: NetInterface, os_branch: str) -> cli.Shell:
def bootup(self, lxd_connection: LXDConnection, os_branch: str) -> cli.Shell:
"""Boot board using TFTP boot. This ensures that board is booted up and ready to accept commands.
lxd_connection: instance of nsfarm.lxd.LXDConnection
device_wan: Wan device to board. This is instance of nsfarm.lxd.NetInterface.
os_branch: Turris OS branch to download medkit from.
Returns instance of cli.Shell
......@@ -78,7 +77,7 @@ class Board(abc.ABC):
# First get U-Boot prompt
uboot = self.uboot()
# Now load image from TFTP
with Container(lxd_connection, "boot", devices=[device_wan, ]) as cont:
with Container(lxd_connection, "boot", self.config.device_map()) as cont:
ccli = cli.Shell(cont.pexpect())
ccli.run(f"prepare_turris_image '{os_branch}'")
uboot.run('setenv ipaddr 192.168.1.142')
......
from .connection import LXDConnection
from .image import Image
from .container import Container
from .device import NetInterface
from . import exceptions
......@@ -32,7 +32,7 @@ def parser(parser):
bootstrap.add_argument(
'IMG',
nargs='*',
help='Image to bootstrap.'
help='Name of image to bootstrap.'
)
bootstrap.add_argument(
'-a', '--all',
......@@ -46,7 +46,18 @@ def parser(parser):
inspect.set_defaults(lxd_op='inspect')
inspect.add_argument(
'IMAGE',
help="""
help="Name of image to inspect."
)
inspect.add_argument(
'-i', '--internet',
action='store_true',
help='Get the Internet access in container even if image specifies no Internet access.'
)
inspect.add_argument(
'-d', '--device',
action='append',
help="""Adds pair to device map. The argument (one pair) has to have format DEVICE=RESOURCE where DEVICE is full
device specification from image and RESOURCE is resource mapped to it.
"""
)
......@@ -102,11 +113,22 @@ def op_bootstrap(args, parser):
sys.exit(0 if success else 1)
def op_inspect(args, _):
def op_inspect(args, parser):
"""Handler for command line operation inspect
"""
kwargs = dict()
if args.internet:
kwargs["internet"] = True
device_map = dict()
if args.device:
for device_spec in args.device:
if '=' not in device_spec:
parser.error(f"Invalid device specifier: {device_spec}")
device, resource = device_spec.split('=', maxsplit=1)
device_map[device] = resource
connection = LXDConnection()
with Container(connection, args.IMAGE) as cont:
with Container(connection, args.IMAGE, device_map=device_map, strict=False, **kwargs) as cont:
sys.exit(subprocess.call(['lxc', 'exec', cont.name, '/bin/sh']))
......
......@@ -5,10 +5,12 @@ import typing
import logging
import pexpect
import itertools
import warnings
from .. import cli
from .connection import LXDConnection
from .image import Image
from .device import Device
from .exceptions import LXDDeviceError
logger = logging.getLogger(__package__)
......@@ -18,15 +20,17 @@ class Container:
"""
# TODO log syslog somehow
def __init__(self, lxd_connection: LXDConnection, image: typing.Union[str, Image],
devices: typing.List[Device] = (), internet: bool = True):
self.image_name = image.name if isinstance(image, Image) else image
def __init__(self, lxd_connection: LXDConnection, image: typing.Union[str, Image], device_map: dict = None,
internet: typing.Optional[bool] = None, strict: bool = True):
self._lxd = lxd_connection
self._internet = internet
self._devices = tuple(devices)
self._internet = False
self._device_map = device_map
self._override_wants_internet = internet
self._strict = strict
self._devices = dict()
self._logger = logging.getLogger(f"{__package__}[{self.image_name}]")
self.image = image if isinstance(image, Image) else Image(lxd_connection, image)
self._image = image if isinstance(image, Image) else Image(lxd_connection, image)
self._logger = logging.getLogger(f"{__package__}[{self._image.name}]")
self.lxd_container = None
......@@ -35,33 +39,39 @@ class Container:
"""
if self.lxd_container is not None:
return
self.image.prepare()
# Collect profiles to be assigned to container
# Collect profiles to be assigned to the container
profiles = [self._lxd.ROOT_PROFILE, ]
if self._internet:
if (self._override_wants_internet is None and self._image.wants_internet) or self._override_wants_internet:
profiles.append(self._lxd.INTERNET_PROFILE)
# Collect devices to attach to container
devices = dict()
for device in itertools.chain(self.image.devices(), self._devices):
devices.update(device.acquire(self))
# Collect devices to be attached to the container
for name, device in self._image.devices().items():
dev = device.acquire(self._device_map)
if dev:
self._devices[name] = dev
continue
if self._strict:
raise LXDDeviceError(name)
warnings.warn(f"Unable to initialize device: {name}")
self._image.prepare()
# Create and start container
self.lxd_container = self._lxd.local.containers.create({
'name': self._container_name(),
'ephemeral': True,
'profiles': profiles,
'devices': devices,
'devices': self._devices,
'source': {
'type': 'image',
'alias': self.image.alias(),
'alias': self._image.alias(),
},
}, wait=True)
self.lxd_container.start(wait=True)
logger.debug("Container prepared: %s", self.lxd_container.name)
def _container_name(self, prefix="nsfarm"):
name = f"{prefix}-{self.image_name}-{os.getpid()}"
name = f"{prefix}-{self._image.name}-{os.getpid()}"
if self._lxd.local.containers.exists(name):
i = 1
while self._lxd.local.containers.exists(f"{name}-{i}"):
......@@ -77,12 +87,6 @@ class Container:
if self.lxd_container is None:
return # No cleanup is required
logger.debug("Removing container: %s", self.lxd_container.name)
# First freeze and remove devices
self.lxd_container.freeze(wait=True)
self.lxd_container.devices = dict()
self.lxd_container.save()
for device in self._devices:
device.release(self)
# Now stop container (Note: container is ephemeral so it is removed automatically after stop)
self.lxd_container.stop()
self.lxd_container = None
......@@ -112,13 +116,22 @@ class Container:
return self.lxd_container.name
@property
def internet(self) -> bool:
"""If host internet connection should be accessible in this instance.
def image(self) -> Image:
"""Allows access to image used for this container.
"""
return self._internet
return self._image
@property
def device_map(self) -> dict:
"""Provides access to device map this container is using. Note that changes performed after container
preparation have no effect.
"""
if self._device_map is None:
return dict()
return self._device_map
@property
def devices(self) -> typing.Tuple[Device]:
"""List of passed devices from host.
"""Dict of passed devices from host.
"""
return tuple(self._devices)
return self._devices
"""Devices management and assigment to containers.
"""
import abc
import collections.abc
class Device(abc.ABC):
"""Generic device handler for LXD container.
This is device handler that can be assigned to containers.
Depending on exclusivity the assigment might be possible only to one container. In such case to acquire device
causes original owner to be automatically frozen.
"""
def __init__(self, exclusive=False):
self._exclusive = exclusive
self._assignment = []
self._def = self._definition()
def __init__(self, value):
self.value = value
def acquire(self, container):
@abc.abstractmethod
def acquire(self, device_map: dict):
"""Acquire device for new container.
Returns LXD device definition for this device.
"""
assert container not in self._assignment
if self._exclusive and self._assignment:
# TODO freeze is not implemented
self._assignment[-1].freeze()
self._assignment.append(container)
return self._def
def release(self, container):
"""Release device from container.
"""
assert container in self._assignment
if self._exclusive and container == self._assignment[-1] and len(self._assignment) > 1:
# TODO unfreeze is not implemented
self._assignment[-2].unfreeze()
self._assignment.remove(container)
@abc.abstractmethod
def _definition(self):
"""This method has to be implemented by child class and should return definition of device that is later
provided to caller as result of calling acquire().
"""
@property
def container(self):
"""Returns handle for container currently assigned to. If device is free then it returns None.
"""
if self._assignment:
return self._assignment[-1]
return None
class NetInterface(Device):
"""Handler to manage single network interface.
"""Handler to manage single network interface using MacVLAN.
"""
def __init__(self, link_name, link_iface):
self._link_name = link_name
self._link_iface = link_iface
super().__init__(exclusive=True)
def _definition(self):
def acquire(self, device_map: dict):
devid = f"net:{self.value}"
if device_map is None or devid not in device_map:
return {}
return {
f"net:{self._link_name}": {
"name": self._link_name,
"nictype": "physical",
"parent": self._link_iface,
"type": "nic"
},
"name": str(self.value),
"nictype": "macvlan",
"parent": str(device_map[devid]),
"type": "nic",
}
......@@ -76,20 +39,11 @@ class CharDevice(Device):
"""Handler to manage character device.
"""
def __init__(self, dev_path, uid=0, gid=0, mode=0o0660):
self._dev_path = dev_path
self._uid = uid
self._gid = gid
self._mode = mode
super().__init__()
def _definition(self):
def acquire(self, device_map: dict):
return {
f"char:{self._dev_path}": {
"source": self._dev_path,
"uid": str(self._uid),
"gid": str(self._gid),
"mode": str(self._mode),
"type": "unix-char"
},
"source": str(self.value),
"uid": "0",
"gid": "0",
"mode": "0660",
"type": "unix-char",
}
......@@ -29,3 +29,12 @@ class LXDImageParameterError(NSFarmLXDError):
def __init__(self, img_name, parameter):
super().__init__(f"The image '{img_name}' has unknown parameter: {parameter}")
class LXDDeviceError(NSFarmLXDError):
"""Image specifies device that wasn't located in device map and thus is not available or there was any other problem
to get full device specification.
"""
def __init__(self, device):
super().__init__(f"The device can't be initialized: {device}")
......@@ -2,15 +2,15 @@
"""
import io
import time
import itertools
import functools
import typing
import logging
import pathlib
import hashlib
import logging
import functools
import pylxd
from .connection import LXDConnection
from .exceptions import LXDImageUndefinedError, LXDImageParentError, LXDImageParameterError
from .device import Device, CharDevice
from .device import Device, NetInterface, CharDevice
logger = logging.getLogger(__package__)
......@@ -49,15 +49,25 @@ class Image:
raise LXDImageParentError(self.name, parent)
attributes = {
"internet": lambda value: True,
"net": NetInterface,
"char": CharDevice,
}
self._devices = []
self._devices = dict()
for param in params:
devtype, value = param.split(':', maxsplit=1)
split_param = param.split(':', maxsplit=1)
negate = split_param[0][0] == '!'
devtype = split_param[0].lstrip('!')
value = split_param[1] if len(split_param) > 1 else None
if devtype in attributes:
self._devices.append(attributes[devtype](value))
if not negate:
self._devices[param] = attributes[devtype](value)
else:
self._devices.pop(param, None)
else:
raise LXDImageParameterError(self.name, param)
self._wants_internet = self._devices.pop(
"internet", self._parent.wants_internet if isinstance(self._parent, Image) else False)
@functools.lru_cache(maxsize=1)
def hash(self) -> str:
......@@ -109,13 +119,21 @@ class Image:
img_hash = self.hash()
return f"nsfarm/{self.name}/{img_hash}"
def devices(self) -> list:
def devices(self) -> typing.Dict[str, Device]:
"""Returns tuple with additional devices to be included in container.
These are not-exclusive devices.
"""
parent_devices = self._parent.devices() if isinstance(self._parent, Image) else tuple()
# TODO what to do with duplicates?
return parent_devices + tuple(self._devices)
devices = dict()
if isinstance(self._parent, Image):
devices.update(self._parent.devices())
devices.update(self._devices)
return devices
@property
def wants_internet(self) -> bool:
"""If container based on this image should have access to the Internet.
"""
return self._wants_internet
def is_prepared(self, img_hash: str = None) -> bool:
"""Check if image we need is prepared.
......
......@@ -98,6 +98,15 @@ class Target:
"""
return name in self._conf
def device_map(self):
"""Provides full device map for all devices for LXD containers.
"""
return {
"net:wan": self.wan,
"net:lan1": self.lan1,
"net:lan2": self.lan2,
}
def __str__(self):
representation = {"name": self._name}
for attr in ("board", "serial_number", "serial", "wan", "lan1", "lan2"):
......
......@@ -8,10 +8,11 @@ def test_new_container(connection):
"""Try to create container for BASE_IMG.
"""
container = Container(connection, BASE_IMG)
assert container.image_name == BASE_IMG
assert isinstance(container.image, Image)
assert container.internet # In default internet should be enabled
assert container.devices == tuple()
assert container.image.name == BASE_IMG
assert not container.image.wants_internet # Base image should not have Internet enabled as a baseline
assert container.device_map == dict() # We provided no device map thus it has to be empty
assert container.devices == dict() # Base image has no devices assigned
def test_new_container_image(connection):
......@@ -19,7 +20,8 @@ def test_new_container_image(connection):
"""
image = Image(connection, BASE_IMG)
container = Container(connection, image)
assert container.image_name == BASE_IMG
assert isinstance(container.image, Image)
assert container.image.name == BASE_IMG
assert container.image is image # This is intentionally 'is' as it should be the same instance
......
......@@ -10,7 +10,7 @@ from .test_image import BASE_IMG
def container(connection):
"""Base container to be used for testing.
"""
with Container(connection, BASE_IMG) as cont:
with Container(connection, BASE_IMG, internet=True) as cont:
shell = Shell(cont.pexpect())
shell.run("wait4network")
yield shell
......
import pytest
def pytest_generate_tests(metafunc):
if "target" not in metafunc.fixturenames:
return
targets = [target for target in metafunc.config.targets.values() if target.is_available()]
if targets:
metafunc.parametrize("target", targets)
else:
metafunc.parametrize("target", [pytest.param("no-target", marks=pytest.mark.skip)])
from pathlib import Path
import pytest
def test_valid(target):
"""Simply check if we consider this target valid using verify method.
"""
assert target.check()
@pytest.mark.parametrize("interface", [
"wan",
"lan1",
"lan2",
])
def test_network_up(target, interface):
"""Interfaces have to be up for macvlan to work. LXD at the moment won't bring them up automatically.
"""
if not target.is_configured(interface):
pytest.skip(f"Interface '{interface}' is not configured for this target.")
pth = Path("/sys/class/net")
with open(pth / target.device_map()[f"net:{interface}"] / "carrier", "r") as file:
assert file.readline() == "1\n"
......@@ -21,11 +21,6 @@ def pytest_addoption(parser):
help="Run tests on one of the targets with given BOARD unless exact target is specified.",
metavar="BOARD",
)
parser.addoption(
"-C", "--targets-config",
help="Path to configuration file with additional targets.",
metavar="PATH",
)
parser.addoption(
"-B", "--branch",
default="hbk",
......@@ -35,17 +30,15 @@ def pytest_addoption(parser):
def pytest_configure(config):
# Parse target configuration
targets = nsfarm.target.Targets(config.getoption("-C") or (), rootdir=config.rootdir)