Verified Commit 627357cf authored by Jan Miksik's avatar Jan Miksik
Browse files

nsfarm: removed nsfarm.connection

Removed LXDConnection and refactored.
Connection replaced wit pylxd.Client()
and variables and logging moved into
lxd.__init__
parent e3eaf7ad
......@@ -4,11 +4,12 @@ import sys
import abc
import time
import logging
import pylxd
import serial
import serial.tools.miniterm
from pexpect import fdpexpect
from .. import cli
from ..lxd import LXDConnection, Container
from ..lxd import Container
from ..target.target import Target
......@@ -68,10 +69,9 @@ class Board(abc.ABC):
self._pexpect.sendline("")
return cli.Uboot(self._pexpect)
def bootup(self, lxd_connection: LXDConnection, os_branch: str) -> cli.Shell:
def bootup(self, lxd_client, os_branch: str) -> cli.Shell:
"""Boot board using TFTP boot. This ensures that board is booted up and ready to accept commands.
lxd_connection: instance of nsfarm.lxd.LXDConnection
os_branch: Turris OS branch to download medkit from.
Returns instance of cli.Shell
......@@ -79,7 +79,7 @@ class Board(abc.ABC):
# First get U-Boot prompt
uboot = self.uboot()
# Now load image from TFTP
with Container(lxd_connection, "boot", self.config.device_map()) as cont:
with Container(lxd_client, "boot", self.config.device_map()) as cont:
ccli = cli.Shell(cont.pexpect())
ccli.run(f"prepare_turris_image '{os_branch}'", timeout=120)
uboot.run('setenv ipaddr 192.168.1.142')
......
from .connection import LXDConnection
import logging
from .image import Image
from .container import Container
from . import exceptions
IMAGE_REPO = "https://images.linuxcontainers.org"
PROFILE_ROOT = "nsfarm-root"
PROFILE_INTERNET = "nsfarm-internet"
# Supppress logging of components pylxd uses (we are not interested that much in them)
logging.getLogger('ws4py').setLevel(logging.ERROR)
logging.getLogger('urllib3').setLevel(logging.ERROR)
import sys
import argparse
import pylxd
import subprocess
import dateutil.relativedelta
from . import utils
from . import Container, LXDConnection
from . import Container
def parser(parser):
......@@ -148,8 +149,8 @@ def op_inspect(args, parser):
device, resource = device_spec.split('=', maxsplit=1)
device_map[device] = resource
connection = LXDConnection()
with Container(connection, args.IMAGE, device_map=device_map, strict=False, **kwargs) as cont:
lxd_client = pylxd.Client()
with Container(lxd_client, args.IMAGE, device_map=device_map, strict=False, **kwargs) as cont:
if args.proxy:
for proxy in args.proxy:
el = proxy.split(':', maxsplit=2)
......@@ -164,7 +165,6 @@ def op_inspect(args, parser):
sys.exit(subprocess.call(['lxc', 'exec', cont.name, '/bin/sh']))
def handle_args(args, parser_ret):
handles = {
'clean': op_clean,
......
"""LXD connection for NSFarm.
"""
import logging
import pylxd
class LXDConnection:
"""This is generic connection handler for LXD handling both connections to local and images server.
"""
IMAGES_SOURCE = "https://images.linuxcontainers.org" # this is needed for remote access via simplestream
ROOT_PROFILE = "nsfarm-root"
INTERNET_PROFILE = "nsfarm-internet"
def __init__(self):
# Suppress logging of pylxd components
logging.getLogger('ws4py').setLevel(logging.ERROR)
logging.getLogger('urllib3').setLevel(logging.ERROR)
self.local = pylxd.Client()
......@@ -6,11 +6,11 @@ import logging
import os
import typing
import warnings
import pylxd
import pexpect
from .. import cli
from .connection import LXDConnection
from .. import cli, lxd
from .device import Device
from .exceptions import LXDDeviceError
from .image import Image
......@@ -26,21 +26,21 @@ class Container:
def __init__(
self,
lxd_connection: LXDConnection,
lxd_client: pylxd.Client,
image: typing.Union[str, Image],
device_map: dict = None,
internet: typing.Optional[bool] = None,
strict: bool = True,
name: typing.Optional[str] = None,
):
self._lxd = lxd_connection
self._lxd = lxd_client
self._device_map = device_map
self._override_wants_internet = internet
self._strict = strict
self._devices = dict()
self._network = None
self._image = image if isinstance(image, Image) else Image(lxd_connection, image)
self._image = image if isinstance(image, Image) else Image(self._lxd, image)
self._logger = logging.getLogger(f"{__package__}[{self._image.name if name is None else name}]")
self.lxd_container = None
......@@ -51,9 +51,9 @@ class Container:
return
# Collect profiles to be assigned to the container
profiles = [self._lxd.ROOT_PROFILE]
profiles = [lxd.PROFILE_ROOT]
if (self._override_wants_internet is None and self._image.wants_internet) or self._override_wants_internet:
profiles.append(self._lxd.INTERNET_PROFILE)
profiles.append(lxd.PROFILE_INTERNET)
# Collect devices to be attached to the container
for name, device in self._image.devices().items():
dev = device.acquire(self._device_map)
......@@ -67,7 +67,7 @@ class Container:
self._image.prepare()
# Create and start container
self.lxd_container = self._lxd.local.containers.create(
self.lxd_container = self._lxd.containers.create(
{
"name": self._container_name(),
"ephemeral": True,
......@@ -90,7 +90,7 @@ class Container:
# cleanup algorithm). Make sure that you update them when you do changes in this code.
name = f"{prefix}-{self._image.name}-{os.getpid()}"
i = 1
while self._lxd.local.containers.exists(f"{name}x{i}"):
while self._lxd.containers.exists(f"{name}x{i}"):
i += 1
name = f"{name}x{i}"
return name
......
......@@ -9,9 +9,9 @@ import hashlib
import functools
import platform
import pylxd
from .connection import LXDConnection
from .exceptions import LXDImageUndefinedError, LXDImageParentError, LXDImageParameterError
from .device import Device, NetInterface, CharDevice
from .. import lxd
logger = logging.getLogger(__package__)
......@@ -22,9 +22,9 @@ class Image:
IMAGE_INIT_PATH = "/nsfarm-init.sh" # Where we deploy initialization script for image
IMGS_DIR = pathlib.Path(__file__).parents[2] / "imgs"
def __init__(self, lxd_connection: LXDConnection, img_name: str):
def __init__(self, lxd_client: pylxd.Client, img_name: str):
self.name = img_name
self._lxd = lxd_connection
self._lxd = lxd_client
self._dir_path = self.IMGS_DIR / img_name
self._file_path = self._dir_path.with_suffix(self._dir_path.suffix + ".sh")
......@@ -43,10 +43,10 @@ class Image:
image_type, image_alias = parent.split(':', maxsplit=1)
if image_type == "nsfarm":
self._parent = Image(lxd_connection, image_alias)
self._parent = Image(self._lxd, image_alias)
elif image_type == "images":
self._parent = self._lxd.local.images.create_from_simplestreams(
self._lxd.IMAGES_SOURCE, image_alias + "/" + self.architecture(), auto_update=True)
self._parent = self._lxd.images.create_from_simplestreams(
lxd.IMAGE_REPO, image_alias + "/" + self.architecture(), auto_update=True)
else:
raise LXDImageParentError(self.name, parent)
......@@ -143,7 +143,7 @@ class Image:
img_hash: specific hash (not latest one) to be checked.
"""
return self._lxd.local.images.exists(self.alias(img_hash), alias=True)
return self._lxd.images.exists(self.alias(img_hash), alias=True)
def prepare(self):
"""Prepare image. It creates it if necessary and populates lxd_image attribute.
......@@ -151,7 +151,7 @@ class Image:
if self.lxd_image is not None:
return
if self.is_prepared(self.hash()):
self.lxd_image = self._lxd.local.images.get_by_alias(self.alias())
self.lxd_image = self._lxd.images.get_by_alias(self.alias())
return
logger.debug("Want to bootstrap image: %s", self.alias())
......@@ -170,7 +170,7 @@ class Image:
logger.warning("Bootstrapping image '%s': %s", self.alias(), container_name)
try:
container = self._lxd.local.containers.create({
container = self._lxd.containers.create({
'name': container_name,
'profiles': ['nsfarm-root', 'nsfarm-internet'],
'source': image_source
......@@ -180,7 +180,7 @@ class Image:
raise
logger.warning("Other instance is already bootsrapping image probably. "
"Waiting for following container to go away: %s", container_name)
while self._lxd.local.containers.exists(container_name):
while self._lxd.containers.exists(container_name):
time.sleep(1)
self.prepare() # possibly get created image or try again
return
......
......@@ -4,7 +4,7 @@ import os
import logging
from datetime import datetime
import dateutil.parser
from .connection import LXDConnection
import pylxd
from .container import Container
from .image import Image
......@@ -21,11 +21,11 @@ def clean_images(delta: dateutil.relativedelta.relativedelta, dry_run: bool = Fa
Returns list of (to be) removed images.
"""
connection = LXDConnection()
lxd_client = pylxd.Client()
since = datetime.today() - delta
removed = []
for img in connection.local.images.all():
for img in lxd_client.images.all():
# We remove our images and images without alias as those are pulled images
if img.aliases and not any(alias["name"].startswith("nsfarm/") for alias in img.aliases):
continue
......@@ -57,11 +57,11 @@ def clean_containers(dry_run=False):
Returns list of (to be) removed containers.
"""
connection = LXDConnection()
lxd_client = pylxd.Client()
since = datetime.today() - BOOTSTRAP_LIMIT
removed = []
for cont in connection.local.instances.all():
for cont in lxd_client.instances.all():
if not cont.name.startswith("nsfarm-"):
continue
if cont.name.startswith("nsfarm-bootstrap-"):
......@@ -95,7 +95,7 @@ def all_images():
return (imgf[:-3] for imgf in os.listdir(Image.IMGS_DIR) if imgf.endswith(".sh"))
def bootstrap(imgs=None):
def bootstrap(lxd_client, imgs=None):
"""Bootstrap all defined images.
imgs: list of images to bootstrap
......@@ -103,8 +103,7 @@ def bootstrap(imgs=None):
Returns True if all images were bootstrapped correctly.
"""
success = True
connection = LXDConnection()
for img in all_images() if imgs is None else imgs:
logger.info("Trying to bootstrap: %s", img)
Image(connection, img).prepare()
Image(lxd_client, img).prepare()
return success
......@@ -3,11 +3,11 @@
import contextlib
import subprocess
import typing
import pylxd
import selenium.webdriver
from ..lxd import Container as LXDContainer
from ..lxd import LXDConnection
IMAGE = "selenium"
......@@ -27,12 +27,12 @@ class Container(LXDContainer):
def __init__(
self,
lxd_connection: LXDConnection,
lxd_client: pylxd.Client,
device_map: dict = None,
internet: typing.Optional[bool] = None,
strict: bool = True,
):
super().__init__(lxd_connection, IMAGE, device_map, internet, strict)
super().__init__(lxd_client, IMAGE, device_map, internet, strict)
self._viewer = None
self._viewer_port = None
......
......@@ -65,8 +65,8 @@ class TestOpenWrt(Common):
"""These are tests in OpenWrt (ash) shell."""
@pytest.fixture(scope="class", autouse=True)
def container(self, lxd_connection):
with Container(lxd_connection, "openwrt") as container:
def container(self, lxd_client):
with Container(lxd_client, "openwrt") as container:
yield container
......@@ -74,6 +74,6 @@ class TestAlpine(Common):
"""These are tests in Alpine Linux (ash) shell."""
@pytest.fixture(scope="class", autouse=True)
def container(self, lxd_connection):
with Container(lxd_connection, "base-alpine") as container:
def container(self, lxd_client):
with Container(lxd_client, "base-alpine") as container:
yield container
import pytest
from nsfarm import lxd
import pylxd
@pytest.fixture(name="lxd_connection", scope="package")
def fixture_lxd_connection():
return lxd.LXDConnection()
@pytest.fixture(name="lxd_client", scope="package")
def fixture_lxd_client():
return pylxd.Client()
......@@ -4,10 +4,10 @@ from nsfarm.lxd import Container, Image
from .test_image import BASE_IMG
def test_new_container(lxd_connection):
def test_new_container(lxd_client):
"""Try to create container for BASE_IMG.
"""
container = Container(lxd_connection, BASE_IMG)
container = Container(lxd_client, BASE_IMG)
assert isinstance(container.image, Image)
assert container.image.name == BASE_IMG
assert not container.image.wants_internet # Base image should not have Internet enabled as a baseline
......@@ -15,49 +15,49 @@ def test_new_container(lxd_connection):
assert container.devices == dict() # Base image has no devices assigned
def test_new_container_image(lxd_connection):
def test_new_container_image(lxd_client):
"""Try to create container for BASE_IMG using Image instance.
"""
image = Image(lxd_connection, BASE_IMG)
container = Container(lxd_connection, image)
image = Image(lxd_client, BASE_IMG)
container = Container(lxd_client, image)
assert isinstance(container.image, Image)
assert container.image.name == BASE_IMG
assert container.image is image # This is intentionally 'is' as it should be the same instance
def test_start_stop(lxd_connection):
def test_start_stop(lxd_client):
"""Prepare and cleanup container and check that it really creates and removes container.
"""
container = Container(lxd_connection, BASE_IMG)
container = Container(lxd_client, BASE_IMG)
assert container.name is None
container.prepare()
assert container.name is not None
assert lxd_connection.local.containers.exists(container.name)
assert lxd_client.containers.exists(container.name)
container.cleanup()
# It takes some time before it disappears but it should go away
for _ in range(10):
if not lxd_connection.local.containers.exists(container.name):
if not lxd_client.containers.exists(container.name):
return
time.sleep(1)
assert not lxd_connection.local.containers.exists(container.name)
assert not lxd_client.containers.exists(container.name)
def test_context(lxd_connection):
def test_context(lxd_client):
"""Check if we can correctly work with context.
"""
with Container(lxd_connection, BASE_IMG) as container:
with Container(lxd_client, BASE_IMG) as container:
container.prepare()
assert lxd_connection.local.containers.exists(container.name)
assert lxd_client.containers.exists(container.name)
# It takes some time before it disappears but it should go away
for _ in range(10):
if not lxd_connection.local.containers.exists(container.name):
if not lxd_client.containers.exists(container.name):
return
time.sleep(1)
assert not lxd_connection.local.containers.exists(container.name)
assert not lxd_client.containers.exists(container.name)
# TODO add tests for enabled and disabled internet and for devices
......@@ -5,25 +5,25 @@ BASE_IMG = "base-alpine"
NOEX_IMG = "no-such-image"
def test_new_image(lxd_connection):
def test_new_image(lxd_client):
"""Test public attributes set by Image.__init__
"""
img = Image(lxd_connection, BASE_IMG)
img = Image(lxd_client, BASE_IMG)
assert img.name == BASE_IMG
assert img.lxd_image is None
def test_image_prepare(lxd_connection):
def test_image_prepare(lxd_client):
"""Test image preparation and lxd_image attribute it should set.
"""
img = Image(lxd_connection, BASE_IMG)
img = Image(lxd_client, BASE_IMG)
img.prepare()
assert img.lxd_image is not None
assert img.is_prepared()
def test_nonexisting_image(lxd_connection):
def test_nonexisting_image(lxd_client):
"""Try to initialize Image for undefined image name.
"""
with pytest.raises(exceptions.LXDImageUndefinedError):
Image(lxd_connection, NOEX_IMG)
Image(lxd_client, NOEX_IMG)
......@@ -6,11 +6,11 @@ from nsfarm.cli import Shell
from .test_image import BASE_IMG
@pytest.fixture(scope="module")
def container(lxd_connection):
@pytest.fixture(name="container", scope="module")
def fixture_container(lxd_client):
"""Base container to be used for testing.
"""
with Container(lxd_connection, BASE_IMG, internet=True) as cont:
with Container(lxd_client, BASE_IMG, internet=True) as cont:
shell = Shell(cont.pexpect())
shell.run("wait4network")
yield shell
......
import pytest
import pylxd
from nsfarm import lxd
@pytest.mark.parametrize("profile", [
lxd.LXDConnection.ROOT_PROFILE,
lxd.LXDConnection.INTERNET_PROFILE,
lxd.PROFILE_ROOT,
lxd.PROFILE_INTERNET,
])
def test_profiles_exists(lxd_connection, profile):
def test_profiles_exists(lxd_client, profile):
"""Check that all profiles we need are configured in LXD.
"""
assert lxd_connection.local.profiles.exists(profile)
assert lxd_client.profiles.exists(profile)
def test_profile_root(lxd_connection):
def test_profile_root(lxd_client):
"""Minimal sanity check of root profile.
"""
profile = lxd_connection.local.profiles.get(lxd.LXDConnection.ROOT_PROFILE)
profile = lxd_client.profiles.get(lxd.PROFILE_ROOT)
assert any(dev for dev in profile.devices.values() if dev["type"] == "disk")
def test_profile_internet(lxd_connection):
def test_profile_internet(lxd_client):
"""Minimal sanity check of internet profile.
"""
profile = lxd_connection.local.profiles.get(lxd.LXDConnection.INTERNET_PROFILE)
profile = lxd_client.profiles.get(lxd.PROFILE_INTERNET)
assert any(dev for dev in profile.devices.values() if dev["type"] == "nic" and dev["name"] == "internet")
......@@ -6,10 +6,10 @@ from nsfarm.web import Container, BROWSERS
@pytest.fixture(name="webcontainer", scope="module")
def fixture_webcontainer(lxd_connection):
def fixture_webcontainer(lxd_client):
"""Provides nsfarm.web.Container instance.
"""
with Container(lxd_connection, internet=True, strict=False) as container:
with Container(lxd_client, internet=True, strict=False) as container:
yield container
......
......@@ -2,7 +2,7 @@ import time
import random
import string
import pytest
import warnings
import pylxd
import nsfarm.board
import nsfarm.cli
import nsfarm.lxd
......@@ -81,11 +81,11 @@ def fixture_board(request):
brd.power(False)
@pytest.fixture(name="lxd", scope="session")
def fixture_lxd():
"""Provides access to nsfarm.lxd.LXDConnection instance.
@pytest.fixture(name="lxd_client", scope="session")
def fixture_lxd_client():
"""Provides access to pylxd.Client() instance.
"""
return nsfarm.lxd.LXDConnection()
return pylxd.Client()
@pytest.fixture(name="device_map", scope="session")
......@@ -99,12 +99,12 @@ def fixture_device_map(request):
# Boot and setup fixtures ##############################################################################################
@pytest.fixture(name="board_serial", scope="package")
def fixture_board_serial(request, lxd, board):
def fixture_board_serial(lxd_client, request, board):
"""Boot board to Shell.
Provides instance of nsfarm.cli.Shell()
"""
request.addfinalizer(lambda: board.reset(True))
return board.bootup(lxd, request.config.target_branch)
return board.bootup(lxd_client, request.config.target_branch)
@pytest.fixture(name="board_root_password", scope="package")
......@@ -146,28 +146,28 @@ def fixture_client_board(board, board_serial, board_root_password, lan1_client):
# Common containers ####################################################################################################
@pytest.fixture(name="isp_container", scope="package")
def fixture_isp_container(lxd, device_map):
def fixture_isp_container(lxd_client, device_map):
"""Minimal ISP container used to provide the Internet access for the most of the tests.
"""
with nsfarm.lxd.Container(lxd, 'isp-common', device_map) as container:
with nsfarm.lxd.Container(lxd_client, 'isp-common', device_map) as container:
container.shell.run('wait4network')
yield container
@pytest.fixture(name="lan1_client", scope="package")
def fixture_lan1_client(lxd, device_map):
def fixture_lan1_client(lxd_client, device_map):
"""Starts client container with static IP address 192.168.1.10/24 on LAN1 and provides it.
"""
with nsfarm.lxd.Container(lxd, 'client', {"net:lan": device_map["net:lan1"]}) as container:
with nsfarm.lxd.Container(lxd_client, 'client', {"net:lan": device_map["net:lan1"]}) as container:
container.shell.run('wait4boot')
yield container
@pytest.fixture(name="lan1_webclient", scope="package")
def fixture_lan1_webclient(lxd, device_map):
def fixture_lan1_webclient(lxd_client, device_map):
"""Starts web-client container on LAN1 and provides it.
"""
with nsfarm.web.Container(lxd, {"net:lan": device_map["net:lan1"]}) as container:
with nsfarm.web.Container(lxd_client, {"net:lan": device_map["net:lan1"]}) as container:
container.shell.run('wait4boot')
yield container
......
......@@ -44,13 +44,14 @@ class DHCPv4Common(abc.ABC):