Merge branch 'manager-tooling-fixes' into 'manager'

manager: tooling fixes & ci integration

See merge request !1249

.gitlab-ci.yml

@@ -346,8 +346,9 @@ pkgtest:
     variables:
       - $SKIP_CI == "1"
 
-manager:  # TODO better CI integration
+manager:
   stage: test
+  needs: []
   trigger:
     include: manager/.gitlab-ci.yml
     strategy: depend
@@ -360,6 +361,7 @@ manager:  # TODO better CI integration
       - $SKIP_CI == "1"
 
 pytests:
+  <<: *common
   # these are executed on LXC runners to increase stability
   image: $CI_REGISTRY/knot/knot-resolver/ci/lxc-debian-11:knot-$KNOT_VERSION
   needs: []

ci/images/manager/Dockerfile

+# SPDX-License-Identifier: GPL-3.0-or-later
+
+FROM fedora:35
+MAINTAINER Knot Resolver <knot-resolver@labs.nic.cz>
+
+WORKDIR /root
+CMD ["/bin/bash"]
+ENV PATH="/root/.local/bin:${PATH}"
+
+# Install Python
+RUN dnf install -y git diffutils python3.6 python3.7 python3.8 python3.9 python3.10 python3-gobject pkg-config cairo-devel gcc python3-devel gobject-introspection-devel cairo-gobject-devel which \
+	&& dnf clean all
+
+# Install Poetry
+RUN python3 -m pip install -U pip \
+	&& curl -sSL https://install.python-poetry.org | python3 - \
+	# not exactly required, but helpful
+	&& python3 -m pip install poethepoet

manager/.gitignore

@@ -11,4 +11,5 @@ dist/
 .vscode/
 /pkg
 .podman-cache/
-docs/_build/*
\ No newline at end of file
+docs/_build/*
+*junit.xml

manager/.gitlab-ci.yml

 stages:
   - check
-  - packaging
 
 default:
-  image: registry.nic.cz/knot/knot-resolver-manager/knot-manager:ci
+  image: registry.nic.cz/knot/knot-resolver/ci/manager:knot-$KNOT_VERSION
   before_script:
-    - pushd manager
-    # make sure Poetry is in $PATH
-    - source $HOME/.poetry/env
-    # there is already a pyproject.toml with installed dependencies in the root
-    # it has its own virtualenv and we want to use that env in a different directory
-    # so let's create a new one and replace it by the already existing one
-    # we don't care about destroying the environment in process, because it's going to be discarded anyway
-    - poetry env use $(which python3.6); ourpath=$(poetry env info -p); upperpath=$( (cd ..; poetry env info -p) ); rm -rf "$ourpath"; cp -a "$upperpath" "$ourpath"
-    # the virtualenv we recycled might be slightly out of date. Let's quickly update it
+    - cd manager
+    - poetry env use $PYTHON_INTERPRETER
     - poetry install
-    # fix podman; see https://gitlab.nic.cz/labs/lxc-gitlab-runner#nesting-with-podman
-    - unset TMPDIR
   tags:
-    - lxc
+    - docker
+    - linux
     - amd64
 
-lint:
+lint:py3.10:
   stage: check
   script:
     - poe check
+  variables:
+    PYTHON_INTERPRETER: python3.10
 
-test:
+
+.unit: &unit
   stage: check
   script:
     - poe test
-    - poetry run coverage xml
+    # the following command makes sure that the source root of the coverage file is at $gitroot
+    - poetry run bash -c "cd ..; coverage combine manager/.coverage; coverage xml"
   artifacts:
     reports:
       cobertura: coverage.xml
+      junit: manager/unit.junit.xml
+    paths:
+      - manager/unit.junit.xml
 
-integration:
-  stage: check
-  script:
-    - poe integration
+unit:py3.6:
+  <<: *unit
+  variables:
+    PYTHON_INTERPRETER: python3.6
 
+unit:py3.7:
+  <<: *unit
+  variables:
+    PYTHON_INTERPRETER: python3.7
 
-package-debian-10:
-  when: manual
-  stage: packaging
-  image: registry.nic.cz/labs/lxc-gitlab-runner/debian-10
-  before_script:
-    - pushd manager
-  script:
-    - bash scripts/make-package.sh
-  artifacts:
-    paths:
-      - knot-resolver/pkg/pkgs/
-      - knot-resolver/pkg/srcpkgs/
-    expire_in: 1 week
+unit:py3.8:
+  <<: *unit
+  variables:
+    PYTHON_INTERPRETER: python3.8
 
-package-fedora-34:
-  when: manual
-  stage: packaging
-  image: registry.nic.cz/labs/lxc-gitlab-runner/fedora-34
-  before_script:
-    - pushd manager
-  script:
-    - bash scripts/make-package.sh
-  artifacts:
-    paths:
-      - knot-resolver/pkg/pkgs/
-      - knot-resolver/pkg/srcpkgs/
-    expire_in: 1 week
+unit:py3.9:
+  <<: *unit
+  variables:
+    PYTHON_INTERPRETER: python3.9
+
+unit:py3.10:
+  <<: *unit
+  variables:
+    PYTHON_INTERPRETER: python3.10

manager/README.md

@@ -11,7 +11,6 @@ Because we want to support multiple versions of Python with one codebase, we dev
 Install these tools:
 * [pyenv](https://github.com/pyenv/pyenv#installation) - a tool for switching between Python versions without affecting the system (can be installed using distro's package manager)
 * [Poetry](https://python-poetry.org/docs/#installation) - dependency management (note: do not install the package via pip, follow instructions in Poetry's official documentation)
-* NodeJS+NPM - used for our type-checker, version packaged in your distro should work just fine
 
 Be careful, that you need the latest version of Poetry. The setup was tested with Poetry version 1.1.7. Due to it's ability to switch between Python versions, it has to be installed separately to work correctly. Make sure to follow [the latest setup guide](https://python-poetry.org/docs/#installation).
 
@@ -24,7 +23,6 @@ pyenv install 3.8.7
 pyenv install 3.9.1
 poetry env use $(pyenv which python)
 poetry install
-npm install
 ```
 
 With this environment, **everything else should just work**. You can run the same checks the CI runs, all commands listed bellow should pass. If something fails and you did all the steps above, please [open a new issue](https://gitlab.nic.cz/knot/knot-resolver-manager/-/issues/new).
@@ -74,14 +72,10 @@ Short answer - mainly for managing other dependencies. By using dependency manag
     * A dependency management system for Python libraries. Normally, all libraries in Python are installed system-wide and dependent on system's Python version. By using virtual environments managed by Poetry, configured to use a the correct Python version through pyenv, we can specify versions of the dependencies in any way we like.
     * Follows PEP 518 and uses the `pyproject.toml` file for all of it's configuration.
     * Written in Python, therefore it's problematic if installed system-wide as an ordinary Python package (because it would be unavailable in its own virtual environment).
-  * Yarn or NPM
-    * Dependency management systems from JavaScript development.
-    * Used for installing pyright - the type checker we use.
 * automatically managed dependencies
   * PoeThePoet - A task management system, or in other words glorified switch statement calling other tools. Used for simplifying interractions with the project.
   * pytest, pytest-cov - unit testing
   * pylint, flake8 - linting
-  * pyright - type checking, compatible with VSCode using the Pylance extension
   * black - autoformatter (might be removed in the future if not used in practice)
   * tox - testing automation
   * tox-pyenv - plugin for tox that makes use of pyenv provided Python binaries
@@ -89,14 +83,3 @@ Short answer - mainly for managing other dependencies. By using dependency manag
 ### Why Poetry? Why should I learn a new tool?
 
 This blog post explains it nicely - https://muttdata.ai/blog/2020/08/21/a-poetic-apology.html.
-
-### Why do we need JavaScript in Python project?
-
-We would like to use a type checker. As of writing this, there are 4 possible options (I was aware of):
-
-* [mypy](http://mypy-lang.org/) - oldest, reports correct code as broken, no advanced features, written in Python, works
-* [pytype](https://github.com/google/pytype) - supports type inference, written in Python, does not work in Python 3.9, Python versions > 3.7 are not yet supported
-* [pyre](https://pyre-check.org/) - supports type inference, contains security focused static analysis tool, written in Python, does not work in Python 3.6
-* [pyright](https://github.com/Microsoft/pyright) - not that advanced as pyre and pytype, reports correct code as broken, basic type inference when no typehints are provided, written in TypeScript, great integration with VSCode, works regardless of current Python version
-
-Type inference is really handy when it comes to libraries without type hints or when you use internal functions without specifiing their types. This is why we use pyright instead of the more classical approach with mypy.

manager/containers/README.md

-# Container definition files
-
-All containers build configurations evolve from the `dev` container. If you want to change something, please do it mainly there. We consider **the `dev` container as a reference container**
-
-## Naming
-
-All containers defined here are named `knot-manager`. The directory name is the container's tag. So, for example - `knot-manager:dev` is the reference development container.
-
-## Building
-
-```sh
-for tag in 
-```
\ No newline at end of file

manager/containers/ci/Containerfile

-FROM registry.nic.cz/labs/lxc-gitlab-runner/fedora-34:podman
-
-# Install Python and NodeJS
-RUN dnf install -y python3.6 nodejs python3-gobject pkg-config cairo-devel gcc python3-devel gobject-introspection-devel cairo-gobject-devel which \
-  && dnf clean all
-
-# Install Poetry
-RUN python3 -m pip install -U pip \
-  && curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python3 - \
-  && source $HOME/.poetry/env \
-  # not exactly required, but helpful
-  && python3 -m pip install poethepoet
-
-# force Poetry to use local .venv/ directory that we can cache
-ENV POETRY_NO_INTERACTION=1 \
-  # python:
-  PYTHONFAULTHANDLER=1 \
-  PYTHONUNBUFFERED=1 \
-  PYTHONHASHSEED=random \
-  PYTHONDONTWRITEBYTECODE=1
-
-RUN dnf install -y knot-resolver procps-ng
-
-# How does this work?
-# ===================
-#
-# NPM dependencies are installed globally. There is no problem with that.
-#
-# Python dependencies are however installed into a virtualenv created by Poetry. Why you might ask?
-# Because we can't change the default python interpreter without virtualenv. This creates a problem,
-# that the virtualenv is created for a directory different than the one, where CI will run.
-#
-# Yup, that's a slight issue, that has to be fixed before running anything. This migration step is however
-# quick. It's just copying files locally and there's not a ton of them. This virtualenv migration step is
-# therefore done every time a CI job starts.
-#
-# How does it speed up CI?
-# ========================
-#
-# We do not have to install the dependencies every single time. They are cached in the container itself and 
-# we can rebuild it only when it's definition or the list of dependencies changes.
-
-COPY pyproject.toml poetry.lock package.json .
-RUN source $HOME/.poetry/env \
-  && poetry config --list \
-  && poetry env use $(which python3.6) \
-  && poetry env info \
-  && poetry install --no-interaction --no-ansi \
-  && npm install -g $(python -c "import json; print(*(k for k in json.loads(open('package.json').read())['dependencies']))")
\ No newline at end of file

manager/containers/debian-supervisord/Containerfile

-FROM registry.nic.cz/knot/knot-resolver-manager/knot-manager:debian
-
-# Remove systemd
-# RUN apt-get remove -y systemd
-#
-# Well, we can't do that... The command above also uninstalls knot-resolver. Which
-# is kind of stupid. So let's just keep systemd installed and not start it.
-
-# install supervisord
-RUN python3 -m pip install supervisor
-
-# install tini init
-RUN apt-get update \
-  && apt-get install --no-install-recommends -y tini \
-  && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
-  && apt-get clean -y && rm -rf /var/lib/apt/lists/*
-
-# replace systemd with dummy init
-ENTRYPOINT ["/usr/bin/tini", "--"]
-CMD ["/bin/sleep", "inf"]
\ No newline at end of file

manager/containers/debian/Containerfile

-FROM docker.io/debian:latest
-
-ENV \
-  # build:
-  BUILD_ONLY_PACKAGES='wget' \
-  # python:
-  PYTHONFAULTHANDLER=1 \
-  PYTHONUNBUFFERED=1 \
-  PYTHONHASHSEED=random \
-  PYTHONDONTWRITEBYTECODE=1 \
-  # pip:
-  PIP_NO_CACHE_DIR=off \
-  PIP_DISABLE_PIP_VERSION_CHECK=on \
-  PIP_DEFAULT_TIMEOUT=100 \
-  # poetry:
-  POETRY_VERSION=1.1.5 \
-  POETRY_NO_INTERACTION=1 \
-  POETRY_VIRTUALENVS_CREATE=false \
-  POETRY_CACHE_DIR='/var/cache/pypoetry' \
-  PATH="$PATH:/root/.poetry/bin" \
-  NODE_VERSION=node_14.x
-ENV LC_ALL=C.UTF-8
-
-# System deps:
-RUN apt-get update \
-  && apt-get install --no-install-recommends -y \
-    bash \
-    build-essential git ca-certificates \
-    python3 python3-pip python3-dev python3-setuptools python3-wheel \
-    libcairo2-dev libgirepository1.0-dev \
-    gettext \
-    systemd \
-    curl \
-    dbus \
-    libcairo2-dev libgirepository1.0-dev \
-    # Defining build-time-only dependencies:
-    $BUILD_ONLY_PACKAGES \
-  # Install Knot Resolver
-  && wget https://secure.nic.cz/files/knot-resolver/knot-resolver-release.deb \
-  && dpkg -i knot-resolver-release.deb \
-  && rm knot-resolver-release.deb \
-  && apt-get update && apt-get install -y --no-install-recommends knot-resolver \
-  # Installing `poetry` package manager:
-  # https://github.com/python-poetry/poetry
-  && curl -sSL 'https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py' | python3 \
-  && poetry --version \
-  # Install test dependencies
-  && apt-get install --no-install-recommends procps curl \
-  && pip3 install requests requests-unixsocket \
-  # Removing build-time-only dependencies:
-  && apt-get remove -y $BUILD_ONLY_PACKAGES \
-  # Cleaning cache:
-  && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
-  && apt-get clean -y && rm -rf /var/lib/apt/lists/*
-
-# Create knot-resolver-manager systemd service
-COPY ./config/knot-resolver-manager.service /etc/systemd/system
-
-# Copy knot-resolver-manager YAML configuration file
-COPY ./config/kres-manager.yaml /etc/knot-resolver
-
-# Copy only requirements, to cache them in docker layer
-# no poetry.lock, because here we have a different python version
-COPY ./pyproject.toml /code/
-
-WORKDIR /code
-
-# Install project dependencies
-RUN poetry --version \
-  # Aghghgh! The packaged pip is buggy and just plainly fails installing dependencies
-  # so here we update it
-  && python3 -m pip --version \
-  && python3 -m pip install -U pip \
-  && python3 -m pip --version \
-  # and install the dependencies
-  && poetry install --no-dev --no-interaction --no-ansi
-
-# Here, we would copy the remaining code if we wanted to permanently keep it in the container. We don't do that, we use read-only bind mounts
-# COPY . /code
-
-CMD ["/bin/systemd"]
\ No newline at end of file

manager/containers/dev/Containerfile

-# source: https://github.com/wemake-services/wemake-django-template/blob/master/%7B%7Bcookiecutter.project_name%7D%7D/docker/django/Dockerfile
-
-# This Dockerfile uses multi-stage build to customize DEV and PROD images:
-# https://docs.docker.com/develop/develop-images/multistage-build/
-
-FROM docker.io/python:3.6.13-slim-buster
-
-# Agh, this would ideally be an ARG command, but podman has problems caching the build
-ENV KNOT_ENV=dev
-
-ENV \
-  # build:
-  BUILD_ONLY_PACKAGES='wget lsb-release gnupg' \
-  # python:
-  PYTHONFAULTHANDLER=1 \
-  PYTHONUNBUFFERED=1 \
-  PYTHONHASHSEED=random \
-  PYTHONDONTWRITEBYTECODE=1 \
-  # pip:
-  PIP_NO_CACHE_DIR=off \
-  PIP_DISABLE_PIP_VERSION_CHECK=on \
-  PIP_DEFAULT_TIMEOUT=100 \
-  # poetry:
-  POETRY_VERSION=1.1.5 \
-  POETRY_NO_INTERACTION=1 \
-  POETRY_VIRTUALENVS_CREATE=false \
-  POETRY_CACHE_DIR='/var/cache/pypoetry' \
-  PATH="$PATH:/root/.poetry/bin" \
-  NODE_VERSION=node_14.x
-
-
-# System deps:
-RUN apt-get update \
-  && apt-get install --no-install-recommends -y \
-    bash \
-    build-essential \
-    curl \
-    gettext \
-    git \
-    systemd \
-    dbus \
-    libcairo2-dev libgirepository1.0-dev \
-    # Defining build-time-only dependencies:
-    $BUILD_ONLY_PACKAGES \
-  # Install Knot Resolver
-  && wget https://secure.nic.cz/files/knot-resolver/knot-resolver-release.deb \
-  && dpkg -i knot-resolver-release.deb \
-  && rm knot-resolver-release.deb \
-  && apt-get update && apt-get install -y --no-install-recommends knot-resolver \
-  # Installing NodeJS
-  && curl -sSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
-  && echo "deb https://deb.nodesource.com/$NODE_VERSION $(lsb_release -s -c) main" | tee /etc/apt/sources.list.d/nodesource.list \
-  && echo "deb-src https://deb.nodesource.com/$NODE_VERSION $(lsb_release -s -c) main" | tee -a /etc/apt/sources.list.d/nodesource.list \
-  && apt-get update && apt-get install --no-install-recommends --no-install-suggests -y nodejs \
-  # Installing `poetry` package manager:
-  # https://github.com/python-poetry/poetry
-  && curl -sSL 'https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py' | python \
-  && poetry --version \
-  # Removing build-time-only dependencies:
-  && apt-get remove -y $BUILD_ONLY_PACKAGES \
-  # Cleaning cache:
-  && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
-  && apt-get clean -y && rm -rf /var/lib/apt/lists/*
-
-# Create knot-resolver-manager systemd service
-COPY ./config/knot-resolver-manager.service /etc/systemd/system
-
-# Copy knot-resolver-manager YAML configuration file
-COPY ./config/kres-manager.yaml /etc/knot-resolver
-
-# Copy only requirements, to cache them in docker layer
-COPY ./poetry.lock ./pyproject.toml ./package.json /code/
-
-WORKDIR /code
-
-# Install project dependencies
-RUN echo "Running in $KNOT_ENV" \
-  && poetry install \
-    $(if [ "$KNOT_ENV" != 'dev' ]; then echo '--no-dev'; fi) \
-    --no-interaction --no-ansi \
-  && if test "$KNOT_ENV" = "dev"; then \
-   npm install -g $(python -c "import json; print(*(k for k in json.loads(open('package.json').read())['dependencies']))"); fi
-
-# Here, we would copy the remaining code if we wanted to permanently keep it in the container. We don't do that, we use read-only bind mounts
-# COPY . /code
-
-CMD ["/bin/systemd"]
\ No newline at end of file

manager/package.json

-{
-  "dependencies": {
-    "pyright": "^1.1.108"
-  }
-}

manager/pyproject.toml

@@ -41,12 +41,13 @@ mypy = "^0.930"
 types-click = "^7.1.8"
 types-Jinja2 = "^2.11.9"
 types-dataclasses = "^0.6.4"
+poetry = "^1.1.12"
 
 [tool.poe.tasks]
 run = { cmd = "scripts/run", help = "Run the manager" }
 run-debug = { cmd = "scripts/run-debug", help = "Run the manager under debugger" }
 docs = { cmd = "scripts/docs", help = "Create HTML documentation" }
-test = { shell = "env PYTHONPATH=. pytest --cov=knot_resolver_manager --show-capture=all tests/unit/", help = "Run tests" }
+test = { shell = "env PYTHONPATH=. pytest --junitxml=unit.junit.xml --cov=knot_resolver_manager --show-capture=all tests/unit/", help = "Run tests" }
 check = { cmd = "scripts/codecheck", help = "Run static code analysis" }
 format = { shell = "black knot_resolver_manager/ tests/ scripts/; isort .", help = "Run code formatter" }
 fixdeps = { shell = "poetry install; npm install; npm update", help = "Install/update dependencies according to configuration files"}

manager/scripts/_env.sh

@@ -6,6 +6,7 @@ red="\033[0;31m"
 yellow="\033[0;33m"
 green="\033[0;32m"
 bright_black="\033[0;90m"
+blue="\033[0;34m"
 reset="\033[0m"
 
 # ensure consistent top level directory
@@ -14,7 +15,7 @@ if test -z "$gitroot"; then
 	echo -e "${red}This command can be run only in a git repository tree.${reset}"
 	exit 1
 fi
-cd $gitroot
+cd $gitroot/manager
 
 # ensure consistent environment with virtualenv
 if test -z "$VIRTUAL_ENV" -a "$CI" != "true" -a -z "$KNOT_ENV"; then
@@ -31,4 +32,4 @@ fi
 PATH="$PATH:$gitroot/node_modules/.bin"
 
 # fail even on unbound variables
-set -o nounset
\ No newline at end of file
+set -o nounset

manager/scripts/codecheck

@@ -34,12 +34,6 @@ flake8 knot_resolver_manager
 check_rv $?
 echo
 
-# check types with pyright
-echo -e "${yellow}Type checking using pyright...${reset}"
-pyright knot_resolver_manager
-check_rv $?
-echo
-
 # check types with mypy
 echo -e "${yellow}Type checking using mypy...${reset}"
 mypy knot_resolver_manager