ci: docker image setup

manager/.gitlab-ci.yml

 stages:
+  - build
   - check
 
-image: debian:latest
+image: $CI_REGISTRY/knot/knot-resolver-manager/ci/devenv:latest
+
+build:
+  image: docker:19.03.12
+  stage: build
+  services:
+    - docker:19.03.12-dind
+  variables:
+    IMAGE_TAG: $CI_REGISTRY/knot/knot-resolver-manager/ci/devenv:latest
+  script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker build -t $IMAGE_TAG ci/devenv
+    - docker push $IMAGE_TAG
 
 # FIXME - use custom Docker image, so that we don't rebuild it all every single time
 lint:
@@ -11,45 +24,12 @@ lint:
     # Python by default uses ascii encoding, which does not work, because source files are in UTF-8. Here is a potential solution, but for newer Python https://www.python.org/dev/peps/pep-0540/
     LC_ALL: C.UTF-8
 
-  # this job uses messy tricks to install dependencies quickly, if something goes wrong, it should kill itself reasonably fast
-  timeout: 7 minutes
-  
-  # FIXME cache does not seem to have any effect when running locally, it might be the same in CI
   cache:
-    key: "always-the-same-cache"
+    key: "check:always-the-same-cache"
     paths:
       - node_modules/
-      # FIXME caching these would help, but gitlab refuses to cache anything outside of project's directory
-      #- /root/.pyenv/  
-      #- /root/.poetry/
-      #- /root/.nvm/
-      #- /var/cache/
     policy: pull-push
   script:
-    # pyenv setup deps
-    - apt-get update && apt-get install --no-install-recommends --no-install-suggests -y build-essential git ca-certificates
-    # python build deps
-    - apt-get update && apt-get install --no-install-recommends --no-install-suggests -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev
-    # Yarn and NodeJS
-    - curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh | bash
-    - export NVM_DIR="$HOME/.nvm"
-    - \[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"  # This loads nvm
-    - \[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion"  # This loads nvm bash_completion
-    - nvm install --lts node
-    - nvm use node
-    - nvm install-latest-npm
-    # install pyenv
-    - unset PYENV_ROOT
-    - curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
-    - export PATH="$HOME/.pyenv/bin:$PATH"
-    - eval "$(pyenv init -)"
-    - eval "$(pyenv virtualenv-init -)"
-    # install python via pyenv
-    - pyenv install
-    - pyenv global 3.6.12
-    # install poetry
-    - curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python -
-    - export PATH="$HOME/.poetry/bin:$PATH"
     # run the actual tests
     - poetry install
     - npm install

manager/.python-version

-3.6.12
\ No newline at end of file
+3.6.12
+3.7.9
+3.8.7
+3.9.1

manager/README.md

@@ -11,16 +11,21 @@ Install these tools:
 * [Poetry](https://python-poetry.org/docs/#installation)
 * [Yarn](https://yarnpkg.com/) (See FAQ for why do we need JS in Python project) or NPM
 
-The actual fully-featured development environment can be setup using these commands:
+Be careful, that you need the latest version of Poetry. The setup was tested with Poetry version 1.1.4. Due to it's ability to switch between Python versions, it has to be installed separately to work correctly. Make sure to follow [the latest setup guide](https://python-poetry.org/docs/#installation).
+
+After installing the tools above, the actual fully-featured development environment can be setup using these commands:
 
 ```sh
-pyenv install 3.6.12 3.7.9 3.8.7 3.9.1
+pyenv install 3.6.12
+pyenv install 3.7.9
+pyenv install 3.8.7
+pyenv install 3.9.1
 poetry env use $(pyenv which python)
 poetry install
 yarn install # or "npm install"
 ```
 
-With this environment, everything else should just work. You can run the same checks the CI runs, all commands listed bellow should pass.
+With this environment, **everything else should just work**. You can run the same checks the CI runs, all commands listed bellow should pass.
 
 ### Minimal development environment
 
@@ -32,7 +37,7 @@ After setting up the environment, you should be able to interract with the proje
 
 * `poe run` - runs the manager from the source
 * `poe test` - unit tests
-* `poe tox` - unit tests in all supported Python versions
+* `poe tox` - unit tests in all supported Python versions (must not be run outside of virtualenv, otherwise it fails to find multiple versions of Python)
 * `poe check` - static code analysis
 * `poe fixdeps` - update installed dependencies according to the project's configuration
 * `poe clean` - cleanup the repository from unwanted files

manager/ci/devenv/Dockerfile

+FROM debian:latest
+
+ENV LC_ALL=C.UTF-8
+
+# pyenv setup deps
+RUN apt-get update && apt-get install --no-install-recommends --no-install-suggests -y build-essential git ca-certificates
+
+# python build deps
+RUN apt-get update && apt-get install --no-install-recommends --no-install-suggests -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev
+
+# Yarn and NodeJS (following https://github.com/nodesource/distributions/blob/master/README.md#debmanual)
+RUN apt-get update && apt-get install --no-install-recommends --no-install-suggests -y gnupg lsb-release
+RUN curl -sSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -
+ENV VERSION=node_14.x
+RUN echo "deb https://deb.nodesource.com/$VERSION $(lsb_release -s -c) main" | tee /etc/apt/sources.list.d/nodesource.list
+RUN echo "deb-src https://deb.nodesource.com/$VERSION $(lsb_release -s -c) main" | tee -a /etc/apt/sources.list.d/nodesource.list
+RUN apt-get update && apt-get install --no-install-recommends --no-install-suggests -y nodejs
+RUN npm install -g yarn
+
+# Add non-root user
+RUN useradd -m -s /bin/bash user
+USER user
+WORKDIR /home/user
+
+# install pyenv
+RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
+ENV PATH="/home/user/.pyenv/bin:$PATH"
+# These would should be run in the shell that will be running pyenv. But it can hopefully work without it
+# RUN eval "$(pyenv init -)"
+# RUN eval "$(pyenv virtualenv-init -)"
+
+# install all required versions of python via pyenv
+RUN pyenv install 3.6.12
+RUN pyenv install 3.7.9
+RUN pyenv install 3.8.7
+RUN pyenv install 3.9.1
+
+# install poetry
+USER root
+RUN apt-get update && apt-get install --no-install-recommends --no-install-suggests -y python3
+USER user
+
+RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python3 -
+# Poetry installer manages to change the PATH somehow, so we do not need this
+# ENV PATH="/home/user/.poetry/bin:$PATH"