Commit 278069e2 authored by Jakub Ružička's avatar Jakub Ružička
Browse files

build-dep: support templates, srcpkgs, archives

A comprehensive refactor of build-dep and related pkgstyle functionality
to install or list (-l/--list) build deps from templates as well as
from source packages.

The new default approach of parsing build deps from templates has
following advantages:

* works directly on project source without archive/srcpkg build
* requires less tools to work
* is faster

Alternatively, -s/--srcpkg makes apkg parse build deps from source
package which is slower but more robust in some cases.

Build deps can also be extracted from archive containing proper distro/
files with -a/--archive option in both template and --srcpkg mode.

-u/--upstream option uses upstream archive (get-archive) to get build
deps for maximum convenience.

Fixes: #34
parent 4a6ea875
Pipeline #80524 passed with stage
in 2 minutes and 13 seconds
......@@ -7,12 +7,12 @@ Usage: apkg <command> [<args>...]
Commands:
status show status of current project
make-archive create dev archive from current project state
make-archive create dev archive from current project
get-archive download upstream source archive
srcpkg create source package (files to build packages from)
build build package
build-dep install build dependencies
install install packages
srcpkg create source package (to build packages from)
build build packages
build-dep install or list build dependencies
install install local or distro packages
Options:
-h --help show help screen, can be used after a command
......
"""
install build dependencies
install or list build dependencies
Usage: apkg build-dep [-u] [-s | -a] [<file> | -F <file-list>]...
Usage: apkg build-dep [-l] [-u] [-s] [-a] [<file> | -F <file-list>]...
[-d <distro>] [-y]
Arguments:
......@@ -10,12 +10,15 @@ Arguments:
use '-' to read from stdin
Options:
-u, --upstream use upstream archive / apkg get-archive
default: dev archive / apkg make-archive
-s, --srcpkg use source package <file>s
-a, --archive use archive <files>s
default: use dev archvie / apkg make-source
-d, --distro <distro> set target distro
-l, --list list buid deps only, don't install
default: install deps
-u, --upstream use upstream template / archive / srcpkg
default: use dev template / archive / srcpkg
-s, --srcpkg use source package
default: use template
-a, --archive use template (/build srcpkg) from archive
default: use dev template
-d, --distro <distro> override target distro
default: current distro
-y, --yes non-interactive mode
default: interactive (distro tool defualts)
......@@ -23,21 +26,18 @@ Options:
from docopt import docopt
from apkg.lib import build
from apkg.lib import deps
def run_command(cargs):
args = docopt(__doc__, argv=cargs)
# XXX: this will be refactored in following patch, disabling for now
# https://gitlab.nic.cz/packaging/apkg/-/issues/34
# pylint: disable=unreachable
raise NotImplementedError
return build.install_build_deps(
return deps.build_dep(
list_only=args['--list'],
upstream=args['--upstream'],
srcpkg=args['--srcpkg'],
archive=args['--archive'],
upstream=args['--upstream'],
version=args['--version'],
release=args['--release'],
input_files=args['<file>'],
input_file_lists=args['--file-list'],
distro=args['--distro'],
interactive=not args['--yes'],
)
......@@ -7,7 +7,7 @@ from apkg import adistro
from apkg.cache import file_checksum
from apkg import ex
from apkg.lib import srcpkg as _srcpkg
from apkg.lib import common
from apkg.lib import common, deps
from apkg.log import getLogger
from apkg.project import Project
import apkg.util.shutil35 as shutil
......@@ -74,9 +74,11 @@ def build_package(
else:
# install build deps if requested
try:
install_build_deps(
srcpkg=srcpkg_path,
distro=distro)
deps.build_dep(
srcpkg=True,
input_files=[srcpkg_path],
distro=distro,
project=proj)
except ex.DistroNotSupported as e:
log.warning("%s - SKIPPING", e)
......@@ -122,47 +124,3 @@ def build_package(
cache_name, cache_key, fns)
return pkgs
def install_build_deps(
srcpkg=None,
archive=None,
upstream=False,
version=None,
release=None,
distro=None,
interactive=False):
log.bold('installing build deps')
proj = Project()
distro = adistro.distro_arg(distro)
log.info("target distro: %s", distro)
# fetch pkgstyle (deb, rpm, arch, ...)
template = proj.get_template_for_distro(distro)
pkgstyle = template.pkgstyle
if not hasattr(pkgstyle, 'install_build_deps'):
msg = "build deps installation isn't supported on distro: %s"
raise ex.DistroNotSupported(msg % distro)
if srcpkg:
# use existing source package
srcpkg_path = Path(srcpkg)
if not srcpkg_path.exists():
raise ex.SourcePackageNotFound(
srcpkg=srcpkg, type=distro)
log.info("using existing source package: %s", srcpkg_path)
else:
# make source package
srcpkg_path = _srcpkg.make_srcpkg(
archive=archive,
version=version,
release=release,
distro=distro,
upstream=upstream)[0]
pkgstyle.install_build_deps(
srcpkg_path,
distro=distro,
interactive=interactive)
from contextlib import contextmanager
from pathlib import Path
import sys
import tempfile
from apkg import ex
from apkg.log import getLogger
......@@ -84,6 +86,24 @@ def ensure_input_files(infiles):
raise ex.InvalidInput(
fail="no input file(s) specified")
for f in infiles:
if not f.exists():
if not f or not f.exists():
raise ex.InvalidInput(
fail="input file not found: %s" % f)
@contextmanager
def text_tempfile(text, prefix='apkg_tmp_'):
"""
write text to a new temporary file and return its path
file is deleted after use
"""
f = tempfile.NamedTemporaryFile(
prefix=prefix, mode='w+t', delete=False)
path = Path(f.name)
f.write(text)
f.close()
try:
yield path
finally:
path.unlink()
"""
apkg lib for handling (build) dependencies
"""
from apkg import adistro
from apkg.pkgstyle import call_pkgstyle_fun
from apkg.lib import ar
from apkg.lib import common
from apkg.lib import srcpkg as _srcpkg
from apkg.log import getLogger
from apkg.project import Project
from apkg.util.archive import unpack_archive
log = getLogger(__name__)
def build_dep(
upstream=False,
srcpkg=False,
archive=False,
input_files=None,
input_file_lists=None,
list_only=False,
distro=None,
interactive=False,
project=None):
action = 'listing' if list_only else 'installing'
log.bold('%s build deps', action)
proj = project or Project()
distro = adistro.distro_arg(distro)
log.info("target distro: %s", distro)
# fetch pkgstyle (deb, rpm, arch, ...)
template = proj.get_template_for_distro(distro)
pkgstyle = template.pkgstyle
infiles = common.parse_input_files(input_files, input_file_lists)
if srcpkg:
# use source package to determine deps
if archive or not infiles:
# build source package
srcpkg_files = _srcpkg.make_srcpkg(
archive=archive,
distro=distro,
input_files=input_files,
input_file_lists=input_file_lists,
upstream=upstream,
project=proj)
else:
# use specified source package
srcpkg_files = infiles
common.ensure_input_files(srcpkg_files)
srcpkg_path = srcpkg_files[0]
log.info("build deps from srcpkg: %s", srcpkg_path)
deps = call_pkgstyle_fun(
pkgstyle, 'get_build_deps_from_srcpkg',
srcpkg_path)
else:
# use tempalte to determine deps
if archive:
archive_files = infiles
if upstream:
archive = True
archive_files = ar.get_archive(project=proj)
if archive:
common.ensure_input_files(archive_files)
archive_path = archive_files[0]
log.info("unpacking archive: %s", archive_path)
unpack_path = unpack_archive(
archive_path, proj.unpacked_archive_path)
log.info("loading template from archive: %s", unpack_path)
# load project with input_path from archive
aproj = Project(path=unpack_path)
template = aproj.get_template_for_distro(distro)
log.info("build deps from template: %s", template.path)
deps = call_pkgstyle_fun(
pkgstyle, 'get_build_deps_from_template',
template.path, distro=distro)
if list_only:
for dep in deps:
print(dep)
return deps
log.info("installing %s build deps...", len(deps))
return call_pkgstyle_fun(
pkgstyle, 'install_build_deps',
deps,
distro=distro,
interactive=interactive)
......@@ -5,9 +5,14 @@ dynamically added because apkg.pkgstyles is PEP 420 namespace package.
import importlib
import pkgutil
from apkg import ex
from apkg.log import getLogger
import apkg.pkgstyles
log = getLogger(__name__)
def iter_pkgstyles():
return pkgutil.iter_modules(
apkg.pkgstyles.__path__,
......@@ -45,4 +50,22 @@ def get_pkgstyle(style):
return PKGSTYLES.get(style)
def ensure_pkgstyle_fun(pkgstyle, fun):
f = getattr(pkgstyle, fun, None)
if not f:
msg = "%s pkgstyle is missing required function: %s"
elif not callable(f):
msg = "%s pkgstyle error: not a function: %s"
else:
return f
raise ex.DistroNotSupported(
msg % (pkgstyle.__name__, fun))
def call_pkgstyle_fun(pkgstyle, fun, *args, **kwargs):
f = ensure_pkgstyle_fun(pkgstyle, fun)
log.verbose("calling %s function: %s", pkgstyle.__name__, fun)
return f(*args, **kwargs)
PKGSTYLES = import_pkgstyles()
......@@ -9,9 +9,11 @@ apkg package style for **Arch** linux.
"""
import glob
from pathlib import Path
import sys
from apkg import ex
from apkg.log import getLogger
from apkg import pkgtemplate
from apkg.util.run import cd, run, sudo
import apkg.util.shutil35 as shutil
......@@ -33,16 +35,11 @@ def is_valid_template(path):
def get_template_name(path):
return _parse_pkgbuild(path / 'PKGBUILD', '$pkgname')
return parse_pkgbuild_(path / 'PKGBUILD', 'echo "$pkgname"')
def get_srcpkg_nvr(path):
return _parse_pkgbuild(path, '$pkgname-$pkgver-$pkgrel')
def _parse_pkgbuild(pkgbuild, bash):
return run('bash', '-c', '. "%s" && echo "%s"'
% (pkgbuild, bash), log_cmd=False)
return parse_pkgbuild_(path, 'echo "$pkgname-$pkgver-$pkgrel"')
def build_srcpkg(
......@@ -92,23 +89,75 @@ def build_packages(
return pkgs
def install_custom_packages(
def install_distro_packages(
packages,
**kwargs):
interactive = kwargs.get('interactive', False)
cmd = ['pacman', '-U']
cmd = ['pacman', '-S']
if not interactive:
cmd += ['--noconfirm']
cmd += packages
sudo(*cmd, direct=True)
def install_distro_packages(
def install_custom_packages(
packages,
**kwargs):
interactive = kwargs.get('interactive', False)
cmd = ['pacman', '-S']
cmd = ['pacman', '-U']
if not interactive:
cmd += ['--noconfirm']
cmd += packages
sudo(*cmd, direct=True)
def install_build_deps(
deps,
**kwargs):
# no special handling for build deps on arch
install_distro_packages(deps, **kwargs)
def get_build_deps_from_template(
template_path,
**kwargs):
"""
parse depends from packaging template
"""
distro = kwargs.get('distro')
# render PKGBUILD
this_style = sys.modules[__name__]
t = pkgtemplate.PackageTemplate(template_path, style=this_style)
env = pkgtemplate.DUMMY_ENV.copy()
if distro:
env['distro'] = distro
pkgbuild_text = t.render_file_content('PKGBUILD', env=env)
deps = parse_pkgbuild_content_(
pkgbuild_text,
'printf \'%s\n\' "${depends[@]}"')
return deps.splitlines()
def get_build_deps_from_srcpkg(
srcpkg_path,
**_):
"""
parse depends from source package (PKGBUILD)
"""
deps = parse_pkgbuild_(
srcpkg_path,
'printf \'%s\n\' "${depends[@]}"')
return deps.splitlines()
# functions bellow with _ postfix are specific to this pkgstyle
def parse_pkgbuild_(pkgbuild, bash):
return run('bash', '-c', '. "%s" && %s'
% (pkgbuild, bash), log_cmd=False)
def parse_pkgbuild_content_(pkgbuild_text, bash):
pkgb = '%s\n%s' % (pkgbuild_text, bash)
return run('bash', '-s', input=pkgb, log_cmd=False)
......@@ -12,10 +12,13 @@ or `--isolated` using `pbuilder`
import glob
from pathlib import Path
import re
import sys
import tempfile
from apkg import ex
from apkg.log import getLogger
from apkg import parse
from apkg import pkgtemplate
from apkg.util.run import cd, run, sudo
from apkg.util.archive import unpack_archive
import apkg.util.shutil35 as shutil
......@@ -33,6 +36,15 @@ SUPPORTED_DISTROS = [
RE_PKG_NAME = r'Source:\s*(\S+)'
# orbital regexp cannon to parse Build-Depends from debian/control
RE_BUILD_DEPENDS = (
r'(?:\n|\A)Build-Depends:[ \t]*' # no whitespace before
r'(?:\n[ \t]+)?' # optional leading newline with whitespace
r'((?:[^,\n]+)' # first build dep
r'(?:,(?:[ \t]*' # comma separator and optional whitespace
r'(?:\n[ \t]+)?' # optional newline starting with whitespace
r'[^,\n]+))*)' # 0-N other build deps
)
def is_valid_template(path):
......@@ -57,10 +69,11 @@ def get_srcpkg_nvr(path):
return nvr
def _copy_srcpkg_files(src_path, dst_path):
def copy_srcpkg_files(src_path, dst_path):
# not part of pkgstyle interface yet but probably should be
for pattern in [
'*.dsc',
'*_source.*',
'*_source.*', # questionable, some tools need these
'*.debian.tar.*',
'*.orig.tar.*',
'*.diff.*']:
......@@ -69,13 +82,15 @@ def _copy_srcpkg_files(src_path, dst_path):
shutil.copyfile(f, dst_path / srcp.name)
# pylint: disable=too-many-locals
def build_srcpkg(
build_path,
out_path,
archive_paths,
template,
env):
"""
build debian source package
"""
archive_path = archive_paths[0]
nv, _ = parse.split_archive_ext(archive_path.name)
source_path = build_path / nv
......@@ -109,7 +124,7 @@ def build_srcpkg(
log.info("copying source package to result dir: %s", out_path)
out_path.mkdir(parents=True)
_copy_srcpkg_files(build_path, out_path)
copy_srcpkg_files(build_path, out_path)
fns = glob.glob('%s/*' % out_path)
# make sure .dsc is first
for i, fn in enumerate(fns):
......@@ -127,6 +142,9 @@ def build_packages(
out_path,
srcpkg_paths,
**kwargs):
"""
build .deb packages from source package
"""
srcpkg_path = srcpkg_paths[0]
build_path.mkdir(parents=True)
out_path.mkdir(parents=True)
......@@ -172,19 +190,18 @@ def build_packages(
return pkgs
def install_build_deps(
srcpkg_path,
def install_distro_packages(
packages,
**kwargs):
interactive = kwargs.get('interactive', False)
log.info("installing build deps using apt-get build-dep")
cmd = ['apt-get', 'build-dep']
cmd = ['apt-get', 'install']
env = {}
if not interactive:
cmd.append('-y')
env['DEBIAN_FRONTEND'] = 'noninteractive'
cmd += ['-y']
cmd.append(srcpkg_path.resolve())
cmd += packages
sudo(*cmd, env=env, direct=True)
......@@ -194,7 +211,7 @@ def install_custom_packages(
def local_path(pkg):
"""
apt install is able to install local packages
apt-get is able to install local packages
as long as they use full path or relative including ./
"""
p = str(pkg)
......@@ -204,7 +221,7 @@ def install_custom_packages(
interactive = kwargs.get('interactive', False)
cmd = ['apt', 'install']
cmd = ['apt-get', 'install']
env = {}
if not interactive:
env['DEBIAN_FRONTEND'] = 'noninteractive'
......@@ -214,16 +231,108 @@ def install_custom_packages(
sudo(*cmd, env=env, direct=True)
def install_distro_packages(
packages,
def install_build_deps(
deps,
**kwargs):
"""
install debian build deps
Debian Build-Depends can contain strings not handled by
`apt-get install` such as "(>= 9~)"
New `apt-get satisfy` command handles Build-Depends strings fine
but it isn't available on current.
Try to use `apt-get satisfy` if available,
otherwise revert to stripping special strings and use `install`.
"""
interactive = kwargs.get('interactive', False)
cmd = ['apt-get', 'install']
env = {}
if not interactive:
env['DEBIAN_FRONTEND'] = 'noninteractive'
cmd += ['-y']
if has_aptget_satisfy_():
# unlike install, satisfy can handle versioned deps
cmd = ['apt-get', 'satisfy']
env = {}
if not interactive:
env['DEBIAN_FRONTEND'] = 'noninteractive'
cmd += ['-y']
cmd += deps
sudo(*cmd, env=env, direct=True)
else:
# satisfy not available, strip special strings and use install
packages = [strip_dep_(d) for d in deps]
install_distro_packages(packages, **kwargs)
cmd += packages
sudo(*cmd, env=env, direct=True)
def get_build_deps_from_template(
template_path,
**kwargs):
"""
parse Build-Depends from packaging template
"""
distro = kwargs.get('distro')
# render control file
this_style = sys.modules[__name__]
t = pkgtemplate.PackageTemplate(template_path, style=this_style)
env = pkgtemplate.DUMMY_ENV.copy()
if distro:
env['distro'] = distro
control_text = t.render_file_content('control', env=env)
return get_build_deps_from_control_(control_text)
def get_build_deps_from_srcpkg(
srcpkg_path,
**_):
"""
parse Build-Depends from source package
"""
debar_path = get_srcpkg_debian_archive_(srcpkg_path.parent)
log.info("unpacking debian archive: %s", debar_path)
with tempfile.TemporaryDirectory(prefix='apkg_deb_') as td:
unpack_path = unpack_archive(debar_path, td)
control_path = unpack_path / 'control'
control_text = control_path.open().read()
return get_build_deps_from_control_(control_text)
# functions bellow with _ postfix are specific to this pkgstyle
def get_build_deps_from_control_(control_text):
"""