lint: new command to invoke native distro linters

lintian for deb, rpmlint for rpm.

Support common options for a relatively consistent use.

Improve `apkg system-setup` on the occasion of adding --lint-dep to
install linting dependencies.

Fixes: #94

apkg/commands/lint.py

+import click
+
+from apkg import adistro
+from apkg.commands.build import build
+from apkg.commands.srcpkg import srcpkg as make_srcpkg
+from apkg.commands.system_setup import system_setup
+from apkg import ex
+from apkg.log import getLogger
+from apkg import pkgstyle
+from apkg.util import common
+
+
+log = getLogger(__name__)
+
+
+@click.command(name="lint")
+@click.argument('input_files', nargs=-1)
+@click.option('-p', '--pedantic', is_flag=True,
+              help="enable extra / all warnings and infos")
+@click.option('-i', '--info', is_flag=True,
+              help="give detailed info about warnings")
+@click.option('-s', '--strict', is_flag=True,
+              help="treat all messages as errors")
+@click.option('-d', '--distro',
+              help="override target distro  [default: current]")
+@click.option('-L', '--lint-dep', is_flag=True,
+              help="install linting dependencies on host"
+                   " (apkg system-setup --lint)")
+@click.option('--cache/--no-cache', default=True, show_default=True,
+              help="enable/disable cache")
+@click.option('-F', '--file-list', 'input_file_lists', multiple=True,
+              help=("specify text file listing one input file per line"
+                    ", use '-' to read from stdin"))
+@click.help_option('-h', '--help',
+                   help="show this help message")
+def cli_lint(*args, **kwargs):
+    """
+    run native distro linter on packages
+
+    Default: build packages from source and lint them
+
+    You can supply files to lint as arguments or use --file-list.
+
+    Use --pedantic to run extra / all checks.
+
+    Use --info to get details explanations about individual messages.
+
+    Use --strict to treat all messages as errors (useful for pedantic CI).
+
+    To install linter packages on host system use --lint-dep which calls
+
+        apkg system-setup --lint
+
+    before linting.
+    """
+    r = lint(*args, **kwargs)
+    if r != 0:
+        raise ex.LintingFailed(
+            fail=f"linter returned error code {r}")
+    else:
+        log.success("linting successful")
+    return r
+
+
+def lint(
+        input_files=None,
+        input_file_lists=None,
+        pedantic=False,
+        info=False,
+        strict=False,
+        distro=None,
+        lint_dep=False,
+        cache=True):
+    """
+    run native distro linter on packages
+    """
+    log.bold("linting packages")
+
+    distro = adistro.distro_arg(distro)
+    log.info("target distro: %s", distro)
+
+    ps = pkgstyle.get_pkgstyle_for_distro(distro)
+    if not ps:
+        raise ex.DistroNotSupported(distro=distro)
+    log.info("target pkgstyle: %s", ps.name)
+
+    if lint_dep:
+        system_setup(lint=True)
+
+    infiles = common.parse_input_files(input_files, input_file_lists)
+
+    if not infiles:
+        # default: use srcpkg and build to get packages to lint
+        infiles = make_srcpkg(
+            distro=distro,
+            cache=cache)
+        infiles += build(
+            distro=distro,
+            cache=cache)
+
+    try:
+        result = pkgstyle.call_pkgstyle_fun(
+            ps, 'lint',
+            infiles,
+            pedantic=pedantic,
+            info=info,
+            strict=strict,
+            distro=distro)
+    except ex.CommandNotFound as e:
+        msg = str(e) + ("\n\nInstall lint deps:\n\n"
+                        "    apkg system-setup --lint\n\n"
+                        "or:\n\n"
+                        "    apkg lint --lint-dep")
+        raise ex.CommandNotFound(msg=msg)
+
+    return result
+
+
+APKG_CLI_COMMANDS = [cli_lint]

apkg/commands/system_setup.py

@@ -10,30 +10,71 @@ log = getLogger(__name__)
 
 
 @click.command(name='system-setup')
+@click.option('-c', '--core', is_flag=True,
+              help="install core packages for direct package builds  [default]")
 @click.option('-I', '--isolated', is_flag=True,
-              help="also install packages for isolated build")
+              help="install packages for isolated package builds")
+@click.option('-L', '--lint', is_flag=True,
+              help="install packages for linting (apkg lint)")
+@click.option('-a', '--all', is_flag=True,
+              help="install all of above (-cIL)")
 @click.option('-d', '--distro',
               help="override target distro  [default: current]")
 @click.option('--ask/--no-ask', 'interactive',
               default=False, show_default=True,
               help="enable/disable interactive mode")
+@click.help_option('-h', '--help',
+                   help="show this help message")
 def cli_system_setup(*args, **kwargs):
     """
     setup system for packaging
+
+    Install native distro packages required for packaging.
+
+    Select desired packages with a combination of
+
+    -c / --core: core packages for direct package builds  [default]
+
+    -I / --isolated`: packages for isolated package builds
+
+    -L / --lint`: packages for linting (apkg lint)
+
+    or
+
+    -a / --all to select all above.
+
+    Defaults to --core when no options are supplied.
     """
     return system_setup(*args, **kwargs)
 
 
 def system_setup(
+        core=False,
         isolated=False,
+        lint=False,
+        all=False,
         distro=None,
         interactive=False):
     """
     setup system for packaging
-
-    Install native distro packages required for packaging.
     """
-    log.bold("system setup for packaging")
+    cats_opts = {
+        'core': core,
+        'isolated': isolated,
+        'lint': lint,
+    }
+    if all:
+        # install all categories
+        cats = cats_opts.keys()
+    else:
+        # nstall only selected categories
+        cats = [cat for cat, e in cats_opts.items() if e]
+        if not cats:
+            # default to --core
+            cats = ['core']
+
+    cats_txt = ", ".join(cats)
+    log.bold("system setup: %s", cats_txt)
 
     distro = adistro.distro_arg(distro)
     log.info("target distro: %s", distro)
@@ -44,9 +85,10 @@ def system_setup(
     log.info("target pkgstyle: %s", style.name)
 
     distro_reqs = getattr(style, 'DISTRO_REQUIRES', {})
-    reqs = distro_reqs.get('core', [])
-    if isolated:
-        reqs += distro_reqs.get('isolated', [])
+    reqs = []
+    for cat in cats:
+        reqs += distro_reqs.get(cat, [])
+
     if reqs:
         pkgstyle.call_pkgstyle_fun(
             style, 'install_distro_packages',
@@ -56,7 +98,7 @@ def system_setup(
     else:
         log.info("no distro packages required")
 
-    log.success("system ready for packaging")
+    log.success("system setup successful: %s", cats_txt)
 
 
 APKG_CLI_COMMANDS = [cli_system_setup]

apkg/ex.py

@@ -115,6 +115,10 @@ class ParsingFailed(ApkgException):
     msg_fmt = "Unable to parse: {fail}"
     returncode = 42
 
+class LintingFailed(ApkgException):
+    msg_fmt = "Linting failed: {fail}"
+    returncode = 48
+
 
 # 50-59: remote failures
 class FileDownloadFailed(ApkgException):

apkg/pkgstyles/deb.py

@@ -48,6 +48,7 @@ SUPPORTED_DISTROS = [
 DISTRO_REQUIRES = {
     'core': ['build-essential'],
     'isolated': ['pbuilder'],
+    'lint': ['lintian'],
 }
 
 
@@ -346,6 +347,28 @@ def get_build_deps_from_srcpkg(
     return get_build_deps_from_control_(control_text)
 
 
+def lint(
+        pkg_paths,
+        pedantic=False,
+        info=False,
+        strict=False,
+        **kwargs):
+    """
+    lint files using lintian
+    """
+    log.info('linting %s files with lintian', len(pkg_paths))
+    cmd = ['lintian']
+    if pedantic:
+        cmd += ['--display-info', '--pedantic']
+    if info:
+        cmd += ['--info']
+    if strict:
+        cmd += ['--fail-on', 'pedantic']
+    cmd += pkg_paths
+    o = run(cmd, check=False, direct=True)
+    return o.returncode
+
+
 # functions bellow with _ postfix are specific to this pkgstyle
 
 

apkg/pkgstyles/rpm.py

@@ -53,6 +53,7 @@ SUPPORTED_DISTROS = sorted(EL_FAMILY_DISTROS + [
 DISTRO_REQUIRES = {
     'core': ['rpm-build'],
     'isolated': ['mock'],
+    'lint': ['rpmlint'],
 }
 
 
@@ -261,6 +262,27 @@ def get_build_deps_from_srcpkg(
     return get_build_deps_from_spec_(spec_text)
 
 
+def lint(
+        pkg_paths,
+        info=False,
+        strict=False,
+        distro=None,
+        **kwargs):
+    """
+    lint files using rpmlint
+    """
+    log.info('linting %s files with rpmlint', len(pkg_paths))
+    suse = distro and distro.id == 'opensuse'
+    cmd = ['rpmlint']
+    if info:
+        cmd += ['--info']
+    if strict and not suse:
+        cmd += ['--strict']
+    cmd += pkg_paths
+    o = run(cmd, check=False, direct=True)
+    return o.returncode
+
+
 # functions bellow with _ postfix are specific to this pkgstyle
 
 

apkg/util/run.py

@@ -114,10 +114,18 @@ def run(cmd,
         except FileNotFoundError:
             raise ex.CommandNotFound(cmd=cmd_str)
     else:
+        if direct:
+            # reuse host process stdout/stderr directly
+            stdout, stderr = None, None
+        else:
+            # this is quivalent of capture_output=True,
+            # but we support old pythons
+            stdout, stderr = subprocess.PIPE, subprocess.PIPE
         try:
             result = subprocess.run(
                 cmd,
-                capture_output=not direct,
+                stdout=stdout,
+                stderr=stderr,
                 check=False,
                 universal_newlines=True,
                 **kwargs)

docs/commands.md

@@ -105,6 +105,13 @@ Resulting packages are cached if [cache.local](config.md#cachelocal) is enabled.
 {{ 'install' | cmd_help }}
 
 
+## lint
+
+{{ 'lint' | cmd_help }}
+
+{{ added_in_version('0.6.0') }}
+
+
 ## clean
 
 {{ 'clean' | cmd_help }}