From 52c701bc4267ae454d29b361367649876d6cd158 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ru=C5=BEi=C4=8Dka?= <jakub.ruzicka@nic.cz>
Date: Tue, 25 Feb 2025 16:11:53 +0100
Subject: [PATCH 1/2] util.toml: new robust TOML lib selector / wrapper
python3-toml package is marked as obsolete in EL 10 and Python 3.11
provides builtin TOML loading module tomllib.
apkg.util.toml selects the best available TOML lib for load and dump
while only raising error when TOML functionality is actually used when
no suitable module is available.
---
apkg/commands/info.py | 7 ++--
apkg/ex.py | 5 +++
apkg/project.py | 4 +-
apkg/util/common.py | 10 +++++
apkg/util/serial.py | 0
apkg/util/toml.py | 94 +++++++++++++++++++++++++++++++++++++++++++
pyproject.toml | 1 -
setup.cfg | 1 -
8 files changed, 115 insertions(+), 7 deletions(-)
create mode 100644 apkg/util/serial.py
create mode 100644 apkg/util/toml.py
diff --git a/apkg/commands/info.py b/apkg/commands/info.py
index 1f4ca919..ba1d340e 100644
--- a/apkg/commands/info.py
+++ b/apkg/commands/info.py
@@ -1,5 +1,4 @@
import json
-import toml
import click
import distro as distro_mod
@@ -9,6 +8,8 @@ from apkg.pkgstyle import PKGSTYLES
from apkg import pkgtemplate
from apkg.log import getLogger, T
from apkg.project import Project
+from apkg.util import common
+from apkg.util import toml
log = getLogger(__name__)
@@ -118,7 +119,7 @@ def template_variables(distro=None, custom=False):
if not custom:
tvars = {'distro': distro}
tvars = template.template_vars(tvars)
- print(toml.dumps(tvars))
+ print(toml.dumps(common.serialize(tvars)))
return
# custom variables
@@ -127,7 +128,7 @@ def template_variables(distro=None, custom=False):
for vsrc in proj.variables_sources:
print("# variables from %s: %s" % (vsrc.src_attr, vsrc.src_val))
custom_tvars = vsrc.get_variables(tvars)
- print(toml.dumps(custom_tvars))
+ print(toml.dumps(common.serialize(custom_tvars)))
tvars.update(custom_tvars)
diff --git a/apkg/ex.py b/apkg/ex.py
index f753b50a..92049979 100644
--- a/apkg/ex.py
+++ b/apkg/ex.py
@@ -100,6 +100,11 @@ class MissingPackagingTemplate(ApkgException):
returncode = 32
+class MissingRequiredModule(ApkgException):
+ msg_fmt = "Missing required module: {mod}"
+ returncode = 34
+
+
class ArchiveNotFound(ApkgException):
msg_fmt = "{type} archive not found: {ar}"
returncode = 36
diff --git a/apkg/project.py b/apkg/project.py
index a098aa88..86078eab 100644
--- a/apkg/project.py
+++ b/apkg/project.py
@@ -4,7 +4,6 @@ from pathlib import Path
import re
import jinja2
-import toml
try:
from functools import cached_property
except ImportError:
@@ -19,6 +18,7 @@ from apkg import pkgtemplate
from apkg import pkgtest
from apkg.util.archive import unpack_archive
from apkg.util.git import git
+from apkg.util import toml
from apkg.util import upstreamversion
@@ -132,7 +132,7 @@ class Project:
"""
if self.path.config.exists():
log.verbose("loading project config: %s", self.path.config)
- self.config = toml.load(self.path.config.open())
+ self.config = toml.loadp(self.path.config)
return True
else:
log.verbose("project config not found: %s", self.path.config)
diff --git a/apkg/util/common.py b/apkg/util/common.py
index dad28f88..908997aa 100644
--- a/apkg/util/common.py
+++ b/apkg/util/common.py
@@ -154,6 +154,16 @@ def fnmatch_any(filename, patterns):
return False
+def serialize(obj):
+ if isinstance(obj, (list, tuple)):
+ return [serialize(v) for v in obj]
+ if isinstance(obj, dict):
+ return {k: serialize(v) for k, v in obj.items()}
+ if isinstance(obj, (str, bool, int, float)):
+ return obj
+ return str(obj)
+
+
class SortReversor:
"""
use this with multi-key sort() to reverse individual keys
diff --git a/apkg/util/serial.py b/apkg/util/serial.py
new file mode 100644
index 00000000..e69de29b
diff --git a/apkg/util/toml.py b/apkg/util/toml.py
new file mode 100644
index 00000000..f776cd18
--- /dev/null
+++ b/apkg/util/toml.py
@@ -0,0 +1,94 @@
+"""
+wrapper to select and provide access to available TOML libraries
+"""
+from apkg import ex
+
+
+LOAD_LIB = None
+DUMP_LIB = None
+
+LOAD_MODE = 'rb'
+
+
+# get load() and loads()
+try:
+ from tomllib import load, loads
+ LOAD_LIB = 'tomllib'
+except ModuleNotFoundError:
+ pass
+
+if not LOAD_LIB:
+ try:
+ from tomli import load, loads
+ LOAD_LIB = 'tomli'
+ except ModuleNotFoundError:
+ pass
+
+if not LOAD_LIB:
+ try:
+ from toml import load, loads
+ LOAD_LIB = 'toml'
+ LOAD_MODE = 'r'
+ except ModuleNotFoundError:
+ pass
+
+if not LOAD_LIB:
+ try:
+ from tomlkit import load, loads
+ LOAD_LIB = 'tomlkit'
+ except ModuleNotFoundError:
+ pass
+
+
+# get dump() and dumps()
+try:
+ from tomli_w import dump, dumps
+ DUMP_LIB = 'tomli_w'
+except ModuleNotFoundError:
+ pass
+
+if not DUMP_LIB:
+ try:
+ from tomlkit import dump, dumps
+ DUMP_LIB = 'tomlkit'
+ except ModuleNotFoundError:
+ pass
+
+if not DUMP_LIB:
+ try:
+ from toml import dump, dumps
+ DUMP_LIB = 'toml'
+ except ModuleNotFoundError:
+ pass
+
+
+def missing_toml_load_module(*args, **kwargs):
+ msg=("Requested operation requires TOML load module but none was found.\n\n"
+ "Please install one of following Python modules:\n\n"
+ "- tomli\n- toml\n- tomlkit")
+ raise ex.MissingRequiredModule(msg=msg)
+
+
+def missing_toml_dump_module(*args, **kwargs):
+ msg=("Requested operation requires TOML dump module but none was found.\n\n"
+ "Please install one of following Python modules:\n\n"
+ "- tomli_w\n- tomlkit\n- toml")
+ raise ex.MissingRequiredModule(msg=msg)
+
+
+# only fail when required functions are called
+if not LOAD_LIB:
+ load = missing_toml_load_module
+ loads = missing_toml_load_module
+
+if not DUMP_LIB:
+ dump = missing_toml_dump_module
+ dumps = missing_toml_dump_module
+
+
+def loadp(path):
+ """
+ Load data from path to TOML file
+ """
+ with open(path, LOAD_MODE) as f:
+ return load(f)
diff --git a/pyproject.toml b/pyproject.toml
index fb6514e9..6bc219d0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -34,7 +34,6 @@ dependencies = [
"jinja2",
"packaging",
"requests",
- "toml",
]
dynamic = ["version"]
diff --git a/setup.cfg b/setup.cfg
index 99999261..90540573 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -35,7 +35,6 @@ install_requires =
jinja2
packaging
requests
- toml
[options.packages.find]
exclude =
--
GitLab
From 8f6f66a37bbb946892232a73ac6145e6b13cdfef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ru=C5=BEi=C4=8Dka?= <jakub.ruzicka@nic.cz>
Date: Wed, 26 Feb 2025 17:11:51 +0100
Subject: [PATCH 2/2] info: new apkg-deps sub-command
Shows colored info about apkg runtime deps and Python version.
Also shows if run from venv.
---
apkg/commands/info.py | 79 +++++++++++++++++++++++++++++++++++++++++++
apkg/terminal.py | 3 ++
2 files changed, 82 insertions(+)
diff --git a/apkg/commands/info.py b/apkg/commands/info.py
index ba1d340e..7ad8bbec 100644
--- a/apkg/commands/info.py
+++ b/apkg/commands/info.py
@@ -2,12 +2,16 @@ import json
import click
import distro as distro_mod
+import importlib
+from pathlib import Path
+import sys
from apkg import adistro
from apkg.pkgstyle import PKGSTYLES
from apkg import pkgtemplate
from apkg.log import getLogger, T
from apkg.project import Project
+from apkg import terminal
from apkg.util import common
from apkg.util import toml
@@ -15,6 +19,21 @@ from apkg.util import toml
log = getLogger(__name__)
+RUNTIME_DEPS = [
+ 'bs4',
+ 'click',
+ 'distro',
+ 'jinja2',
+ 'packaging',
+ 'requests',
+]
+
+BUILD_DEPS = [
+ 'build',
+ 'setuptools',
+]
+
+
@click.group(name='info')
@click.help_option('-h', '--help', help='show command help')
def cli_info():
@@ -23,6 +42,34 @@ def cli_info():
"""
+@cli_info.command()
+@click.help_option('-h', '--help', help='show command help')
+def apkg_deps():
+ """
+ show apkg dependencies info
+ """
+ print("{t.bold}Python interpreter{t.normal}:".format(t=T))
+ py_cmd = Path(sys.executable).stem
+ s = ("{t.green}{c}{t.normal} {t.cyan}{v}{t.normal}: "
+ "{t.bold}{p}{t.normal}")
+ if sys.prefix != sys.base_prefix:
+ s += ' - {t.magenta}venv{t.normal}'
+ print(s.format(c=py_cmd, v=sys.version, p=sys.executable, t=T))
+
+ print("\n{t.bold}core dependencies{t.normal}:".format(t=T))
+ for dep in RUNTIME_DEPS:
+ print(modinfo_t(dep))
+
+ print("\n{t.bold}dynamic dependencies{t.normal}:".format(t=T))
+ print(modinfo_t(toml.LOAD_LIB, 'TOML load'))
+ print(modinfo_t(toml.DUMP_LIB, 'TOML dump', failc='yellow'))
+ print(modinfo_t(terminal.COLOR_LIB, 'terminal colors', failc='yellow'))
+
+ print("\n{t.bold}build dependencies{t.normal}:".format(t=T))
+ for dep in BUILD_DEPS:
+ print(modinfo_t(dep))
+
+
@cli_info.command()
@click.help_option('-h', '--help', help='show command help')
def cache():
@@ -143,4 +190,36 @@ def upstream_version():
print(msg.format(v=proj.upstream_version, t=T))
+def modinfo_t(modname, desc=None, failc='red'):
+ if not modname:
+ s = "{d}: {t.%s}no supported module found{t.normal}" % (failc)
+ s = s.format(d=desc, t=T)
+ return s
+
+ mod = None
+ err = None
+ try:
+ mod = importlib.import_module(modname)
+ except Exception as e:
+ err = e
+
+ if mod:
+ s = ''
+ if desc:
+ s += '%s: ' % desc
+ s += "{t.green}{m}{t.normal}"
+ if hasattr(mod, '__version__'):
+ ver = getattr(mod, '__version__')
+ s += " {t.cyan}%s{t.normal}" % ver
+ s += ": {t.bold}{p}{t.normal}"
+ return s.format(m=modname, p=Path(mod.__file__).parent, t=T)
+
+ s = ''
+ if desc:
+ s += '%s: ' % desc
+ s += ("{t.red}{m}{t.normal}: {t.bold_red}{et}{t.normal}: "
+ "{t.normal}{e}{t.normal}")
+ return s.format(m=modname, et=err.__class__.__name__, e=err, t=T)
+
+
APKG_CLI_COMMANDS = [cli_info]
diff --git a/apkg/terminal.py b/apkg/terminal.py
index 92d066c9..4355e0f5 100644
--- a/apkg/terminal.py
+++ b/apkg/terminal.py
@@ -23,11 +23,14 @@ class PlainTerminal:
COLOR_TERMINAL = False
+COLOR_LIB = None
try:
try:
import blessed
+ COLOR_LIB = 'blessed'
except Exception:
import blessings as blessed
+ COLOR_LIB = 'blessings'
# this can throw _curses.error: setupterm: could not find terminal
# better find out now
blessed.Terminal()
--
GitLab