Commit ade04df5 authored by Jakub Ružička's avatar Jakub Ružička
Browse files

Merge branch 'py35path-nuke' into 'master'

refactor: get rid of compat.py35path

Closes #36

See merge request !33
parents a4b52ab9 878ef8f6
Pipeline #78467 passed with stages
in 3 minutes and 5 seconds
......@@ -6,7 +6,6 @@ import hashlib
import json
from pathlib import Path
from apkg.compat import py35path
from apkg.log import getLogger
......@@ -15,7 +14,7 @@ log = getLogger(__name__)
def file_checksum(path):
chsum = hashlib.sha256()
with open(py35path(path), "rb") as f:
with Path(path).open("rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
chsum.update(chunk)
return chsum.hexdigest()[:20]
......
"""
python version compatibility helpers
"""
import sys
# python 3.5 doesn't have pathlib fully integrated so str() conversion is
# needed in some places - use py35path() for easy removal later
if sys.version_info.minor <= 5:
def py35path(s):
return str(s)
else:
def py35path(s):
return s
"""
apkg lib for handling source archives
"""
import os
from pathlib import Path
import shutil
import requests
from apkg import exception
from apkg.lib import common
from apkg.log import getLogger
from apkg.compat import py35path
from apkg.parse import split_archive_fn, parse_version
from apkg.project import Project
from apkg.util.run import run
import apkg.util.shutil35 as shutil
log = getLogger(__name__)
......@@ -74,8 +72,8 @@ def make_archive(
log.info(msg, archive_fn)
archive_path = ar_base_path / archive_fn
log.info("copying archive to: %s", archive_path)
os.makedirs(py35path(ar_base_path), exist_ok=True)
shutil.copy(py35path(in_archive_path), py35path(archive_path))
ar_base_path.mkdir(parents=True, exist_ok=True)
shutil.copy(in_archive_path, archive_path)
log.success("made archive: %s", archive_path)
results = [archive_path]
if use_cache:
......@@ -126,7 +124,7 @@ def get_archive(
_, _, archive_fn = archive_url.rpartition('/')
archive_path = ar_base_path / archive_fn
log.info('saving archive to: %s', archive_path)
os.makedirs(py35path(ar_base_path), exist_ok=True)
ar_base_path.mkdir(parents=True, exist_ok=True)
archive_path.open('wb').write(r.content)
log.success('downloaded archive: %s', archive_path)
results = [archive_path]
......
......@@ -2,7 +2,6 @@
apkg lib for handling package builds
"""
from pathlib import Path
import shutil
from apkg import adistro
from apkg.cache import file_checksum
......@@ -11,6 +10,7 @@ from apkg.lib import srcpkg as _srcpkg
from apkg.lib import common
from apkg.log import getLogger
from apkg.project import Project
import apkg.util.shutil35 as shutil
log = getLogger(__name__)
......
import os
from pathlib import Path
import shutil
import sys
from apkg.log import getLogger
from apkg.compat import py35path
from apkg import exception
import apkg.util.shutil35 as shutil
log = getLogger(__name__)
......@@ -16,7 +14,7 @@ def copy_paths(paths, dst):
utility to copy a list of paths to dst
"""
if not dst.exists():
os.makedirs(py35path(dst), exist_ok=True)
dst.mkdir(parents=True, exist_ok=True)
dst_full = dst.resolve()
new_paths = []
for p in paths:
......@@ -25,7 +23,7 @@ def copy_paths(paths, dst):
else:
p_dst = dst / p.name
log.verbose("copying file: %s -> %s", p, p_dst)
shutil.copy(py35path(p), py35path(p_dst))
shutil.copy(p, p_dst)
new_paths.append(p_dst)
return new_paths
......
"""
apkg lib for handling source archives
"""
import os
from pathlib import Path
import shutil
from apkg import adistro
from apkg.cache import file_checksum
from apkg.compat import py35path
from apkg import exception
from apkg.lib import ar
from apkg.lib import common
from apkg.log import getLogger
from apkg.project import Project
from apkg.util.archive import unpack_archive
import apkg.util.shutil35 as shutil
log = getLogger(__name__)
......@@ -113,8 +111,8 @@ def make_srcpkg(
# prepare new build dir
if build_path.exists():
log.info("removing existing build dir: %s", build_path)
shutil.rmtree(py35path(build_path))
os.makedirs(py35path(build_path), exist_ok=True)
shutil.rmtree(build_path)
build_path.mkdir(parents=True, exist_ok=True)
# ensure output dir doesn't exist unless it was specified
if not result_dir and out_path.exists():
log.info("removing existing result dir: %s", out_path)
......
......@@ -9,13 +9,11 @@ apkg package style for **Arch** linux.
"""
import glob
from pathlib import Path
import os
import shutil
from apkg import exception
from apkg.log import getLogger
from apkg.compat import py35path
from apkg.util.run import cd, run, sudo
import apkg.util.shutil35 as shutil
log = getLogger(__name__)
......@@ -59,7 +57,7 @@ def build_srcpkg(
out_archive = out_path / archive_path.name
log.info("building arch source package: %s", in_pkgbuild)
template.render(build_path, env or {})
os.makedirs(out_path)
out_path.mkdir(parents=True)
log.info("copying PKGBUILD and archive to: %s", out_path)
shutil.copyfile(in_pkgbuild, out_pkgbuild)
shutil.copyfile(archive_path, out_archive)
......@@ -78,7 +76,7 @@ def build_packages(
% srcpkg_path.name)
isolated = kwargs.get('isolated')
log.info("copying source package to build dir: %s", build_path)
shutil.copytree(py35path(srcpkg_path.parent), py35path(build_path))
shutil.copytree(srcpkg_path.parent, build_path)
# build package using makepkg
if not isolated:
msg = "arch doesn't support direct host build - using isolated"
......@@ -87,12 +85,12 @@ def build_packages(
with cd(build_path):
run('makepkg', direct='auto')
log.info("copying built packages to result dir: %s", out_path)
os.makedirs(py35path(out_path), exist_ok=True)
out_path.mkdir(parents=True, exist_ok=True)
pkgs = []
# find and copy resulting packages
for src_pkg in glob.iglob('%s/*.zst' % build_path):
dst_pkg = out_path / Path(src_pkg).name
shutil.copyfile(py35path(src_pkg), py35path(dst_pkg))
shutil.copyfile(src_pkg, dst_pkg)
pkgs.append(dst_pkg)
return pkgs
......
......@@ -10,16 +10,14 @@ and its many clones such as Ubuntu or Mint.
or `--isolated` using `pbuilder`
"""
import glob
import os
from pathlib import Path
import re
import shutil
from apkg.compat import py35path
from apkg import exception
from apkg.log import getLogger
from apkg import parse
from apkg.util.run import cd, run, sudo
import apkg.util.shutil35 as shutil
log = getLogger(__name__)
......@@ -62,7 +60,7 @@ def _copy_srcpkg_files(src_path, dst_path):
for pattern in ['*.dsc', '*.debian.tar.*', '*.orig.tar.*', '*.diff.*']:
for f in glob.iglob('%s/%s' % (src_path, pattern)):
srcp = Path(f)
shutil.copyfile(py35path(f), py35path(dst_path / srcp.name))
shutil.copyfile(f, dst_path / srcp.name)
# pylint: disable=too-many-locals
......@@ -77,7 +75,7 @@ def build_srcpkg(
source_path = build_path / nv
log.info("building deb source package: %s", nv)
log.info("unpacking archive: %s", archive_path)
os.makedirs(py35path(source_path))
source_path.mkdir(parents=True)
run('aunpack', '-X', build_path, archive_path)
if not source_path.exists():
# NOTE: if this happens oftern (it shouldn't), consider using
......@@ -92,7 +90,7 @@ def build_srcpkg(
debian_ar = "%s_%s.orig%s" % (env['name'], env['version'], ext)
debian_ar_path = build_path / debian_ar
log.info("copying archive into source package: %s", debian_ar_path)
shutil.copyfile(py35path(archive_path), py35path(debian_ar_path))
shutil.copyfile(archive_path, debian_ar_path)
log.info("building deb source-only package...")
with cd(source_path):
......@@ -106,7 +104,7 @@ def build_srcpkg(
direct='auto')
log.info("copying source package to result dir: %s", out_path)
os.makedirs(py35path(out_path))
out_path.mkdir(parents=True)
_copy_srcpkg_files(build_path, out_path)
fns = glob.glob('%s/*' % out_path)
# make sure .dsc is first
......@@ -126,8 +124,8 @@ def build_packages(
srcpkg_paths,
**kwargs):
srcpkg_path = srcpkg_paths[0]
os.makedirs(py35path(build_path))
os.makedirs(py35path(out_path))
build_path.mkdir(parents=True)
out_path.mkdir(parents=True)
isolated = kwargs.get('isolated')
if isolated:
log.info("starting isolated build using pbuilder")
......@@ -164,7 +162,7 @@ def build_packages(
log.info("copying built packages to result dir: %s", out_path)
for src_pkg in glob.iglob('%s/*.deb' % build_path):
dst_pkg = out_path / Path(src_pkg).name
shutil.copyfile(py35path(src_pkg), py35path(dst_pkg))
shutil.copyfile(src_pkg, dst_pkg)
pkgs.append(dst_pkg)
return pkgs
......
......@@ -10,16 +10,14 @@ such as Fedora, CentOS, SUSE, RHEL.
or `--isolated` using `mock`
"""
import glob
import os
from pathlib import Path
import re
import shutil
import subprocess
from apkg.compat import py35path
from apkg import exception
from apkg.log import getLogger
from apkg.util.run import run, sudo
import apkg.util.shutil35 as shutil
log = getLogger(__name__)
......@@ -108,20 +106,20 @@ def build_srcpkg(
log.info("copying archive files into SOURCES: %s", rpmbuild_src)
for src_path in archive_paths:
dst_path = rpmbuild_src / src_path.name
shutil.copyfile(py35path(src_path), py35path(dst_path))
shutil.copyfile(src_path, dst_path)
log.info("building .src.rpm using rpmbuild")
out = run('rpmbuild', '-bs',
'--define', '_topdir %s' % rpmbuild_topdir.resolve(),
spec_path)
log.info("copying .src.rpm to result dir: %s", out_path)
os.makedirs(py35path(out_path))
out_path.mkdir(parents=True)
srcpkgs = []
for m in re.finditer(RE_RPMBUILD_OUT_SRPM, out):
srpm = m.group(1)
src_srpm = Path(srpm)
dst_srpm = out_path / src_srpm.name
shutil.copyfile(py35path(src_srpm), py35path(dst_srpm))
shutil.copyfile(src_srpm, dst_srpm)
srcpkgs.append(dst_srpm)
if not srcpkgs:
raise exception.ParsingFailed(
......@@ -148,11 +146,11 @@ def build_packages(
preserve_env=True,
direct='auto')
log.info("copying built packages to result dir: %s", out_path)
os.makedirs(py35path(out_path))
out_path.mkdir(parents=True)
for rpm in glob.iglob('%s/*.rpm' % build_path):
src_pkg = Path(rpm)
dst_pkg = out_path / src_pkg.name
shutil.copyfile(py35path(src_pkg), py35path(dst_pkg))
shutil.copyfile(src_pkg, dst_pkg)
pkgs.append(dst_pkg)
else:
log.info("starting direct host .rpm build using rpmbuild")
......@@ -162,12 +160,12 @@ def build_packages(
'--define', '_topdir %s' % rpmbuild_topdir.resolve(),
srcpkg_path)
log.info("copying built packages to result dir: %s", out_path)
os.makedirs(py35path(out_path))
out_path.mkdir(parents=True)
for m in re.finditer(RE_RPMBUILD_OUT_RPM, out):
rpm = m.group(1)
src_pkg = Path(rpm)
dst_pkg = out_path / src_pkg.name
shutil.copyfile(py35path(src_pkg), py35path(dst_pkg))
shutil.copyfile(src_pkg, dst_pkg)
pkgs.append(dst_pkg)
if not pkgs:
raise exception.ParsingFailed(
......
"""
module for handling and rendering apkg package templates
"""
import os
from pathlib import Path
import jinja2
from apkg.compat import py35path
from apkg.log import getLogger
from apkg import pkgstyle as _pkgstyle
import apkg.util.shutil35 as shutil
log = getLogger(__name__)
......@@ -32,13 +31,13 @@ class PackageTemplate:
if out_path.exists():
log.verbose("template render dir exists: %s", out_path)
else:
os.makedirs(py35path(out_path), exist_ok=True)
out_path.mkdir(parents=True, exist_ok=True)
# recursively render all files
for d, _, files in os.walk(py35path(self.path)):
for d, _, files in shutil.walk(self.path):
rel_dir = Path(d).relative_to(self.path)
dst_dir = out_path / rel_dir
os.makedirs(py35path(dst_dir), exist_ok=True)
dst_dir.mkdir(parents=True, exist_ok=True)
for fn in files:
dst = out_path / rel_dir / fn
......
"""
apkg archive (tarball) utils
"""
import os
from pathlib import Path
from apkg import exception
......@@ -19,7 +18,7 @@ def unpack_archive(archive_path, out_path):
def root_dir(ps):
return Path(ps).parts[0]
os.makedirs(out_path, exist_ok=True)
out_path.mkdir(parents=True, exist_ok=True)
o = run('aunpack', '-X', out_path, archive_path)
# parse output and make sure there's only a single root dir
root_dirs = set(map(root_dir, o.split("\n")))
......
......@@ -4,7 +4,6 @@ import os
import subprocess
import sys
from apkg.compat import py35path
from apkg import exception
from apkg.log import getLogger, T, LOG_LEVEL, INFO
......@@ -133,7 +132,7 @@ def cd(newdir):
Temporarily change current directory.
"""
olddir = os.getcwd()
os.chdir(os.path.expanduser(py35path(newdir)))
os.chdir(os.path.expanduser(str(newdir)))
try:
yield
finally:
......
"""
A wrapper around selected shutil and os functions in order
to ensure compatibility with pathlib.Path in Python 3.5.
TODO: Once Python 3.5 support is dropped, replace all
import apkg.util.shutil35 as shutil
with
import shutil
and remove this compat module.
"""
import shutil
import os
def copy(src, dst, **kwargs):
return shutil.copy(str(src), str(dst), **kwargs)
def copyfile(src, dst, **kwargs):
return shutil.copyfile(str(src), str(dst), **kwargs)
def copytree(src, dst, **kwargs):
return shutil.copytree(str(src), str(dst), **kwargs)
def rmtree(path, **kwargs):
return shutil.rmtree(str(path), **kwargs)
def walk(top, **kwargs):
return os.walk(str(top), **kwargs)
"""
shared apkg testing functions
"""
import os
from pathlib import Path
import shutil
from apkg.compat import py35path
import apkg.util.shutil35 as shutil
def init_testing_repo(repo_path, test_path, ignore_dirs=[]):
......@@ -21,10 +19,10 @@ def inject_tree(src_path, dst_path, ignore_dirs=[]):
overwrite existing files
"""
if not dst_path.exists():
os.makedirs(py35path(dst_path), exist_ok=True)
dst_path.mkdir(parents=True, exist_ok=True)
# recursively copy all files
for d, subdirs, files in os.walk(py35path(src_path)):
for d, subdirs, files in shutil.walk(src_path):
if ignore_dirs:
# ignore selected dirs
ignored = []
......@@ -36,12 +34,12 @@ def inject_tree(src_path, dst_path, ignore_dirs=[]):
rel_dir = Path(d).relative_to(src_path)
dst_dir = dst_path / rel_dir
os.makedirs(py35path(dst_dir), exist_ok=True)
dst_dir.mkdir(parents=True, exist_ok=True)
for fn in files:
dst = dst_dir / fn
src = Path(d) / fn
shutil.copy(py35path(src), py35path(dst))
shutil.copy(src, dst)
def log_contains(string, caplog):
......
......@@ -4,9 +4,8 @@ integration tests for Knot Resolver
import glob
from pathlib import Path
import pytest
import shutil
import apkg.util.shutil35 as shutil
from apkg.compat import py35path
from apkg.util import test
from apkg.util.run import cd
from apkg.util.git import git
......@@ -40,7 +39,7 @@ def kresd_path(kresd_git, tmpdir):
"""
# NOTE(py35): use tmp_path once py3.5 is dropped
p = '%s/knot-resolver' % tmpdir
shutil.copytree(py35path(kresd_git), p, symlinks=True)
shutil.copytree(kresd_git, p, symlinks=True)
return Path(p)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment