Add a new "auto_checksum" config option which, when set, will make
pmbootstrap fix checksums for local sources in-place before building a
package.
This _drastically_ speeds up development when working with files that
are local to pmaports.
It isn't done automatically for remote sources as there is still some
value in having these error out (e.g. catching corrupt downloads).
Additionally, those sources shouldn't change regularly (if you're doing
local development like this, please use "pmbootstrap build --src"!)
Enable this option by default, it doesn't introduce any additional risk.
Suggested-by: Oliver Smith <ollieparanoid@postmarketos.org>
Signed-off-by: Caleb Connolly <kc@postmarketos.org>
---
pmb/build/_package.py | 3 +++
pmb/build/checksum.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++
pmb/config/__init__.py | 5 +++++
pmb/parse/_apkbuild.py | 8 ++++++++
4 files changed, 71 insertions(+)
diff --git a/pmb/build/_package.py b/pmb/build/_package.py
index 884c931fb6ec..9f69bdc3369c 100644
--- a/pmb/build/_package.py
+++ b/pmb/build/_package.py
@@ -500,6 +500,9 @@ def package(args, pkgname, arch=None, force=False, strict=False,
skip_init_buildenv, src):
return
+ if args.auto_checksum:
+ pmb.build.checksum.fix_local(args, pkgname)
+
# Build and finish up
(output, cmd, env) = run_abuild(args, apkbuild, arch, strict, force, cross,
suffix, src)
diff --git a/pmb/build/checksum.py b/pmb/build/checksum.py
index 921ed6763fc0..e49e7079ce35 100644
--- a/pmb/build/checksum.py
+++ b/pmb/build/checksum.py
@@ -1,12 +1,67 @@
# Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import logging
+import hashlib
+import os
+import fileinput
import pmb.chroot
import pmb.build
import pmb.helpers.run
import pmb.helpers.pmaports
+def fix_local(args, pkgname, path=None):
+ """
+ Fix the checksums of all local sources. Returns True if something was
+ changed, False otherwise.
+
+ :param pkgname: name of the package
+ :param path: path to the APKBUILD file
+ """
+
+ if not path:
+ path = os.path.join(pmb.helpers.pmaports.find(args, pkgname, True), "APKBUILD")
+ apkbuild = pmb.parse.apkbuild(path)
+ checksums = apkbuild["sha512sums"]
+ pmaportdir = pmb.helpers.pmaports.find(args, pkgname)
+ if not pmaportdir:
+ raise RuntimeError("Package '" + pkgname + "': Could not find package!")
+
+ changed = False
+ # Use the filename from the checksums, it will be a proper relative path
+ # whereas the sources array might have fun subshell stuff in it
+ for file in checksums.keys():
+ srcpath = os.path.join(pmaportdir, file)
+ if not os.path.exists(srcpath):
+ continue
+ # Maybe some way to optimise this by buffering the file?
+ # we aren't dealing with huge files here though
+ checksum = hashlib.sha512(open(srcpath, "rb").read()).hexdigest()
+ if checksum != checksums[file]:
+ logging.info("Fixing checksum for " + srcpath)
+ checksums[file] = checksum
+ changed = True
+
+ if not changed:
+ return False
+
+ with fileinput.input(path, inplace=True) as file:
+ in_checksums = False
+ for line in file:
+ if line.startswith("sha512sums=\""):
+ in_checksums = True
+ elif in_checksums and line.startswith("\""):
+ in_checksums = False
+ elif in_checksums:
+ _src = line.split(" ")[-1].strip()
+ # Is a silent failure here ok?
+ if _src in checksums:
+ print(f"{checksums[_src]} {_src}")
+ continue
+
+ print(line, end="")
+
+ return True
def update(args, pkgname):
""" Fetch all sources and update the checksums in the APKBUILD. """
diff --git a/pmb/config/__init__.py b/pmb/config/__init__.py
index d0dac15f4786..aac6248c0c8a 100644
--- a/pmb/config/__init__.py
+++ b/pmb/config/__init__.py
@@ -73,6 +73,7 @@ def sudo(cmd: List[str]) -> List[str]:
# Keys saved in the config file (mostly what we ask in 'pmbootstrap init')
config_keys = [
"aports",
+ "auto_checksum",
"boot_size",
"build_default_device_arch",
"build_pkgs_on_install",
@@ -106,6 +107,9 @@ config_keys = [
# overridden on the commandline)
defaults = {
"aports": "$WORK/cache_git/pmaports",
+ # Automatically correct checksums for local files in APKBUILD
+ # sources (NOT web sources).
+ "auto_checksum": True,
"ccache_size": "5G",
"is_default_channel": True,
"cipher": "aes-xts-plain64",
@@ -713,6 +717,7 @@ apkbuild_attributes = {
"pkgver": {},
"subpackages": {},
"url": {},
+ "sha512sums": {"array": True, "pairs": True},
# cross-compilers
"makedepends_build": {"array": True},
diff --git a/pmb/parse/_apkbuild.py b/pmb/parse/_apkbuild.py
index 39b27868a7e7..57806afcccdb 100644
--- a/pmb/parse/_apkbuild.py
+++ b/pmb/parse/_apkbuild.py
@@ -210,6 +210,14 @@ def _parse_attributes(path, lines, apkbuild_attributes, ret):
if options.get("array", False):
# Split up arrays, delete empty strings inside the list
ret[attribute] = list(filter(None, ret[attribute].split(" ")))
+ if options.get("pairs", False):
+ # Convert an array of ["k1", "v1", "k2", "v2"] to
+ # [("k1", "v1"), ("k2", "v2")]
+ # The only user is sha512sum which uses "value key" pairs
+ # so flip them, for now
+ right = ret[attribute][::2]
+ left = ret[attribute][1::2]
+ ret[attribute] = dict(zip(left, right))
if options.get("int", False):
if ret[attribute]:
ret[attribute] = int(ret[attribute])
--
2.41.0
With automated checksums, the last piece of the puzzle for quick
development of local packages is to fix the pkgver automatically too.
Add a new --dirty flag to pmbootstrap build which will append the current
datetime to the pkgver. This has two main purposes:
1. Ensures that the newly built package will get picked up by apk
2. Makes it clear that the package is from a local "dirty" build and not
from any binary repo.
The dirty flag only applies when either the local checksums are out of
date, or if the local package source dir has been modified since the
last build. This ensures that only packages the user is working on will
get built when this flag is set.
With this logic in place, it is now sensible to add a '--build' flag to
pmbootstrap sideload, the behaves pretty much like a shortcut for
"pmbootstrap build --dirty foobar && pmbootstrap sideload foobar", just
combined for speed. A flag like this didn't make much sense before as
local modification necessitated fixing the checksum and bumping the
pkg{ver,rel}.
Signed-off-by: Caleb Connolly <kc@postmarketos.org>
---
pmb/build/_package.py | 57 ++++++++++++++++++++++++++++++++++++++-----------
pmb/helpers/frontend.py | 5 +++++
pmb/helpers/pmaports.py | 31 +++++++++++++++++++++++++++
pmb/parse/arguments.py | 6 ++++++
4 files changed, 87 insertions(+), 12 deletions(-)
diff --git a/pmb/build/_package.py b/pmb/build/_package.py
index 9f69bdc3369c..6372c8bdf928 100644
--- a/pmb/build/_package.py
+++ b/pmb/build/_package.py
@@ -228,7 +228,7 @@ def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None,
return True
-def get_pkgver(original_pkgver, original_source=False, now=None):
+def get_pkgver(original_pkgver, clean=False, now=None):
"""
Get the original pkgver when using the original source. Otherwise, get the
pkgver with an appended suffix of current date and time. For example:
@@ -237,11 +237,10 @@ def get_pkgver(original_pkgver, original_source=False, now=None):
replaced.
:param original_pkgver: unmodified pkgver from the package's APKBUILD.
- :param original_source: the original source is used instead of overriding
- it with --src.
+ :param clean: clean building, use original pkgver
:param now: use a specific date instead of current date (for test cases)
"""
- if original_source:
+ if clean:
return original_pkgver
# Append current date
@@ -256,8 +255,6 @@ def override_source(args, apkbuild, pkgver, src, suffix="native"):
Mount local source inside chroot and append new functions (prepare() etc.)
to the APKBUILD to make it use the local source.
"""
- if not src:
- return
# Mount source in chroot
mount_path = "/mnt/pmbootstrap-source-override/"
@@ -358,8 +355,14 @@ def link_to_git_dir(args, suffix):
"/home/pmos/build/.git"], suffix)
+def make_dirty(args, apkuild):
+ """
+ Update the pkgver with the datetime suffix
+ """
+
+
def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
- suffix="native", src=None):
+ suffix="native", src=None, dirty=False):
"""
Set up all environment variables and construct the abuild command (all
depending on the cross-compiler method and target architecture), copy
@@ -378,8 +381,12 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
" cross-compiling in the native chroot. This will"
" probably fail!")
+ # We don't care about the dirty build logic if we're using --src
+ # as that does most of the same stuff anyway
+ dirty = dirty and src is None
+
# Pretty log message
- pkgver = get_pkgver(apkbuild["pkgver"], src is None)
+ pkgver = get_pkgver(apkbuild["pkgver"], not dirty)
output = (arch + "/" + apkbuild["pkgname"] + "-" + pkgver +
"-r" + apkbuild["pkgrel"] + ".apk")
message = "(" + suffix + ") build " + output
@@ -426,7 +433,13 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
# Copy the aport to the chroot and build it
pmb.build.copy_to_buildpath(args, apkbuild["pkgname"], suffix)
- override_source(args, apkbuild, pkgver, src, suffix)
+ if src:
+ override_source(args, apkbuild, pkgver, src, suffix)
+ elif dirty:
+ # Overwrite the pkgver with the dirty pkgver
+ pmb.chroot.user(args,
+ ["sed", "-i",
+ f"s/pkgver=.*/pkgver={pkgver}/g", "/home/pmos/build/APKBUILD"], suffix)
link_to_git_dir(args, suffix)
pmb.chroot.user(args, cmd, suffix, "/home/pmos/build", env=env)
return (output, cmd, env)
@@ -459,7 +472,7 @@ def finish(args, apkbuild, arch, output, strict=False, suffix="native"):
def package(args, pkgname, arch=None, force=False, strict=False,
- skip_init_buildenv=False, src=None):
+ skip_init_buildenv=False, src=None, dirty=False):
"""
Build a package and its dependencies with Alpine Linux' abuild.
@@ -494,17 +507,37 @@ def package(args, pkgname, arch=None, force=False, strict=False,
# Detect the build environment (skip unnecessary builds)
if not check_build_for_arch(args, pkgname, arch):
return
+
+ # For a dirty build we want to force the build if either
+ # the checksums mismatch OR the package includes unstaged changes
+ if dirty:
+ path = pmb.helpers.pmaports.find(args, pkgname, True)
+
+ # Fix checksums in-place
+ force = pmb.build.checksum.fix_local(args, pkgname, os.path.join(path, "APKBUILD"))
+ # if the checksums were up to date, check if the package has been modified since
+ # the last build, and force if it has been
+ if not force:
+ build_time = pmb.helpers.pmaports.get_last_build_date(args, pkgname, arch)
+ mtime = pmb.helpers.pmaports.get_last_mtime(args, path)
+ force = build_time < mtime
+ if force:
+ logging.debug(f"{pkgname}: doing dirty build because package has been"
+ " modified since last build")
+
suffix = pmb.build.autodetect.suffix(apkbuild, arch)
cross = pmb.build.autodetect.crosscompile(args, apkbuild, arch, suffix)
if not init_buildenv(args, apkbuild, arch, strict, force, cross, suffix,
skip_init_buildenv, src):
return
- if args.auto_checksum:
+ # If not a dirty build (ie pkgver was bumped) then update checksums
+ # now we the build was intentional
+ if args.auto_checksum and not dirty:
pmb.build.checksum.fix_local(args, pkgname)
# Build and finish up
(output, cmd, env) = run_abuild(args, apkbuild, arch, strict, force, cross,
- suffix, src)
+ suffix, src, dirty)
finish(args, apkbuild, arch, output, strict, suffix)
return output
diff --git a/pmb/helpers/frontend.py b/pmb/helpers/frontend.py
index 903476f7524d..ecea5a21219d 100644
--- a/pmb/helpers/frontend.py
+++ b/pmb/helpers/frontend.py
@@ -138,6 +138,11 @@ def sideload(args):
arch = args.arch
user = args.user
host = args.host
+ if args.build:
+ for package in args.packages:
+ arch_package = args.arch or pmb.build.autodetect.arch(args, package)
+ if not pmb.build.package(args, package, arch_package, dirty=True):
+ logging.debug(f"'{package}' is up to date, not dirty-building")
pmb.sideload.sideload(args, user, host, args.port, arch, args.install_key,
args.packages)
diff --git a/pmb/helpers/pmaports.py b/pmb/helpers/pmaports.py
index 510ec4d0112b..8427c6d47509 100644
--- a/pmb/helpers/pmaports.py
+++ b/pmb/helpers/pmaports.py
@@ -287,3 +287,34 @@ def get_channel_new(channel):
logging.verbose(f"Legacy channel '{channel}' translated to '{ret}'")
return ret
return channel
+
+
+def get_last_build_date(args, pkgname, arch):
+ """ Get the last build date of a package in the binary repository.
+
+ :param pkgname: package name
+ :param arch: package architecture
+ :returns: date as string in the format "YYYY-MM-DD" or None
+ """
+ # Get the last build date from the APKINDEX
+ index_data = pmb.parse.apkindex.package(args, pkgname, arch, False)
+ if not index_data:
+ return None
+
+ return int(index_data["timestamp"])
+
+def get_last_mtime(args, pkg):
+ """
+ Get the last modified time of a package in pmaports.
+
+ :param pkg: package name or absolute path to package dir
+ """
+
+ path = pkg
+ if not os.path.isabs(path):
+ path = find(args, pkg)
+
+ if not path:
+ raise RuntimeError(f"Could not find package {pkg}")
+
+ return int(os.path.getmtime(path))
\ No newline at end of file
diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py
index 3ba3651002f8..bc7ab8ce5098 100644
--- a/pmb/parse/arguments.py
+++ b/pmb/parse/arguments.py
@@ -220,6 +220,9 @@ def arguments_sideload(subparser):
ret.add_argument("--install-key", help="install the apk key from this"
" machine if needed",
action="store_true", dest="install_key")
+ ret.add_argument("--build", action="store_true", help="dirty-build the package"
+ " before sideloading it. Shortcut for 'pmbootstrap build --dirty"
+ " xyz && pmbootstrap sideload xyz'")
return ret
@@ -819,6 +822,9 @@ def arguments():
build.add_argument("--strict", action="store_true", help="(slower) zap and"
" install only required depends when building, to"
" detect dependency errors")
+ build.add_argument("--dirty", action="store_true", help="append the current"
+ " datetime to the pkgver, to make sure it is always"
+ " newer than the last build")
build.add_argument("--src", help="override source used to build the"
" package with a local folder (the APKBUILD must"
" expect the source to be in $builddir, so you might"
--
2.41.0