Skip to content
Snippets Groups Projects
Commit 9bb8ab29 authored by Marek Behun's avatar Marek Behun
Browse files

compile_pkgs: speedup compilation by caching built host tools and toolchain

In order to speed up subsequent builds try to cache built tools and
toolchain.

The resulting tarball is stored in the dl directory and has a name in
format:
  turris-build-cache-BRANCH-TARGET-BOARD-MD5SUM.tar.zst
where
  BRANCH is hb?/crashlab (determined from $GIT_BRANCH, and if empty,
         from git rev-parse --abbrev-ref HEAD. If this returns something
	 different from hb?/crashlab, cached version won't be used),
  TARGET is tools or toolchain,
  BOARD is turris1x, omnia or mox,
  MD5SUM is md5 checksum of the content of .config and the TARGET
  	 directory (tools or toolchain), so any change in these will
	 force a rebuild. I.e. if a patch is added/remove/renamed for
	 automake, all tools will be rebuilt. If .config changes, both
	 tools and toolchain will be rebuilt (this is because we can't
	 determine in a simple way whether the .config settings changed
	 for tools/toolchain).

When a build of tools/toolchain is forced (the cached tarball name has
different MD5SUM or is nonexistent), the old tarball are removed from dl
directory and compilation is invoked.

After the compilation unneeded sources are removed to make the tarball
smaller. This includes all regular files in build_dir/host (or
build_dir/toolchain) except files starting with ".built" or ".prepared"
and files with name ".configured" - this is done to make OpenWRT's build
system think that the sources are there and has not changed, so it won't
try to rebuild.

Afterwards the build_dir/TARGET_DIR and staging_dir/TARGET_DIR are
packed and stored.

On subsequent run if the tarball with given name exists, instead of
compiling again the cached version will be used.

Testing showed that toolchain tarball has around 370 MB and tools
tarball around 40 MB. Zstd compression is used because of speed (cca 5
seconds to compress 1.2 GB of toolchain to 370 MB).
parent 35d7d5ed
No related merge requests found
......@@ -469,10 +469,60 @@ _compile() {
fi
}
_compile_cached() {
local checksum tarball="" tarball_prefix="" rev must_build
if [ -n "$GIT_BRANCH" ]; then
rev=$(echo "$GIT_BRANCH" | sed -e "s|origin/||")
else
rev=$(git -C "$src_dir" rev-parse --abbrev-ref HEAD)
fi
case "$rev" in
hb?|crashlab)
checksum=$(( cat .config; find "$1"; find "$1" -type f -exec cat {} \; ) | md5sum | cut -d ' ' -f 1)
tarball_prefix="turris-build-cache-${rev}-${1}-${TARGET_BOARD}-"
tarball="dl/${tarball_prefix}${checksum}.tar.zst"
;;
esac
must_build="yes"
if [ -n "$tarball" ] && [ -f "$tarball" ]; then
report "Previously compiled $1 found, extracting"
rm -rf build_dir/${2} staging_dir/$2
if tar -xf "$tarball"; then
must_build=""
else
report "Extraction of $1 failed, building forced"
fi
fi
if [ -n "$must_build" ]; then
if [ -n "$tarball_prefix" ]; then
# remove old tarballs
find dl -name "${tarball_prefix}*" -print -delete
fi
report "Compiling $1"
_compile $1/compile
if [ -n "$tarball" ]; then
report "Removing $1 sources"
find build_dir/${2} -type f ! -name ".built*" -a ! -name .configured -a ! -name ".prepared*" -delete
report "Tarballing compiled $1 for future"
rm -f "$tarball"
tar --zstd -cf "$tarball" build_dir/$2 staging_dir/$2
report "Tarball size $(du -hs "$tarball" | awk '{ print $1 }')"
fi
fi
}
available_commands+=( ["compile_tools"]="Compile host tools" )
compile_tools() {
report "Compiling tools"
_compile tools/compile toolchain/compile
_compile_cached tools host
_compile_cached toolchain "toolchain*"
}
available_commands+=( ["compile_target"]="Compile target specific software (Linux kernel)" )
......@@ -491,6 +541,8 @@ available_commands+=( ["compile"]=" Compile tools, target and packages" )
compile() {
compile_tools
compile_target
# _compile package/feeds/packages/ruby/compile V=s
# exit 1
compile_packages
}
......@@ -508,6 +560,12 @@ clean() {
available_commands+=( ["prepare"]=" Prepare build but don't build it (Implies: checkout clean patch_openwrt repatch_feeds gen_version configure set_ccache)" )
prepare() {
# report "clean ccache"
# report "$(du -hs ${CCACHE_HOST_DIR})"
# clean_ccache
# report "$(du -hs ${CCACHE_HOST_DIR})"
# report "remove ruby patch"
# rm ${src_dir}/patches/packages/wip/0001-ruby-do-not-check-for-diag-disable-compiler-flag.patch
checkout
clean
patch_openwrt
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment