compile_pkgs

#!/bin/bash -e
# OpenWRT compilation script
# (C) 2018-2020 CZ.NIC, z.s.p.o.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# Step list that is run when no commands are specified
DEFAULT_STEPS=( "prepare" "compile" "sign" "store_hash" "pkgsrepo" "stats" "gen_junit" )

# Paths to OpenWRT build and turris-build sources
src_dir="$(dirname "$(readlink -f "$0")")"
build_dir="$(readlink -f "$(pwd)")"
if [ "$build_dir" == "$src_dir" ]; then
	# Protect against turris-build repository wipe
	mkdir -p build
	cd build
	build_dir="$(pwd)"
fi

# Include common utilities
. "$src_dir/helpers/common.sh"

# Sets various variables to match the specified target, just a helper
# It is defined here so it can be used from configuration
set_target() {
	report "Setting target as $1"
	case "$1" in
		omnia)
			TARGET_BOARD=omnia
			TARGET_ARCH=armv7l
			;;
		turris1x)
			TARGET_BOARD=turris1x
			TARGET_ARCH=ppcspe
			;;
		mox)
			TARGET_BOARD=mox
			TARGET_ARCH=aarch64
			;;
		*)
			echo "Invalid target board!!! Use -t [turris1x|omnia|mox]!!!"
			exit 1
			;;
	esac
}

# Get OpenWRT URL and branch from feeds.conf file
openwrt_feed() {
	# OpenWRT git repository URL
	OPENWRT_URL="$(sed -n 's/# openwrt \([^;^]\+\).*/\1/p' "$src_dir/feeds.conf")"
	# Git reference used to checkout OpenWRT repository
	OPENWRT_BRANCH="$(sed -n 's/# openwrt [^;^]\+[;^]\([^\s]\+\).*/\1/p' "$src_dir/feeds.conf")"
	# Check if ^ is used to separate URL and branch and in such case it is hash not branch
	grep -E '^# openwrt [^;^]+\^.+' "$src_dir/feeds.conf" && OPENWRT_BRANCH="#$OPENWRT_BRANCH"
	[ -n "$OPENWRT_BRANCH" ] || OPENWRT_BRANCH="master"
}

# Configuration variables
. "$src_dir/defaults.sh"
FORCE="" # Force build
CLONE_DEEP="" # Set this variable to clone OpenWRT tree in full depth not just latest commit
BUILD_ARGS=() # Additional arguments passed to OpenWRT make call (note that this is for all make calls in OpenWRT)
BUILD_JOBS="$(nproc)" # Number of jobs to be passed to make calls
SIGN_KEY="" # Path to private signing key
GIT_MIRROR="" # Path to git mirror directory
DL_MIRROR="" # Path to downloads mirror directory
CCACHE_HOST_DIR="" # Path to ccache directory for host compilations
CCACHE_TARGET_DIR="" # Path to ccache directory for target compilations
OUTPUT_DIR="./pkgsrepo" # Output directory for pkgsrepo command
openwrt_feed

# Load configurations
[ -f "${src_dir}"/turris-build.conf ] && . "${src_dir}"/turris-build.conf
[ -f ~/.turris-build ] && . ~/.turris-build
[ -f ./turris-build.conf ] && . ./turris-build.conf

##################################################################################

# Technical variables
declare -A available_commands # Set of all commands defined in this script
mirror_updated="" # Variable signaling that mirrors were update in this run already
# Variables set by _enable_debug function
perl_debug=""
sh_debug=""
make_debug=""
owrt_debug=""

# OpenWRT directory for temporally data
export TMPDIR="${build_dir}/tmp"
mkdir -p "$TMPDIR"

# Git that should be used for commands such as: git commit or git am
_git() {
	git \
		-c "commit.gpgsign=false" \
		-c "user.email=auto-build@example.com" \
		-c "user.name=Build system automate" \
		"$@"
}

_sh() {
	sh $sh_debug "$@"
}

_perl() {
	perl $perl_debug "$@"
}

_make() {
	make $make_debug "$@"
}

# Make for OpenWRT
_openwrt_make() {
	local args=( "IGNORE_ERRORS=m" "BUILD_LOG=1" )
	[ -z "$owrt_debug" ] || args+=( "$owrt_debug" )
	[ "${#BUILD_ARGS[@]}" -eq 0 ] || args+=( "${BUILD_ARGS[@]}" )
	_make "${args[@]}" "$@"
}

_enable_debug() {
	set -x
	perl_debug="-d:Trace"
	make_debug="-d"
	sh_debug="-x"
	owrt_debug="V=s"
}

# Definition of avaliable commands ###############################################

available_commands+=( ["gen_version"]="Generates Turris version package from NEWS" )
gen_version() {
	report "Generating turris-version package"
	"${src_dir}/helpers/turris-version.sh" --output "${build_dir}/package/system/turris-version" package
	pushd "${build_dir}"
	git add "${build_dir}/package/system/turris-version"
	_git commit -m 'turris-version: Create a package with release notes'
	popd
}

available_commands+=( ["conflicts"]="Show packages that we are overriding in Turris OS packages" )
conflicts() {
	report "Showing conflicting packages"
	echo "Following packages are being overridden by Turris packages:"
	echo
	mkdir -p tmp
	find feeds/turrispackages -name Makefile | sed 's|.*/\([^/]*\)/Makefile|\1|' | sort > tmp/turris-packages.list
	find feeds/ -name Makefile | grep -v '^feeds/turrispackages' | sed 's|.*/\([^/]*\)/Makefile|\1|' | sort > tmp/other-packages.list
	comm -12 tmp/turris-packages.list tmp/other-packages.list | sed 's|^| * |'
	echo
	rm -f tmp/turris-packages.list tmp/other-packages.list
}

available_commands+=( ["stats"]="	Prints statistics regarding number of built packages" )
stats() {
	mkdir -p logs
	report "Reporting statistics"
	{
		echo "Statistics of the build:"
		echo
		echo " * $(find bin/packages -name '*.ipk' | wc -l) binary packages built"
		[ -f "logs/package/error.txt" ] && \
			echo -n " * $(wc -l <logs/package/error.txt)" || \
			echo -n " * 0"
		echo "/$(find logs/package/ -name 'compile.txt' | wc -l) source packages failed"
	} | tee logs/stats
}

available_commands+=( ["configure"]="Recreates configuration for target boards" )
configure() {
	report "Creating default configuration"
	[ -n "${TARGET_BOARD}" ] || _die "No board selected!"
	cat "${src_dir}"/configs/common/* "${src_dir}/configs/${TARGET_BOARD}"/* > ./.config
	{
		[ -n "$DL_MIRROR" ] && echo "CONFIG_DOWNLOAD_FOLDER=$DL_MIRROR"
		echo "CONFIG_VERSION_REPO=\"https://repo.turris.cz/$PUBLISH_BRANCH/$TARGET_BOARD/packages\""
		echo "CONFIG_VERSION_NUMBER=\"$("${src_dir}/helpers/turris-version.sh" version)\""
		echo "CONFIG_VERSION_CODE=\"$(_git_remote_hash)\""
	} >> .config
	_make defconfig diffconfig
}

available_commands+=( ["update_mirror"]="Updates all local mirrors" )
update_mirror() {
	[ -n "$GIT_MIRROR" ] || return 0
	[ -d "$GIT_MIRROR" ] || return 0
	[ -z "$mirror_updated" ] || return 0

	_report "Updating local mirrors"
	flock --exclusive "$GIT_MIRROR" "$SHELL" -s "$GIT_MIRROR" <<"EOF"
		cd "$1"
		for mirror in ./*; do
			[ -d "$mirror" ] || continue
			cd "$mirror"
			git remote update --prune
			cd "$1"
		done
EOF
	mirror_updated="yes"
}

_checkout_init() {
	report "Checking out clean OpenWRT repository"
	update_mirror
	if [ -n "$GIT_MIRROR" ]; then
		mkdir -p "$GIT_MIRROR"
		[ -d "$GIT_MIRROR/openwrt" ] || \
			git clone --mirror "$OPENWRT_URL" "$GIT_MIRROR/openwrt"
		OPENWRT_URL="$GIT_MIRROR/openwrt"
	fi
	rm -rf .git
	git init
	git remote add origin "$OPENWRT_URL"
}

_checkout_clean() {
	BUILD_SETTINGS="$(cat turris-build.conf 2> /dev/null || true)"
	git clean -dff
	find . -name '*.rej' -delete
	[ -z "$BUILD_SETTINGS" ] || echo "$BUILD_SETTINGS" > turris-build.conf
	git rev-parse --short HEAD > version
	echo "/turris-build.conf" >> ./.gitignore
	echo "/version" >> ./.gitignore
	_git commit -m 'gitignore: Ignore some more files' ./.gitignore
	if [ -n "$DL_MIRROR" ]; then
		mkdir -p "$DL_MIRROR"
		rm -rf dl && ln -s "$DL_MIRROR" dl
	fi
}

available_commands+=( ["checkout"]="Start everything from scratch - all changes deleted and fresh copy of OpenWRT gets checked out" )
checkout() {
	report "Starting out fresh!"
	_checkout_init
	local githash
	githash="$(echo "$OPENWRT_BRANCH" | sed -n 's|^#||p')"
	if [ -n "$githash" ]; then
		git fetch origin
		git checkout -f "$githash"
	else
		git fetch ${CLONE_DEEP+--depth 1} origin "$OPENWRT_BRANCH"
		git checkout -f "origin/$OPENWRT_BRANCH"
	fi
	_checkout_clean
	cp "$src_dir"/feeds.conf .
}

available_commands+=( ["repo_checkout"]="Start everything from scratch relative to repository on repo.turris.cz - all changes deleted and copy of OpenWRT gets checked out with same hashes as repo.turris.cz" )
repo_checkout() {
	report "Starting out $TARGET_BOARD-$PUBLISH_BRANCH!"
	curl "https://repo.turris.cz/$PUBLISH_BRANCH/$TARGET_BOARD/packages/git-hash" | tail -n +2 | grep -v '^$' | sed 's/^ \* //;s/: /:/' > repo-git-hashes
	_checkout_init
	git fetch
	git checkout -f "$(awk -F : '/^openwrt:/{print $2}' repo-git-hashes)"
	GIT_HASHES="$(cat repo-git-hashes)"
	_checkout_clean
	echo "$GIT_HASHES" > repo-git-hashes
	rm -f feeds.conf
	local line githash
	while read -r line; do
		# TODO clean this awk
		githash="$(awk -F : "/^feeds\/$(echo "$line" | awk '{print $2}')\:/{print \$2}" repo-git-hashes)"
		if [ -z "$githash" ]; then
			echo "$line" >> feeds.conf
		else
			echo "$line" | sed "s/[;^].*$//;s/$/\^$githash/" >> feeds.conf
		fi
	done < "$src_dir/feeds.conf"
}

available_commands+=( ["clean_ccache"]="Clean persistent ccache paths" )
clean_ccache() {
	if [ "$(which ccache)" ]; then
		report "Cleaning ccache"
		[ -z "$CCACHE_HOST_DIR" ]   || CCACHE_DIR="$CCACHE_HOST_DIR"   ccache -C
		[ -z "$CCACHE_TARGET_DIR" ] || CCACHE_DIR="$CCACHE_TARGET_DIR/$TARGET_ARCH" ccache -C
	else
		report "Not cleaning ccache as you don't have ccache installed"
	fi
}

available_commands+=( ["set_ccache"]="Set persistent ccache paths" )
set_ccache() {
	[ -z "$CCACHE_SET" ] || return 0
	report "Setting ccache paths"
	CCACHE_SET=y
	[ -z "$CCACHE_HOST_DIR" ] || sed -i 's|$(STAGING_DIR_HOST)/ccache|'"$CCACHE_HOST_DIR|" include/host-build.mk
	[ -z "$CCACHE_TARGET_DIR" ] || [ -z "$TARGET_ARCH" ] || sed -i 's|$(STAGING_DIR)/ccache|'"$CCACHE_TARGET_DIR/$TARGET_ARCH|" include/package.mk
	[ -z "$(git diff include/host-build.mk include/package.mk)" ] || _git commit -m "include: ccache settings" include/host-build.mk include/package.mk
}

available_commands+=( ["set_local_feeds"]="Change feed URL to their mirror counterparts" )
set_local_feeds() {
	[ -n "$GIT_MIRROR" ] || return 0
	report "Setting feeds to their local counterparts"
	while read -r vcs name url rest; do
		if [ "$vcs" = src-git ] && [ -d "$GIT_MIRROR/$name" ]; then
			local branch=""
			[[ "$url" != *^* ]] || branch="^${url#*^}"
			[[ "$url" != *\;* ]] || branch=";${url#*;}"
			echo "$vcs $name file://$GIT_MIRROR/$name$branch $rest"
		else
			echo "$vcs $name $url $rest"
		fi
	done < feeds.conf > feeds.conf.new
	mv feeds.conf.new feeds.conf
}

available_commands+=( ["mirror_feeds"]="Creates initial mirrors of all configured feeds" )
mirror_feeds() {
	[ -n "$GIT_MIRROR" ] || return 0
	pushd "$GIT_MIRROR" >/dev/null
	cat "$src_dir"/feeds.conf "$build_dir"/feeds.conf 2> /dev/null | while read -r vcs name url rest; do
		if [ "$vcs" = src-git ] && [ ! -d "$GIT_MIRROR/$name" ]; then
			git clone --mirror "${url%[;^]*}" "$name"
		fi
	done
	popd >/dev/null
	update_mirror
}

available_commands+=( ["patch_openwrt"]="Patch the main OpenWRT repository" )
patch_openwrt() {
	report "Patching OpenWRT repository"
	for patch in "$src_dir/patches/openwrt"/*/*.patch; do
		_git am --whitespace=nowarn --reject "$patch"
	done
	if [ -d "$src_dir/src" ]; then
		cp -r "$src_dir"/src/* .
		find "$src_dir"/src -printf "%P\\0" | xargs --null git add
		_git commit -m 'Add additional files'
	fi
}

_git_remote_hash() {
	local dir="${1:-}"
	git -C "$dir" rev-list HEAD | \
		git -C "$dir" name-rev --stdin --refs='origin/*' | \
		awk '$2 != "" { print $1; exit }'
}

available_commands+=( ["store_hash"]="Stores hashes of current build" )
store_hash() {
	report "Storing hashes"
	mkdir -p bin
	{
	echo "Project was build from following repositories:"
	echo
	echo " * turris-build: $(_git_remote_hash "${src_dir}")"
	echo " * openwrt: $(_git_remote_hash "${build_dir}")"
	for feed in feeds/*; do
		[ -d "$feed/.git" ] || continue
		echo " * $feed: $(_git_remote_hash "${feed}")"
	done
	echo
	} | tee bin/git-hash
}

available_commands+=( ["repatch_feeds"]="Cleanup feeds, update them and patch them" )
repatch_feeds() {
	report "Getting feeds"
	update_mirror
	set_local_feeds
	_perl ./scripts/feeds clean -a
	_perl ./scripts/feeds update -a

	report "Patching feeds"
	for feed in "$src_dir"/patches/*; do
		[ -d "$feed" ] || continue
		feed_name="$(basename "$feed")"
		[ -d "$build_dir/feeds/$feed_name" ] || continue
		for patch in "$feed"/*/*.patch; do
			[ -f "$patch" ] || continue
			pushd "$build_dir/feeds/$feed_name" >/dev/null
			_git am --whitespace=nowarn --reject "$patch"
			popd >/dev/null
		done
	done

	report "Installing feeds packages"
	_perl ./scripts/feeds update -a -i
	_perl ./scripts/feeds install -a
}

available_commands+=( ["prefetch"]="Runs make download" )
prefetch() {
	_openwrt_make -j"$BUILD_JOBS" download
}

available_commands+=( ["gen_junit"]="Generates junit output from build logs" )
gen_junit() {
	_sh "${src_dir}"/helpers/generate_junit.sh
}

available_commands+=( ["pkgsrepo"]="Extract packages from OpenWRT tree and place them to repository format as they are deployed to Turris servers" )
pkgsrepo() {
	report "Extracting packages to pkgsrepo directory: $OUTPUT_DIR"
	rm -rf "$OUTPUT_DIR"
	mkdir -p "$OUTPUT_DIR"
	mv bin/packages/*/* "$OUTPUT_DIR"
	mv bin/targets/*/*/packages "$OUTPUT_DIR/core"
	mv bin/targets/*/*/config.buildinfo "$OUTPUT_DIR"
	mv bin/git-hash "$OUTPUT_DIR"
}

# Function calling make in OpenWRT tree that is designed to handle force option in
# compact way.
_compile() {
	if [ -z "$FORCE" ]; then
		_openwrt_make -j"$BUILD_JOBS" "$@"
	else
		local success=""
		local force_jobs="$BUILD_JOBS"
		while [ "$force_jobs" -gt 0 ] && [ -z "$success" ]; do
			local countdown="$FORCE"
			while [ $countdown -gt 0 ] && [ -z "$success" ]; do
				if _openwrt_make -j"$force_jobs" "$@"; then
					success="YES!!!"
				else
					countdown="$((countdown - 1))"
					report "Build job with -j$force_jobs failed (trying $((FORCE - countdown))/$FORCE)..."
				fi
			done
			if [ -z "$success" ]; then
				report "Build job with -j$force_jobs failed, decreasing parallelism..."
				force_jobs="$((force_jobs / 2))"
			fi
		done
		if [ "$force_jobs" -le 1 ] && [ -z "$success" ]; then
			_openwrt_make -j1 V=s "$@"
		fi
	fi
}

available_commands+=( ["compile_tools"]="Compile host tools" )
compile_tools() {
	report "Compiling tools"
	_compile tools/compile toolchain/compile
}

available_commands+=( ["compile_target"]="Compile target specific software (Linux kernel)" )
compile_target() {
	report "Compiling target"
	_compile target/compile
}

available_commands+=( ["compile_packages"]="Compile packages" )
compile_packages() {
	report "Compiling packages"
	_compile package/compile
}

available_commands+=( ["compile"]="	Compile tools, target and packages" )
compile() {
	compile_tools
	compile_target
	compile_packages
}

available_commands+=( ["sign"]="	Generate packages index and sign it" )
sign() {
	_compile package/index BUILD_KEY="$SIGN_KEY"
}

available_commands+=( ["clean"]="	Clean current build directory" )
clean() {
	report "Cleaning up current build directory"
	rm -rf ./build_dir ./tmp ./staging_dir ./logs ./bin
	mkdir -p tmp
}

available_commands+=( ["prepare"]="	Prepare build but don't build it (Implies: checkout clean patch_openwrt repatch_feeds gen_version configure set_ccache)" )
prepare() {
	checkout
	clean
	patch_openwrt
	repatch_feeds
	gen_version
	configure
	set_ccache
}

available_commands+=( ["prepare_tools"]="Prepare build and build tools (Impies: prepare compile_tools compile_target)" )
prepare_tools() {
	prepare
	compile_tools
	compile_target
}

available_commands+=( ["repo_prepare"]="Same as prepare but instead of checkout uses repo_checkout" )
repo_prepare() {
	repo_checkout
	clean
	patch_openwrt
	repatch_feeds
	gen_version
	configure
	set_ccache
}

available_commands+=( ["autopkg"]="	Deploy autopkg scripts that can be used for rolling software development" )
autopkg() {
	report "Deploying autopkg scripts"
	cp "$src_dir"/helpers/autopkg/* "$build_dir"/include/
	git add include/autopkg-*.mk
	_git commit -m 'autopkg: Add autopkg scripts'
}

##################################################################################

print_help() {
	echo "Usage: ${0} [OPTION].. [COMMAND].."
	echo "Compile TurrisOS packages. This script fetches, patches and configures OpenWRT tree."
	echo
	echo "Primary options:"
	echo "	-t BOARD	Set target board to BOARD. This is required for almost all commands."
	echo "	-jNUM		Number of jobs in paralel to be run. In default number of CPUs is used."
	echo "	-a ARG		Add build arguments - passed directly to make. This option can be specified multiple times."
	echo "	-f[NUM]		Try hard to get stuff to compile (optional number specifies number of tries)"
	echo "	-o PATH		Path to output directory where repository is prepared. In default ./pkgsrepo is used."
	echo "	-s KEY		Sign packges with given private key. Key has to be generated using using tool."
	echo "	-h, --help	Show this help text"
	echo
	echo "Advanced options:"
	echo "	-p BRANCH	Name of the target build alias. This is used to set OPKG feeds and for repo_checkout. If not specified '$PUBLISH_BRANCH' is used."
	echo "	-l		Do not update local git mirrors. Use them as they are to fetch OpenWRT and its feeds."
	echo "	-d		Do not use shallow checkouts"
	echo "	-x		Enable debug mode"
	echo
	echo "Available commands are:"
	for cmd in "${!available_commands[@]}"; do
		echo "	$cmd	${available_commands[$cmd]}"
	done
	echo
	echo "Default commands are: ${DEFAULT_STEPS[*]}"
}

steps=()
# Parse arguments
while [ $# -gt 0 ]; do
	case "$1" in
		-e)
			echo "-e option is obsoleted and execution now always runs as with this option." >&2
			;;
		-j*)
			BUILD_JOBS="${1#-j}"
			;;
		-f*)
			FORCE="${1#-f}"
			[ -n "$FORCE" ] || FORCE="1"
			;;
		-t)
			shift
			set_target "$1"
			;;
		-a)
			shift
			BUILD_ARGS+=( "$1" )
			;;
		-p)
			shift
			PUBLISH_BRANCH="$1"
			;;
		-l)
			mirror_updated="override"
			OPENWRT_URL="$GIT_MIRROR/openwrt"
			;;
		-o)
			shift
			OUTPUT_DIR="$1"
			;;
		-s)
			shift
			SIGN_KEY="$1"
			;;
		-d)
			CLONE_DEEP="y"
			;;
		-x)
			_enable_debug
			;;
		-h|--help)
			print_help
			exit 0
			;;
		*)
			[ -n "${available_commands[$1]}" ] || \
				die "There is no such option or command: $1"
			steps+=( "$1" )
			;;
	esac
	shift
done

[ "$(id -u)" != 0 ] || _die "Build should not be run under root!"

[ ${#steps[@]} -gt 0 ] || steps=( "${DEFAULT_STEPS[@]}" )
for step in "${steps[@]}"; do
	"$step"
done