compile_fw

#!/bin/bash -e

# OpenWRT compilation script
# (C) 2018 CZ.NIC, z.s.p.o.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

BUILD_DIR="$(pwd)"
SRC_DIR="$(dirname "$0")"
SRC_DIR="$(cd "$SRC_DIR"; pwd)"
CMD="${0}"
OPENWRT_URL="https://git.openwrt.org/openwrt/openwrt.git"
OPENWRT_BRANCH=master
DEBUG=""
KEY=""
EVERYTHING=""
PERL_DEBUG=""
SH_DEBUG=""
MAKE_DEBUG=""
PUBLISH_BRANCH=""
DEPTH="--depth 1"
FORCE=""
DEFAULT_STEPS="checkout clean patch_openwrt get_feeds patch_feeds gen_version configure build gen_lists store_hash sign stats gen_junit"

if [ "$BUILD_DIR" == "$SRC_DIR" ]; then
	mkdir -p build
	cd build
	BUILD_DIR="$(pwd)"
fi

export TMPDIR="${BUILD_DIR}/tmp"
mkdir -p "$TMPDIR"

[ \! -f "${SRC_DIR}"/turris-build.conf ] || . "${SRC_DIR}"/turris-build.conf
[ \! -f ~/.turris-build ] || . ~/.turris-build
[ \! -f ./turris-build.conf ] || . ./turris-build.conf

MIRROR_UPDATED=""


_report() {
	echo -e '\033[0;34m'"$1"'\033[0m' >&2
}

_die() {
	echo -e '\033[0;31m'"$1"'\033[0m' >&2
	exit 1
}

_get_version() {
	# If first grep fails - version is not first non-empty, add 999 to the end
	# Be aware that Condition in if also outputs the results
	if ! grep . "${SRC_DIR}"/NEWS | head -n 1 | grep '^[0-9.]\+$'; then
		echo "$(grep '^[0-9.]\+$' "${SRC_DIR}"/NEWS | head -n 1).999"
	fi
}

gen_version_help="Generates Turris version package from NEWS"
gen_version() {
	_report "Generating turris-version package"
	# Find the current version
	VERSION="$(_get_version)"
	# If top of the NEWS is not a version number, find previous one and add 99
	# Also try to figure out where are the actual NEWS for this version
	if expr "$VERSION" : '999$' > /dev/null; then
		START=1
		END="$(grep -n '^[0-9.]\+$' "${SRC_DIR}"/NEWS | head -n 1 | sed 's|:.*||')"
		END="$(expr $END - 1)"
	else
		START="$(grep -n '^[0-9.]\+$' "${SRC_DIR}"/NEWS | head -n 1 | sed 's|:.*||')"
		START="$(expr $START + 1)"
		END="$(grep -n '^[0-9.]\+$' "${SRC_DIR}"/NEWS | sed -n '2 s|:.*||p')"
		[ -n "$END" ] || END="\$"
	fi
	# Generate package
	mkdir -p "${BUILD_DIR}"/package/system/turris-version
cat > "${BUILD_DIR}"/package/system/turris-version/Makefile << EOF
#
## Copyright (C) $(date +%Y) CZ.NIC z.s.p.o. (http://www.nic.cz/)
#
## This is free software, licensed under the GNU General Public License v2.
# See /LICENSE for more information.
# #
#
include \$(TOPDIR)/rules.mk

PKG_NAME:=turris-version
PKG_VERSION:=$VERSION

PKG_MAINTAINER:=CZ.NIC <packaging@nic.cz>

include \$(INCLUDE_DIR)/package.mk

define Package/turris-version
	TITLE:=turris-version
endef

define Build/Prepare
endef

define Build/Compile
endef

define Package/turris-version/postinst
#!/bin/sh
# Danger: spaces are not ordinary spaces, but special unicode ones
[ -n "\$\$IPKG_INSTROOT" ] || {
create_notification -s news "$(sed -n "$START,$END p" "${SRC_DIR}"/NEWS | grep . | sed 's|^[[:blank:]]*\*[[:blank:]]*| • |')"
}
endef

define Package/turris-version/install
	\$(INSTALL_DIR) \$(1)/etc
	echo \$(PKG_VERSION) > \$(1)/etc/turris-version
endef

\$(eval \$(call BuildPackage,turris-version))
EOF
	pushd "${BUILD_DIR}"
	git add "${BUILD_DIR}"/package/system/turris-version
	git commit -m 'turris-version: Create a package with release notes'
	popd
}

conflicts_help="Show packages that we are overriding in Turris OS packages"
conflicts() {
	_report "Showing conflicting packages"
	echo "Following packages are being overridden by Turris packages:"
	echo
	mkdir -p tmp
	find feeds/turrispackages -name Makefile | sed 's|.*/\([^/]*\)/Makefile|\1|' | sort > tmp/turris-packages.list
	find feeds/ -name Makefile | grep -v '^feeds/turrispackages' | sed 's|.*/\([^/]*\)/Makefile|\1|' | sort > tmp/other-packages.list
	comm -12 tmp/turris-packages.list tmp/other-packages.list | sed 's|^| * |'
	echo
	rm -f tmp/turris-packages.list tmp/other-packages.list
}

stats_help="	Prints statistics regarding number of built packages"
stats() {
	mkdir -p logs
	_report "Reporting statistics"
	{
	echo "Statistics of the build:"
	echo
	echo " * $(find bin/packages -name '*.ipk' | wc -l) binary packages built"
	echo " * $(cat logs/package/error.txt | wc -l)/$(find logs/package/ -name 'compile.txt' | wc -l) source packages failed"
	if [ -n "$BUILD_START" ] && [ -n "$BUILD_END" ]; then
		BUILD_TIME="$(expr $BUILD_END - $BUILD_START)"
		echo " * build time $(expr $BUILD_TIME / 3600):$(expr \( $BUILD_TIME % 3600 \) / 60):$(expr $BUILD_TIME % 60)"
	fi
	} | tee logs/stats
}

configure_help="Recreates configuration for target boards"
configure() {
	_report "Creating default configuration"
	[ -n "${TARGET_BOARD}" ] || _die "No board selected!"
	cat "${SRC_DIR}"/configs/common/* "${SRC_DIR}"/configs/${TARGET_BOARD}/* > ./.config
	echo "CONFIG_DEVEL=y" >> .config
	echo "CONFIG_CCACHE=y" >> .config
	echo "CONFIG_DOWNLOAD_FOLDER=$DL_MIRROR" >> .config
	if [ -n "${PUBLISH_BRANCH}" ]; then
		echo "CONFIG_VERSION_REPO=\"https://repo.turris.cz/${TARGET_BOARD}-${PUBLISH_BRANCH}\"" >> .config
	else
		echo "CONFIG_VERSION_REPO=\"https://repo.turris.cz/${TARGET_BOARD}\"" >> .config
	fi
	echo "CONFIG_VERSION_NUMBER=\"$(_get_version)\"" >> .config
	if [ "$EVERYTHING" = yes ]; then
		echo "CONFIG_ALL_KMODS=y" >> .config
		echo "CONFIG_ALL=y" >> .config
		echo "CONFIG_IB=y" >> .config
		echo "CONFIG_IB_STANDALONE=y" >> .config
		echo "CONFIG_SDK=y" >> .config
	else
		echo "CONFIG_ALL=n" >> .config
		echo "CONFIG_IB=n" >> .config
		echo "CONFIG_IB_STANDALONE=n" >> .config
		echo "CONFIG_SDK=n" >> .config
	fi
	make $MAKE_DEBUG defconfig
}

update_mirror_help="Updates all local mirrors"
update_mirror() {
	[ -n "$GIT_MIRROR" ] || return 0
	[ -z "$MIRROR_UPDATED" ] || return 0
	
	_report "Updating local mirrors"
	mkdir -p "$GIT_MIRROR"
	pushd "$GIT_MIRROR"
	if [ \! -d openwrt ]; then
		mkdir openwrt
		cd openwrt && git init --bare && git remote add --mirror=fetch origin "$OPENWRT_URL"
	fi
	cd "$GIT_MIRROR"
	OPENWRT_URL="$GIT_MIRROR/openwrt"
	for mirror in ./*; do
		cd "$mirror" || continue
		if ! git fetch --all; then
			sleep $(( 1 + $RANDOM % 30 ))
			git fetch --all
		fi
		cd "$GIT_MIRROR"
	done
	popd
	MIRROR_UPDATED="yes"
}

sign_help="	Signs generated lists and resigns packages"
sign() {
	[ -x staging_dir/host/bin/usign ] || make package/usign/host/compile
	[ -x staging_dir/host/bin/usign ] || _die "Don't have usign, can't sign"
	[ -n "$KEY" ] || KEY=key-build
	[ -f "$KEY" ] || _die "No key available"
	for i in bin/packages/lists/*.lua bin/packages/*/*/Packages bin/targets/*/*/packages/Packages; do
		[ -f "$i" ] || continue
		rm -f "$i".sig
		staging_dir/host/bin/usign -S -m "$i" -s "$KEY"
	done
}

checkout_help="Start everything from scratch - all changes deleted and fresh copy of OpenWRT gets checked out"
checkout() {
	_report "Starting out fresh!"
	_report "Checking out clean OpenWRT repository"
	update_mirror
	rm -rf .git
	git init
	git remote add origin "$OPENWRT_URL"
	HASH="$(echo "$OPENWRT_BRANCH" | sed -n 's|^#||p')"
	if [ -n "$HASH" ]; then
		git fetch origin
		git checkout -f $HASH 
	else
		git fetch $DEPTH origin "$OPENWRT_BRANCH"
		git checkout -f "origin/$OPENWRT_BRANCH"
	fi
	BUILD_SETTINGS="$(cat turris-build.conf 2> /dev/null || true)"
	git clean -dff
	[ -z "$BUILD_SETTINGS" ] || echo "$BUILD_SETTINGS" > turris-build.conf
	git config --local commit.gpgsign false
	echo "/turris-build.conf" >> ./.gitignore
	echo "/version" >> ./.gitignore
	git commit -m 'gitignore: Ignore some more files' ./.gitignore
	[ \! -f "$SRC_DIR"/feeds.conf ] || cp "$SRC_DIR"/feeds.conf .
	git log -n 1 --format="%h" > version
	if [ -n "$DL_MIRROR" ]; then
		mkdir -p "$DL_MIRROR"
		rm -rf dl && ln -s "$DL_MIRROR" dl
	fi
}

patch_feeds_help="Apply patches to the feeds"
patch_feeds() {
	_report "Patching feeds"
	pushd "$SRC_DIR"/patches
	for feed in *; do
		[ -d "$BUILD_DIR"/feeds/$feed ] || continue
		cd "$SRC_DIR"/patches/$feed
		for patch in */*.patch; do
			[ -f "$patch" ] || continue
			cd "$BUILD_DIR"/feeds/$feed
			git config --local commit.gpgsign false
			git am --reject "$SRC_DIR"/patches/$feed/$patch
			cd "$SRC_DIR"/patches/$feed
		done
	done
	popd
}

clean_ccache_help="Set persistent ccache paths"
clean_ccache() {
	if [ "$(which ccache)" ]; then
		_report "Cleaning ccache"
		[ -z "$CCACHE_HOST_DIR" ]   || CCACHE_DIR="$CCACHE_HOST_DIR"   ccache -C
		[ -z "$CCACHE_TARGET_DIR" ] || CCACHE_DIR="$CCACHE_TARGET_DIR" ccache -C
	else
		_report "Not cleaning ccache as you don't have ccache installed"
	fi
}

set_ccache_help="Set persistent ccache paths"
set_ccache() {
	[ -z "$CCACHE_SET" ] || return 0
	_report "Setting ccache paths"
	CCACHE_SET=y
	[ -z "$CCACHE_HOST_DIR" ] || sed -i 's|\(export CCACHE_DIR:=\)[^),]*|\1'"$CCACHE_HOST_DIR|" include/host-build.mk
	[ -z "$CCACHE_TARGET_DIR" ] || [ -z "$TARGET_ARCH" ] || sed -i 's|\(export CCACHE_DIR:=\)[^),]*|\1'"$CCACHE_TARGET_DIR/$TARGET_ARCH)|" include/package.mk
	[ -z "$(git diff include/host-build.mk include/package.mk)" ] || git commit -m "include: ccache settings" include/host-build.mk include/package.mk
}

set_local_feeds_help="Change feed URL to their mirror counterparts"
set_local_feeds() {
	[ -n "$GIT_MIRROR" ] || return 0
	_report "Setting feeds to their local counterparts"
	while read vcs name url rest; do
		if [ "$vcs" = src-git ] && [ -d "$GIT_MIRROR"/$name ]; then
			feed_url="$(echo "$url" | sed 's|[[:blank:]^].*||')"
			sed -i "s|$feed_url|file://$GIT_MIRROR/$name|" feeds.conf
		fi
	done < feeds.conf
}

mirror_feeds_help="Creates initial mirrors of all configured feeds"
mirror_feeds() {
	[ -n "$GIT_MIRROR" ] || return 0
	pushd "$GIT_MIRROR"
	cat "$SRC_DIR"/feeds.conf "$BUILD_DIR"/feeds.conf 2> /dev/null | while read vcs name url rest; do
		if [ "$vcs" = src-git ] && [ \! -d "$GIT_MIRROR"/$name ]; then
			feed_url="$(echo "$url" | sed 's|[[:blank:]^].*||')"
			mkdir "$name"
			cd "$name" && git init --bare && git remote add --mirror=fetch origin "$feed_url" && git fetch --all
			cd "$GIT_MIRROR"
		fi
	done
	popd
	update_mirror
}

patch_openwrt_help="Patch the main OpenWRT repository"
patch_openwrt() {
	_report "Patching OpenWRT repository"
	pushd "$SRC_DIR"/patches/openwrt
	for patch in */*.patch; do
		cd "$BUILD_DIR"
		git am --reject "$SRC_DIR"/patches/openwrt/$patch
	done
	popd
	if [ -n "$(ls -1 "$SRC_DIR"/src/)" ]; then
		cp -r "$SRC_DIR"/src/* .
		find "$SRC_DIR"/src | sed 's|^'"$SRC_DIR"'/src/|git add |' | sh
		git commit -m 'Add additional files'
	fi
	set_ccache
}

_git_remote_hash() {
	pushd "$1" > /dev/null
	br="$(LANG=C git status --long | sed -n 's|HEAD detached from \(.*\)|\1|p')"
	[ -n "$br" ] || br="$(LANG=C git status --long | sed -n "s|Your branch is up to date with '\\([^']*\\)'.*|\1|p")"
	[ -n "$br" ] || br="$(LANG=C git status --long | sed -n "s|Your branch is ahead of '\\([^']*\\)'.*|\1|p")"
	[ -n "$br" ] || br=HEAD
	git log -n1 --pretty=%H	"$br"
	popd > /dev/null
}

store_hash_help="Stores hashes of current build"
store_hash() {
	_report "Storing hashes"
	mkdir -p bin
	{
	echo "Project was build from following repositories:"
	echo
	echo " * turris-build: $(_git_remote_hash "${SRC_DIR}")"
	echo " * openwrt: $(_git_remote_hash "${BUILD_DIR}")"
	for feed in feeds/*; do
		[ -d "$feed/.git" ] || continue
		echo " * $feed: $(_git_remote_hash "${feed}")"
	done
	echo
	} | tee bin/git-hash
}

get_feeds_help="Recreate configured feeds"
get_feeds() {
	_report "Getting feeds"
	update_mirror
	set_local_feeds
	perl $PERL_DEBUG ./scripts/feeds clean -a
	perl $PERL_DEBUG ./scripts/feeds update -a
	perl $PERL_DEBUG ./scripts/feeds install -a
	set +e
	DISABLED_PACKAGES="$(cat "${SRC_DIR}"/disabled_packages/common "${SRC_DIR}"/disabled_packages/${TARGET_BOARD} 2> /dev/null)"
	DISABLED_PACKAGES="$(echo "$DISABLED_PACKAGES" | sed 's|#.*||' | grep .)"
	set -e
	[ -z "$DISABLED_PACKAGES" ] || perl $PERL_DEBUG ./scripts/feeds uninstall $DISABLED_PACKAGES
}

repatch_feeds_help="Cleanup feeds, update them and patch them"
repatch_feeds() {
	get_feeds
	patch_feeds
}

prefetch_help="Runs make download"
prefetch() {
	make $MAKE_DEBUG $BUILD_ARGS $OWRT_DEBUG download
}

gen_junit_help="Generates junit output from build logs"
gen_junit() {
	sh $SH_DEBUG "${SRC_DIR}"/helpers/generate_junit.sh
}

gen_lists_help="Generate updater package lists"
gen_lists() {
	_report "Generating package lists for updater"
	[ -n "${TARGET_BOARD}" ] || _die "You need to specify target board!"
	mkdir -p bin/packages/lists
	"${SRC_DIR}"/helpers/generate_userlists.sh --branch "${PUBLISH_BRANCH}" --src "${SRC_DIR}/lists" bin/packages/lists
}

build_help="	Builds everything"
build() {
	_report "Starting real build"
	set_ccache
	BUILD_START="$(date +%s)"
	if [ -z "$FORCE" ]; then
		make $MAKE_DEBUG IS_TTY=1 BUILD_LOG=1 $BUILD_ARGS $OWRT_DEBUG
	else
		JOBS="$(echo " $BUILD_ARGS " | sed -n 's|.*-j\([0-9]\+\)|\1\n|p' | head -n 1)"
		[ -n "$JOBS" ] || JOBS=0
		SUCCESS=""
		while [ "$JOBS" -gt 0 ] && [ -z "$SUCCESS" ]; do
			COUNTDOWN="$FORCE"
			while [ $COUNTDOWN -gt 0 ] && [ -z "$SUCCESS" ]; do
				if make $MAKE_DEBUG IGNORE_ERRORS=m IS_TTY=1 BUILD_LOG=1 $BUILD_ARGS $OWRT_DEBUG -j${JOBS}; then
					SUCCESS="YES!!!"
				else
					COUNTDOWN="$(expr $COUNTDOWN - 1 || true)"
					_report "Build job with -j$JOBS failed (try $(expr $FORCE - $COUNTDOWN || true)/$FORCE)..."
				fi
			done
			if [ -z "$SUCCESS" ]; then
				_report "Build job with -j$JOBS failed, decreasing parallelism..."
				JOBS="$(expr $JOBS / 2 || true)"
			fi
		done
		if [ "$JOBS" -le 1 ] && [ -z "$SUCCESS" ]; then
			make $MAKE_DEBUG IGNORE_ERRORS=m IS_TTY=1 BUILD_LOG=1 $BUILD_ARGS $OWRT_DEBUG -j1 V=s
		fi
	fi
	BUILD_END="$(date +%s)"
}

clean_help="	Clean current build directory"
clean() {
	_report "Cleaning up current build directory"
	rm -rf ./build_dir ./tmp ./staging_dir ./logs ./bin
	mkdir -p tmp
}

prepare_help="	Prepare build but don't build it"
prepare() {
	checkout
	clean
	patch_openwrt
	repatch_feeds
	gen_version
	configure
}

# Sets various variables to match the specified target, just a helper
_set_target() {
	_report "Setting target as $1"
	case "$1" in
		omnia)
			TARGET_BOARD=omnia
			TARGET_ARCH=armv7l
			;;
		turris)
			TARGET_BOARD=turris
			TARGET_ARCH=ppcspe
			;;
		mox)
			TARGET_BOARD=mox
			TARGET_ARCH=aarch64
			;;
		*)
			echo "Invalid target board!!! Use -t [turris|omnia|mox]!!!"
			exit 1
			;;
	esac
}

help_help="	Displays text with help"
help() {
	echo "Usage: $CMD [options] [command] [command] ..."
	echo
	echo "Available options are:"
	echo "	-x		Enable debug mode"
	echo "	-e		Build everything, not just minimal set"
	echo "	-f[num]	Try hard to get stuff to compile (optional number specifies number of tries)"
	echo "	-d		Do not use shallow checkouts"
	echo "	-t board	Set target board to _board_"
	echo "	-a \"arg1 arg2\"	Set build arguments - passed directly to make"
	echo "	-b branch	Branch to checkout"
	echo "	-p branch	Name of the branch that is being build"
	echo "	-l		Do not update local git mirrors"
	echo "	-h		Show help"
	echo
	echo "Available commands are:"
	declare -F | sed -n 's|declare -f \([a-z]\)|\1|p' | while read func; do
		echo "	$func	$(eval echo \"\$${func}_help\")"
	done
	echo
	echo "Default commands are: $DEFAULT_STEPS"
	echo
}

[ -z "$BOARD" ] || _set_target "$BOARD"

while expr x$1 : x- > /dev/null; do
	cmd="x$1"
	shift
	case $cmd in
		x-x)
			set -x
			DEBUG="yes"
			PERL_DEBUG="-d:Trace"
			MAKE_DEBUG="-n"
			SH_DEBUG="-x"
			OWRT_DEBUG="V=s"
			;;
		x-e)
			EVERYTHING="yes"
			;;
		x-d)
			DEPTH=""
			;;
		x-f*)
			FORCE="$(echo $cmd | sed -n 's|^x-f\([0-9]*\)$|\1|p')"
			[ -n "$FORCE" ] || FORCE="1"
			;;
		x-t)
			_set_target "$1"
			shift
			;;
		x-h|x--help)
			help
			exit 0
			;;
		x-a)
			BUILD_ARGS="$1"
			shift
			;;
		x-p)
			PUBLISH_BRANCH="$1"
			shift
			;;
		x-b)
			OPENWRT_BRANCH="$1"
			shift
			;;
		x-l)
			MIRROR_UPDATED="override"
			OPENWRT_URL="$GIT_MIRROR/openwrt"
			;;
	esac
done

[ -n "$1" ] || set $DEFAULT_STEPS

while [ -n "$1" ]; do
	eval "$1"
	shift
done