Initial version of build scripts

Commiting basic building script. Some functionality is already there, what is
missing is handling branches and freezing stuff, correct version setup (no need
in development for now), userlist generation and medkit creation (no need till
we build at least a big part of packages).
Signed-off-by: Michal Hrusecky <Michal@Hrusecky.net>

.gitignore

+/README.html

README.asciidoc

+Simple Turris OS building script
+================================
+
+Getting started
+---------------
+
+Create an empty directory you want to build Turris OS in, enter that directory
+and from it call `compile_fw` script. In most basic use-case, this should be
+enough. Be aware - *it will delete all previous content of the current
+directory*. You can repeat individual steps of the compilation separately,
+check `compile_fw help` for a list of available commands, generic options and
+their brief descriptions.
+
+Advanced usage
+--------------
+
+Patching
+~~~~~~~~
+
+If you need to patch OpenWRT or any of the feeds, put a git formatted patch
+into respective subdirectory of patches directory. In patches directory, there
+is openwrt directory and directory for each feed we are patching. Inside those
+directories, patches are also divided into subdirectories like `to-upstream`,
+`wip` or `hack` to somehow distinguish between quality of patches.
+
+Configuration
+~~~~~~~~~~~~~
+
+Default config for OpenWRT is put in `configs` directory either in file
+`common` or in a file named after the target board. Those two files gets merged
+and will form the final `.config` file. Some parts of `compile_fw` might modify
+it further.

compile_fw

+#!/bin/bash -e
+
+# OpenWRT compilation script
+# (C) 2018 CZ.NIC, z.s.p.o.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+BUILD_DIR="$(pwd)"
+SRC_DIR="$(dirname "$0")"
+SRC_DIR="$(cd "$SRC_DIR"; pwd)"
+CMD="${0}"
+OPENWRT_URL="https://git.openwrt.org/openwrt/openwrt.git"
+BRANCH=master
+DEBUG=""
+PERL_DEBUG=""
+MAKE_DEBUG=""
+DEPTH="--depth 1"
+FORCE=""
+DEFAULT_STEPS="checkout patch_openwrt get_feeds patch_feeds configure build"
+
+export TMPDIR="${BUILD_DIR}/tmp"
+mkdir -p "$TMPDIR"
+
+[ \! -f "${SRC_DIR}"/turris-build.conf ] || . "${SRC_DIR}"/turris-build.conf
+[ \! -f ~/.turris-build ] || . ~/.turris-build
+[ \! -f ./turris-build.conf ] || . ./turris-build.conf
+
+MIRROR_UPDATED=""
+
+
+_report() {
+	echo -e '\033[0;34m'"$1"'\033[0m' >&2
+}
+
+_die() {
+	echo -e '\033[0;31m'"$1"'\033[0m' >&2
+	exit 1
+}
+
+configure_help="Recreates configuration for target boards"
+configure() {
+	_report "Creating default configuration"
+	[ -n "${TARGET_BOARD}" ] || _die "No board selected!"
+	set -x
+	cat "${SRC_DIR}"/configs/common "${SRC_DIR}"/configs/${TARGET_BOARD} > ./.config
+	echo "CONFIG_CCACHE=y" >> .config
+	make $MAKE_DEBUG defconfig
+}
+
+update_mirror_help="Updates all local mirrors"
+update_mirror() {
+	[ -n "$GIT_MIRROR" ] || return 0
+	[ -z "$MIRROR_UPDATED" ] || return 0
+	
+	_report "Updating local mirrors"
+	pushd "$GIT_MIRROR"
+	if [ \! -d openwrt ]; then
+		mkdir openwrt
+		cd openwrt && git init --bare && git remote add --mirror=fetch origin "$OPENWRT_URL"
+	fi
+	OPENWRT_URL="$GIT_MIRROR/openwrt"
+	for mirror in ./*; do
+		cd "$mirror" && git fetch --all
+		cd "$GIT_MIRROR"
+	done
+	popd
+	MIRROR_UPDATED="yes"
+}
+
+checkout_help="Start everything from scratch - all changes deleted and fresh copy of OpenWRT gets checked out"
+checkout() {
+	_report "Starting out fresh!"
+	_report "Checking out clean OpenWRT repository"
+	update_mirror
+	rm -rf .git
+	git init
+	git remote add origin "$OPENWRT_URL"
+	git fetch $DEPTH origin "$BRANCH"
+	git checkout -f "origin/$BRANCH"
+	BUILD_SETTINGS="$(cat turris-build.conf 2> /dev/null)"
+	git clean -dff
+	[ -z "$BUILD_SETTINGS" ] || echo "$BUILD_SETTINGS" > turris-build.conf
+	git config --local commit.gpgsign false
+	echo "/turris-build.conf" >> ./.gitignore
+	echo "/version" >> ./.gitignore
+	git commit -m 'gitignore: Ignore some more files' ./.gitignore
+	[ \! -f "$SRC_DIR"/feeds.conf ] || cp "$SRC_DIR"/feeds.conf .
+	if [ -n "$DL_MIRROR" ]; then
+		rm -rf dl && ln -s "$DL_MIRROR" dl
+	fi
+	git log -n 1 --format="%h" > version
+}
+
+patch_feeds_help="Apply patches to the feeds"
+patch_feeds() {
+	_report "Patching feeds"
+	pushd "$SRC_DIR"/patches
+	for feed in *; do
+		[ -d "$BUILD_DIR"/feeds/$feed ] || continue
+		cd "$SRC_DIR"/patches/$feed
+		for patch in */*.patch; do
+			[ -f "$patch" ] || continue
+			cd "$BUILD_DIR"/feeds/$feed
+			git config --local commit.gpgsign false
+			git am "$SRC_DIR"/patches/$feed/$patch
+		done
+	done
+	popd
+}
+
+set_ccache_help="Set persistent ccache paths"
+set_ccache() {
+	[ -z "$CCACHE_SET" ] || return 0
+	_report "Setting ccache paths"
+	CCACHE_SET=y
+	[ -z "$CCACHE_HOST_DIR" ] || sed -i 's|\(export CCACHE_DIR:=\).*|\1'"$CCACHE_HOST_DIR|" include/host-build.mk
+	[ -z "$CCACHE_TARGET_DIR" ] || [ -z "$TARGET_ARCH" ] || sed -i 's|\(export CCACHE_DIR:=\).*|\1'"$CCACHE_TARGET_DIR/$TARGET_ARCH|" include/package.mk
+	[ -z "$(git diff include/host-build.mk include/package.mk)" ] || git commit -m "include: ccache settings" include/host-build.mk include/package.mk
+}
+
+set_local_feeds_help="Change feed URL to their mirror counterparts"
+set_local_feeds() {
+	[ -n "$GIT_MIRROR" ] || return 0
+	while read vcs name url rest; do
+		if [ "$vcs" = src-git ] && [ -d "$GIT_MIRROR"/$name ]; then
+			feed_url="$(echo "$url" | sed 's|[[:blank:]^].*||')"
+			sed -i "s|$feed_url|file://$GIT_MIRROR/$name|" feeds.conf
+		fi
+	done < feeds.conf
+}
+
+mirror_feeds_help="Creates initial mirrors of all configured feeds"
+mirror_feeds() {
+	[ -n "$GIT_MIRROR" ] || return 0
+	pushd "$GIT_MIRROR"
+	cat "$SRC_DIR"/feeds.conf "$BUILD_DIR"/feeds.conf 2> /dev/null | while read vcs name url rest; do
+		if [ "$vcs" = src-git ] && [ \! -d "$GIT_MIRROR"/$name ]; then
+			feed_url="$(echo "$url" | sed 's|[[:blank:]^].*||')"
+			mkdir "$name"
+			cd "$name" && git init --bare && git remote add --mirror=fetch origin "$feed_url" && git fetch --all
+			cd "$GIT_MIRROR"
+		fi
+	done
+	popd
+	update_mirror
+}
+
+patch_openwrt_help="Patch the main OpenWRT repository"
+patch_openwrt() {
+	_report "Patching OpenWRT repository"
+	pushd "$SRC_DIR"/patches/openwrt
+	for patch in */*.patch; do
+		cd "$BUILD_DIR"
+		git am "$SRC_DIR"/patches/openwrt/$patch
+	done
+	popd
+	set_ccache
+}
+
+get_feeds_help="Recreate configured feeds"
+get_feeds() {
+	update_mirror
+	set_local_feeds
+	perl $PERL_DEBUG ./scripts/feeds clean -a
+	perl $PERL_DEBUG ./scripts/feeds update -a
+	perl $PERL_DEBUG ./scripts/feeds install -a
+}
+
+repatch_feeds_help="Cleanup feeds, update them and patch them"
+repatch_feeds() {
+	get_feeds
+	patch_feeds
+}
+
+prefetch_help="Runs make download"
+prefetch() {
+	make $MAKE_DEBUG $BUILD_ARGS $OWRT_DEBUG download
+}
+
+build_help="	Builds everything"
+build() {
+	_report "Starting real build"
+	set_ccache
+	if [ -z "$FORCE" ]; then
+		make $MAKE_DEBUG IS_TTY=1 LOGFILE=1 BUILD_LOG=1 $BUILD_ARGS $OWRT_DEBUG
+	else
+		make $MAKE_DEBUG IS_TTY=1 LOGFILE=1 BUILD_LOG=1 $BUILD_ARGS $OWRT_DEBUG || \
+		IGNORE_ERRORS=m make $MAKE_DEBUG IS_TTY=1 LOGFILE=1 BUILD_LOG=1 $BUILD_ARGS $OWRT_DEBUG || \
+		IGNORE_ERRORS=m make $MAKE_DEBUG IS_TTY=1 LOGFILE=1 BUILD_LOG=1 $BUILD_ARGS $OWRT_DEBUG -j1 V=s
+	fi
+}
+
+# Sets various variables to match the specified target, just a helper
+_set_target() {
+	_report "Setting target as $1"
+	case "$1" in
+		omnia)
+			TARGET_BOARD=omnia
+			TARGET_ARCH=armv7l
+			;;
+		turris)
+			TARGET_BOARD=turris
+			TARGET_ARCH=ppcspe
+			;;
+		mox)
+			TARGET_BOARD=mox
+			TARGET_ARCH=aarch64
+			;;
+		*)
+			echo "Invalid target board!!! Use -t [turris|omnia|mox]!!!"
+			exit 1
+			;;
+	esac
+}
+
+help_help="	Displays text with help"
+help() {
+	echo "Usage: $CMD [options] [command] [command] ..."
+	echo
+	echo "Available options are:"
+	echo "	-x		Enable debug mode"
+	echo "	-f		Try hard to get stuff to compile"
+	echo "	-d		Do not use shallow checkouts"
+	echo "	-t board	Set target board to _board_"
+	echo "	-a \"arg1 arg2\"	Set build arguments - passed directly to make"
+	echo "	-l		Do not update local git mirrors"
+	echo
+	echo "Available commands are:"
+	declare -F | sed -n 's|declare -f \([a-z]\)|\1|p' | while read func; do
+		echo "	$func	$(eval echo \"\$${func}_help\")"
+	done
+	echo
+	echo "Default commands are: $DEFAULT_STEPS"
+	echo
+}
+
+[ -z "$BOARD" ] || _set_target "$BOARD"
+
+while expr x$1 : x- > /dev/null; do
+	cmd="x$1"
+	shift
+	case $cmd in
+		x-x)
+			set -x
+			DEBUG="yes"
+			PERL_DEBUG="-d:Trace"
+			MAKE_DEBUG="-n"
+			OWRT_DEBUG="V=s"
+			;;
+		x-d)
+			DEPTH=""
+			;;
+		x-f)
+			FORCE="y"
+			;;
+		x-t)
+			_set_target "$1"
+			shift
+			;;
+		x-a)
+			BUILD_ARGS="$1"
+			shift
+			;;
+		x-b)
+			BRANCH="$1"
+			shift
+			;;
+		x-l)
+			MIRROR_UPDATED="override"
+			OPENWRT_URL="$GIT_MIRROR/openwrt"
+			;;
+	esac
+done
+
+[ -n "$1" ] || set $DEFAULT_STEPS
+
+while [ -n "$1" ]; do
+	eval "$1"
+	shift
+done

configs/common

+#
+# Automatically generated file; DO NOT EDIT.
+# OpenWrt Configuration
+#

feeds.conf

+src-git turrispackages https://gitlab.labs.nic.cz/turris/turris-os-packages.git^origin/cleanup
+src-git packages https://git.lede-project.org/feed/packages.git
+src-git luci https://git.lede-project.org/project/luci.git
+src-git routing https://git.lede-project.org/feed/routing.git
+src-git telephony https://git.lede-project.org/feed/telephony.git
+src-git printing https://github.com/miska/openwrt-printing-packages.git
+src-git openwisp https://github.com/openwisp/openwisp-config.git
+#src-git video https://github.com/openwrt/video.git
+#src-git targets https://github.com/openwrt/targets.git
+#src-git management https://github.com/openwrt-management/packages.git
+#src-git oldpackages http://git.openwrt.org/packages.git
+#src-link custom /usr/src/openwrt/custom-feed

patches/openwrt/to-upstream/0001-build-Allow-to-get-just-a-subset-of-packages-from-fe.patch

+From decaa09ed89d8685ff683526ea75dca8975039a0 Mon Sep 17 00:00:00 2001
+From: Michal Hrusecky <michal.hrusecky@nic.cz>
+Date: Mon, 23 Oct 2017 12:56:44 +0200
+Subject: [PATCH] build: Allow to get just a subset of packages from feeds
+
+Sometimes it is useful to get just some subset of packages from feed. This
+commit changes scripts/feeds in a way that you can specify as the last optional
+argument which packages should be used from the feed.
+
+Signed-off-by: Michal Hrusecky <michal.hrusecky@nic.cz>
+---
+ scripts/feeds | 25 ++++++++++++++++---------
+ 1 file changed, 16 insertions(+), 9 deletions(-)
+
+diff --git a/scripts/feeds b/scripts/feeds
+index f49cdb3..cebe05a 100755
+--- a/scripts/feeds
++++ b/scripts/feeds
+@@ -52,7 +52,7 @@ sub parse_config() {
+ 		chomp;
+ 		s/#.+$//;
+ 		next unless /\S/;
+-		my @line = split /\s+/, $_, 3;
++		my @line = split /\s+/, $_, 4;
+ 		my @src;
+ 		$line++;
+ 
+@@ -65,7 +65,7 @@ sub parse_config() {
+ 		$name{$line[1]} and die "Duplicate feed name '$line[1]', line: $line\n";
+ 		$name{$line[1]} = 1;
+ 
+-		push @feeds, [$line[0], $line[1], \@src];
++		push @feeds, [$line[0], $line[1], \@src, $line[3]];
+ 	}
+ 	close FEEDS;
+ }
+@@ -165,12 +165,13 @@ my %update_method = (
+ # src-git: pull broken
+ # src-cpy: broken if `basename $src` != $name
+ 
+-sub update_feed_via($$$$$) {
++sub update_feed_via($$$$$$) {
+ 	my $type = shift;
+ 	my $name = shift;
+ 	my $src = shift;
+ 	my $relocate = shift;
+ 	my $force = shift;
++	my $filter = shift;
+ 
+ 	my $m = $update_method{$type};
+ 	my $localpath = "./feeds/$name";
+@@ -198,6 +199,11 @@ sub update_feed_via($$$$$) {
+ 		system("cd '$safepath'; $update_cmd") == 0 or return 1;
+ 	}
+ 
++	if($filter) {
++		system("cd '$safepath'; find ./[a-zA-Z0-9]* -name Makefile -not -regex '" . $filter . "' -exec dirname \{\} \\; | xargs -d '\n' rm -rf");
++		system("cd '$safepath'; find ./[a-zA-Z0-9]* -empty -exec rmdir -p \\{\\} \\; 2> /dev/null");
++	}
++
+ 	return 0;
+ }
+ 
+@@ -715,13 +721,14 @@ sub uninstall {
+ 	return 0;
+ }
+ 
+-sub update_feed($$$$$)
++sub update_feed($$$$$$)
+ {
+ 	my $type=shift;
+ 	my $name=shift;
+ 	my $src=shift;
+ 	my $perform_update=shift;
+ 	my $force_update=shift;
++	my $filter=shift;
+ 	my $force_relocate=update_location( $name, "@$src" );
+ 	my $rv=0;
+ 
+@@ -736,7 +743,7 @@ sub update_feed($$$$$)
+ 		my $failed = 1;
+ 		foreach my $feedsrc (@$src) {
+ 			warn "Updating feed '$name' from '$feedsrc' ...\n";
+-			if (update_feed_via($type, $name, $feedsrc, $force_relocate, $force_update) != 0) {
++			if (update_feed_via($type, $name, $feedsrc, $force_relocate, $force_update, $filter) != 0) {
+ 				if ($force_update) {
+ 					$rv=1;
+ 					$failed=0;
+@@ -788,17 +795,17 @@ sub update {
+ 
+ 	if ( ($#ARGV == -1) or $opts{a}) {
+ 		foreach my $feed (@feeds) {
+-			my ($type, $name, $src) = @$feed;
+-			update_feed($type, $name, $src, $perform_update, $opts{f}) == 0 or $failed=1;
++			my ($type, $name, $src, $filter) = @$feed;
++			update_feed($type, $name, $src, $perform_update, $opts{f}, $filter) == 0 or $failed=1;
+ 		}
+ 	} else {
+ 		while ($feed_name = shift @ARGV) {
+ 			foreach my $feed (@feeds) {
+-				my ($type, $name, $src) = @$feed;
++				my ($type, $name, $src, $filter) = @$feed;
+ 				if($feed_name ne $name) {
+ 					next;
+ 				}
+-				update_feed($type, $name, $src, $perform_update, $opts{f}) == 0 or $failed=1;
++				update_feed($type, $name, $src, $perform_update, $opts{f}, $filter) == 0 or $failed=1;
+ 			}
+ 		}
+ 	}
+-- 
+2.15.1
+

patches/openwrt/to-upstream/0001-build-Optionally-provide-file-checksums-in-package-m.patch

+From c3868937dfff17c9e715563da23a394736ec1f14 Mon Sep 17 00:00:00 2001
+From: Michal Hrusecky <Michal@Hrusecky.net>
+Date: Wed, 17 Jan 2018 13:25:43 +0100
+Subject: [PATCH] build: Optionally provide file checksums in package metadata
+
+This may be useful if you don't entirely trust your flash and want to be able
+to check for corruptions.
+
+Signed-off-by: Michal Hrusecky <Michal@Hrusecky.net>
+---
+ config/Config-build.in                  |   8 ++
+ include/package-ipkg.mk                 |   8 ++
+ package/base-files/Makefile             |   2 +
+ package/base-files/files/sbin/pkg_check | 130 ++++++++++++++++++++++++++++++++
+ 4 files changed, 148 insertions(+)
+ create mode 100755 package/base-files/files/sbin/pkg_check
+
+diff --git a/config/Config-build.in b/config/Config-build.in
+index f9987fc..38f725f 100644
+--- a/config/Config-build.in
++++ b/config/Config-build.in
+@@ -68,6 +68,14 @@ menu "Global build settings"
+ 		  This removes all ipkg/opkg status data files from the target directory
+ 		  before building the root filesystem.
+ 
++	config IPK_FILES_CHECKSUMS
++		bool
++		prompt "Record files checksums in package metadata"
++		default n
++		help
++		  This makes file checksums part of package metadata. It increases size
++		  but provides you with pkg_check command to check for flash coruptions.
++
+ 	config INCLUDE_CONFIG
+ 		bool "Include build configuration in firmware" if DEVEL
+ 		default n
+diff --git a/include/package-ipkg.mk b/include/package-ipkg.mk
+index 9e2717e..358aede 100644
+--- a/include/package-ipkg.mk
++++ b/include/package-ipkg.mk
+@@ -194,6 +194,15 @@ $(_endef)
+ 	$(CheckDependencies)
+ 
+ 	$(RSTRIP) $$(IDIR_$(1))
++
++    ifneq ($$(CONFIG_IPK_FILES_CHECKSUMS),)
++	(cd $$(IDIR_$(1)); \
++		( \
++			find . -type f \! -path ./CONTROL/\* -exec sha256sum \{\} \; 2> /dev/null | \
++			sed 's|\([[:blank:]]\)\./|\1/|' > $$(IDIR_$(1))/CONTROL/files-sha256 \
++		) || true \
++	)
++    endif
+ 	(cd $$(IDIR_$(1))/CONTROL; \
+ 		( \
+ 			echo "$$$$CONTROL"; \
+diff --git a/package/base-files/Makefile b/package/base-files/Makefile
+index 0055474..411ed8e 100644
+--- a/package/base-files/Makefile
++++ b/package/base-files/Makefile
+@@ -195,6 +195,8 @@ define Package/base-files/install
+ 		mkdir -p $(1)/etc/opkg; \
+ 		$(call FeedSourcesAppend,$(1)/etc/opkg/distfeeds.conf); \
+ 		$(VERSION_SED) $(1)/etc/opkg/distfeeds.conf)
++	$(if $(CONFIG_IPK_FILES_CHECKSUMS), \
++		rm -f $(1)/sbin/pkg_check,)
+ endef
+ 
+ ifneq ($(DUMP),1)
+diff --git a/package/base-files/files/sbin/pkg_check b/package/base-files/files/sbin/pkg_check
+new file mode 100755
+index 0000000..9968c9b
+--- /dev/null
++++ b/package/base-files/files/sbin/pkg_check
+@@ -0,0 +1,130 @@
++#!/bin/sh
++#
++# Package checksums checking script
++# (C) 2018 CZ.NIC, z.s.p.o.
++#
++# This program is free software: you can redistribute it and/or modify
++# it under the terms of the GNU General Public License as published by
++# the Free Software Foundation, either version 3 of the License, or
++# (at your option) any later version.
++#
++# This program is distributed in the hope that it will be useful,
++# but WITHOUT ANY WARRANTY; without even the implied warranty of
++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
++# GNU General Public License for more details.
++#
++# You should have received a copy of the GNU General Public License
++# along with this program.  If not, see <http://www.gnu.org/licenses/>.
++
++
++ERRFATAL="no"
++QUIET="yes"
++MISSING=""
++SUMMARY=""
++NL="
++"
++
++# Arguments parsing
++while expr "x$1" : "x-" > /dev/null; do
++	if [ "x$1" = "x-s" ]; then
++		ERRFATAL="yes"
++		shift
++	elif [ "x$1" = "x-v" ]; then
++		QUIET="	no"
++		shift
++	else
++		echo "Usage: $(basename $0) [-s] [-v] [pkg1 pkg2 ...]"
++		echo
++		echo "   -s   Stop on first change"
++		echo "   -v   Verbose"
++		if [ "x$1" = "x-h" ]; then
++			exit 0
++		else
++			echo
++			echo "ERROR: Unknown option '$1'"
++			exit 1
++		fi
++	fi
++done
++
++# Check all packages by default
++if [ -z "$1" ]; then
++	set $(cd /usr/lib/opkg/info/; for i in *.files-sha256sum; do basename $i .files-sha256sum; done)
++fi
++
++# Iterate over packages
++while [ "$1" ]; do
++	if [ \! -f "/usr/lib/opkg/info/$1.files-sha256sum" ]; then
++		if [ "$ERRFATAL" = no ]; then
++			echo " * No checksums for $1 - skipping"
++			echo
++		else
++			echo " * No checksums for $1 - exiting"
++			exit 1
++		fi
++		if [ -z "$MISSING" ]; then
++			MISSING="$1"
++		else
++			MISSING="$MISSING, $1"
++		fi
++		shift
++		continue
++	fi
++	[ $QUIET = yes ] || echo " * Checking package $1:"
++	ERR=""
++	CHECK="`sha256sum -c /usr/lib/opkg/info/$1.files-sha256sum 2> /dev/null`"
++
++	# Are the changed files config files?
++	if [ $? -ne 0 ] && [ "`cat "/usr/lib/opkg/info/$1.files-sha256sum"`" ]; then