Document release and steps for new version branch

README.asciidoc

@@ -4,16 +4,54 @@ Simple Turris OS building script
 Getting started
 ---------------
 
-Create an empty directory you want to build Turris OS in, enter that directory
-and from it call `compile_pkgs` script. In most basic use-case, this should be
-enough. Be aware - *it will delete all previous content of the current
-directory*. You can repeat individual steps of the compilation separately,
-check with the following command `compile_pkgs --help` for a list of available commands, generic options, and
-their brief descriptions.
+Create an empty directory you want to build Turris OS in, enter that directory and
+from it call `compile_pkgs` script. In most basic use-case, this should be enough.
+Be aware - *it will delete all previous content of the current directory*. You can
+repeat individual steps of the compilation separately, check with the following
+command `compile_pkgs --help` for a list of available commands, generic options,
+and their brief descriptions.
+
 
 Advanced usage
 --------------
 
+Building a single package
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+To build only one package you have to first prepare build directory. That can be
+achieved by running in target directory.
+--
+ compile_pkgs prepare_tools -t board # <1>
+--
+
+<1> ~-t~ means `-target`, so only valid values are `turris`, `omnia`, `mox`.
+
+Be aware *it removes previous content of current directory*!
+
+Once, you have compiled required tools, you need **enter build directory** and
+then run `make menuconfig`, select the package you want to compile and save it.
+Then you can build it using this command: +
+--
+ make package/name/compile # <2>
+--
+
+<2> ~name~ is package name, which you want to compile.
+
+You can also clean single package just by replacing `compile` with `clean`.
+
+The resulting package is placed in directory **bin/packages/~ARCH~/~REPO~**
+
+;; `ARCH` is target architecture specific string.
+;; `REPO` is the name of package's source repository.
+
+Occasionally, you can meet cases, where you need to run `make
+package/name/compile` with parameters `-j1 V=s` or `-j1 V=sc` to be able to see,
+what went wrong.
+
+
+Development and maintenance
+---------------------------
+
 Configuration
 ~~~~~~~~~~~~~
 
@@ -30,6 +68,37 @@ it lines with new features in the release. This file is used to generate
 `turris-version` package. `compile_pkgs` script reads it, finds the newest
 version and from notes underneath it creates the package with release notes.
 
+Releasing new version
+~~~~~~~~~~~~~~~~~~~~~
+
+When new version is released to Snails (see workflow for explanation) then new
+commit with hashes has to be created and pushed. This commit have to be tagged
+with appropriate version tag in format `vVERSION` where VERSION is released
+version. To make this all simple and to correctly set always all configurations we
+have script `helpers/new_release.sh`. Run this script after release from
+turris-build project root directory and it is going to automatically detect,
+commit and tag new version. You should review commit and tag it created for you
+and then push it using `git push --tags`.
+
+Script `helpers/new_release.sh` can generate various errors and warnings. It is
+advised to run it even before release in `verify` mode to review possible problems
+with release.
+
+Forking new release
+~~~~~~~~~~~~~~~~~~~
+
+According to workflow new releases are forked from parent branch. During this
+process care should be taken to tweak defaults to appropriate values. Following
+list should be taken as a checklist for new release branch.
+
+. Set `PUBLISH_BRANCH` in `defaults.sh`. Master branch should always be set to
+  `hbd` and release branches should be set to `hbs`.
+. Set branches in `feeds.conf`. You should append string like this:
+  `;openwrt-18.06`. This has to be done for all OpenWRT feeds as well for OpenWRT
+  it self. URL used by `compile_pkgs` is specified as first line in `feeds.conf`
+  and with exception of first column it has same format as feeds. Note that that
+  line is intentionally commented out as that is not feed.
+
 Patching
 ~~~~~~~~
 
@@ -60,33 +129,3 @@ manually and afterward, we need to call `git add` on newly patched files. Once
 all conflicts are resolved, `git am --continue` will create real commit that we
 were trying to add using `git am`. Now all that is left is to export it using
 `git format-patch -1` and overwrite patch stored in it of _build_ repository.
-
-Building a single package
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-To build only one package you have to first prepare build directory. That can be
-achieved by running in target directory.
---
- compile_pkgs prepare_tools -t board # <1>
---
-
-<1> ~-t~ means `-target`, so only valid values are `turris`, `omnia`, `mox`.
-
-Be aware *it removes previous content of current directory*!
-
-Once, you have compiled required tools, you need **enter build directory** and then run `make menuconfig`, select the package you want to compile and save it.
-Then you can build it using this command: +
---
- make package/name/compile # <2>
---
-
-<2> ~name~ is package name, which you want to compile.
-
-You can also clean single package just by replacing `compile` with `clean`.
-
-The resulting package is placed in directory **bin/packages/~ARCH~/~REPO~**
-
-;; `ARCH` is target architecture specific string.
-;; `REPO` is the name of package's source repository.
-
-Occasionally, you can meet cases, where you need to run `make package/name/compile` with parameters `-j1 V=s` or `-j1 V=sc` to be able to see, what went wrong.

compile_pkgs

@@ -61,10 +61,19 @@ set_target() {
 	esac
 }
 
+# Get OpenWRT URL and branch from feeds.conf file
+openwrt_feed() {
+	# OpenWRT git repository URL
+	OPENWRT_URL="$(sed -n 's/# openwrt \([^;^]\+\).*/\1/p' "$src_dir/feeds.conf")"
+	# Git reference used to checkout OpenWRT repository
+	OPENWRT_BRANCH="$(sed -n 's/# openwrt [^;^]\+[;^]\([^\s]\+\).*/\1/p' "$src_dir/feeds.conf")"
+	# Check if ^ is used to separate URL and branch and in such case it is hash not branch
+	grep -E '^# openwrt [^;^]+\^.+' "$src_dir/feeds.conf" && OPENWRT_BRANCH="#$OPENWRT_BRANCH"
+	[ -n "$OPENWRT_BRANCH" ] || OPENWRT_BRANCH="master"
+}
+
 # Configuration variables
-OPENWRT_URL="https://git.openwrt.org/openwrt/openwrt.git" # OpenWRT git repository URL
-OPENWRT_BRANCH="openwrt-18.06" # Git reference used to checkout OpenWRT repository
-PUBLISH_BRANCH="hbs" # Target publish branch
+. "$src_dir/defaults.sh"
 FORCE="" # Force build
 EVERYTHING="" # Set this variable to build all packages not only minimal set
 CLONE_DEEP="" # Set this variable to clone OpenWRT tree in full depth not just latest commit
@@ -76,6 +85,7 @@ DL_MIRROR="" # Path to downloads mirror directory
 CCACHE_HOST_DIR="" # Path to ccache directory for host compilations
 CCACHE_TARGET_DIR="" # Path to ccache directory for target compilations
 OUTPUT_DIR="./pkgsrepo" # Output directory for pkgsrepo command
+openwrt_feed
 
 # Load configurations
 [ -f "${src_dir}"/turris-build.conf ] && . "${src_dir}"/turris-build.conf

defaults.sh

+# This file contains project wide defaults. They are here because we have to tweak
+# these defaults every time we fork new release.
+
+# This sets default publish branch
+# Expected values are:
+#   "hbd" for master branch
+#   "hbs" for release branch
+PUBLISH_BRANCH="hbs"

feeds.conf

+## Following line defines used openwrt repository.
+## It is commented out because it is not feed but is used by compile_pkgs script.
+# openwrt https://git.openwrt.org/openwrt/openwrt.git;openwrt-18.06
+
 src-git turrispackages https://gitlab.labs.nic.cz/turris/turris-os-packages.git^origin/master
 src-git packages https://git.openwrt.org/feed/packages.git^origin/openwrt-18.06
 src-git luci https://git.openwrt.org/project/luci.git^origin/openwrt-18.06

generate_medkit

@@ -16,8 +16,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 set -e
 
+. "$(readlink -f "$(dirname "$0")")/defaults.sh"
 export BOARD=
-export BRANCH="hbs"
+export BRANCH="$PUBLISH_BRANCH"
 export UPDATER_BRANCH=
 export L10N=cs,de
 # TODO fill in default lists when we have them selected

helpers/new_release.sh

+#!/bin/bash
+# Turris OS script verifying new release
+# (C) 2019 CZ.NIC, z.s.p.o.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+set -e
+
+# TODO:
+# * Do we want to verify medkit somehow?
+
+# Constants
+REPO="https://repo.turris.cz"
+BOARDS=('mox' 'omnia' 'turris')
+DEFAULT_VERIFY_BRANCH="hbk"
+DEFAULT_RELEASE_BRANCH="hbs"
+
+
+COLOR_RED="\033[0;31m"
+COLOR_GREEN="\033[0;32m"
+COLOR_ORANGE="\033[0;33m"
+COLOR_OFF="\033[0m"
+info() {
+	printf "%s\n" "$@" >&2
+}
+warning() {
+	if [ -t 2 ]; then
+		echo -e "${COLOR_ORANGE}$*${COLOR_OFF}" >&2
+	else
+		echo -e "WARNING: $*" >&2
+	fi
+}
+error() {
+	if [ -t 2 ]; then
+		echo -e "${COLOR_RED}$*${COLOR_OFF}" >&2
+	else
+		echo -e "ERROR: $*" >&2
+	fi
+}
+fail() {
+	error "$@"
+	exit 1
+}
+ask() {
+	echo -e "${COLOR_GREEN}$*${COLOR_OFF}" >&2
+	local answer
+	read -r -p "Write upper case yes to continue: " answer
+	[ "$answer" = "YES" ] || return 1
+}
+
+FETCH_DIR=
+cleanup() {
+	[ -d "$FETCH_DIR" ] && rm -rf "$FETCH_DIR"
+	return 0
+}
+trap cleanup EXIT TERM INT QUIT ABRT
+
+# Fetch all files we need from repository
+fetch_files() {
+	local branch="$1"
+	FETCH_DIR="$(mktemp -d)"
+	GIT_HASH_LISTS="$FETCH_DIR/git-hash-lists"
+	GIT_HASH_PACKAGES="$FETCH_DIR/git-hash-packages"
+	VERSION_LISTS="$FETCH_DIR/version-lists"
+
+	curl -s "$REPO/$branch/lists/git-hash" >"$GIT_HASH_LISTS"
+	curl -s "$REPO/$branch/lists/turris-version" >"$VERSION_LISTS"
+	for board in "${BOARDS[@]}"; do
+		curl -s "$REPO/$branch/packages/$board/git-hash" >"$GIT_HASH_PACKAGES-$board"
+	done
+}
+
+# Get git hash from git-hash file on REPO
+# Expected arguments are: BOARD FEED
+git_hash() {
+	awk -v feed="$2:" '$2 == feed { print $3; exit }' "$GIT_HASH_PACKAGES-$1"
+}
+
+# Get turris-build hash
+# (version is primarily sourced from lists so consider lists as authoritative)
+tb_hash() {
+	local ec=0
+	local l_hash b_hash
+	l_hash="$(cat "$GIT_HASH_LISTS")"
+	for board in "${BOARDS[@]}"; do
+		b_hash="$(git_hash "$board" "turris-build")"
+		[ "$b_hash" = "$l_hash" ] || {
+			error "Turris build used to generate lists is not same as for packages for board: $board ($l_hash / $b_hash)"
+			ec=1
+		}
+	done
+	echo "$l_hash"
+	return $ec
+}
+
+# Get hash for openwrt
+owrt_hash() {
+	local ec=0
+	local hsh="" t_hsh
+	# TODO this is not optimal and better would be to decide which is authoritative (newer?)
+	for board in "${BOARDS[@]}"; do
+		t_hsh="$(git_hash "$board" openwrt)"
+		[ -n "$hsh" ] || hsh="$t_hsh"
+		if [ "$t_hsh" != "$hsh" ]; then
+			error "Different OpenWRT commit build for board: $board ($hsh / $t_hsh)"
+			ec=1
+		fi
+	done
+	echo "$hsh"
+	return $ec
+}
+
+# This function sets dictionary FEEDS by collecting all feeds in git-hash
+# Caller should define associative array FEEDS
+feeds() {
+	local ec=0
+	# TODO this is not optimal and better would be to decide which is authoritative (newer?)
+	for board in "${BOARDS[@]}"; do
+		local lines feed hsh
+		lines="$(sed -En 's|^ \* feeds/([^:]+): ([^\s]+)|\1 \2|p' "$GIT_HASH_PACKAGES-$board")"
+		while read -r feed hsh; do
+			if [[ -v "FEEDS[$feed]" ]]; then
+				if [ "$hsh" != "${FEEDS["$feed"]}" ]; then
+					error "Different feed ($feed) commit compiled for $board (${FEEDS["$feed"]} / $hsh)"
+					ec=1
+				fi
+			else
+				FEEDS["$feed"]="$hsh"
+			fi
+		done <<<"$lines"
+	done
+	return $ec
+}
+
+# Sets given hash as a feed target
+# Arguments: FEED HASH
+feeds_conf_set() {
+	awk -v feed="$1" -v hash="$2" \
+		'$2 == feed { gsub("[;^].*$", "", $3); gsub("$", "^" hash) } { print $0 }' \
+		./feeds.conf > feeds.conf.new
+	mv ./feeds.conf.new ./feeds.conf
+}
+
+check_for_turris_build_root() {
+	[ -e .git -a -f ./feeds.conf ] || \
+		fail "This has to be run from turris-build root"
+}
+
+##########
+verify() {
+	local branch="$1"
+	local ec=0
+	declare -A FEEDS
+	fetch_files "$branch"
+
+	tb_hash >/dev/null || ec=$?
+	owrt_hash >/dev/null || ec=$?
+	feeds || ec=$?
+	[ "$ec" = 0 ] && info "No problems detected in branch: $branch"
+	return $ec
+}
+
+##########
+release() {
+	local branch="$1"
+	local tversion target_hsh owrt_hsh
+	declare -A FEEDS
+	fetch_files "$branch"
+
+	tversion="$(cat "$VERSION_LISTS")"
+	target_hsh="$(tb_hash)" || ask "turris-build hashes do not match. Planning to use: $target_hsh"
+	owrt_hsh="$(owrt_hash)" || ask "openwrt hashes do not match. Planning to use: $owrt_hsh"
+	feeds || ask "feed hashes do not match. Please review difference."
+
+	check_for_turris_build_root
+	local tag="v$tversion"
+	git rev-parse "$tag" &>/dev/null && \
+		fail "Tag for version $tag exists."
+
+	git checkout "$target_hsh"
+	feeds_conf_set openwrt "$owrt_hsh"
+	for feed in "${!FEEDS[@]}"; do
+		feeds_conf_set "$feed" "${FEEDS["$feed"]}"
+	done
+	git add ./feeds.conf
+	git ci -m "Turris OS $tversion"
+	git tag -s -m "Turris OS $tversion release" "v$tversion"
+
+	info "Tag $tag was created. Review changes and push it with: git push --tags origin"
+}
+
+##################################################################################
+print_help() {
+	echo "Usage: $0 [OPTION].. [MODE [BRANCH]]"
+	echo "Turris OS new releases managing tool."
+	echo
+	echo "Options:"
+	echo "  -v  Run script with verbose output"
+	echo "  -h  Print this help text"
+	echo "Modes:"
+	echo "  verify"
+	echo "    Run script in verify mode where BRANCH (in default $DEFAULT_VERIFY_BRANCH) is checked"
+	echo "    for possible release problems. This is default."
+	echo "  release"
+	echo "    Create new commit and tag for release to BRANCH (in default $DEFAULT_RELEASE_BRANCH)"
+	echo
+	echo "Example usage:"
+	echo "  Verify branch to be released: $0 verify"
+	echo "  Commit and tag new release: $0 release"
+}
+
+while getopts ':hd' OPT; do
+    case "$OPT" in
+        h)
+            print_help
+            exit 0
+            ;;
+        d)
+            set -x
+            ;;
+        \?)
+            error "Illegal option '-$OPTARG'"
+            exit 1
+            ;;
+    esac
+done
+shift $(( OPTIND-1 ))
+
+[ $# -le 2 ] || fail "Provided too many arguments. See help \`-h\`"
+
+case "${1:-verify}" in
+	verify)
+		verify "${2:-${DEFAULT_VERIFY_BRANCH}}"
+		;;
+	release)
+		release "${2:-${DEFAULT_RELEASE_BRANCH}}"
+		;;
+	*)
+		fail "Invalid mode: $1"
+		;;
+esac

turris-build.conf.example

@@ -17,8 +17,6 @@ SIGN_KEY="mime.key"
 # Set target board. Allowed arguments are: turris, omnia and mox
 set_target mox
 
-# Git reference used to checkout OpenWRT repository
-#OPENWRT_BRANCH=master
 # Target publish branch
 #PUBLISH_BRANCH="hbs"
 # Force build (set to some number to try build multiple times)
@@ -46,8 +44,6 @@ set_target mox
 #BUILD_ARGS=() 
 
 ## Variables that you most probably want to let in default #######################
-# OpenWRT git repository URL
-#OPENWRT_URL="https://git.openwrt.org/openwrt/openwrt.git" 
 # Set this variable to clone OpenWRT tree in full depth not just latest commit
 #CLONE_DEEP="y"