Create generate_lists as separate script

compile_fw

@@ -31,7 +31,7 @@ MAKE_DEBUG=""
 PUBLISH_BRANCH=""
 DEPTH="--depth 1"
 FORCE=""
-DEFAULT_STEPS="prepare build gen_lists store_hash sign stats gen_junit"
+DEFAULT_STEPS="prepare build store_hash stats gen_junit"
 
 if [ "$BUILD_DIR" == "$SRC_DIR" ]; then
 	mkdir -p build
@@ -230,19 +230,6 @@ update_mirror() {
 	MIRROR_UPDATED="yes"
 }
 
-sign_help="	Signs generated lists and resigns packages"
-sign() {
-	[ -x staging_dir/host/bin/usign ] || make package/usign/host/compile
-	[ -x staging_dir/host/bin/usign ] || _die "Don't have usign, can't sign"
-	[ -n "$KEY" ] || KEY=key-build
-	[ -f "$KEY" ] || _die "No key available"
-	for i in bin/packages/lists/*.lua bin/packages/*/*/Packages bin/targets/*/*/packages/Packages; do
-		[ -f "$i" ] || continue
-		rm -f "$i".sig
-		staging_dir/host/bin/usign -S -m "$i" -s "$KEY"
-	done
-}
-
 _checkout_init() {
 	_report "Checking out clean OpenWRT repository"
 	update_mirror
@@ -434,7 +421,7 @@ repatch_feeds() {
 
 prefetch_help="Runs make download"
 prefetch() {
-	make $MAKE_DEBUG $BUILD_ARGS $OWRT_DEBUG download
+	make $MAKE_DEBUG $BUILD_ARGS $OWRT_DEBUG BUILD_KEY="${KEY:-key-build}" download
 }
 
 gen_junit_help="Generates junit output from build logs"
@@ -442,14 +429,6 @@ gen_junit() {
 	sh $SH_DEBUG "${SRC_DIR}"/helpers/generate_junit.sh
 }
 
-gen_lists_help="Generate updater package lists"
-gen_lists() {
-	_report "Generating package lists for updater"
-	[ -n "${TARGET_BOARD}" ] || _die "You need to specify target board!"
-	mkdir -p bin/packages/lists
-	"${SRC_DIR}"/helpers/generate_userlists.sh --branch "${PUBLISH_BRANCH}" --src "${SRC_DIR}/lists" bin/packages/lists
-}
-
 build_help="	Builds everything"
 build() {
 	_report "Starting real build"

generate_lists

+#!/bin/bash -e
+# Updater-ng configuration lists generating script
+# (C) 2018 CZ.NIC, z.s.p.o.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+output_path=
+fallback=
+sign_key=
+while [ $# -gt 0 ]; do
+	case "$1" in
+		-h|--help)
+			echo "This script generates updater-ng configuration lists from Turris OS build repository."
+			echo "Usage: $0 [OPTION]... OUTPUT_PATH"
+			echo
+			echo "Options:"
+			echo "  --help, -h"
+			echo "    Prints this help text."
+			echo "  --branch BRANCH"
+			echo "    Target branch for which these lists are generated."
+			echo "  --minimal BRANCH"
+			echo "    Generate lists for minimal branch. (This adds BRANCH as a fallback branch)"
+			echo "  --sign KEY"
+			echo "    Sign lists with given KEY and usign utility"
+			echo "  --debug"
+			echo "    Run this script in debug mode"
+			exit
+			;;
+		--branch)
+			shift
+			branch="$1"
+			;;
+		--minimal)
+			shift
+			fallback="$1"
+			;;
+		--sign)
+			shift
+			sign_key="$1"
+			;;
+		--debug)
+			set -x
+			;;
+		*)
+			if [ -z "$output_path" ]; then
+				output_path="$1"
+			else
+				echo "Unknown option: $1"
+				exit 1
+			fi
+			;;
+	esac
+	shift
+done
+
+[ -n "$output_path" ] || output_path="generated_lists"
+[ -n "$branch" ] || {
+	echo "Missing --branch option." >&2
+	exit 1
+}
+tb="$(dirname "$(readlink -f "$0")")"
+[ -d "$tb/lists" -a -f "$tb/feeds.conf" ] || {
+	echo "This script has to be in same direstory as feeds.conf and lists directory." >&2
+	exit 1
+}
+. "$tb/helpers/generate_common.sh"
+
+rm -rf "$output_path"
+mkdir -p "$output_path"
+
+m4args=( "--include=$tb/lists" "-D_INCLUDE_=$tb/lists/" "-D_FEEDS_=$tb/feeds.conf" "-D_BRANCH_=$branch" )
+[ -z "$fallback" ] || m4args+=( "-D _BRANCH_FALLBACK_=$fallback" )
+
+for f in "$tb"/lists/*.lua.m4; do
+	[ -f "$f" ] || continue
+	m4 "${m4args[@]}" "$f" > "$output_path/$(basename "$f" | sed s/\.m4$//)"
+done
+for f in "$tb"/lists/*.lua; do
+	[ -f "$f" ] || continue
+	cp "$f" "$output_path/$(basename "$f")"
+done
+
+if [ -n "$sign_key" ]; then
+	get_usign
+	for list in "$output_path"/*.lua; do
+		"$USIGN" -S -m "$list" -s "$sign_key"
+	done
+fi

helpers/generate_common.sh

+# Common functions for generate_lists and generate_medkit functions
+# (C) 2018 CZ.NIC, z.s.p.o.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+USIGN_VERSION=ef6419142a3b0fbcddcccf536e3c1880302c6f89
+
+SRC_USIGN="https://git.openwrt.org/project/usign.git"
+
+# Git puller helper function (git_pull output_path source_url target)
+git_get() {
+	local REPO="$1"
+	local OUTPUT="$2"
+	local BRANCH="${3:-master}"
+	if [ ! -d "$OUTPUT/.git" ]; then
+		mkdir -p "$(dirname "$OUTPUT")"
+		git clone "$REPO" "$OUTPUT"
+		(
+		cd "$OUTPUT"
+		git checkout "$BRANCH"
+		git submodule update --init --recursive
+		git clean -Xdf
+		)
+		return 1
+	else
+		(
+		cd "$OUTPUT"
+		git fetch
+		local REF="origin/$BRANCH"
+		git show-ref -q "$REF" || REF="$BRANCH" # Is this reference on remote or object
+		if ! git diff --quiet HEAD "$REF"; then
+			git reset --hard "$REF"
+			git submodule update --init --recursive
+			git clean -Xdf
+			return 1
+		fi
+		)
+	fi
+}
+# Wget puller helper function (wget_pull output_file source_url)
+wget_get() {
+	local HREF="$1"
+	local OUTPUT="$2"
+	if [ ! -e "$OUTPUT" ] || [ $(expr $(date -u +%s) - $(stat -c %Z "$OUTPUT")) -gt 86400 ]; then
+		wget "$HREF" -O "$OUTPUT"
+	fi
+}
+
+get_usign() {
+	if ! git_get "$SRC_USIGN" turris-tools/usign "$USIGN_VERSION"; then
+		( cd turris-tools/usign && cmake . )
+		make -C turris-tools/usign
+	fi
+	export USIGN="$(pwd)/turris-tools/usign/usign"
+}

helpers/generate_userlists.sh

-#!/bin/sh
-set -e
-
-LISTS_DIR=
-OUTPUT_PATH=
-FALLBACK=
-while [ $# -gt 0 ]; do
-	case "$1" in
-		-h|--help)
-			echo "This script generates updater-ng userlists from Turris OS repository."
-			echo "Usage: $0 [OPTION]... OUTPUT_PATH"
-			echo
-			echo "Options:"
-			echo "  --help, -h"
-			echo "    Prints this help text."
-			echo "  --branch BRANCH"
-			echo "    Target branch for which this userlist is generated."
-			echo "  --minimal BRANCH"
-			echo "    Generate userlists for minimal branch. (This adds BRANCH as a fallback branch)"
-			echo "  --src PATH"
-			echo "    Source directory with list to process"
-			exit
-			;;
-		--branch)
-			shift
-			BRANCH="$1"
-			;;
-		--minimal)
-			shift
-			FALLBACK="$1"
-			;;
-		--src)
-			shift
-			LISTS_DIR="$1"
-			;;
-		*)
-			if [ -z "$OUTPUT_PATH" ]; then
-				OUTPUT_PATH="$1"
-			else
-				echo "Unknown option: $1"
-				exit 1
-			fi
-			;;
-	esac
-	shift
-done
-
-[ -z "$OUTPUT_PATH" ] && {
-	echo "You have to specify output path." >&2
-	exit 1
-}
-[ -z "$BRANCH" ] && {
-	echo "Missing --branch option." >&2
-	exit 1
-}
-[ -d "$LISTS_DIR" ] || {
-	echo "Valid --src directory has to be specified" >&2
-	exit 1
-}
-[ -f Makefile -a -f feeds.conf ] || {
-	echo "This script has to be run in OpenWRT build directory" >&2
-	exit 1
-}
-
-mkdir -p $OUTPUT_PATH
-
-M4ARGS="--include=$LISTS_DIR -D _INCLUDE_=$LISTS_DIR/ -D _BRANCH_=$BRANCH"
-[ -z "$FALLBACK" ] || M4ARGS="$M4ARGS -D _BRANCH_FALLBACK_=$FALLBACK"
-
-for f in $(find "$LISTS_DIR" -name '*.lua.m4'); do
-	m4 $M4ARGS $f > "$OUTPUT_PATH/$(basename "$f" | sed s/\.m4$//)"
-done
-for f in $(find "$LISTS_DIR" -name '*.lua'); do
-	cp $f "$OUTPUT_PATH/$(basename "$f")"
-done

lists/repository.m4

@@ -3,7 +3,7 @@ dnl We expect this to be include in base.lua just after utils.m4
 divert(-1)
 
 # This is definition of subrepositories
-pushdef(`SUBDIRS',`subdirs = {"base", "core" esyscmd(`awk "/^src-git/{printf \", \\\"%s\\\"\", \$'`2}" feeds.conf')}')
+pushdef(`SUBDIRS',`subdirs = {"base", "core" esyscmd(`awk "/^src-git/{printf \", \\\"%s\\\"\", \$'`2}" '_FEEDS_)}')
 
 divert(0)dnl
 local board
@@ -19,7 +19,7 @@ end
 
 dnl
 dnl Basic turris repository
-Repository("turris", "https://repo.turris.cz/" .. board .. "ifdef(`_BRANCH_',-_BRANCH_)/packages", {
+Repository("turris", "https://repo.turris.cz/_BRANCH_/packages/" .. board, {
 	SUBDIRS
 })
 dnl
@@ -27,7 +27,7 @@ dnl Fallback turris repository for not complete branches
 dnl In testing branches we are compiling just a minimal set of packages to allow
 dnl updater to use all packages we are adding nightly as fallback reposutory.
 ifdef(`_BRANCH_FALLBACK_',
-`Repository("turris-fallback", "https://repo.turris.cz/" .. board .. "-_BRANCH_FALLBACK_/packages", {
+`Repository("turris-fallback", "https://repo.turris.cz/_BRANCH_FALLBACK_/packages/" .. board, {
 	SUBDIRS,
 	priority = 40,
 	ignore = {"missing"}