From 9ab867bcb306509aaab686451c19044e249c2dfc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Karel=20Ko=C4=8D=C3=AD?= <karel.koci@nic.cz>
Date: Mon, 10 Sep 2018 16:46:23 +0200
Subject: [PATCH] Create generate_lists as separate script
---
compile_fw | 25 +--------
generate_lists | 99 +++++++++++++++++++++++++++++++++++
helpers/generate_common.sh | 65 +++++++++++++++++++++++
helpers/generate_userlists.sh | 75 --------------------------
lists/repository.m4 | 6 +--
5 files changed, 169 insertions(+), 101 deletions(-)
create mode 100755 generate_lists
create mode 100644 helpers/generate_common.sh
delete mode 100755 helpers/generate_userlists.sh
diff --git a/compile_fw b/compile_fw
index 1e4f45d6a..1df0c19dc 100755
--- a/compile_fw
+++ b/compile_fw
@@ -31,7 +31,7 @@ MAKE_DEBUG=""
PUBLISH_BRANCH=""
DEPTH="--depth 1"
FORCE=""
-DEFAULT_STEPS="prepare build gen_lists store_hash sign stats gen_junit"
+DEFAULT_STEPS="prepare build store_hash stats gen_junit"
if [ "$BUILD_DIR" == "$SRC_DIR" ]; then
mkdir -p build
@@ -230,19 +230,6 @@ update_mirror() {
MIRROR_UPDATED="yes"
}
-sign_help=" Signs generated lists and resigns packages"
-sign() {
- [ -x staging_dir/host/bin/usign ] || make package/usign/host/compile
- [ -x staging_dir/host/bin/usign ] || _die "Don't have usign, can't sign"
- [ -n "$KEY" ] || KEY=key-build
- [ -f "$KEY" ] || _die "No key available"
- for i in bin/packages/lists/*.lua bin/packages/*/*/Packages bin/targets/*/*/packages/Packages; do
- [ -f "$i" ] || continue
- rm -f "$i".sig
- staging_dir/host/bin/usign -S -m "$i" -s "$KEY"
- done
-}
-
_checkout_init() {
_report "Checking out clean OpenWRT repository"
update_mirror
@@ -434,7 +421,7 @@ repatch_feeds() {
prefetch_help="Runs make download"
prefetch() {
- make $MAKE_DEBUG $BUILD_ARGS $OWRT_DEBUG download
+ make $MAKE_DEBUG $BUILD_ARGS $OWRT_DEBUG BUILD_KEY="${KEY:-key-build}" download
}
gen_junit_help="Generates junit output from build logs"
@@ -442,14 +429,6 @@ gen_junit() {
sh $SH_DEBUG "${SRC_DIR}"/helpers/generate_junit.sh
}
-gen_lists_help="Generate updater package lists"
-gen_lists() {
- _report "Generating package lists for updater"
- [ -n "${TARGET_BOARD}" ] || _die "You need to specify target board!"
- mkdir -p bin/packages/lists
- "${SRC_DIR}"/helpers/generate_userlists.sh --branch "${PUBLISH_BRANCH}" --src "${SRC_DIR}/lists" bin/packages/lists
-}
-
build_help=" Builds everything"
build() {
_report "Starting real build"
diff --git a/generate_lists b/generate_lists
new file mode 100755
index 000000000..2307d24ae
--- /dev/null
+++ b/generate_lists
@@ -0,0 +1,99 @@
+#!/bin/bash -e
+# Updater-ng configuration lists generating script
+# (C) 2018 CZ.NIC, z.s.p.o.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+output_path=
+fallback=
+sign_key=
+while [ $# -gt 0 ]; do
+ case "$1" in
+ -h|--help)
+ echo "This script generates updater-ng configuration lists from Turris OS build repository."
+ echo "Usage: $0 [OPTION]... OUTPUT_PATH"
+ echo
+ echo "Options:"
+ echo " --help, -h"
+ echo " Prints this help text."
+ echo " --branch BRANCH"
+ echo " Target branch for which these lists are generated."
+ echo " --minimal BRANCH"
+ echo " Generate lists for minimal branch. (This adds BRANCH as a fallback branch)"
+ echo " --sign KEY"
+ echo " Sign lists with given KEY and usign utility"
+ echo " --debug"
+ echo " Run this script in debug mode"
+ exit
+ ;;
+ --branch)
+ shift
+ branch="$1"
+ ;;
+ --minimal)
+ shift
+ fallback="$1"
+ ;;
+ --sign)
+ shift
+ sign_key="$1"
+ ;;
+ --debug)
+ set -x
+ ;;
+ *)
+ if [ -z "$output_path" ]; then
+ output_path="$1"
+ else
+ echo "Unknown option: $1"
+ exit 1
+ fi
+ ;;
+ esac
+ shift
+done
+
+[ -n "$output_path" ] || output_path="generated_lists"
+[ -n "$branch" ] || {
+ echo "Missing --branch option." >&2
+ exit 1
+}
+tb="$(dirname "$(readlink -f "$0")")"
+[ -d "$tb/lists" -a -f "$tb/feeds.conf" ] || {
+ echo "This script has to be in same direstory as feeds.conf and lists directory." >&2
+ exit 1
+}
+. "$tb/helpers/generate_common.sh"
+
+rm -rf "$output_path"
+mkdir -p "$output_path"
+
+m4args=( "--include=$tb/lists" "-D_INCLUDE_=$tb/lists/" "-D_FEEDS_=$tb/feeds.conf" "-D_BRANCH_=$branch" )
+[ -z "$fallback" ] || m4args+=( "-D _BRANCH_FALLBACK_=$fallback" )
+
+for f in "$tb"/lists/*.lua.m4; do
+ [ -f "$f" ] || continue
+ m4 "${m4args[@]}" "$f" > "$output_path/$(basename "$f" | sed s/\.m4$//)"
+done
+for f in "$tb"/lists/*.lua; do
+ [ -f "$f" ] || continue
+ cp "$f" "$output_path/$(basename "$f")"
+done
+
+if [ -n "$sign_key" ]; then
+ get_usign
+ for list in "$output_path"/*.lua; do
+ "$USIGN" -S -m "$list" -s "$sign_key"
+ done
+fi
diff --git a/helpers/generate_common.sh b/helpers/generate_common.sh
new file mode 100644
index 000000000..b6bf0c503
--- /dev/null
+++ b/helpers/generate_common.sh
@@ -0,0 +1,65 @@
+# Common functions for generate_lists and generate_medkit functions
+# (C) 2018 CZ.NIC, z.s.p.o.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+USIGN_VERSION=ef6419142a3b0fbcddcccf536e3c1880302c6f89
+
+SRC_USIGN="https://git.openwrt.org/project/usign.git"
+
+# Git puller helper function (git_pull output_path source_url target)
+git_get() {
+ local REPO="$1"
+ local OUTPUT="$2"
+ local BRANCH="${3:-master}"
+ if [ ! -d "$OUTPUT/.git" ]; then
+ mkdir -p "$(dirname "$OUTPUT")"
+ git clone "$REPO" "$OUTPUT"
+ (
+ cd "$OUTPUT"
+ git checkout "$BRANCH"
+ git submodule update --init --recursive
+ git clean -Xdf
+ )
+ return 1
+ else
+ (
+ cd "$OUTPUT"
+ git fetch
+ local REF="origin/$BRANCH"
+ git show-ref -q "$REF" || REF="$BRANCH" # Is this reference on remote or object
+ if ! git diff --quiet HEAD "$REF"; then
+ git reset --hard "$REF"
+ git submodule update --init --recursive
+ git clean -Xdf
+ return 1
+ fi
+ )
+ fi
+}
+# Wget puller helper function (wget_pull output_file source_url)
+wget_get() {
+ local HREF="$1"
+ local OUTPUT="$2"
+ if [ ! -e "$OUTPUT" ] || [ $(expr $(date -u +%s) - $(stat -c %Z "$OUTPUT")) -gt 86400 ]; then
+ wget "$HREF" -O "$OUTPUT"
+ fi
+}
+
+get_usign() {
+ if ! git_get "$SRC_USIGN" turris-tools/usign "$USIGN_VERSION"; then
+ ( cd turris-tools/usign && cmake . )
+ make -C turris-tools/usign
+ fi
+ export USIGN="$(pwd)/turris-tools/usign/usign"
+}
diff --git a/helpers/generate_userlists.sh b/helpers/generate_userlists.sh
deleted file mode 100755
index 990cb7630..000000000
--- a/helpers/generate_userlists.sh
+++ /dev/null
@@ -1,75 +0,0 @@
-#!/bin/sh
-set -e
-
-LISTS_DIR=
-OUTPUT_PATH=
-FALLBACK=
-while [ $# -gt 0 ]; do
- case "$1" in
- -h|--help)
- echo "This script generates updater-ng userlists from Turris OS repository."
- echo "Usage: $0 [OPTION]... OUTPUT_PATH"
- echo
- echo "Options:"
- echo " --help, -h"
- echo " Prints this help text."
- echo " --branch BRANCH"
- echo " Target branch for which this userlist is generated."
- echo " --minimal BRANCH"
- echo " Generate userlists for minimal branch. (This adds BRANCH as a fallback branch)"
- echo " --src PATH"
- echo " Source directory with list to process"
- exit
- ;;
- --branch)
- shift
- BRANCH="$1"
- ;;
- --minimal)
- shift
- FALLBACK="$1"
- ;;
- --src)
- shift
- LISTS_DIR="$1"
- ;;
- *)
- if [ -z "$OUTPUT_PATH" ]; then
- OUTPUT_PATH="$1"
- else
- echo "Unknown option: $1"
- exit 1
- fi
- ;;
- esac
- shift
-done
-
-[ -z "$OUTPUT_PATH" ] && {
- echo "You have to specify output path." >&2
- exit 1
-}
-[ -z "$BRANCH" ] && {
- echo "Missing --branch option." >&2
- exit 1
-}
-[ -d "$LISTS_DIR" ] || {
- echo "Valid --src directory has to be specified" >&2
- exit 1
-}
-[ -f Makefile -a -f feeds.conf ] || {
- echo "This script has to be run in OpenWRT build directory" >&2
- exit 1
-}
-
-mkdir -p $OUTPUT_PATH
-
-M4ARGS="--include=$LISTS_DIR -D _INCLUDE_=$LISTS_DIR/ -D _BRANCH_=$BRANCH"
-[ -z "$FALLBACK" ] || M4ARGS="$M4ARGS -D _BRANCH_FALLBACK_=$FALLBACK"
-
-for f in $(find "$LISTS_DIR" -name '*.lua.m4'); do
- m4 $M4ARGS $f > "$OUTPUT_PATH/$(basename "$f" | sed s/\.m4$//)"
-done
-for f in $(find "$LISTS_DIR" -name '*.lua'); do
- cp $f "$OUTPUT_PATH/$(basename "$f")"
-done
diff --git a/lists/repository.m4 b/lists/repository.m4
index d8f4aaa1f..a3e1d8875 100644
--- a/lists/repository.m4
+++ b/lists/repository.m4
@@ -3,7 +3,7 @@ dnl We expect this to be include in base.lua just after utils.m4
divert(-1)
# This is definition of subrepositories
-pushdef(`SUBDIRS',`subdirs = {"base", "core" esyscmd(`awk "/^src-git/{printf \", \\\"%s\\\"\", \$'`2}" feeds.conf')}')
+pushdef(`SUBDIRS',`subdirs = {"base", "core" esyscmd(`awk "/^src-git/{printf \", \\\"%s\\\"\", \$'`2}" '_FEEDS_)}')
divert(0)dnl
local board
@@ -19,7 +19,7 @@ end
dnl
dnl Basic turris repository
-Repository("turris", "https://repo.turris.cz/" .. board .. "ifdef(`_BRANCH_',-_BRANCH_)/packages", {
+Repository("turris", "https://repo.turris.cz/_BRANCH_/packages/" .. board, {
SUBDIRS
})
dnl
@@ -27,7 +27,7 @@ dnl Fallback turris repository for not complete branches
dnl In testing branches we are compiling just a minimal set of packages to allow
dnl updater to use all packages we are adding nightly as fallback reposutory.
ifdef(`_BRANCH_FALLBACK_',
-`Repository("turris-fallback", "https://repo.turris.cz/" .. board .. "-_BRANCH_FALLBACK_/packages", {
+`Repository("turris-fallback", "https://repo.turris.cz/_BRANCH_FALLBACK_/packages/" .. board, {
SUBDIRS,
priority = 40,
ignore = {"missing"}
--
GitLab