diff --git a/compile_fw b/compile_fw index 1e4f45d6ae8df54d2c8202e477cd5bab214e896b..1df0c19dc8ee2090d0a8c57d51041c778ae2888b 100755 --- a/compile_fw +++ b/compile_fw @@ -31,7 +31,7 @@ MAKE_DEBUG="" PUBLISH_BRANCH="" DEPTH="--depth 1" FORCE="" -DEFAULT_STEPS="prepare build gen_lists store_hash sign stats gen_junit" +DEFAULT_STEPS="prepare build store_hash stats gen_junit" if [ "$BUILD_DIR" == "$SRC_DIR" ]; then mkdir -p build @@ -230,19 +230,6 @@ update_mirror() { MIRROR_UPDATED="yes" } -sign_help=" Signs generated lists and resigns packages" -sign() { - [ -x staging_dir/host/bin/usign ] || make package/usign/host/compile - [ -x staging_dir/host/bin/usign ] || _die "Don't have usign, can't sign" - [ -n "$KEY" ] || KEY=key-build - [ -f "$KEY" ] || _die "No key available" - for i in bin/packages/lists/*.lua bin/packages/*/*/Packages bin/targets/*/*/packages/Packages; do - [ -f "$i" ] || continue - rm -f "$i".sig - staging_dir/host/bin/usign -S -m "$i" -s "$KEY" - done -} - _checkout_init() { _report "Checking out clean OpenWRT repository" update_mirror @@ -434,7 +421,7 @@ repatch_feeds() { prefetch_help="Runs make download" prefetch() { - make $MAKE_DEBUG $BUILD_ARGS $OWRT_DEBUG download + make $MAKE_DEBUG $BUILD_ARGS $OWRT_DEBUG BUILD_KEY="${KEY:-key-build}" download } gen_junit_help="Generates junit output from build logs" @@ -442,14 +429,6 @@ gen_junit() { sh $SH_DEBUG "${SRC_DIR}"/helpers/generate_junit.sh } -gen_lists_help="Generate updater package lists" -gen_lists() { - _report "Generating package lists for updater" - [ -n "${TARGET_BOARD}" ] || _die "You need to specify target board!" - mkdir -p bin/packages/lists - "${SRC_DIR}"/helpers/generate_userlists.sh --branch "${PUBLISH_BRANCH}" --src "${SRC_DIR}/lists" bin/packages/lists -} - build_help=" Builds everything" build() { _report "Starting real build" diff --git a/generate_lists b/generate_lists new file mode 100755 index 0000000000000000000000000000000000000000..2307d24aed387b43a5ce3b75f686ca37312158dc --- /dev/null +++ b/generate_lists @@ -0,0 +1,99 @@ +#!/bin/bash -e +# Updater-ng configuration lists generating script +# (C) 2018 CZ.NIC, z.s.p.o. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +output_path= +fallback= +sign_key= +while [ $# -gt 0 ]; do + case "$1" in + -h|--help) + echo "This script generates updater-ng configuration lists from Turris OS build repository." + echo "Usage: $0 [OPTION]... OUTPUT_PATH" + echo + echo "Options:" + echo " --help, -h" + echo " Prints this help text." + echo " --branch BRANCH" + echo " Target branch for which these lists are generated." + echo " --minimal BRANCH" + echo " Generate lists for minimal branch. (This adds BRANCH as a fallback branch)" + echo " --sign KEY" + echo " Sign lists with given KEY and usign utility" + echo " --debug" + echo " Run this script in debug mode" + exit + ;; + --branch) + shift + branch="$1" + ;; + --minimal) + shift + fallback="$1" + ;; + --sign) + shift + sign_key="$1" + ;; + --debug) + set -x + ;; + *) + if [ -z "$output_path" ]; then + output_path="$1" + else + echo "Unknown option: $1" + exit 1 + fi + ;; + esac + shift +done + +[ -n "$output_path" ] || output_path="generated_lists" +[ -n "$branch" ] || { + echo "Missing --branch option." >&2 + exit 1 +} +tb="$(dirname "$(readlink -f "$0")")" +[ -d "$tb/lists" -a -f "$tb/feeds.conf" ] || { + echo "This script has to be in same direstory as feeds.conf and lists directory." >&2 + exit 1 +} +. "$tb/helpers/generate_common.sh" + +rm -rf "$output_path" +mkdir -p "$output_path" + +m4args=( "--include=$tb/lists" "-D_INCLUDE_=$tb/lists/" "-D_FEEDS_=$tb/feeds.conf" "-D_BRANCH_=$branch" ) +[ -z "$fallback" ] || m4args+=( "-D _BRANCH_FALLBACK_=$fallback" ) + +for f in "$tb"/lists/*.lua.m4; do + [ -f "$f" ] || continue + m4 "${m4args[@]}" "$f" > "$output_path/$(basename "$f" | sed s/\.m4$//)" +done +for f in "$tb"/lists/*.lua; do + [ -f "$f" ] || continue + cp "$f" "$output_path/$(basename "$f")" +done + +if [ -n "$sign_key" ]; then + get_usign + for list in "$output_path"/*.lua; do + "$USIGN" -S -m "$list" -s "$sign_key" + done +fi diff --git a/helpers/generate_common.sh b/helpers/generate_common.sh new file mode 100644 index 0000000000000000000000000000000000000000..b6bf0c50333c07403db306baec9187571b67b3b1 --- /dev/null +++ b/helpers/generate_common.sh @@ -0,0 +1,65 @@ +# Common functions for generate_lists and generate_medkit functions +# (C) 2018 CZ.NIC, z.s.p.o. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +USIGN_VERSION=ef6419142a3b0fbcddcccf536e3c1880302c6f89 + +SRC_USIGN="https://git.openwrt.org/project/usign.git" + +# Git puller helper function (git_pull output_path source_url target) +git_get() { + local REPO="$1" + local OUTPUT="$2" + local BRANCH="${3:-master}" + if [ ! -d "$OUTPUT/.git" ]; then + mkdir -p "$(dirname "$OUTPUT")" + git clone "$REPO" "$OUTPUT" + ( + cd "$OUTPUT" + git checkout "$BRANCH" + git submodule update --init --recursive + git clean -Xdf + ) + return 1 + else + ( + cd "$OUTPUT" + git fetch + local REF="origin/$BRANCH" + git show-ref -q "$REF" || REF="$BRANCH" # Is this reference on remote or object + if ! git diff --quiet HEAD "$REF"; then + git reset --hard "$REF" + git submodule update --init --recursive + git clean -Xdf + return 1 + fi + ) + fi +} +# Wget puller helper function (wget_pull output_file source_url) +wget_get() { + local HREF="$1" + local OUTPUT="$2" + if [ ! -e "$OUTPUT" ] || [ $(expr $(date -u +%s) - $(stat -c %Z "$OUTPUT")) -gt 86400 ]; then + wget "$HREF" -O "$OUTPUT" + fi +} + +get_usign() { + if ! git_get "$SRC_USIGN" turris-tools/usign "$USIGN_VERSION"; then + ( cd turris-tools/usign && cmake . ) + make -C turris-tools/usign + fi + export USIGN="$(pwd)/turris-tools/usign/usign" +} diff --git a/helpers/generate_userlists.sh b/helpers/generate_userlists.sh deleted file mode 100755 index 990cb7630fc859d0acc6e629299c2c87e549ed06..0000000000000000000000000000000000000000 --- a/helpers/generate_userlists.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/sh -set -e - -LISTS_DIR= -OUTPUT_PATH= -FALLBACK= -while [ $# -gt 0 ]; do - case "$1" in - -h|--help) - echo "This script generates updater-ng userlists from Turris OS repository." - echo "Usage: $0 [OPTION]... OUTPUT_PATH" - echo - echo "Options:" - echo " --help, -h" - echo " Prints this help text." - echo " --branch BRANCH" - echo " Target branch for which this userlist is generated." - echo " --minimal BRANCH" - echo " Generate userlists for minimal branch. (This adds BRANCH as a fallback branch)" - echo " --src PATH" - echo " Source directory with list to process" - exit - ;; - --branch) - shift - BRANCH="$1" - ;; - --minimal) - shift - FALLBACK="$1" - ;; - --src) - shift - LISTS_DIR="$1" - ;; - *) - if [ -z "$OUTPUT_PATH" ]; then - OUTPUT_PATH="$1" - else - echo "Unknown option: $1" - exit 1 - fi - ;; - esac - shift -done - -[ -z "$OUTPUT_PATH" ] && { - echo "You have to specify output path." >&2 - exit 1 -} -[ -z "$BRANCH" ] && { - echo "Missing --branch option." >&2 - exit 1 -} -[ -d "$LISTS_DIR" ] || { - echo "Valid --src directory has to be specified" >&2 - exit 1 -} -[ -f Makefile -a -f feeds.conf ] || { - echo "This script has to be run in OpenWRT build directory" >&2 - exit 1 -} - -mkdir -p $OUTPUT_PATH - -M4ARGS="--include=$LISTS_DIR -D _INCLUDE_=$LISTS_DIR/ -D _BRANCH_=$BRANCH" -[ -z "$FALLBACK" ] || M4ARGS="$M4ARGS -D _BRANCH_FALLBACK_=$FALLBACK" - -for f in $(find "$LISTS_DIR" -name '*.lua.m4'); do - m4 $M4ARGS $f > "$OUTPUT_PATH/$(basename "$f" | sed s/\.m4$//)" -done -for f in $(find "$LISTS_DIR" -name '*.lua'); do - cp $f "$OUTPUT_PATH/$(basename "$f")" -done diff --git a/lists/repository.m4 b/lists/repository.m4 index d8f4aaa1ffaebad1e06a06594e04f75fb3a58b89..a3e1d8875ed1e71baa5ca41b0c596d137e6f79e5 100644 --- a/lists/repository.m4 +++ b/lists/repository.m4 @@ -3,7 +3,7 @@ dnl We expect this to be include in base.lua just after utils.m4 divert(-1) # This is definition of subrepositories -pushdef(`SUBDIRS',`subdirs = {"base", "core" esyscmd(`awk "/^src-git/{printf \", \\\"%s\\\"\", \$'`2}" feeds.conf')}') +pushdef(`SUBDIRS',`subdirs = {"base", "core" esyscmd(`awk "/^src-git/{printf \", \\\"%s\\\"\", \$'`2}" '_FEEDS_)}') divert(0)dnl local board @@ -19,7 +19,7 @@ end dnl dnl Basic turris repository -Repository("turris", "https://repo.turris.cz/" .. board .. "ifdef(`_BRANCH_',-_BRANCH_)/packages", { +Repository("turris", "https://repo.turris.cz/_BRANCH_/packages/" .. board, { SUBDIRS }) dnl @@ -27,7 +27,7 @@ dnl Fallback turris repository for not complete branches dnl In testing branches we are compiling just a minimal set of packages to allow dnl updater to use all packages we are adding nightly as fallback reposutory. ifdef(`_BRANCH_FALLBACK_', -`Repository("turris-fallback", "https://repo.turris.cz/" .. board .. "-_BRANCH_FALLBACK_/packages", { +`Repository("turris-fallback", "https://repo.turris.cz/_BRANCH_FALLBACK_/packages/" .. board, { SUBDIRS, priority = 40, ignore = {"missing"}