From ab1dd25dadf5401d50ff4da19c5e73fee82c7ae7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karel=20Ko=C4=8D=C3=AD?= <cynerd@email.cz> Date: Mon, 9 Nov 2020 07:53:56 +0100 Subject: [PATCH] lists: move to separate repository and improve scripts There are two changes in this commit. First most noticable is removal of all updater lists. The second but more significant change is pretty much reformat of all generate scripts. It extracts common git and configuration handling from compile_pkgs to generate_common. The advantage of this is of course code sharing but also that other generate scripts now get features such as local git mirror. --- .gitignore | 4 +- .gitlab-ci.yml | 6 - compile_pkgs | 195 +++-------------- feeds.conf | 3 + generate_lists | 58 +++-- generate_medkits | 5 +- helpers/conf.sh | 71 ++++++ helpers/generate_common.sh | 310 ++++++++++++++++++++++----- lists/base-conditional.lua.m4 | 13 -- lists/base-fix.lua | 178 --------------- lists/base-min.lua.m4 | 117 ---------- lists/base-netboot.lua.m4 | 11 - lists/base.lua.m4 | 23 -- lists/bootstrap.lua | 111 ---------- lists/contracts/cti.lua.m4 | 21 -- lists/contracts/cznic.lua.m4 | 7 - lists/contracts/shield.lua.m4 | 41 ---- lists/drivers/pci.lua | 39 ---- lists/drivers/usb.lua | 127 ----------- lists/foris.lua.m4 | 71 ------ lists/kmod-mox.list | 10 - lists/kmod-omnia.list | 15 -- lists/kmod-turris.list | 11 - lists/kmod.list | 20 -- lists/luci.lua.m4 | 113 ---------- lists/migrate3x.lua | 134 ------------ lists/pkglists/3g.lua.m4 | 15 -- lists/pkglists/atlas.lua.m4 | 7 - lists/pkglists/datacollect.lua.m4 | 24 --- lists/pkglists/dev-detect.lua | 1 - lists/pkglists/drivers.lua.m4 | 15 -- lists/pkglists/dvb.lua.m4 | 6 - lists/pkglists/hardening.lua.m4 | 19 -- lists/pkglists/honeypot.lua | 1 - lists/pkglists/luci-controls.lua | 1 - lists/pkglists/luci_controls.lua.m4 | 42 ---- lists/pkglists/lxc.lua.m4 | 12 -- lists/pkglists/nas.lua.m4 | 68 ------ lists/pkglists/net_monitoring.lua.m4 | 16 -- lists/pkglists/netboot.lua.m4 | 11 - lists/pkglists/netdata.lua.m4 | 6 - lists/pkglists/netmetr.lua | 1 - lists/pkglists/nextcloud.lua.m4 | 6 - lists/pkglists/openvpn.lua.m4 | 10 - lists/pkglists/pakon.lua | 1 - lists/pkglists/printserver.lua | 1 - lists/pkglists/reforis.lua | 1 - lists/pkglists/tor.lua.m4 | 6 - lists/repository.lua.m4 | 27 --- lists/repository.m4 | 7 - lists/terminal-apps.lua.m4 | 25 --- lists/utils.m4 | 64 ------ lists/webapps.lua.m4 | 21 -- turris-build.conf.example | 1 - 54 files changed, 402 insertions(+), 1727 deletions(-) create mode 100644 helpers/conf.sh delete mode 100644 lists/base-conditional.lua.m4 delete mode 100644 lists/base-fix.lua delete mode 100644 lists/base-min.lua.m4 delete mode 100644 lists/base-netboot.lua.m4 delete mode 100644 lists/base.lua.m4 delete mode 100644 lists/bootstrap.lua delete mode 100644 lists/contracts/cti.lua.m4 delete mode 100644 lists/contracts/cznic.lua.m4 delete mode 100644 lists/contracts/shield.lua.m4 delete mode 100644 lists/drivers/pci.lua delete mode 100644 lists/drivers/usb.lua delete mode 100644 lists/foris.lua.m4 delete mode 100644 lists/kmod-mox.list delete mode 100644 lists/kmod-omnia.list delete mode 100644 lists/kmod-turris.list delete mode 100644 lists/kmod.list delete mode 100644 lists/luci.lua.m4 delete mode 100644 lists/migrate3x.lua delete mode 100644 lists/pkglists/3g.lua.m4 delete mode 100644 lists/pkglists/atlas.lua.m4 delete mode 100644 lists/pkglists/datacollect.lua.m4 delete mode 100644 lists/pkglists/dev-detect.lua delete mode 100644 lists/pkglists/drivers.lua.m4 delete mode 100644 lists/pkglists/dvb.lua.m4 delete mode 100644 lists/pkglists/hardening.lua.m4 delete mode 100644 lists/pkglists/honeypot.lua delete mode 100644 lists/pkglists/luci-controls.lua delete mode 100644 lists/pkglists/luci_controls.lua.m4 delete mode 100644 lists/pkglists/lxc.lua.m4 delete mode 100644 lists/pkglists/nas.lua.m4 delete mode 100644 lists/pkglists/net_monitoring.lua.m4 delete mode 100644 lists/pkglists/netboot.lua.m4 delete mode 100644 lists/pkglists/netdata.lua.m4 delete mode 100644 lists/pkglists/netmetr.lua delete mode 100644 lists/pkglists/nextcloud.lua.m4 delete mode 100644 lists/pkglists/openvpn.lua.m4 delete mode 100644 lists/pkglists/pakon.lua delete mode 100644 lists/pkglists/printserver.lua delete mode 100644 lists/pkglists/reforis.lua delete mode 100644 lists/pkglists/tor.lua.m4 delete mode 100644 lists/repository.lua.m4 delete mode 100644 lists/repository.m4 delete mode 100644 lists/terminal-apps.lua.m4 delete mode 100644 lists/utils.m4 delete mode 100644 lists/webapps.lua.m4 diff --git a/.gitignore b/.gitignore index c4aeaf532..a6baef541 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ # compile_pkgs default directory /build +# generate_lists default directory +/lists # generate_lists default directory /generated_lists @@ -11,5 +13,5 @@ /generated_medkits # generate_medkit build tools directory -/turris-tools +/.turris-tools /*-medkit.tar.gz diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e360f5abe..32ad8549c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -59,9 +59,3 @@ lists_generate: name: "Generated package lists" paths: - generated_lists/* - -lists_verify: - stage: verify - needs: [lists_generate] - script: - - find generated_lists -type f -name '*.lua' -printf '-- %P --\n' -exec luac5.1 -p '{}' \; diff --git a/compile_pkgs b/compile_pkgs index 885173004..28d231cea 100755 --- a/compile_pkgs +++ b/compile_pkgs @@ -15,77 +15,17 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. set -e -shopt -s nullglob # Step list that is run when no commands are specified DEFAULT_STEPS=( "prepare" "compile" "sign" "store_hash" "stats" "pkgsrepo" "gen_junit" ) -# Paths to OpenWRT build and turris-build sources +# Paths to turris-build sources and common utilities inclusion src_dir="$(readlink -f "${0%/*}")" -build_dir="$(readlink -f "$(pwd)")" -if [ "$build_dir" == "$src_dir" ]; then - # Protect against turris-build repository wipe - mkdir -p build - cd build - build_dir="$(pwd)" -fi - -# Include common utilities -. "$src_dir/helpers/common.sh" - -# Sets various variables to match the specified target, just a helper -# It is defined here so it can be used from configuration -set_target() { - report "Setting target as $1" - case "$1" in - omnia) - TARGET_BOARD=omnia - TARGET_ARCH=armv7l - ;; - turris1x) - TARGET_BOARD=turris1x - TARGET_ARCH=ppcspe - ;; - mox) - TARGET_BOARD=mox - TARGET_ARCH=aarch64 - ;; - *) - echo "Invalid target board!!! Use -t [turris1x|omnia|mox]!!!" - exit 1 - ;; - esac -} - -# Get OpenWRT URL and branch from feeds.conf file -openwrt_feed() { - # OpenWRT git repository URL - OPENWRT_URL="$(sed -n 's/# openwrt \([^;^]\+\).*/\1/p' "$src_dir/feeds.conf")" - # Git reference used to checkout OpenWRT repository - OPENWRT_BRANCH="$(sed -n 's/# openwrt [^;^]\+[;^]\([^[:blank:]]\+\).*/\1/p' "$src_dir/feeds.conf")" - # Check if ^ is used to separate URL and branch and in such case it is hash not branch - grep -E '^# openwrt [^;^]+\^.+' "$src_dir/feeds.conf" && OPENWRT_BRANCH="#$OPENWRT_BRANCH" - [ -n "$OPENWRT_BRANCH" ] || OPENWRT_BRANCH="master" -} +source "$src_dir/helpers/generate_common.sh" -# Configuration variables -. "$src_dir/defaults.sh" -FORCE="" # Force build -CLONE_DEEP="" # Set this variable to clone OpenWRT tree in full depth not just latest commit -BUILD_ARGS=() # Additional arguments passed to OpenWRT make call (note that this is for all make calls in OpenWRT) -BUILD_JOBS="$(nproc)" # Number of jobs to be passed to make calls -SIGN_KEY="" # Path to private signing key -GIT_MIRROR="" # Path to git mirror directory -DL_MIRROR="" # Path to downloads mirror directory -CCACHE_HOST_DIR="" # Path to ccache directory for host compilations -CCACHE_TARGET_DIR="" # Path to ccache directory for target compilations +# Configuration +source "$src_dir/helpers/conf.sh" OUTPUT_DIR="./pkgsrepo" # Output directory for pkgsrepo command -openwrt_feed - -# Load configurations -[ -f "${src_dir}"/turris-build.conf ] && . "${src_dir}"/turris-build.conf -[ -f ~/.turris-build ] && . ~/.turris-build -[ -f ./turris-build.conf ] && . ./turris-build.conf ################################################################################## @@ -94,19 +34,6 @@ declare -A available_commands # Set of all commands defined in this script mirror_updated="" # Variable signaling that mirrors were update in this run already debug="" # set when debug on executed scripts should be enabled -# OpenWRT directory for temporally data -export TMPDIR="${build_dir}/tmp" -mkdir -p "$TMPDIR" - -# Git that should be used for commands such as: git commit or git am -_git() { - git \ - -c "commit.gpgsign=false" \ - -c "user.email=auto-build@example.com" \ - -c "user.name=Build system automate" \ - "$@" -} - _sh() { sh ${debug:+-x} "$@" } @@ -181,7 +108,7 @@ stats() { available_commands+=( ["configure"]="Recreates configuration for target boards" ) configure() { report "Creating default configuration" - [ -n "${TARGET_BOARD}" ] || die "No board selected!" + [ -n "${TARGET_BOARD:-}" ] || die "No board selected!" cat "${src_dir}"/configs/common/* "${src_dir}/configs/${TARGET_BOARD}"/* > ./.config { [ -n "$DL_MIRROR" ] && echo "CONFIG_DOWNLOAD_FOLDER=$DL_MIRROR" @@ -213,62 +140,29 @@ configure() { available_commands+=( ["update_mirror"]="Updates all local mirrors" ) update_mirror() { - [ -n "$GIT_MIRROR" ] || return 0 - [ -d "$GIT_MIRROR" ] || return 0 - [ -z "$mirror_updated" ] || return 0 - + use_git_mirror || return 0 report "Updating local mirrors" - flock --exclusive "$GIT_MIRROR" "$SHELL" -s "$GIT_MIRROR" <<"EOF" - cd "$1" - for mirror in ./*; do - [ -d "$mirror" ] || continue - cd "$mirror" - git remote update --prune - cd "$1" - done -EOF - mirror_updated="yes" + git_mirror_update_all } available_commands+=( ["checkout"]="Start everything from scratch - all changes deleted and fresh copy of OpenWRT gets checked out" ) checkout() { - local target="${OPENWRT_BRANCH#\#}" - report "Starting out: $target" - + local url + url="$(get_feed "#" "openwrt")" + report "Starting out: $(feed_ref "$url")" report "Checking out OpenWRT repository" - update_mirror - if [ -n "$GIT_MIRROR" ]; then - mkdir -p "$GIT_MIRROR" - [ -d "$GIT_MIRROR/openwrt" ] || \ - git clone --mirror "$OPENWRT_URL" "$GIT_MIRROR/openwrt" - OPENWRT_URL="$GIT_MIRROR/openwrt" - fi - rm -rf .git - git init - git remote add origin "$OPENWRT_URL" - if [ -z "$GIT_MIRROR" -a "${OPENWRT_BRANCH:0:1}" = "#" ]; then - # If we are downloading directly from server we can't fetch specific - # commit so fetch everything - git fetch origin - git checkout -f "$target" - else - git fetch ${CLONE_DEEP+--depth 1} origin "$target" - git checkout -f FETCH_HEAD - fi - - git clean -dff -e '/turris-build.conf' - find . -name '*.rej' -delete + git_clean_checkout "openwrt" "$url" "." \ + "/turris-build.conf" - git rev-parse --short HEAD > version echo "/turris-build.conf" >> ./.gitignore - echo "/version" >> ./.gitignore - _git commit -m 'gitignore: Ignore some more files' ./.gitignore + _git commit -m 'gitignore: Ignore turris-build related files' ./.gitignore cp "$src_dir"/feeds.conf . if [ -n "$DL_MIRROR" ]; then mkdir -p "$DL_MIRROR" - rm -rf dl && ln -s "$DL_MIRROR" dl + rm -rf dl + ln -s "$DL_MIRROR" dl fi } @@ -285,7 +179,7 @@ clean_ccache() { available_commands+=( ["set_ccache"]="Set persistent ccache paths" ) set_ccache() { - [ -z "$CCACHE_SET" ] || return 0 + [ "${CCACHE_SET:-n}" == "y" ] && return 0 report "Setting ccache paths" CCACHE_SET=y [ -z "$CCACHE_HOST_DIR" ] || sed -i 's|$(STAGING_DIR_HOST)/ccache|'"$CCACHE_HOST_DIR|" include/host-build.mk @@ -295,40 +189,22 @@ set_ccache() { available_commands+=( ["set_local_feeds"]="Change feed URL to their mirror counterparts" ) set_local_feeds() { - [ -n "$GIT_MIRROR" ] || return 0 + use_git_mirror || return 0 report "Setting feeds to their local counterparts" while read -r vcs name url rest; do - if [ "$vcs" = src-git ] && [ -d "$GIT_MIRROR/$name" ]; then - local branch="" - [[ "$url" != *^* ]] || branch="^${url#*^}" - [[ "$url" != *\;* ]] || branch=";${url#*;}" - echo "$vcs $name file://$GIT_MIRROR/$name$branch $rest" - else - echo "$vcs $name $url $rest" + if [ "$vcs" = src-git ]; then + git_mirror_update "$name" "$(feed_url "$url")" + url="$(feed_url_replace "$url" "file://$(git_mirror_path "$name")")" fi + echo "$vcs" "$name" "$url" "$rest" done < feeds.conf > feeds.conf.new mv feeds.conf.new feeds.conf } -available_commands+=( ["mirror_feeds"]="Creates initial mirrors of all configured feeds" ) -mirror_feeds() { - [ -n "$GIT_MIRROR" ] || return 0 - pushd "$GIT_MIRROR" >/dev/null - cat "$src_dir"/feeds.conf "$build_dir"/feeds.conf 2> /dev/null | while read -r vcs name url rest; do - if [ "$vcs" = src-git ] && [ ! -d "$GIT_MIRROR/$name" ]; then - git clone --mirror "${url%[;^]*}" "$name" - fi - done - popd >/dev/null - update_mirror -} - available_commands+=( ["patch_openwrt"]="Patch the main OpenWRT repository" ) patch_openwrt() { report "Patching OpenWRT repository" - for patch in "$src_dir/patches/openwrt"/*/*.patch; do - _git am --whitespace=nowarn --reject "$patch" - done + patch_repository "openwrt" if [ -d "$src_dir/src" ]; then cp -r "$src_dir"/src/* . find "$src_dir"/src -printf "%P\\0" | xargs --null git add @@ -363,23 +239,16 @@ store_hash() { available_commands+=( ["repatch_feeds"]="Cleanup feeds, update them and patch them" ) repatch_feeds() { report "Getting feeds" - update_mirror set_local_feeds _perl ./scripts/feeds clean -a _perl ./scripts/feeds update -a report "Patching feeds" - for feed in "$src_dir"/patches/*; do - [ -d "$feed" ] || continue - feed_name="$(basename "$feed")" - [ -d "$build_dir/feeds/$feed_name" ] || continue - for patch in "$feed"/*/*.patch; do - [ -f "$patch" ] || continue - pushd "$build_dir/feeds/$feed_name" >/dev/null - _git am --whitespace=nowarn --reject "$patch" - popd >/dev/null - done - done + while read -r vcs name url rest; do + if [ "$vcs" = src-git ]; then + patch_repository "$name" "$build_dir/feeds/$name" + fi + done < feeds.conf report "Installing feeds packages" _perl ./scripts/feeds update -a -i @@ -527,8 +396,7 @@ while [ $# -gt 0 ]; do PUBLISH_BRANCH="$1" ;; -l) - mirror_updated="override" - OPENWRT_URL="$GIT_MIRROR/openwrt" + set_git_mirrors_updated ;; -o) shift @@ -559,6 +427,13 @@ done [ "$(id -u)" != 0 ] || die "Build should not be run under root!" +# Set build_dir variable +set_protected_build_dir + +# OpenWRT directory for temporally data +export TMPDIR="${build_dir}/tmp" +mkdir -p "$TMPDIR" + [ ${#steps[@]} -gt 0 ] || steps=( "${DEFAULT_STEPS[@]}" ) for step in "${steps[@]}"; do "$step" diff --git a/feeds.conf b/feeds.conf index f4d65797d..d74387a54 100644 --- a/feeds.conf +++ b/feeds.conf @@ -10,3 +10,6 @@ src-git routing https://git.openwrt.org/feed/routing.git;openwrt-19.07 src-git telephony https://git.openwrt.org/feed/telephony.git;openwrt-19.07 src-git sidn https://github.com/SIDN/sidn_openwrt_pkgs.git src-git cesnet https://github.com/CESNET/Nemea-OpenWRT.git + +## Following line is not feed but it is used in generate_lists script. +# updater-lists https://gitlab.nic.cz/turris/os/updater-lists.git;develop diff --git a/generate_lists b/generate_lists index 52912af49..34aef499f 100755 --- a/generate_lists +++ b/generate_lists @@ -16,12 +16,14 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. set -e +FEED_NAME="updater-lists" + src_dir="$(readlink -f "${0%/*}")" -. "$src_dir/helpers/common.sh" -. "$src_dir/helpers/generate_common.sh" +source "$src_dir/helpers/generate_common.sh" output_path= sign_key= +prepare="n" while [ $# -gt 0 ]; do case "$1" in -h|--help) @@ -33,6 +35,8 @@ while [ $# -gt 0 ]; do echo " Prints this help text." echo " --sign KEY" echo " Sign lists with given KEY and usign utility" + echo " --prepare" + echo " Only prepare in current directory. Do not generate lists." echo " --debug" echo " Run this script in debug mode" exit @@ -41,6 +45,9 @@ while [ $# -gt 0 ]; do shift sign_key="$1" ;; + --prepare) + prepare="y" + ;; --debug) set -x ;; @@ -56,30 +63,30 @@ while [ $# -gt 0 ]; do shift done [ -n "$output_path" ] || output_path="generated_lists" +output_path="$(readlink -f "$output_path")" -[ -d "$src_dir/lists" -a -f "$src_dir/feeds.conf" ] || \ - die "This script has to be in the same direstory as feeds.conf and lists directory." +[ -f "$src_dir/feeds.conf" ] || \ + die "This script has to be in the same directory as feeds.conf file." -rm -rf "$output_path" -mkdir -p "$output_path" +set_protected_build_dir "lists" -"$src_dir/helpers/turris-version.sh" version > "$output_path/turris-version" -( cd "$src_dir" && git rev-parse HEAD > "$output_path/git-hash" ) +url="$(get_feed "#" "$FEED_NAME" "$src_dir/feeds.conf")" +tos_version="$("$src_dir/helpers/turris-version.sh" version)" -lists_dir="$src_dir/lists" -m4args=( "--include=$lists_dir" "-D_INCLUDE_=$lists_dir/" "-D_FEEDS_=$src_dir/feeds.conf" ) -m4args+=( "-D_TURRIS_OS_VERSION_=$("${0%/*}/helpers/turris-version.sh" version)" ) +git_checkout "$FEED_NAME" "$url" "$build_dir" +patch_repository "$FEED_NAME" "$build_dir" -find "$lists_dir" -name '*.lua.m4' -print0 | while read -r -d '' f; do - output="$output_path/${f##$lists_dir/}" - mkdir -p "${output%/*}" - m4 "${m4args[@]}" "$f" > "${output%.m4}" -done -find "$lists_dir" -name '*.lua' -print0 | while read -r -d '' f; do - output="$output_path/${f##$lists_dir/}" - mkdir -p "${output%/*}" - cp "$f" "${output}" -done +( +cd "$build_dir" +./configure \ + -f "$src_dir/feeds.conf" \ + -v "$tos_version" \ + -d "$output_path" +) + +[ "$prepare" = "n" ] || exit 0 + +make -C "$build_dir" if [ -n "$sign_key" ]; then get_usign @@ -87,3 +94,12 @@ if [ -n "$sign_key" ]; then "$USIGN" -S -m "$f" -s "$sign_key" done fi + +echo "$tos_version" > "$output_path/turris-version" +cat > "$output_path/git-hash" <<EOF +Project was build from following repositories:" + + * turris-build: $(git_remote_hash "$src_dir") + * $FEED_NAME: $(git_remote_hash "$output_path") + +EOF diff --git a/generate_medkits b/generate_medkits index 38c8b5803..aab6c277a 100755 --- a/generate_medkits +++ b/generate_medkits @@ -17,9 +17,8 @@ set -eu src_dir="$(readlink -f "${0%/*}")" -. "$src_dir/helpers/common.sh" -. "$src_dir/helpers/generate_common.sh" -. "$src_dir/defaults.sh" +source "$src_dir/helpers/generate_common.sh" +source "$src_dir/defaults.sh" output_path= board="omnia" diff --git a/helpers/conf.sh b/helpers/conf.sh new file mode 100644 index 000000000..61767736e --- /dev/null +++ b/helpers/conf.sh @@ -0,0 +1,71 @@ +# Turris-build configuration +# (C) 2018-2021 CZ.NIC, z.s.p.o. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +[ -n "${src_dir:-}" ] || { + echo "${0##*/}: Invalid conf.sh usage. Variable src_dir has to be defined!" >&2 + exit 1 +} + + +# Sets various variables to match the specified target, just a helper +# It is defined here so it can be used from configuration +set_target() { + report "Setting target as $1" + case "$1" in + omnia) + TARGET_BOARD=omnia + TARGET_ARCH=armv7l + ;; + turris1x) + TARGET_BOARD=turris1x + TARGET_ARCH=ppcspe + ;; + mox) + TARGET_BOARD=mox + TARGET_ARCH=aarch64 + ;; + *) + echo "Invalid target board!!! Use -t [turris1x|omnia|mox]!!!" + exit 1 + ;; + esac +} + +# Configuration variables +source "$src_dir/defaults.sh" + +BUILD_JOBS="$(nproc)" # Number of jobs to be passed to make calls +SIGN_KEY="" # Path to private signing key +FORCE="" # Force build + +GIT_MIRROR="" # Path to git mirror directory +CLONE_DEEP="" # Set this variable to clone OpenWRT tree in full depth not just latest commit +DL_MIRROR="" # Path to downloads mirror directory +CCACHE_HOST_DIR="" # Path to ccache directory for host compilations +CCACHE_TARGET_DIR="" # Path to ccache directory for target compilations + +BUILD_ARGS=() # Additional arguments passed to OpenWRT make call (note that this is for all make calls in OpenWRT) + +# Load configurations +configs=( + "${src_dir}"/turris-build.conf + ~/.turris-build + ./turris-build.conf +) +for config in "${configs[@]}"; do + if [ -f "$config" ]; then + source "$config" + fi +done diff --git a/helpers/generate_common.sh b/helpers/generate_common.sh index ae6c117a1..1deb78343 100644 --- a/helpers/generate_common.sh +++ b/helpers/generate_common.sh @@ -1,5 +1,5 @@ -# Common functions for generate_lists and generate_medkit functions -# (C) 2018-2020 CZ.NIC, z.s.p.o. +# Common functions for compile_pkgs, generate_lists and generate_medkit scripts +# (C) 2018-2021 CZ.NIC, z.s.p.o. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -13,85 +13,277 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -USIGN_VERSION=f1f65026a94137c91b5466b149ef3ea3f20091e9 -UPDATER_VERSION=master +set -eu +shopt -s nullglob +[ -n "${src_dir:-}" ] || { + echo "${0##*/}: Invalid generate_common.sh usage. Variable src_dir has to be defined!" >&2 + exit 1 +} +source "$src_dir/helpers/common.sh" -SRC_USIGN="https://git.openwrt.org/project/usign.git" +DEFAULT_UPDATER_VERSION=master +SRC_USIGN="https://git.openwrt.org/project/usign.git^f1f65026a94137c91b5466b149ef3ea3f20091e9" SRC_UPDATER="https://gitlab.nic.cz/turris/updater/updater.git" +# Git that should be used for commands such as: git commit or git am +_git() { + git \ + -c "commit.gpgsign=false" \ + -c "user.email=auto-build@example.com" \ + -c "user.name=Turris build system" \ + "$@" +} + +# Set build_dir variable to current working directory unless that is the src_dir. +# This also protect against removal of source and thus when build directory is the +# source directory it creates new build subdirectory to be used instead. +# This sets global variable but it makes this important code common for all +# generate scripts. +set_protected_build_dir() { + local alt="${1:-build}" + build_dir="$(readlink -f "$(pwd)")" + if [ "$build_dir" == "$src_dir" ]; then + # Protect against turris-build repository wipe + mkdir -p "$alt" + cd "$alt" + build_dir="$(pwd)" + fi +} + +# Helper function to locate specific line in feeds.conf +# First argument is feed's VCS. That is in general "src-git" for real feeds or +# "#" for non-feed repositories. +# Second argument is feed's name. +# Echoes URL as stored in feeds.conf file +get_feed() { + local req_vcs="$1" + local req_name="$2" + local feed_file="${3:-$src_dir/feeds.conf}" + while read -r vcs name url rest; do + if [[ "$vcs" == "$req_vcs" && "$name" == "$req_name" ]]; then + echo "$url" + return + fi + done < "$feed_file" +} + +# Removes optional branch or hash from the end of the url. +# Feed url has to be passed as first argument. +feed_url() { + echo "${1%[;^]*}" +} + +# Removes url from feed url and that way provides only reference +# It provides "master" instead if there was no reference in feed url. +# Feed url has to be provided as first argument. +feed_ref() { + local res="${1##*[;^]}" + if [[ -n "$res" ]]; then + echo "$res" + else + echo "master" + fi +} + +# Checks if feed url reference points to branch or rather to specific commit +# You have to provide feed url as first argument. +feed_ref_is_branch() { + [[ "$1" == *\;* ]] +} + +# Replaces URL in feed's url. +feed_url_replace() { + local old_url="$1" + local new="$2" + if feed_ref_is_branch "$old_url"; then + echo "$new;$(feed_ref "$old_url")" + else + echo "$new^$(feed_ref "$old_url")" + fi +} + +# Applies patched to git repository. +# First argument is name of patch directory +# Second argument is optional path to repository to apply patches to. By default +# current working directory is used. +patch_repository() { + local patchdir="$1" + local repo="${2:-./}" + ( + cd "$repo" + for patch in "$src_dir/patches/$patchdir"/*/*.patch; do + _git am --whitespace=nowarn --reject "$patch" + done + ) +} + + +# Check if git mirror is configured +use_git_mirror() { + [[ -n "${GIT_MIRROR:-}" ]] +} + +# Assembles path to git mirror +# Expects one argument and that tail part of path. +git_mirror_path() { + echo "$GIT_MIRROR/$1" +} + +# Protect given command operating on mirrors using flock +# This should be always when manipulating with local mirror +_git_mirror_lock() { + if use_git_mirror; then + flock --exclusive "$GIT_MIRROR" "$@" + else + "$@" + fi +} + +_updated_mirrors_override="" +declare -A _updated_mirrors + +# Update and initialize mirror repository +# First argument is name of repository (used to identify it in mirror directory) +# Second argument is URL to upstream repository (URL not feed's url) +git_mirror_update() { + use_git_mirror || return 0 + local repo="$1" + local url="$2" + [[ -z "${_updated_mirrors[$repo]}" && -z "${_updated_mirrors_override}" ]] \ + || return 0 + local mirror="$GIT_MIRROR/$repo" + if [ -d "$repo" ]; then + _git_mirror_lock "$SHELL" -eus "$mirror" "$url" <<-"EOF" + git -C "$1" remote set-url origin "$2" + git -C "$1" remote update --prune + EOF + else + _git_mirror_lock \ + _git -C "$GIT_MIRROR" clone --mirror "$url" + fi +} + +# Update all available mirrors +git_mirror_update_all() { + use_git_mirror || return 0 + for mirror in "$GIT_MIRROR"/*; do + _git_mirror_lock \ + _git -C "$mirror" remote update --prune + done +} + +# Makes it so all mirrors are considered as updated +set_git_mirrors_updated() { + _updated_mirrors_override="__override__" +} + # Git puller helper function (git_pull output_path source_url target) -git_get() { - local REPO="$1" - local OUTPUT="$2" - local BRANCH="${3:-master}" - if [ ! -d "$OUTPUT/.git" ]; then - mkdir -p "$(dirname "$OUTPUT")" - git clone "$REPO" "$OUTPUT" - ( - cd "$OUTPUT" - git checkout "$BRANCH" - git submodule update --init --recursive - git clean -Xdf - ) - return 1 +# First argument has to be unique name of repository. +# Second argument is URL of feed (that is including optional reference) +# Third argument is local repository directory (in default current one). +git_checkout() ( + local name="$1" + local url="$2" + local outdir="${3:-.}" + + mkdir -p "$outdir" + cd "$outdir" + + if [ -d .git ]; then + _git remote set-url origin "$(feed_url "$url")" else - ( - cd "$OUTPUT" - git fetch - local REF="origin/$BRANCH" - git show-ref -q "$REF" || REF="$BRANCH" # Is this reference on remote or object - if ! git diff --quiet HEAD "$REF"; then - git reset --hard "$REF" - git submodule update --init --recursive - git clean -Xdf - return 1 - fi - ) + _git init + _git remote add origin "$(feed_url "$url")" fi + + if use_git_mirror || feed_ref_is_branch "$url"; then + git_mirror_update "$name" "$(feed_url "$url")" + _git_mirror_lock \ + _git fetch ${CLONE_DEEP+--depth 1} origin "$(feed_ref "$url")" + else + # If we are downloading directly from server we can't fetch specific + # commit so fetch everything + _git fetch origin + fi + _git checkout -f "$(feed_ref "$url")" + _git submodule update --init --recursive +) + +# The git_checkout function variant that performs cleanup. The all three arguments +# of git_checkout are required. +# Any additional arguments are file exclusion rules for files to not be removed by +# cleanup. +git_clean_checkout() { + local outdir="${3:-.}" + rm -rf "$outdir/.git" + git_checkout "$@" + shift 3 + _git -C "$outdir" clean -dff "${@/#/--exclude=}" + find "$outdir" -name '*.rej' -delete } -# Wget puller helper function (wget_pull output_file source_url) -wget_get() { - local HREF="$1" - local OUTPUT="$2" - if [ ! -e "$OUTPUT" ] || [ $(expr $(date -u +%s) - $(stat -c %Z "$OUTPUT")) -gt 86400 ]; then - wget "$HREF" -O "$OUTPUT" + +git_remote_hash() { + local dir="${1:-.}" + _git -C "$dir" rev-list HEAD | \ + _git -C "$dir" name-rev --stdin --refs='origin/*' | \ + awk '$2 != "" { print $1; exit }' +} + +# Provides commit for given branch (or HEAD if empty) for repository in given directory. This +# won't fail if repository does not exist and instead provides dummy hash. +# (Note this hash is also used by git for the same purpose) +git_local_hash() { + local dir="$1" + local branch="${2:-HEAD}" + if [ -d "$dir/.git" ]; then + _git -C "$dir" rev-parse "$branch" + else + echo "0000000000000000000000000000000000000000" fi } get_usign() { - if ! git_get "$SRC_USIGN" turris-tools/usign "$USIGN_VERSION"; then - ( cd turris-tools/usign && cmake . ) - make -C turris-tools/usign - fi - export USIGN="$(pwd)/turris-tools/usign/usign" + local dir=".turris-tools/usign" + export USIGN="$(pwd)/$dir/usign" + local orig_commit + orig_commit="$(git_local_hash "$dir")" + git_checkout "usign" "$SRC_USIGN" "$dir" + if [ "$orig_commit" != "$(git_local_hash "$dir")" ] || [ -x "$USIGN" ]; then ( + cd "$dir" + cmake . + make + ) fi } -# Automaticaly detect updater-ng's version from repository +# Automatically detect updater-ng's version from repository # This function expects that tag of v* variant exists and that version of package # is of format *-[09]+ # To be pedantic you should provide target board as second argument but if not # provided then mox is used. updater_ng_repodetect() { - local REPO="$1" - local BOARD="${2:-mox}" - local VERSION="$(curl "https://repo.turris.cz/$REPO/$BOARD/packages/turrispackages/Packages" | \ + local repo="$1" + local board="${2:-mox}" + local version="$(curl "https://repo.turris.cz/$repo/$board/packages/turrispackages/Packages" | \ awk '/^Package: updater-ng$/,/^$/ { if ($1 == "Version:") { gsub("-.*$","",$2); print $2 } }')" - if [ -z "$VERSION" ]; then - warn "Detection of updater-ng version from repository failed. Using $UPDATER_VERSION instead." + if [ -z "$version" ]; then + warn "Detection of updater-ng version from repository failed. Using $DEFAULT_UPDATER_VERSION instead." else - UPDATER_VERSION="v$VERSION" + UPDATER_VERSION="v$version" fi } get_updater_ng() { - if ! git_get "$SRC_UPDATER" turris-tools/updater-ng "$UPDATER_VERSION"; then - ( - cd turris-tools/updater-ng - ./bootstrap - ./configure --disable-tests --disable-linters --disable-docs --disable-valgrind - make - ) - fi - export PKGUPDATE="$PWD/turris-tools/updater-ng/pkgupdate" - export PKGTRANSACTION="$PWD/turris-tools/updater-ng/pkgtransaction" + local dir=".turris-tools/updater-ng" + export PKGUPDATE="$(pwd)/$dir/pkgupdate" + export PKGTRANSACTION="$(pwd)/$dir/pkgtransaction" + local orig_commit + orig_commit="$(git_local_hash "$dir")" + git_checkout "updater" "$SRC_UPDATER^$UPDATER_VERSION" "$dir" + if [ "$orig_commit" != "$(git_local_hash "$dir")" ] \ + || [ ! -x "$PKGUPDATE" ] || [ ! -x "$PKGTRANSACTION" ]; then ( + cd "$dir" + ./bootstrap + ./configure --disable-tests --disable-linters --disable-docs --disable-valgrind + make + ) fi } diff --git a/lists/base-conditional.lua.m4 b/lists/base-conditional.lua.m4 deleted file mode 100644 index 217f4bb92..000000000 --- a/lists/base-conditional.lua.m4 +++ /dev/null @@ -1,13 +0,0 @@ --- Packages extensions and integrations with other with indirect dependencies -include(utils.m4)dnl - -_FEATURE_GUARD_ - --- Install replacement link for python when python-base is not installed and python3-base is -Install("python3-python", { condition = {Not("python-base"), "python3-base"}, priority = 40 }) - --- Reload of OpenVPN on network restart to ensure fast reconnect -Install("openvpn-hotplug", { priority = 30, condition = "openvpn" }) - - -_END_FEATURE_GUARD_ diff --git a/lists/base-fix.lua b/lists/base-fix.lua deleted file mode 100644 index 8bbe80eda..000000000 --- a/lists/base-fix.lua +++ /dev/null @@ -1,178 +0,0 @@ --- Fixes and hacks to migrate from older setups - --- ABI changed in libubus with version 2019-12-27 -if not version_match or not installed or - (installed["libubus"] and version_match(installed["libubus"].version, "<2019-12-27")) then - Package("libubus", { abi_change = true }) -end - - ---[[ -Fix packages triggers. -Following code installs in various cases special fix packages. Those packages are -provided primarily as a way to run script during update process. -Fix packages should be installed and then with replan removed. This means that -trigger that install them should be in general applicable only before fix is -applied. This means reading configuration but current updater languages allows us -only limited access so in general we rely on update of packages that are included -in appropriate release or on Turris OS release version. -]] - --- Migrate from Samba3 to Samba4 -if installed and installed["samba36-server"] and not installed["samba4-server"] then - -- This effectively detects that users has Samba3 installed and is installing Samba4 - -- In such case we want to also install fix package to migrate samba config. - local extra = {} - if features.request_condition then - extra.condition = "samba4-server" - end - Install("fix-samba-migrate-to-samba4", extra) - Package("fix-samba-migrate-to-samba4", { replan = "finished" }) - --[[ - We do here hack. If updater is not supporting request conditions then we just - install Samba4 server possibly just to migrate to it and remove it later. If - it is requested then it stays installed. Later updater is going to detect - migration correctly and run this fix. - ]] -end - --- Fix package alternatives with updater version 65.0 ---[[ -if not version_match or not installed or - (installed["updater-ng"] and version_match(installed["updater-ng"].version, "<65.0")) then - Install("fix-updater-v65.0-alternatives-update") - Package("fix-updater-v65.0-alternatives-update", { replan = "finished" }) -end -]] --- For now keep this fix in place. The problem is with updater not yet handling --- busybox not providing alternatives fully. We can return to previous version --- once that is handled in updater appropriately. -Install("fix-updater-v65.0-alternatives-update") - --- Migrate Quad9 DNS config (it was renamed/split) -if not version_match or not installed or - (installed["resolver-conf"] and version_match(installed["resolver-conf"].version, "<0.0.1-32")) then - Install("fix-dns-forward-quad9-split") - Package("fix-dns-forward-quad9-split", { replan = "finished" }) -end - --- Migrate ends buffer size value in resolver config. --- It was changed because of DNS flag day 2020. -if not version_match or not installed or - (installed["resolver-conf"] and version_match(installed["resolver-conf"].version, "<0.0.2-8")) then - Install("fix-edns-buffer-size") - Package("fix-edns-buffer-size", { replan = "finished" }) -end - --- Migrate original pkglists to separate config with options in place -if not version_match or not installed or - (installed["pkglists"] and version_match(installed["pkglists"].version, "<1.3")) then - Install("fix-pkglists-options") - Package("fix-pkglists-options", { replan = "finished" }) -end - --- Remove no longer generated task log from Updater -if not version_match or not installed or - (installed["updater-supervisor"] and version_match(installed["updater-supervisor"].version, "<1.3.2")) then - Install("fix-updater-rm-log") -end - --- Restore previous non-empty version of /etc/config/foris --- With Turris OS 5.1.0 there was a bug, which removed content of /etc/config/foris and it slipped --- through testing. This fix just reverts older version of affected file from --- snapshots. -if root_dir == "/" and version_match(os_release.VERSION, "<5.1.1") then - Install("fix-config-foris-restore") - Package("fix-config-foris-restore", { replan = "finished" }) -end - --- Contracted routers have in boot environment set contract variable that is used --- in boot arguments. This variable should be preserved but due to bug in rescue --- could have been corrupted on factory reset. This fix should recover it. --- We apply it in 5.1.2 but because that version is already in RC we have to keep --- it installed for that version in system. --- We request reboot as contract is applied only after reboot. -if root_dir == "/" and version_match(os_release.VERSION, "<=5.1.2") then - Install("fix-corrupted-contract-by-rescue") - Package("fix-corrupted-contract-by-rescue", { replan = "finished", reboot = "delayed" }) -end - --- Default configuration on Turris Shield was invalid in factory (Turris OS 5.0). --- Only three LAN ports were correctly assigned. Fourth port was unassigned. This --- uses shield-support package to detect old version of Shield and fix it. -if version_match and installed and installed["shield-support"] and - version_match(installed["shield-support"].version, "<2.2.0") then - Install("fix-all-lan-ports-in-lan") - Package("fix-all-lan-ports-in-lan", { replan = "finished" }) -end - --- Transmission previously implemented multiple variants but that was later --- abandoned in favor of single SSL variant. This covers previously defined --- variants of packages and simply marks them as virtual and request installation --- of package instead. -for _, ssl in pairs({"openssl", "mbedtls"}) do - for _, pkg in pairs({"transmission-daemon", "transmission-cli", "transmission-remote"}) do - Package(pkg .. "-" .. ssl, { virtual = true, deps = pkg }) - end -end - --- With Turris OS 5.2.0 we removed long time obsolete package cznic-cacert-bundle. --- This package was storing its certificates to backup storage. We have to remove --- them from there. -if version_match and installed and installed["cznic-cacert-bundle"] then - Install("fix-cleanup-cert-backup") - Package("fix-cleanup-cert-backup", { replan = "finished" }) -end - --- With Turris OS 5.2.0 packages luci-lighttpd and turris-webapps integration --- included in luci-base was merged to single dedicated package --- turris-webapps-luci. Problem is that luci-base was handled by patch that did --- caused no version change so we can end up potentially with conflict between --- previous luci-base and new turris-webapps-luci. This simply requests reinstall --- when luci-lighttpd is installed. It is not the cause but it should be removed --- at the same time as this error is being resolved. -if installed and installed["luci-lighttpd"] and not installed["turris-webapps-luci"] then - -- Note: condition here makes this request prety much ignored and so we won't - -- interfere with install priorities. - Install("luci-base", { reinstall = true, condition = "luci-base" }) -end - --- With uboot-envtools version 2018.03-4 environment configuration was fixed. Problem --- is that it is not applied in default as script checks for existence of --- /etc/config/ubootenv file and does nothing. --- In case of Mox we also move fw_env.config from mox-support package. Because of --- that we have to update mox-support first so we would not remove generated file. -if not version_match or not installed or - (installed["uboot-envtools"] and version_match(installed["uboot-envtools"].version, "<2018.03-4")) then - Package("uboot-envtools", { deps = "fix-uboot-env-reset" }) - if board == "mox" then - Package("uboot-envtools", { deps = "mox-support" }) - end -end - --- Package dhparam was removed and replaced by turris-cagen ability to generate --- dhparam instead. These files are expected to be in different locations so we --- have to fix paths in existing server configurations. This does exactly that. --- With Turris OS 5.2.0 this fix was released and it turns out that it was way --- cautious. We removed some unnecessary checks and we reapply it with 5.2.1 --- version again. -if installed and os_release and (installed["foris-controller-openvpn-module"] and version_match(os_release['VERSION'], "<5.2.1")) then - Install("fix-dhparam-to-cagen") - Package("fix-dhparam-to-cagen", { replan = "finished" }) -end - --- pkglists package 1.6.0 moved content of hardening package list to separate --- options. This fix just enables those option, that are not in default enabled, --- to users with hardening enabled. -if not version_match or not installed or - (installed["pkglists"] and version_match(installed["pkglists"].version, "<1.6.0")) then - Install("fix-pkglists-hardening-options") -end - --- Fix empty nextcloud config if it was created by nextcloud cron. --- If empty config is detected it will remove it. -if not version_match or not installed or - (installed["nextcloud"] and version_match(installed["nextcloud"].version, "<19.0.3-3")) then - Install("fix-nextcloud-conf") - Package("fix-nextcloud-conf", { replan = "finished" }) -end diff --git a/lists/base-min.lua.m4 b/lists/base-min.lua.m4 deleted file mode 100644 index 3ccde3b24..000000000 --- a/lists/base-min.lua.m4 +++ /dev/null @@ -1,117 +0,0 @@ -include(utils.m4)dnl -include(repository.m4)dnl - -list_script('base-fix.lua') -list_script('base-conditional.lua') - --- Updater itself -Install('updater-ng', 'updater-supervisor', { critical = true }) -Package('updater-ng', { replan = 'finished' }) - --- Kernel -Package("kernel", { reboot = "delayed" }) -Package("kmod-mac80211", { reboot = "delayed" }) -forInstallCritical(kmod,file2args(kmod.list)) -if board == "mox" then - forInstallCritical(kmod,file2args(kmod-mox.list)) - Install("mox-support", { critical = true }) - Install("zram-swap", { priority = 40 }) -elseif board == "omnia" then - forInstallCritical(kmod,file2args(kmod-omnia.list)) - Install("omnia-support", { critical = true }) -elseif board == "turris1x" then - forInstallCritical(kmod,file2args(kmod-turris.list)) - Install("turris1x-support", { critical = true }) -end -Install("fstools", { critical = true }) - --- Critical minimum -Install("base-files", "busybox", "procd", "ubus", "uci", { critical = true }) -Install("netifd", "firewall", "dns-resolver", { critical = true}) - --- OpenWrt minimum -Install("ebtables", "dnsmasq-full", "odhcpd", "odhcp6c", { priority = 40 }) -Install("urandom-seed", { priority = 40 }) -Install("opkg", "libustream-openssl", { priority = 40 }) -Uninstall("wget-nossl", { priority = 40 }) -- opkg required SSL variant only - --- Turris minimum -Install("turris-defaults", { priority = 40 }) -Install("cronie", { priority = 40 }) -Install("syslog-ng", "logrotate", { priority = 40 }) -if board == "turris1x" then - Install("unbound", "unbound-anchor", { priority = 40 }) - Install("turris1x-btrfs", { priority = 40 }) -- Currently only SD card root is supported -else - Install("knot-resolver", { priority = 40 }) -end - --- Certificates -Install("dnssec-rootkey", "cznic-repo-keys", { critical = true }) --- Note: We don't ensure safety of these CAs -Install("ca-certificates", { priority = 40 }) - --- Network protocols -Install("ppp", "ppp-mod-pppoe", { priority = 40 }) - -_FEATURE_GUARD_ - --- Updater utility -Install("updater-drivers", { priority = 40 }) -Install("updater-opkg-wrapper", { priority = 40 }) -Install('switch-branch', { priority = 40 }) - -Package('updater-drivers', { replan = 'finished' }) -Package('l10n-supported', { replan = 'finished' }) -Package('updater-opkg-wrapper', { replan = 'finished' }) -Package('localrepo', { replan = 'finished' }) -Package('switch-branch', { replan = 'finished' }) - --- Network tools -Install("ip-full", "tc", "genl", "ip-bridge", "ss", "nstat", "devlink", "rdma", { priority = 40 }) -Install("iputils-ping", "iputils-ping6", "iputils-tracepath", "iputils-tracepath6", "iputils-traceroute6", { priority = 40 }) -Install("iptables", "ip6tables", "conntrack", { priority = 40 }) -Install("bind-client", "bind-dig", { priority = 40 }) -Install("umdns", { priority = 40 }) - --- Admin utilities -Install("shadow", "shadow-utils", "uboot-envtools", "i2c-tools", { priority = 40 }) -Install("openssh-server", "openssh-sftp-server", "openssh-moduli", { priority = 40 }) -Uninstall("dropbear", { priority = 40 }) -Install("pciutils", "usbutils", "lsof", "btrfs-progs", { priority = 40 }) -Install("lm-sensors", { priority = 40 }) -if board == "turris1x" or board == "omnia" then - Install("haveged", { priority = 40 }) -end - --- Turris utility -Install("turris-version", "start-indicator", { priority = 40 }) -Install("turris-utils", "user-notify", "watchdog_adjust", { priority = 40 }) -Install("turris-diagnostics", { priority = 40 }) -if for_l10n then - for_l10n("user-notify-l10n-") -end -if board == "mox" then - Install("mox-otp", { priority = 40 }) -elseif board == "omnia" then - Install("turris-rainbow", { priority = 40 }) - Install("libatsha204", { priority = 40 }) -elseif board == "turris1x" then - Install("turris-rainbow", { priority = 40 }) - Install("libatsha204", "update_mac", { priority = 40 }) -end -if board ~= "turris1x" then - Install("schnapps", { priority = 40 }) -end - - --- Wifi -Install("hostapd-common", "wireless-tools", "wpad-openssl", "iw", "iwinfo", { priority = 40 }) -if board == "mox" then - Install("kmod-ath10k-ct", { priority = 40 }) - Install("mwifiex-sdio-firmware", "ath10k-firmware-qca988x-ct", { priority = 40 }) -else - Install("ath10k-firmware-qca988x", { priority = 40 }) -end - -_END_FEATURE_GUARD_ diff --git a/lists/base-netboot.lua.m4 b/lists/base-netboot.lua.m4 deleted file mode 100644 index f2454d4de..000000000 --- a/lists/base-netboot.lua.m4 +++ /dev/null @@ -1,11 +0,0 @@ -include(utils.m4)dnl -include(repository.m4)dnl - -list_script('base-min.lua') - -_FEATURE_GUARD_ - -Install("turris-netboot-data", { priority = 40 }) -Install("foris-controller", { priority = 40 }) - -_END_FEATURE_GUARD_ diff --git a/lists/base.lua.m4 b/lists/base.lua.m4 deleted file mode 100644 index f8f9496ac..000000000 --- a/lists/base.lua.m4 +++ /dev/null @@ -1,23 +0,0 @@ -include(utils.m4)dnl -include(repository.m4)dnl - -list_script('base-min.lua') - -_FEATURE_GUARD_ - -list_script('luci.lua') -list_script('foris.lua') -list_script('terminal-apps.lua') -list_script('webapps.lua') - - --- Install various timezone information so local time would work -local zoneinfo = {"core", "atlantic", "asia", "africa", "australia-nz", "europe", "northamerica", "india", "pacific", "poles", "simple", "southamerica"} -for _, zone in pairs(zoneinfo) do - Install("zoneinfo-" .. zone, { priority = 40 }) -end - --- IPv6 -Install("ds-lite", "6in4", "6rd", "6to4", { priority = 40 }) - -_END_FEATURE_GUARD_ diff --git a/lists/bootstrap.lua b/lists/bootstrap.lua deleted file mode 100644 index 7348d55df..000000000 --- a/lists/bootstrap.lua +++ /dev/null @@ -1,111 +0,0 @@ ---[[ -Root script for updater-ng configuration used for bootstrapping new root. - -This script expects following variables to be possibly defined in environment: - BOOTSTRAP_BOARD: board name (Mox|Omnia|Turris) - BOOTSTRAP_BASE: optional specification of base script. 'base' is used in default. - BOOTSTRAP_L10N: commas separated list of languages to be installed in root. - BOOTSTRAP_PKGLISTS: commas separated list of package lists to be included in - root. To specify options you can optionally add parentheses at the end of - package name and list pipe separated options inside them. (ex: - foo(opt1|opt2),fee) - BOOTSTRAP_DRIVERS: comamas separated list of driver flags. Every driver class is - combination of bus ('usb' or 'pci') and class itself separated by colon (ex: - usb:foo,pci:fee). Special class 'all' can be used to include all drivers. - BOOTSTRAP_CONTRACT: name of contract medkit is generated for. Do not specify for - standard medkits - BOOTSTRAP_TESTKEY: if defined non-empty then test kyes are included in - installation - BOOTSTRAP_INITIAL_CONFIG: if defined non-empty then package initial-config is - included. -]] - --- Sanity checks -if root_dir == "/" then - DIE("Bootstrap is not allowed on root.") -end - - --- Get target board -model = os.getenv('BOOTSTRAP_BOARD') -if not model or model == "" then - DIE("Target model has to be provided by BOOTSTRAP_BOARD environment variable.") -end -Export('model') - - --- Load requested localizations -l10n = {} -local env_l10n = os.getenv('BOOTSTRAP_L10N') -if env_l10n then - for lang in env_l10n:gmatch('[^,]+') do - table.insert(l10n, lang) - end -end -Export('l10n') - - --- Always include base script -local base = os.getenv('BOOTSTRAP_BASE') -if not base or base == "" then - base = "base" -end -Script(base .. '.lua') - --- Include any additional lists -local env_pkglists = os.getenv('BOOTSTRAP_PKGLISTS') -if env_pkglists then - for list in env_pkglists:gmatch('[^,]+') do - local list_name = list:match('^[^(]+') - local list_options = list:match('%((.*)%)$') - options = {} - Export("options") - if list_options then - for opt in list_options:gmatch('[^|]+') do - options[opt] = true - end - end - Script(list_name .. '.lua') - end -end - --- Include any additional driver classes -local env_devices = os.getenv('BOOTSTRAP_DRIVERS') -if env_devices then - local usb_devices = {} - local pci_devices = {} - for device in env_devices:gmatch('[^,]+') do - local tp, class = device:match('([^:]+):(.*)') - if tp == "usb" then - table.insert(usb_devices, class) - elseif tp == "pci" then - table.insert(pci_devices, class) - else - WARN("Invalid device type, ignoring: " .. device) - end - end - devices = usb_devices - Export('devices') - Script('drivers/usb.lua') - devices = pci_devices - Script('drivers/pci.lua') - Unexport('drivers') -end - --- Include contract if specified -local env_contract = os.getenv('BOOTSTRAP_CONTRACT') -if env_contract and env_contract ~= "" then - Script('contracts/' .. env_contract .. '.lua') -end - - -local env_testkey = os.getenv('BOOTSTRAP_TESTKEY') -if env_testkey and env_testkey ~= "" then - Install('cznic-repo-keys-test') -end - - -local env_initial_config = os.getenv('BOOTSTRAP_INITIAL_CONFIG') -if env_initial_config and env_initial_config ~= "" then - Install('initial-config') -end diff --git a/lists/contracts/cti.lua.m4 b/lists/contracts/cti.lua.m4 deleted file mode 100644 index 21f87aa2b..000000000 --- a/lists/contracts/cti.lua.m4 +++ /dev/null @@ -1,21 +0,0 @@ -include(utils.m4)dnl -_FEATURE_GUARD_ - --- Extra security -Install('common_passwords') - --- Data collection -Install('sentinel-i_agree_with_eula') - -options = { - ["dynfw"] = true, - ["haas"] = true, - ["survey"] = true, - ["nikola"] = true, - ["minipot"] = true, -} -Export("options") -Script("../pkglists/datacollect.lua") -Unexport("options") - -_END_FEATURE_GUARD_ diff --git a/lists/contracts/cznic.lua.m4 b/lists/contracts/cznic.lua.m4 deleted file mode 100644 index 18616d4c8..000000000 --- a/lists/contracts/cznic.lua.m4 +++ /dev/null @@ -1,7 +0,0 @@ -include(utils.m4)dnl -_FEATURE_GUARD_ - -Install("sentinel-dynfw-client") -Install("sentinel-minipot", "sentinel-nikola") - -_END_FEATURE_GUARD_ diff --git a/lists/contracts/shield.lua.m4 b/lists/contracts/shield.lua.m4 deleted file mode 100644 index 7cdf9ea35..000000000 --- a/lists/contracts/shield.lua.m4 +++ /dev/null @@ -1,41 +0,0 @@ -include(utils.m4)dnl -_FEATURE_GUARD_ - --- No more Foris and LuCI and data collect UI -Uninstall("foris", "luci", "luci-base", "turris-webapps", { priority = 45 }) - --- Remove some more advanced reForis plugins -Uninstall("reforis-storage-plugin", { priority = 45 }) - --- Alternative versions of packages -Install("shield-support", { priority = 45 }) - --- Extra usability packages -Install("ip_autoselector", "firewall-redirect-192-168-1-1", { priority = 45 }) - --- Extra security -Install('common_passwords', { priority = 45 }) -Install('sentinel-i_agree_with_eula', { priority = 45 }) - -options = { - ["dynfw"] = true, - ["haas"] = true, - ["survey"] = true, - ["nikola"] = true, - ["minipot"] = true, -} -Export("options") -Script("../pkglists/datacollect.lua") -Unexport("options") - --- Extra software -options = { - ["netmetr"] = true, - ["dev_detect"] = true, -} -Export("options") -Script("../pkglists/net_monitoring.lua") -Unexport("options") -Script("../pkglists/openvpn.lua") - -_END_FEATURE_GUARD_ diff --git a/lists/drivers/pci.lua b/lists/drivers/pci.lua deleted file mode 100644 index 35c7c0389..000000000 --- a/lists/drivers/pci.lua +++ /dev/null @@ -1,39 +0,0 @@ ---[[ -Following table is database of PCI devices identifiers and drivers in OpenWrt to -support them. Feel free to expand it with additional devices. - -Every device has to have numerical 'vendor' and 'device' identifier and list of -'packages' defined. Please also add comment with device name. -]] -local db = { - { -- Qualcomm Atheros AR9287 Wireless Network Adapter - vendor = 0x168c, - device = 0x002e, - packages = {"kmod-ath9k"} - }, - { -- Qualcomm Atheros AR93xx Wireless Network Adapter (rev 01) - vendor = 0x168c, - device = 0x0030, - packages = {"kmod-ath9k"} - }, - { -- Qualcomm Atheros QCA986x/988x 802.11ac Wireless Network Adapter - vendor = 0x168c, - device = 0x003c, - packages = {"kmod-ath10k", "ath10k-firmware-qca988x"} - }, - { -- MEDIATEK Corp. Device 7612 - vendor = 0x14c3, - device = 0x7612, - packages = {"kmod-mt76"} - }, -} - ----------------------------------------------------------------------------------- -for _, device in pairs(devices) do - for _, dbdev in pairs(db) do - if (type(device) == "string" and device == "all") or - (type(device) == "table" and device.vendor == dbdev.vendor and device.device == dbdev.device) then - Install(unpack(dbdev.packages), { priority = 40 }) - end - end -end diff --git a/lists/drivers/usb.lua b/lists/drivers/usb.lua deleted file mode 100644 index b3895c30e..000000000 --- a/lists/drivers/usb.lua +++ /dev/null @@ -1,127 +0,0 @@ ---[[ -Following table is database of USB devices identifiers and drivers in OpenWrt to -support them. Feel free to expand it with additional devices. - -We do not essentially want to have here full list. It is not desirable to include -devices that are in main used as hotplug (such as USB flash stick) and devices -with generic driver (such as mass storage). It is rather desirable to have this as -list of devices such as modems, tunners and so on. - -Every device has to have numberical 'vendor' and 'product' identifier and list of -'packages' defined. Please also comment with device name. There is also optional -'class' table. Those are used to enable selected subset of packages no matter if -the are present in device or not. - -This expects variable 'devices' to be exported. It has to contain array. Elements -in array can be either string or table. String is considered as flag and any -device with matching flag is installed. Special flag is "all" that matches all -devices in database. Other option is table where keys are "vendor" and "product". -Values of those fields are compared with values in database and if they match then -appropriate packages are requested. -]] -local db = { - -- DVB tunners --------------------------------------------------------------- - { -- TechnoTrend AG TT-connect CT-3650 CI - vendor = 0x0b48, - product = 0x300d, - packages = {"kmod-dvb-usb-ttusb2", "kmod-dvb-tda10023", "kmod-dvb-tda10048", "kmod-media-tuner-tda827x"}, - class = {"dvb"} - }, - { -- TechniSat Digital GmbH CableStar Combo HD CI - vendor = 0x14f7, - product = 0x0003, - packages = {"kmod-dvb-usb-ttusb2", "kmod-dvb-drxk", "kmod-dvb-usb-v2", "kmod-dvb-usb-az6007", "kmod-media-tuner-mt2063"}, - -- Note: additional not packaged firmware is required - class = {"dvb"} - }, - { -- Computer & Entertainment, Inc. Astrometa DVB-T/T2/C FM & DAB receiver [RTL2832P] - vendor = 0x15f4, - product = 0x0131, - packages = {"kmod-dvb-usb", "kmod-dvb-cxd2841er", "kmod-dvb-mn88473", "kmod-dvb-rtl2832", "kmod-dvb-usb-rtl28xxu", "kmod-media-tuner-r820t", "kmod-media-tuner-r820t"}, - class = {"dvb"} - }, - { -- Microsoft Corporation Xbox One Digital TV Tuner - vendor = 0x045e, - product = 0x02d5, - packages = {"kmod-dvb-mn88472", "kmod-media-tuner-tda18250", "kmod-dvb-usb-dib0700"}, - -- Note: additional not packaged firmware is required - class = {"dvb"} - }, - -- LTE modems ---------------------------------------------------------------- - { -- Qualcomm, Inc. Acer Gobi 2000 Wireless Modem - vendor = 0x05c6, - product = 0x9215, - packages = {"kmod-usb-net-qmi-wwan", "kmod-usb-serial-qualcomm"}, - class = {"broadband"} - }, - { -- Huawei Technologies Co., Ltd. K5150 LTE modem - vendor = 0x12d1, - product = 0x1f16, - packages = {"comgt-ncm", "umbim", "kmod-usb-net-cdc-ether"}, - class = {"broadband"} - }, - { -- TCL Communication Ltd Alcatel OneTouch L850V / Telekom Speedstick LTE - vendor = 0x1bbb, - product = 0x0195, - packages = {"kmod-usb-net-rndis"}, - class = {"broadband"} - }, - -- WiFi dongles -------------------------------------------------------------- - { -- Ralink Technology, Corp. RT2870/RT3070 Wireless Adapter - vendor = 0x148f, - product = 0x3070, - packages = {"kmod-rt2800-usb"}, - class = {"wifi"} - }, - { -- NETGEAR, Inc. WN111(v2) RangeMax Next Wireless [Atheros AR9170+AR9101] - vendor = 0x0846, - product = 0x9001, - packages = {"kmod-carl9170"}, - class = {"wifi"} - }, - -- Serial -------------------------------------------------------------------- - { -- Future Technology Devices International Limited, Bridge(I2C/SPI/UART/FIFO) - vendor = 0x0403, - product = 0x6015, - packages = {"kmod-usb-acm"}, - class = {"serial"} - }, - { -- Future Technology Devices International, Ltd FT232 USB-Serial (UART) IC - vendor = 0x0403, - product = 0x6001, - packages = {"kmod-usb-serial-ftdi"}, - class = {"serial"} - }, - { -- Sigma Designs, Inc. Aeotec Z-Stick Gen5 (ZW090) - UZB - vendor = 0x0658, - product = 0x0200, - packages = {"kmod-usb-acm"}, - class = {"serial", "z-wave"} - }, - -- Random number generator --------------------------------------------------- - { -- OpenMoko, Inc. USBtrng hardware random number generator - vendor = 0x1d50, - product = 0x60c6, - packages = {"kmod-chaoskey"}, - class = {"random"} - }, -} ----------------------------------------------------------------------------------- --- First convert class arrays to set -for _, dbdev in pairs(db) do - local class_set = {} - for _, cl in pairs(dbdev.class or {}) do - class_set[cl] = true - end - dbdev.class = class_set -end - --- Now request packages for requested devices -for _, device in pairs(devices) do - for _, dbdev in pairs(db) do - if (type(device) == "string" and (device == "all" or dbdev.class[device])) or - (type(device) == "table" and device.vendor == dbdev.vendor and device.product == dbdev.product) then - Install(unpack(dbdev.packages), { priority = 40 }) - end - end -end diff --git a/lists/foris.lua.m4 b/lists/foris.lua.m4 deleted file mode 100644 index 4c9068a71..000000000 --- a/lists/foris.lua.m4 +++ /dev/null @@ -1,71 +0,0 @@ -include(utils.m4)dnl -_FEATURE_GUARD_ - -local foris_plugins = { - ["diagnostics"] = "turris-diagnostics", - ["netmetr"] = "netmetr", - ["openvpn"] = "openvpn", - ["pakon"] = "pakon", - ["storage"] = false, - ["subordinates"] = "turris-netboot-tools", -} - -local reforis_plugins = { - ["data-collection"] = "sentinel-proxy", - ["haas"] = "haas-proxy", - ["diagnostics"] = "turris-diagnostics", - ["netboot"] = "turris-netboot-tools", - ["netmetr"] = "netmetr", - ["openvpn"] = "openvpn", - ["remote-access"] = false, - ["remote-devices"] = false, - ["remote-wifi-settings"] = false, - ["snapshots"] = "schnapps", -} - ----------------------------------------------------------------------------------- - -Install("foris", "foris-storage-plugin", { priority = 40 }) -Install("reforis", "reforis-storage-plugin", { priority = 40 }) - -for plugin, condition in pairs(foris_plugins) do - local fplugin = "foris-" .. plugin .. "-plugin" - if condition ~= false then - Install(fplugin, { priority = 40, condition = condition }) - end - for _, lang in pairs(l10n or {}) do - Install(fplugin .. "-l10n-" .. lang, { - priority = 10, - optional = true, - condition = fplugin - }) - end -end - -for plugin, condition in pairs(reforis_plugins) do - local refplugin = "reforis-" .. plugin .. "-plugin" - if condition ~= false then - Install(refplugin, { priority = 40, condition = condition or nil }) - end - for _, lang in pairs(l10n or {}) do - Install(refplugin .. "-l10n-" .. lang, { - priority = 10, - optional = true, - condition = refplugin - }) - end -end - -if for_l10n then - for_l10n("foris-l10n-") - for_l10n("reforis-l10n-") - for_l10n('pkglists-l10n-') -end - -Install("lighttpd-https-cert", { priority = 40 }) - --- Workaround how to install foris-controller-nextcloud-module --- because there is no nextcloud-plugin -Install("foris-controller-nextcloud-module", { condition = {"nextcloud", "foris-controller-storage-module"} }) - -_END_FEATURE_GUARD_ diff --git a/lists/kmod-mox.list b/lists/kmod-mox.list deleted file mode 100644 index 4cd6ecd7d..000000000 --- a/lists/kmod-mox.list +++ /dev/null @@ -1,10 +0,0 @@ -# Real time clock -rtc-ds1307 -# Button and led on A module -gpio-button-hotplug -ledtrig-heartbeat - -# SDIO Wi-Fi and BT -mwlwifi -mwifiex-sdio -btmrvl diff --git a/lists/kmod-omnia.list b/lists/kmod-omnia.list deleted file mode 100644 index 5b806674c..000000000 --- a/lists/kmod-omnia.list +++ /dev/null @@ -1,15 +0,0 @@ -# Turris Omnia specific drivers -switch-mvsw61xx -usb-serial-qualcomm -gpio-pca953x -i2c-mux-pca954x -eeprom-at24 - -# Various led triggers -ledtrig-default-on -ledtrig-gpio -ledtrig-heartbeat -ledtrig-netdev -ledtrig-oneshot -ledtrig-timer -ledtrig-transient diff --git a/lists/kmod-turris.list b/lists/kmod-turris.list deleted file mode 100644 index b1013c94a..000000000 --- a/lists/kmod-turris.list +++ /dev/null @@ -1,11 +0,0 @@ -# Turris 1.x specific drivers -button-hotplug -hwmon-lm90 - -leds-gpio -input-gpio-keys - -rtc-ds1307 - -# For Turris 1.1 -usb3 diff --git a/lists/kmod.list b/lists/kmod.list deleted file mode 100644 index ddfee4db8..000000000 --- a/lists/kmod.list +++ /dev/null @@ -1,20 +0,0 @@ -# Wi-Fi drivers -ath9k -ath10k - -# Firewall -ipt-offload -nf-flow -nf-nathelper -br-netfilter - -# Other hardware -usb2 - -# Add support for USB Mass Storage devices -usb-storage -usb-storage-uas -usb-storage-extras - -# Kernel configuration via /proc/config.gz -ikconfig diff --git a/lists/luci.lua.m4 b/lists/luci.lua.m4 deleted file mode 100644 index 844648bb3..000000000 --- a/lists/luci.lua.m4 +++ /dev/null @@ -1,113 +0,0 @@ -include(utils.m4)dnl -_FEATURE_GUARD_ - -local luci_apps = { - "acme", - "adblock", - "advanced-reboot", - "ahcp", - "aria2", - "attendedsysupgrade", - "banip", - "bcp38", - "bird1-ipv4", - "bird1-ipv6", - "bmx6", - "bmx7", - "cjdns", - "clamav", - "commands", - "cshark", - "dcwapd", - "ddns", - "diag-core", - "dnscrypt-proxy", - "dump1090", - "dynapoint", - "e2guardian", - "firewall", - "fwknopd", - "hd-idle", - "hnet", - "https-dns-proxy", - "ipfixprobe", - "ksmbd", - "lxc", - "minidlna", - "mjpg-streamer", - "mosquitto", - "mwan3", - "nextdns", - "nft-qos", - "nlbwmon", - "noddos", - "ntpc", - "nut", - "ocserv", - "olsr", - "olsr-services", - "olsr-viz", - "openvpn", - "opkg", - "p910nd", - "pagekitec", - "polipo", - "privoxy", - "qos", - "radicale", - "radicale2", - "rainbow", - "rosy-file-server", - "rp-pppoe-server", - "samba", - "samba4", - "shadowsocks-libev", - "shairplay", - "siitwizard", - "simple-adblock", - "snmpd", - "splash", - "sqm", - "squid", - "statistics", - "tinyproxy", - "transmission", - "travelmate", - "ttyd", - "udpxy", - "uhttpd", - "unbound", - "upnp", - "vnstat", - "vpn-policy-routing", - "vpnbypass", - "watchcat", - "wifischedule", - "wireguard", - "wol", -} - --- Conditional install requests for language packages -for _, lang in pairs({"en", unpack(l10n or {})}) do - for _, name in pairs(luci_apps) do - Install("luci-i18n-" .. name .. "-" .. lang, { - priority = 10, - optional = true, - condition = "luci-app-" .. name - }) - end -end - - -Install("luci", "luci-base", { priority = 40 }) -if for_l10n then - Install("luci-i18n-base-en", { optional = true, priority = 10 }) - for_l10n("luci-i18n-base-") -end - -Install("luci-app-commands", { priority = 40 }) -Install("luci-proto-ipv6", "luci-proto-ppp", { priority = 40 }) --- Install resolver-debug for DNS debuging -Install("resolver-debug", { priority = 40 }) - -_END_FEATURE_GUARD_ diff --git a/lists/migrate3x.lua b/lists/migrate3x.lua deleted file mode 100644 index f95282205..000000000 --- a/lists/migrate3x.lua +++ /dev/null @@ -1,134 +0,0 @@ ---[[ -This is migration script used to migrate/update system from Turris OS 3.x to 4.x. - -We have to update updater first so we do immediate replan and update only updater -itself. -]] - -if not version_match or not self_version or version_match(self_version, "<64.0") then - - local board - if model:match("[Oo]mnia") then - board = "omnia" - elseif model:match("^[Tt]urris$") then - board = "turris1x" - else - DIE("Unsupported Turris model: " .. tostring(model)) - end - - --[[ - We provide access to only HBS repository and to only minimal set of feeds. We - don't need anything more to update updater. - ]] - Repository("turris", "https://repo.turris.cz/hbs/" .. board .. "/packages", { - priority = 60, - subdirs = { "base", "core", "packages", "turrispackages"} - }) - - Install('updater-ng', { critical = true }) - Package('updater-ng', { - replan = 'immediate', - deps = { 'libgcc', 'busybox' } - }) - --[[ - Updater package does not depend on libgcc but it requires it and dependency - breaks otherwise. - ]] - -else - - if version_match(installed["tos3to4"].version, "<2.0.0") then - --[[ - The first phase is to install new version of updater-ng. This is a second - phase where we install tos3to4-early that migrates updater's configuration - before we perform full system update. - The tos3to4-early is downloaded from HBS no matter what branch user has - configured as migration itself is performed by it later on and in case of - no settings HBS is the default. - With Turris OS 5.2.0 the package lists were migrated to separate file. The - fixup scripts were created to migrate it but trigger for them won't work - for us here as we install latest version of pkglists as part of this - immediate replan. - ]] - Install('tos3to4-early', 'fix-pkglists-hardening-options', { critical = true, priority = 100 }) - Package('tos3to4-early', { replan = 'immediate' }) - Package('fix-pkglists-hardening-options', { - replan = 'immediate', - deps = 'fix-pkglists-options' - }) - Package("fix-pkglists-options", { - deps = 'tos3to4-early' - }) - - --[[ - tos3to4-early requires switch-branch but that depends on - updater-supervisor and that pulls in load of dependencies including - cronie. The issue is that cronie collides with vixe-cron. By making - updater-supervisor virtual we skip it for now and thus we prevent the - collision. - It has to be virtual till we install package we are doing replan on here. - ]] - if not installed["fix-pkglists-hardening-options"] then - Package("updater-supervisor", { virtual = true }) - end - end - - --[[ - The process of update is that we first update only updater and minimal system - dependencies with it. That replaces libc and pretty much breaks whole system. - We do it by just including minimal subset of latest feeds together with Turris - OS 3.x feeds. With next replan all Turris OS 3.x feeds are no longer used that - unfortunately can cause update fault because there might be requests for - packages that were available in Turris OS 3.x but are no more. In such - situation system stays in broken state. - - Setting that all installs are optional solves this and ensures that update is - going to proceed and that result is working system. - - This does not solve problem ultimately. It instead moves problem for user to - solve later when migration is completed. User have to modify its requests. - ]] - Mode("optional_installs") - - --[[ - We install newer version of tos3to4 that migrates switch configuration. The - migration creates situation where we are running on kernel without DSA support - and without switch-config tools and thus we can't configure LAN in any way. - The solution is to just unconditionally reboot router after migration is - finished. - ]] - Package("tos3to4", { reboot = "finished" }) - -end - ---[[ -Ubus is used by init scripts to communicate with procd. The problem this creates -is that new version of ubus is unable to communicate with old ubusd. This means -that we have to restart ubusd as soon as possible. We do it in postinst of ubus -package but there can be multiple service restarts before that and every such call -to ubus takes few minutes because of ubus timeouts. It also causes action to not -be executed. This uses replan to trick updater to update ubus as soon as possible -without updating anything else. First updater is updated. Then ubus is updated and -then rest of the system. -]] -if version_match and installed and installed["ubus"] and - version_match(self_version, ">=63.0") and - version_match(installed["ubus"].version, "<2018") then - Package("ubus", { replan = 'immediate' }) -end - ---[[ -This same package as in Turris OS 3.x which contains script pulling in this -script. The difference is that Turris OS 4.x contains version 2.x of this package -to ensure update and that it does not contain noted script. Instead it contains -script migrating system configuration. That way configuration is updated and at -the same time this script is left out of all following updater executions because -tos3to4 is going to be removed. -]] -Install("tos3to4") -Package("tos3to4", { replan = "finished" }) - ---[[ -We are potentially migrating from uClibc so reinstall everything depending on it. -]] -Package("libc", { abi_change_deep = true }) diff --git a/lists/pkglists/3g.lua.m4 b/lists/pkglists/3g.lua.m4 deleted file mode 100644 index fe35dc8c2..000000000 --- a/lists/pkglists/3g.lua.m4 +++ /dev/null @@ -1,15 +0,0 @@ -include(utils.m4)dnl -_FEATURE_GUARD_ - --- 3G -Install("br2684ctl", "comgt", "uqmi", { priority = 40 }) -Install("ppp-mod-pppoe", "pptpd", { priority = 40 }) -Install("usb-modeswitch", { priority = 40 }) - --- Kernel -forInstall(kmod,nf-nathelper-extra,usb-net-rndis,usb-net-qmi-wwan,usb-serial-option,usb-serial-qualcomm) - --- Luci -Install("luci-proto-3g", "luci-proto-qmi", { priority = 40 }) - -_END_FEATURE_GUARD_ diff --git a/lists/pkglists/atlas.lua.m4 b/lists/pkglists/atlas.lua.m4 deleted file mode 100644 index dc4d2f657..000000000 --- a/lists/pkglists/atlas.lua.m4 +++ /dev/null @@ -1,7 +0,0 @@ -include(utils.m4)dnl -_FEATURE_GUARD_ - -Install("atlas-sw-probe", { priority = 40 }) - -_END_FEATURE_GUARD_ - diff --git a/lists/pkglists/datacollect.lua.m4 b/lists/pkglists/datacollect.lua.m4 deleted file mode 100644 index ad84382de..000000000 --- a/lists/pkglists/datacollect.lua.m4 +++ /dev/null @@ -1,24 +0,0 @@ -include(utils.m4)dnl -_FEATURE_GUARD_ - -if not options or options.survey ~= false then - Install("turris-survey", { priority = 40 }) -end - -if not options or options.dynfw ~= false then - Install("sentinel-dynfw-client", { priority = 40 }) -end - -if not options or options.nikola ~= false then - Install("sentinel-nikola", { priority = 40 }) -end - -if not options or options.minipot ~= false then - Install("sentinel-minipot", { priority = 40 }) -end - -if options and options.haas then - Install("haas-proxy", { priority = 40 }) -end - -_END_FEATURE_GUARD_ diff --git a/lists/pkglists/dev-detect.lua b/lists/pkglists/dev-detect.lua deleted file mode 100644 index 66f362969..000000000 --- a/lists/pkglists/dev-detect.lua +++ /dev/null @@ -1 +0,0 @@ --- This list is obsoleted and superseded by option in net_monitoring list. diff --git a/lists/pkglists/drivers.lua.m4 b/lists/pkglists/drivers.lua.m4 deleted file mode 100644 index 451cf77f4..000000000 --- a/lists/pkglists/drivers.lua.m4 +++ /dev/null @@ -1,15 +0,0 @@ -include(utils.m4)dnl -_FEATURE_GUARD_ -if options then - -if options.ath10k_ct or options.ath10k_ct_htt then - Install("kmod-ath10k-ct", { priority = 40 }) - if options.ath10k_ct_htt then - Install("ath10k-firmware-qca988x-ct-htt", { priority = 40 }) - else - Install("ath10k-firmware-qca988x-ct", { priority = 40 }) - end -end - -end -_END_FEATURE_GUARD_ diff --git a/lists/pkglists/dvb.lua.m4 b/lists/pkglists/dvb.lua.m4 deleted file mode 100644 index 4bcf5d5bb..000000000 --- a/lists/pkglists/dvb.lua.m4 +++ /dev/null @@ -1,6 +0,0 @@ -include(utils.m4)dnl -_FEATURE_GUARD_ - -Install("tvheadend", { priority = 40 }) - -_END_FEATURE_GUARD_ diff --git a/lists/pkglists/hardening.lua.m4 b/lists/pkglists/hardening.lua.m4 deleted file mode 100644 index 5b8ac26fc..000000000 --- a/lists/pkglists/hardening.lua.m4 +++ /dev/null @@ -1,19 +0,0 @@ -include(utils.m4)dnl -_FEATURE_GUARD_ - --- Common passwords Foris filter -- -if not options or options.common_passwords ~= false then - Install('common_passwords', { priority = 40 }) -end - --- ujail utils -- -if options and options.ujail then - Install("procd-ujail", { priority = 40 }) -end - --- Seccomp -- -if options and options.seccomp and board ~= "turris1x" then - Install("libseccomp", "scmp_sys_resolver", { priority = 40 }) -end - -_END_FEATURE_GUARD_ diff --git a/lists/pkglists/honeypot.lua b/lists/pkglists/honeypot.lua deleted file mode 100644 index 775becf57..000000000 --- a/lists/pkglists/honeypot.lua +++ /dev/null @@ -1 +0,0 @@ --- This list is obsoleted and superseded by option in datacollect list. diff --git a/lists/pkglists/luci-controls.lua b/lists/pkglists/luci-controls.lua deleted file mode 100644 index 88938da75..000000000 --- a/lists/pkglists/luci-controls.lua +++ /dev/null @@ -1 +0,0 @@ --- This list is obsoleted and superseded by luci_controls diff --git a/lists/pkglists/luci_controls.lua.m4 b/lists/pkglists/luci_controls.lua.m4 deleted file mode 100644 index 6cfa6229d..000000000 --- a/lists/pkglists/luci_controls.lua.m4 +++ /dev/null @@ -1,42 +0,0 @@ -include(utils.m4)dnl -_FEATURE_GUARD_ - -if board == "omnia" or board == "turris1x" then - Install("luci-app-rainbow", { priority = 40 }) -end - --- Additional protocols -forInstall(luci-app,ahcp,bcp38) -Install("luci-proto-relay", { priority = 40 }) - - -if options and options.adblock then - Install("luci-app-adblock", { priority = 40 }) -end - -if options and options.sqm then - Install("luci-app-sqm", { priority = 40 }) -end - -if options and options.tinyproxy then - Install("luci-app-tinyproxy", { priority = 40 }) -end - -if options and options.upnp then - Install("luci-app-upnp", { priority = 40 }) -end - -if options and options.printserver then - Install("kmod-usb-printer", { priority = 40 }) - Install("luci-app-p910nd", { priority = 40 }) -end - -if options and options.statistics then - Install("luci-app-statistics", { priority = 40 }) -end - -if options and options.wireguard then - Install("luci-app-wireguard", { priority = 40 }) -end - -_END_FEATURE_GUARD_ diff --git a/lists/pkglists/lxc.lua.m4 b/lists/pkglists/lxc.lua.m4 deleted file mode 100644 index b4623afb6..000000000 --- a/lists/pkglists/lxc.lua.m4 +++ /dev/null @@ -1,12 +0,0 @@ -include(utils.m4)dnl -_FEATURE_GUARD_ - -Install("lxc", { priority = 40 }) -forInstall(lxc,attach,auto,console,copy,create,destroy,freeze,info,ls,monitor,monitord,snapshot,start,stop,unfreeze) - -Install("luci-app-lxc", { priority = 40 }) - -Install("kmod-veth", { priority = 40 }) -Install("gnupg", "gnupg-utils", "getopt", "tar", "wget", { priority = 40 }) - -_END_FEATURE_GUARD_ diff --git a/lists/pkglists/nas.lua.m4 b/lists/pkglists/nas.lua.m4 deleted file mode 100644 index c4b8ae1e5..000000000 --- a/lists/pkglists/nas.lua.m4 +++ /dev/null @@ -1,68 +0,0 @@ -include(utils.m4)dnl -_FEATURE_GUARD_ - --- Kernel -- -Install("kmod-ata-ahci", { priority = 40 }) --- File systems -forInstall(kmod-fs,autofs4,btrfs,cifs,exfat,exportfs,ext4,hfs,hfsplus,msdos,nfs,nfsd,ntfs,vfat,xfs) --- Native language support -forInstall(kmod-nls,cp1250,cp1251,cp437,cp775,cp850,cp852,cp862,cp864,cp866,cp932,iso8859-1,iso8859-13,iso8859-15,iso8859-2,iso8859-6,iso8859-8,koi8r,utf8) --- Disk maintenance -Install("blkdiscard", "fstrim", { priority = 40 }) --- Fix for SATA card sent with Omnia NAS pack -if board == "omnia" then - Install("asm1062-fix", { priority = 40 }) -end - --- Tools -- -Install("mount-utils", "losetup", "lsblk", "blkid", "file", { priority = 40 }) -Install("fdisk", "cfdisk", "hdparm", "resize2fs", "partx-utils", { priority = 40 }) -Install("acl", "attr", { priority = 40 }) -Install("blockd" , "smartd", "smartmontools", { priority = 40 }) -Install("swap-utils", { priority = 40 }) - --- File systems userspace utilities -Install("lvm2", "dosfstools", "mkhfs", "btrfs-progs", "davfs2", "e2fsprogs", "fuse-utils", "xfs-mkfs", { priority = 40 }) -Install("block-mount", "badblocks", "cifsmount", "hfsfsck", "xfs-fsck", "xfs-growfs", { priority = 40 }) -Install("nfs-kernel-server", "nfs-kernel-server-utils", { priority = 40 }) -Install("ntfs-3g", "ntfs-3g-utils", { priority = 40 }) -Install("sshfs", { priority = 40 }) - --- Network -Install("wget", "rsync", "rsyncd", { priority = 40 }) - --- Luci -Install("luci-app-hd-idle", { priority = 40 }) - --- Samba -- -if options and options.samba then - forInstall(samba4,client,server,admin,utils) - Install("luci-app-samba4", { priority = 40 }) -end - --- DLNA -- -if options and options.dlna then - Install("luci-app-minidlna", { priority = 40 }) -end - --- Transmission -- -if options and options.transmission then - Install("transmission-daemon", { priority = 40 }) - Install("luci-app-transmission", { priority = 40 }) -end - --- Raid -- -if options and options.raid then - forInstall(kmod-md,linear,multipath,raid0,raid1,raid10,raid456) - Install("mdadm", { priority = 40 }) -end - --- Encryption -- -if options and options.encrypt then - Install("cryptsetup", "kmod-cryptodev", "kmod-crypto-user", { priority = 40 }) - forInstall(kmod-crypto,cbc,ctr,pcbc,des,ecb,xts) - forInstall(kmod-crypto,cmac,crc32c,sha1,sha256,sha512,md4,md5,hmac) - forInstall(kmod-crypto,seqiv,ccm,deflate) -end - -_END_FEATURE_GUARD_ diff --git a/lists/pkglists/net_monitoring.lua.m4 b/lists/pkglists/net_monitoring.lua.m4 deleted file mode 100644 index a7fef863e..000000000 --- a/lists/pkglists/net_monitoring.lua.m4 +++ /dev/null @@ -1,16 +0,0 @@ -include(utils.m4)dnl -_FEATURE_GUARD_ - -if options and options.netmetr then - Install("netmetr", { priority = 40 }) -end - -if options and options.dev_detect then - Install("dev-detect", { priority = 40 }) -end - -if options and options.pakon then - Install("pakon", { priority = 40 }) -end - -_END_FEATURE_GUARD_ diff --git a/lists/pkglists/netboot.lua.m4 b/lists/pkglists/netboot.lua.m4 deleted file mode 100644 index aa1fc4ff0..000000000 --- a/lists/pkglists/netboot.lua.m4 +++ /dev/null @@ -1,11 +0,0 @@ -include(utils.m4)dnl -_FEATURE_GUARD_ - -Install("turris-netboot-tools", { priority = 40 }) - -Install("foris-subordinates-plugin", { priority = 40 }) - -Install("reforis-remote-access-plugin", { priority = 40 }) -Install("reforis-remote-devices-plugin", "reforis-remote-wifi-settings-plugin", { priority = 40 }) - -_END_FEATURE_GUARD_ diff --git a/lists/pkglists/netdata.lua.m4 b/lists/pkglists/netdata.lua.m4 deleted file mode 100644 index 77b216756..000000000 --- a/lists/pkglists/netdata.lua.m4 +++ /dev/null @@ -1,6 +0,0 @@ -include(utils.m4)dnl -_FEATURE_GUARD_ - -Install("netdata", { priority = 40 }) - -_END_FEATURE_GUARD_ diff --git a/lists/pkglists/netmetr.lua b/lists/pkglists/netmetr.lua deleted file mode 100644 index 66f362969..000000000 --- a/lists/pkglists/netmetr.lua +++ /dev/null @@ -1 +0,0 @@ --- This list is obsoleted and superseded by option in net_monitoring list. diff --git a/lists/pkglists/nextcloud.lua.m4 b/lists/pkglists/nextcloud.lua.m4 deleted file mode 100644 index 6c151f328..000000000 --- a/lists/pkglists/nextcloud.lua.m4 +++ /dev/null @@ -1,6 +0,0 @@ -include(utils.m4)dnl -_FEATURE_GUARD_ - -Install("nextcloud", "nextcloud-install", { priority = 40 }) - -_END_FEATURE_GUARD_ diff --git a/lists/pkglists/openvpn.lua.m4 b/lists/pkglists/openvpn.lua.m4 deleted file mode 100644 index f1be3772c..000000000 --- a/lists/pkglists/openvpn.lua.m4 +++ /dev/null @@ -1,10 +0,0 @@ -include(utils.m4)dnl -_FEATURE_GUARD_ - --- OpenVPN itself -Install("openvpn-crypto", { priority = 40 }) - --- CA management for OpenVPN server -Install("turris-cagen", { priority = 40 }) - -_END_FEATURE_GUARD_ diff --git a/lists/pkglists/pakon.lua b/lists/pkglists/pakon.lua deleted file mode 100644 index 66f362969..000000000 --- a/lists/pkglists/pakon.lua +++ /dev/null @@ -1 +0,0 @@ --- This list is obsoleted and superseded by option in net_monitoring list. diff --git a/lists/pkglists/printserver.lua b/lists/pkglists/printserver.lua deleted file mode 100644 index fcc8fa463..000000000 --- a/lists/pkglists/printserver.lua +++ /dev/null @@ -1 +0,0 @@ --- This list is obsoleted and superseded by option in luci_controls list. diff --git a/lists/pkglists/reforis.lua b/lists/pkglists/reforis.lua deleted file mode 100644 index a7f241af2..000000000 --- a/lists/pkglists/reforis.lua +++ /dev/null @@ -1 +0,0 @@ --- This list is obsoleted and its content was moved to base installation diff --git a/lists/pkglists/tor.lua.m4 b/lists/pkglists/tor.lua.m4 deleted file mode 100644 index a1527f80a..000000000 --- a/lists/pkglists/tor.lua.m4 +++ /dev/null @@ -1,6 +0,0 @@ -include(utils.m4)dnl -_FEATURE_GUARD_ - -Install("tor", "tor-hs", { priority = 40 }) - -_END_FEATURE_GUARD_ diff --git a/lists/repository.lua.m4 b/lists/repository.lua.m4 deleted file mode 100644 index 29213f4f5..000000000 --- a/lists/repository.lua.m4 +++ /dev/null @@ -1,27 +0,0 @@ -include(utils.m4)dnl - -local feeds = { "base", "core" esyscmd(`awk "/^src-git/{printf \", \\\"%s\\\"\", \$'`2}" '_FEEDS_)} - -local rroot -if features.relative_uri then - rroot = ".." -else - rroot = (repo_base_uri or "https://repo.turris.cz/hbs") .. "/" .. board -end - -for _, feed in ipairs(feeds) do - -- Standard Turris OS package repository - Repository(feed, rroot .. "/packages/" .. feed) -dnl Note: to test just package lists you can replace preceding line with: -dnl Repository(feed, "https://repo.turris.cz/hbs/" .. board .. "/packages/" .. feed, { -dnl pubkey = { -dnl -- Turris release key -dnl "data:base64,dW50cnVzdGVkIGNvbW1lbnQ6IFR1cnJpcyByZWxlYXNlIGtleSBnZW4gMQpSV1Rjc2c1VFhHTGRXOWdObEdITi9vZmRzTTBLQWZRSVJCbzVPVlpJWWxWVGZ5STZGR1ZFT0svZQo=", -dnl -- Turris development key -dnl "data:base64,dW50cnVzdGVkIGNvbW1lbnQ6IFR1cnJpcyBPUyBkZXZlbCBrZXkKUldTMEZBMU51bjdKRHQwTDhTalJzRFJKR0R2VUNkRGRmczIxZmVpVytxcEdITk1oVlo5MzBoa3kK", -dnl } -dnl }) -end - -INFO("Target Turris OS: _TURRIS_OS_VERSION_") -DBG("Current Turris OS: " .. tostring(os_release.VERSION)) diff --git a/lists/repository.m4 b/lists/repository.m4 deleted file mode 100644 index 1f57a6fd8..000000000 --- a/lists/repository.m4 +++ /dev/null @@ -1,7 +0,0 @@ -dnl This is common repository include -dnl It guards it against multiple inclusion in subscripts. -if not turris_repo_included then - list_script("repository.lua") - turris_repo_included = true - Export('turris_repo_included') -end diff --git a/lists/terminal-apps.lua.m4 b/lists/terminal-apps.lua.m4 deleted file mode 100644 index 6135264b1..000000000 --- a/lists/terminal-apps.lua.m4 +++ /dev/null @@ -1,25 +0,0 @@ -include(utils.m4)dnl -_FEATURE_GUARD_ - --- Turris message of the day -Install("turris-motd", { priority = 40 }) - --- Core shell and utils -Install("bash", "terminfo", "vim-full", { priority = 40 }) -Install("coreutils", "diffutils", { priority = 40 }) -Install("openssl-util", { priority = 40 }) - --- Process monitoring -Install("htop", "psmisc", "procps-ng-top", "procps-ng-ps", { priority = 40 }) - --- Network -Install("curl", "wget-ssl", "tcpdump", "rsync", { priority = 40 }) - --- Filesystems -Install("partx-utils", "blkid", "lsblk", { priority = 40 }) - --- SSH Clients additions -Install("openssh-client", "openssh-client-utils", "openssh-sftp-client", { priority = 40 }) - - -_END_FEATURE_GUARD_ diff --git a/lists/utils.m4 b/lists/utils.m4 deleted file mode 100644 index 24af48183..000000000 --- a/lists/utils.m4 +++ /dev/null @@ -1,64 +0,0 @@ -divert(-1) - -# Transform lines in file to comma separated arguments -# Usage: file2args(`FILE') -define(`file2args',`syscmd(test -f _INCLUDE_`'$1)ifelse(sysval,0,,`errprint(File $1 is missing!)m4exit(`1')')dnl -esyscmd(`sed "/^#/d;s/\s//g;/^\s*\$/d" '_INCLUDE_`$1 | paste -sd "," | tr -d "\n"')') - -# Expand second argument for all arguments after second one defined as macro -# with name of first argument. -# Usage: foreach(X,TEXT(X),a,b) -define(`foreach',`ifelse(eval($#>2),1,`pushdef(`$1',`$3')$2`'popdef(`$1')`'ifelse(eval($#>3),1,`$0(`$1',`$2',shift(shift(shift($@))))')')') - -# Expand second argument for all arguments after third one defined as macro with -# name of fist argument. Every argument is then joined by third argument. -# Usage: foreach_join(X,TEXT(X),Y,a,b) -define(`foreach_join',`ifelse(eval($#>3),1,`pushdef(`$1',`$4')$2`'ifelse(eval($#>4),1,`$3')`'popdef(`$1')`'ifelse(eval($#>4),1,`$0(`$1',`$2',`$3',shift(shift(shift(shift($@)))))')')') - -# Generate Install command with given PKGBASE and PKGPARTs joined: PKGBASE-PKGPART -# Usage: forInstall(PKGBASE, PKGPARTa, PKGPARTb) -define(`forInstall',`Install(foreach_join(PKGPART,`"$1-PKGPART"',`, ',shift($@)), { priority = 40 })') -define(`forInstallCritical',`Install(foreach_join(PKGPART,`"$1-PKGPART"',`, ',shift($@)), { critical = true })') - -# Feature guard -# Some packages might not be installable without some features. Skipping every -# additional packages ensures that at least updater is updated. -define(`_FEATURE_GUARD_', `if features.request_condition then -- Advanced dependencies guard') -define(`_END_FEATURE_GUARD_', `end') -# This is empty for now because minimal version check is enought for now but is left there for future reuse. - -divert(0)dnl -dnl We have minimal updater-ng version requirements -if not version_match or not self_version or version_match(self_version, "<60.0.1") then - DIE("Minimal required version of Updater-ng for Turris repository is 60.0.1!") -end - -if not board then - local model = model or os_release["OPENWRT_DEVICE_PRODUCT"] or os_release["LEDE_DEVICE_PRODUCT"] - if model:match("[Mm]ox") then - board = "mox" - elseif model:match("[Oo]mnia") then - board = "omnia" - elseif model:match("^[Tt]urris$") or model:match("[Tt]urris ?1%.?x") then - board = "turris1x" - else - DIE("Unsupported Turris model: " .. tostring(model)) - end -end - --- Script simplifying lists inclusion when older version of updater is used -function list_script(list) - if features["relative_uri"] then - Script(list) - else - Script((repo_base_uri or "https://repo.turris.cz/hbs") .. "/" .. board .. "/lists/" .. list) - end -end - --- Our own version of for_l10n to override priority -function for_l10n(fragment) - for _, lang in pairs(l10n or {}) do - Install(fragment .. lang, {optional = true, priority = 10}) - end -end ----------------------------------------------------------------------------------- diff --git a/lists/webapps.lua.m4 b/lists/webapps.lua.m4 deleted file mode 100644 index 88849fdb6..000000000 --- a/lists/webapps.lua.m4 +++ /dev/null @@ -1,21 +0,0 @@ -include(utils.m4)dnl -_FEATURE_GUARD_ - -local webapps = { - ["luci"] = "luci-base", - ["netdata"] = "netdata", - ["nextcloud"] = "nextcloud", - ["transmission"] = "transmission-web", - ["tvheadend"] = "tvheadend", -} - -Install("turris-webapps", { priority = 40 }) - -for webapp, condition in pairs(webapps) do - Install("turris-webapps-" .. webapp, { - condition = condition or nil, - priority = 40 - }) -end - -_END_FEATURE_GUARD_ diff --git a/turris-build.conf.example b/turris-build.conf.example index 2acfa3fee..45c9e1410 100644 --- a/turris-build.conf.example +++ b/turris-build.conf.example @@ -19,7 +19,6 @@ set_target mox #FORCE=1 # Path to git mirror directory. -# Once set, you can init it with: `compile_fw mirror_feeds` #GIT_MIRROR="" # Path to downloads mirror directory. # Useful when multiple builds are being run on same machine or if you don't want t -- GitLab