Newer
Older
#!/bin/bash -e
# OpenWRT compilation script
# (C) 2018 CZ.NIC, z.s.p.o.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
declare -A available_commands
BUILD_DIR="$(pwd)"
SRC_DIR="$(dirname "$0")"
SRC_DIR="$(cd "$SRC_DIR"; pwd)"
CMD="${0}"
OPENWRT_URL="https://git.openwrt.org/openwrt/openwrt.git"
DEFAULT_STEPS=(
"prepare",
"build",
"store_hash",
"stats gen_junit",
)
if [ "$BUILD_DIR" == "$SRC_DIR" ]; then
mkdir -p build
cd build
BUILD_DIR="$(pwd)"
fi
export TMPDIR="${BUILD_DIR}/tmp"
mkdir -p "$TMPDIR"
[ \! -f "${SRC_DIR}"/turris-build.conf ] || . "${SRC_DIR}"/turris-build.conf
[ \! -f ~/.turris-build ] || . ~/.turris-build
[ \! -f ./turris-build.conf ] || . ./turris-build.conf
MIRROR_UPDATED=""
_git() {
git \
-c "commit.gpgsign=false" \
-c "user.email=auto-build@example.com" \
-c "user.name=Build system automate" \
"$@"
_report() {
echo -e '\033[0;34m'"$1"'\033[0m' >&2
}
_die() {
echo -e '\033[0;31m'"$1"'\033[0m' >&2
exit 1
}
_enable_debug() {
set -x
DEBUG="yes"
PERL_DEBUG="-d:Trace"
MAKE_DEBUG="-n"
SH_DEBUG="-x"
OWRT_DEBUG="V=s"
}
_get_version() {
# If first grep fails - version is not first non-empty, add 999 to the end
# Be aware that Condition in if also outputs the results
if ! grep . "${SRC_DIR}"/NEWS | head -n 1 | grep '^[0-9.]\+$'; then
echo "$(grep '^[0-9.]\+$' "${SRC_DIR}"/NEWS | head -n 1).999"
fi
}
available_commands+=( ["gen_version"]="Generates Turris version package from NEWS" )
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
gen_version() {
_report "Generating turris-version package"
# Find the current version
VERSION="$(_get_version)"
# If top of the NEWS is not a version number, find previous one and add 99
# Also try to figure out where are the actual NEWS for this version
if expr "$VERSION" : '999$' > /dev/null; then
START=1
END="$(grep -n '^[0-9.]\+$' "${SRC_DIR}"/NEWS | head -n 1 | sed 's|:.*||')"
END="$(expr $END - 1)"
else
START="$(grep -n '^[0-9.]\+$' "${SRC_DIR}"/NEWS | head -n 1 | sed 's|:.*||')"
START="$(expr $START + 1)"
END="$(grep -n '^[0-9.]\+$' "${SRC_DIR}"/NEWS | sed -n '2 s|:.*||p')"
[ -n "$END" ] || END="\$"
fi
# Generate package
mkdir -p "${BUILD_DIR}"/package/system/turris-version
cat > "${BUILD_DIR}"/package/system/turris-version/Makefile << EOF
#
## Copyright (C) $(date +%Y) CZ.NIC z.s.p.o. (http://www.nic.cz/)
#
## This is free software, licensed under the GNU General Public License v2.
# See /LICENSE for more information.
# #
#
include \$(TOPDIR)/rules.mk
PKG_NAME:=turris-version
PKG_VERSION:=$VERSION
PKG_MAINTAINER:=CZ.NIC <packaging@nic.cz>
include \$(INCLUDE_DIR)/package.mk
define Package/turris-version
TITLE:=turris-version
endef
define Build/Prepare
endef
define Build/Compile
endef
define Package/turris-version/postinst
#!/bin/sh
# Danger: spaces are not ordinary spaces, but special unicode ones
[ -n "\$\$IPKG_INSTROOT" ] || {
create_notification -s news "$(sed -n "$START,$END p" "${SRC_DIR}"/NEWS | grep . | sed 's|^[[:blank:]]*\*[[:blank:]]*| • |')"
}
endef
define Package/turris-version/install
\$(INSTALL_DIR) \$(1)/etc
echo \$(PKG_VERSION) > \$(1)/etc/turris-version
endef
\$(eval \$(call BuildPackage,turris-version))
EOF
pushd "${BUILD_DIR}"
git add "${BUILD_DIR}"/package/system/turris-version
_git commit -m 'turris-version: Create a package with release notes'
popd
}
available_commands+=( ["conflicts_help"]="Show packages that we are overriding in Turris OS packages" )
conflicts() {
_report "Showing conflicting packages"
echo "Following packages are being overridden by Turris packages:"
echo
mkdir -p tmp
find feeds/turrispackages -name Makefile | sed 's|.*/\([^/]*\)/Makefile|\1|' | sort > tmp/turris-packages.list
find feeds/ -name Makefile | grep -v '^feeds/turrispackages' | sed 's|.*/\([^/]*\)/Makefile|\1|' | sort > tmp/other-packages.list
comm -12 tmp/turris-packages.list tmp/other-packages.list | sed 's|^| * |'
echo
rm -f tmp/turris-packages.list tmp/other-packages.list
}
available_commands+=( ["stats"]=" Prints statistics regarding number of built packages" )
mkdir -p logs
_report "Reporting statistics"
{
echo " * $(find bin/packages -name '*.ipk' | wc -l) binary packages built"
echo " * $(cat logs/package/error.txt | wc -l)/$(find logs/package/ -name 'compile.txt' | wc -l) source packages failed"
if [ -n "$BUILD_START" ] && [ -n "$BUILD_END" ]; then
BUILD_TIME="$(expr $BUILD_END - $BUILD_START)"
echo " * build time $(expr $BUILD_TIME / 3600):$(expr \( $BUILD_TIME % 3600 \) / 60):$(expr $BUILD_TIME % 60)"
fi
available_commands+=( ["configure"]="Recreates configuration for target boards" )
configure() {
_report "Creating default configuration"
[ -n "${TARGET_BOARD}" ] || _die "No board selected!"
cat "${SRC_DIR}"/configs/common/* "${SRC_DIR}"/configs/${TARGET_BOARD}/* > ./.config
echo "CONFIG_DEVEL=y" >> .config
echo "CONFIG_DOWNLOAD_FOLDER=$DL_MIRROR" >> .config
if [ -n "${PUBLISH_BRANCH}" ]; then
echo "CONFIG_VERSION_REPO=\"https://repo.turris.cz/${TARGET_BOARD}-${PUBLISH_BRANCH}\"" >> .config
else
echo "CONFIG_VERSION_REPO=\"https://repo.turris.cz/${TARGET_BOARD}\"" >> .config
fi
echo "CONFIG_VERSION_NUMBER=\"$(_get_version)\"" >> .config
if [ "$EVERYTHING" = yes ]; then
echo "CONFIG_ALL_KMODS=y" >> .config
echo "CONFIG_ALL=y" >> .config
echo "CONFIG_IB=y" >> .config
echo "CONFIG_IB_STANDALONE=y" >> .config
echo "CONFIG_SDK=y" >> .config
else
echo "CONFIG_ALL=n" >> .config
echo "CONFIG_IB=n" >> .config
echo "CONFIG_IB_STANDALONE=n" >> .config
echo "CONFIG_SDK=n" >> .config
fi
OPENWRT_ARCH="$(sed -n 's|^CONFIG_TARGET_BOARD="\([^"]*\)"|\1|p' .config)"
EXTRA_KERNEL="$(cat "${SRC_DIR}"/configs/common/* "${SRC_DIR}/configs/${TARGET_BOARD}"/* | sed -n 's|^[[:blank:]]*CONFIG_KERNEL|CONFIG|p')"
TO_COMMIT=""
for config in target/linux/"${OPENWRT_ARCH}"/config-*; do
echo "${EXTRA_KERNEL}" >> "$config"
TO_COMMIT="$TO_COMMIT $config"
done
_git commit -m 'kernel: Add customized kernel options' $TO_COMMIT
available_commands+=( ["update_mirror"]="Updates all local mirrors" )
update_mirror() {
[ -n "$GIT_MIRROR" ] || return 0
[ -z "$MIRROR_UPDATED" ] || return 0
_report "Updating local mirrors"
pushd "$GIT_MIRROR"
if [ \! -d openwrt ]; then
git clone --mirror "$OPENWRT_URL" openwrt
fi
OPENWRT_URL="$GIT_MIRROR/openwrt"
for mirror in ./*; do
cd "$GIT_MIRROR"
done
popd
MIRROR_UPDATED="yes"
}
_report "Checking out clean OpenWRT repository"
update_mirror
rm -rf .git
git init
git remote add origin "$OPENWRT_URL"
}
_checkout_clean() {
BUILD_SETTINGS="$(cat turris-build.conf 2> /dev/null || true)"
[ -z "$BUILD_SETTINGS" ] || echo "$BUILD_SETTINGS" > turris-build.conf
echo "/turris-build.conf" >> ./.gitignore
echo "/version" >> ./.gitignore
_git commit -m 'gitignore: Ignore some more files' ./.gitignore
rm -rf dl && ln -s "$DL_MIRROR" dl
fi
}
available_commands+=( ["checkout"]="Start everything from scratch - all changes deleted and fresh copy of OpenWRT gets checked out" )
checkout() {
_report "Starting out fresh!"
_checkout_init
HASH="$(echo "$OPENWRT_BRANCH" | sed -n 's|^#||p')"
if [ -n "$HASH" ]; then
git fetch origin
git checkout -f $HASH
else
git fetch $DEPTH origin "$OPENWRT_BRANCH"
git checkout -f "origin/$OPENWRT_BRANCH"
fi
_checkout_clean
cp "$SRC_DIR"/feeds.conf .
}
available_commands+=( ["repo_checkout"]="Start everything from scratch relative to repository on repo.turris.cz - all changes deleted and copy of OpenWRT gets checked out with same hashes as repo.turris.cz" )
repo_checkout() {
_report "Starting out $TARGET_BOARD-$PUBLISH_BRANCH!"
curl "https://repo.turris.cz/$TARGET_BOARD-$PUBLISH_BRANCH/git-hash" | tail -n +2 | grep -v '^$' | sed 's/^ \* //;s/: /:/' > repo-git-hashes
_checkout_init
git fetch
git checkout -f "$(awk -F : '/^openwrt:/{print $2}' repo-git-hashes)"
GIT_HASHES="$(cat repo-git-hashes)"
_checkout_clean
echo "$GIT_HASHES" > repo-git-hashes
rm -f feeds.conf
while read LINE; do
local HASH="$(awk -F : "/^feeds\/$(echo "$LINE" | awk '{print $2}')\:/{print \$2}" repo-git-hashes)"
if [ -z "$HASH" ]; then
echo "$LINE" >> feeds.conf
else
echo "$LINE" | sed "s/\^.*$//;s/$/\^$HASH/" >> feeds.conf
fi
done < "$SRC_DIR/feeds.conf"
}
available_commands+=( ["patch_feeds"]="Apply patches to the feeds" )
patch_feeds() {
_report "Patching feeds"
pushd "$SRC_DIR"/patches
for feed in *; do
[ -d "$BUILD_DIR"/feeds/$feed ] || continue
cd "$SRC_DIR"/patches/$feed
for patch in */*.patch; do
[ -f "$patch" ] || continue
cd "$BUILD_DIR"/feeds/$feed
_git am --reject "$SRC_DIR"/patches/$feed/$patch
available_commands+=( ["clean_ccache"]="Clean persistent ccache paths" )
clean_ccache() {
if [ "$(which ccache)" ]; then
_report "Cleaning ccache"
[ -z "$CCACHE_HOST_DIR" ] || CCACHE_DIR="$CCACHE_HOST_DIR" ccache -C
[ -z "$CCACHE_TARGET_DIR" ] || CCACHE_DIR="$CCACHE_TARGET_DIR/$TARGET_ARCH" ccache -C
else
_report "Not cleaning ccache as you don't have ccache installed"
fi
}
available_commands+=( ["set_ccache"]="Set persistent ccache paths" )
set_ccache() {
[ -z "$CCACHE_SET" ] || return 0
_report "Setting ccache paths"
CCACHE_SET=y
[ -z "$CCACHE_HOST_DIR" ] || sed -i 's|$(STAGING_DIR_HOST)/ccache|'"$CCACHE_HOST_DIR|" include/host-build.mk
[ -z "$CCACHE_TARGET_DIR" ] || [ -z "$TARGET_ARCH" ] || sed -i 's|$(STAGING_DIR)/ccache|'"$CCACHE_TARGET_DIR/$TARGET_ARCH|" include/package.mk
[ -z "$(git diff include/host-build.mk include/package.mk)" ] || _git commit -m "include: ccache settings" include/host-build.mk include/package.mk
available_commands+=( ["set_local_feeds"]="Change feed URL to their mirror counterparts" )
set_local_feeds() {
[ -n "$GIT_MIRROR" ] || return 0
_report "Setting feeds to their local counterparts"
while read vcs name url rest; do
if [ "$vcs" = src-git ] && [ -d "$GIT_MIRROR"/$name ]; then
feed_url="$(echo "$url" | sed 's|[[:blank:]^].*||')"
sed -i "s|$feed_url|file://$GIT_MIRROR/$name|" feeds.conf
fi
done < feeds.conf
}
available_commands+=( ["mirror_feeds"]="Creates initial mirrors of all configured feeds" )
mirror_feeds() {
[ -n "$GIT_MIRROR" ] || return 0
pushd "$GIT_MIRROR"
cat "$SRC_DIR"/feeds.conf "$BUILD_DIR"/feeds.conf 2> /dev/null | while read vcs name url rest; do
if [ "$vcs" = src-git ] && [ \! -d "$GIT_MIRROR"/$name ]; then
feed_url="$(echo "$url" | sed 's|[[:blank:]^].*||')"
git clone --mirror "$feed_url" "$name"
fi
done
popd
update_mirror
}
available_commands+=( ["patch_openwrt"]="Patch the main OpenWRT repository" )
patch_openwrt() {
_report "Patching OpenWRT repository"
pushd "$SRC_DIR"/patches/openwrt
for patch in */*.patch; do
cd "$BUILD_DIR"
_git am --reject "$SRC_DIR"/patches/openwrt/$patch
if [ -n "$(ls -1 "$SRC_DIR"/src/)" ]; then
cp -r "$SRC_DIR"/src/* .
find "$SRC_DIR"/src | sed 's|^'"$SRC_DIR"'/src/|git add |' | sh
_git commit -m 'Add additional files'
_git_remote_hash() {
pushd "$1" > /dev/null
br="$(LANG=C git status --long | sed -n 's|HEAD detached from \(.*\)|\1|p')"
[ -n "$br" ] || br="$(LANG=C git status --long | sed -n "s|Your branch is up to date with '\\([^']*\\)'.*|\1|p")"
[ -n "$br" ] || br="$(LANG=C git status --long | sed -n "s|Your branch is ahead of '\\([^']*\\)'.*|\1|p")"
[ -n "$br" ] || br=HEAD
git log -n1 --pretty=%H "$br"
popd > /dev/null
}
available_commands+=( ["store_hash"]="Stores hashes of current build" )
store_hash() {
_report "Storing hashes"
mkdir -p bin
{
echo "Project was build from following repositories:"
echo
echo " * turris-build: $(_git_remote_hash "${SRC_DIR}")"
echo " * openwrt: $(_git_remote_hash "${BUILD_DIR}")"
for feed in feeds/*; do
[ -d "$feed/.git" ] || continue
echo " * $feed: $(_git_remote_hash "${feed}")"
done
echo
} | tee bin/git-hash
}
available_commands+=( ["get_feeds"]="Recreate configured feeds" )
update_mirror
set_local_feeds
perl $PERL_DEBUG ./scripts/feeds clean -a
perl $PERL_DEBUG ./scripts/feeds update -a
perl $PERL_DEBUG ./scripts/feeds install -a
DISABLED_PACKAGES="$(cat "${SRC_DIR}"/disabled_packages/common "${SRC_DIR}"/disabled_packages/${TARGET_BOARD} 2> /dev/null)"
DISABLED_PACKAGES="$(echo "$DISABLED_PACKAGES" | sed 's|#.*||' | grep .)"
set -e
[ -z "$DISABLED_PACKAGES" ] || perl $PERL_DEBUG ./scripts/feeds uninstall $DISABLED_PACKAGES
available_commands+=( ["repatch_feeds"]="Cleanup feeds, update them and patch them (Implies: get_feeds patch_feeds)" )
repatch_feeds() {
get_feeds
patch_feeds
}
available_commands+=( ["prefetch"]="Runs make download" )
make $MAKE_DEBUG $BUILD_ARGS $OWRT_DEBUG download
available_commands+=( ["gen_junit"]="Generates junit output from build logs" )
available_commands+=( ["build"]=" Builds everything" )
build() {
_report "Starting real build"
set_ccache
make $MAKE_DEBUG IS_TTY=1 BUILD_LOG=1 $BUILD_ARGS $OWRT_DEBUG
JOBS="$(echo " $BUILD_ARGS " | sed -n 's|.*-j\([0-9]\+\)|\1\n|p' | head -n 1)"
[ -n "$JOBS" ] || JOBS=0
SUCCESS=""
while [ "$JOBS" -gt 0 ] && [ -z "$SUCCESS" ]; do
COUNTDOWN="$FORCE"
while [ $COUNTDOWN -gt 0 ] && [ -z "$SUCCESS" ]; do
if make $MAKE_DEBUG IGNORE_ERRORS=m IS_TTY=1 BUILD_LOG=1 $BUILD_ARGS $OWRT_DEBUG -j${JOBS}; then
COUNTDOWN="$(expr $COUNTDOWN - 1 || true)"
_report "Build job with -j$JOBS failed (try $(expr $FORCE - $COUNTDOWN || true)/$FORCE)..."
_report "Build job with -j$JOBS failed, decreasing parallelism..."
if [ "$JOBS" -le 1 ] && [ -z "$SUCCESS" ]; then
make $MAKE_DEBUG IGNORE_ERRORS=m IS_TTY=1 BUILD_LOG=1 $BUILD_ARGS $OWRT_DEBUG -j1 V=s
available_commands+=( ["clean"]=" Clean current build directory" )
clean() {
_report "Cleaning up current build directory"
rm -rf ./build_dir ./tmp ./staging_dir ./logs ./bin
mkdir -p tmp
}
available_commands+=( ["prepare"]=" Prepare build but don't build it (Implies: checkout clean patch_openwrt repatch_feeds gen_version configure)" )
prepare() {
checkout
clean
patch_openwrt
repatch_feeds
gen_version
configure
}
available_commands+=( ["prepare_tools"]="Prepare build and build tools" )
make $MAKE_DEBUG IS_TTY=1 BUILD_LOG=1 $BUILD_ARGS $OWRT_DEBUG tools/compile toolchain/compile target/compile
available_commands+=( ["repo_prepare"]="Same as prepare but instead of checkout uses repo_checkout" )
repo_prepare() {
repo_checkout
clean
patch_openwrt
repatch_feeds
gen_version
configure
}
available_commands+=( ["autopkg"]=" Deploy autopkg scripts that can be used for rolling software development" )
_report "Deploying autopkg scripts"
cp "$SRC_DIR"/helpers/autopkg/* "$BUILD_DIR"/include/
git add include/autopkg-*.mk
_git commit -m 'autopkg: Add autopkg scripts'
# Sets various variables to match the specified target, just a helper
_set_target() {
_report "Setting target as $1"
case "$1" in
omnia)
TARGET_BOARD=omnia
TARGET_ARCH=armv7l
;;
turris)
TARGET_BOARD=turris
TARGET_ARCH=ppcspe
;;
mox)
TARGET_BOARD=mox
TARGET_ARCH=aarch64
;;
*)
echo "Invalid target board!!! Use -t [turris|omnia|mox]!!!"
exit 1
;;
esac
}
print_help() {
echo "Usage: $CMD [option].. [command].."
echo
echo "Available options are:"
echo " -x Enable debug mode"
echo " -e Build everything, not just minimal set"
echo " -f[num] Try hard to get stuff to compile (optional number specifies number of tries)"
echo " -d Do not use shallow checkouts"
echo " -t board Set target board to _board_"
echo " -a \"arg1 arg2\" Set build arguments - passed directly to make"
echo " -b branch Branch to checkout"
echo " -p branch Name of the branch that is being build"
echo " -l Do not update local git mirrors"
echo " -s key Sign packges with given key"
echo
echo "Available commands are:"
for cmd in "${!available_commands[@]}"; do
echo " $cmd ${available_commands[$cmd]}"
echo "Default commands are: ${DEFAULT_STEPS[*]}"
steps=()
# Parse arguments
while [ $# -gt 0 ]; do
case "$1" in
-h|--help)
print_help
exit 0
;;
-x)
_enable_debug
-f*)
FORCE="${1#-f}"
_set_target "$1"
BUILD_ARGS="$BUILD_ARGS $1"
PUBLISH_BRANCH="$1"
OPENWRT_BRANCH="$1"
MIRROR_UPDATED="override"
OPENWRT_URL="$GIT_MIRROR/openwrt"
;;
BUILD_ARGS="$BUILD_ARGS BUILD_KEY=$1"
;;
*)
[ -n "${available_commands[$1]}" ] || \
_die "There is no such option or command: $1"
steps+=( "$1" )
[ -z "$BOARD" ] || _set_target "$BOARD"
[ ${#steps[@]} -gt 0 ] || steps=( "${DEFAULT_STEPS[@]}" )
for step in "${steps[@]}"; do
"$step"