Newer
Older
#!/bin/bash -e
# OpenWRT compilation script
# (C) 2018 CZ.NIC, z.s.p.o.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
BUILD_DIR="$(pwd)"
SRC_DIR="$(dirname "$0")"
SRC_DIR="$(cd "$SRC_DIR"; pwd)"
CMD="${0}"
OPENWRT_URL="https://git.openwrt.org/openwrt/openwrt.git"
DEFAULT_STEPS="prepare build gen_lists store_hash sign stats gen_junit"
if [ "$BUILD_DIR" == "$SRC_DIR" ]; then
mkdir -p build
cd build
BUILD_DIR="$(pwd)"
fi
export TMPDIR="${BUILD_DIR}/tmp"
mkdir -p "$TMPDIR"
[ \! -f "${SRC_DIR}"/turris-build.conf ] || . "${SRC_DIR}"/turris-build.conf
[ \! -f ~/.turris-build ] || . ~/.turris-build
[ \! -f ./turris-build.conf ] || . ./turris-build.conf
MIRROR_UPDATED=""
trap _cleanup 0 2 3 15
_git_config() {
if [ "$1" ]; then
pushd "$1" > /dev/null
EXTRA_ARGS=""
else
pushd "$BUILD_DIR" > /dev/null
EXTRA_ARGS="$(echo feeds/*/.git)"
fi
for i in ./.git $EXTRA_ARGS; do
pushd "$(dirname "$i")" > /dev/null
git config --local commit.gpgsign false
git config --local user.email "auto-build@example.com"
git config --local user.name "Build system automate"
popd > /dev/null
done
popd > /dev/null
}
_git_unconfig() {
pushd "$BUILD_DIR" > /dev/null
for i in ./.git feeds/*/.git; do
pushd "$(dirname "$i")" > /dev/null
git config --local --unset commit.gpgsign
git config --local --unset user.email
git config --local --unset user.name
popd > /dev/null
done
popd > /dev/null
}
_report() {
echo -e '\033[0;34m'"$1"'\033[0m' >&2
}
_die() {
echo -e '\033[0;31m'"$1"'\033[0m' >&2
exit 1
}
_cleanup() {
_report "Cleaning up..."
_git_unconfig
}
_get_version() {
# If first grep fails - version is not first non-empty, add 999 to the end
# Be aware that Condition in if also outputs the results
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
if ! grep . "${SRC_DIR}"/NEWS | head -n 1 | grep '^[0-9.]\+$'; then
echo "$(grep '^[0-9.]\+$' "${SRC_DIR}"/NEWS | head -n 1).999"
fi
}
gen_version_help="Generates Turris version package from NEWS"
gen_version() {
_report "Generating turris-version package"
# Find the current version
VERSION="$(_get_version)"
# If top of the NEWS is not a version number, find previous one and add 99
# Also try to figure out where are the actual NEWS for this version
if expr "$VERSION" : '999$' > /dev/null; then
START=1
END="$(grep -n '^[0-9.]\+$' "${SRC_DIR}"/NEWS | head -n 1 | sed 's|:.*||')"
END="$(expr $END - 1)"
else
START="$(grep -n '^[0-9.]\+$' "${SRC_DIR}"/NEWS | head -n 1 | sed 's|:.*||')"
START="$(expr $START + 1)"
END="$(grep -n '^[0-9.]\+$' "${SRC_DIR}"/NEWS | sed -n '2 s|:.*||p')"
[ -n "$END" ] || END="\$"
fi
# Generate package
mkdir -p "${BUILD_DIR}"/package/system/turris-version
cat > "${BUILD_DIR}"/package/system/turris-version/Makefile << EOF
#
## Copyright (C) $(date +%Y) CZ.NIC z.s.p.o. (http://www.nic.cz/)
#
## This is free software, licensed under the GNU General Public License v2.
# See /LICENSE for more information.
# #
#
include \$(TOPDIR)/rules.mk
PKG_NAME:=turris-version
PKG_VERSION:=$VERSION
PKG_MAINTAINER:=CZ.NIC <packaging@nic.cz>
include \$(INCLUDE_DIR)/package.mk
define Package/turris-version
TITLE:=turris-version
endef
define Build/Prepare
endef
define Build/Compile
endef
define Package/turris-version/postinst
#!/bin/sh
# Danger: spaces are not ordinary spaces, but special unicode ones
[ -n "\$\$IPKG_INSTROOT" ] || {
create_notification -s news "$(sed -n "$START,$END p" "${SRC_DIR}"/NEWS | grep . | sed 's|^[[:blank:]]*\*[[:blank:]]*| • |')"
}
endef
define Package/turris-version/install
\$(INSTALL_DIR) \$(1)/etc
echo \$(PKG_VERSION) > \$(1)/etc/turris-version
endef
\$(eval \$(call BuildPackage,turris-version))
EOF
pushd "${BUILD_DIR}"
git add "${BUILD_DIR}"/package/system/turris-version
git commit -m 'turris-version: Create a package with release notes'
popd
}
conflicts_help="Show packages that we are overriding in Turris OS packages"
conflicts() {
_report "Showing conflicting packages"
echo "Following packages are being overridden by Turris packages:"
echo
mkdir -p tmp
find feeds/turrispackages -name Makefile | sed 's|.*/\([^/]*\)/Makefile|\1|' | sort > tmp/turris-packages.list
find feeds/ -name Makefile | grep -v '^feeds/turrispackages' | sed 's|.*/\([^/]*\)/Makefile|\1|' | sort > tmp/other-packages.list
comm -12 tmp/turris-packages.list tmp/other-packages.list | sed 's|^| * |'
echo
rm -f tmp/turris-packages.list tmp/other-packages.list
}
stats_help=" Prints statistics regarding number of built packages"
mkdir -p logs
_report "Reporting statistics"
{
echo " * $(find bin/packages -name '*.ipk' | wc -l) binary packages built"
echo " * $(cat logs/package/error.txt | wc -l)/$(find logs/package/ -name 'compile.txt' | wc -l) source packages failed"
if [ -n "$BUILD_START" ] && [ -n "$BUILD_END" ]; then
BUILD_TIME="$(expr $BUILD_END - $BUILD_START)"
echo " * build time $(expr $BUILD_TIME / 3600):$(expr \( $BUILD_TIME % 3600 \) / 60):$(expr $BUILD_TIME % 60)"
fi
configure_help="Recreates configuration for target boards"
configure() {
_report "Creating default configuration"
[ -n "${TARGET_BOARD}" ] || _die "No board selected!"
cat "${SRC_DIR}"/configs/common/* "${SRC_DIR}"/configs/${TARGET_BOARD}/* > ./.config
echo "CONFIG_DEVEL=y" >> .config
echo "CONFIG_DOWNLOAD_FOLDER=$DL_MIRROR" >> .config
if [ -n "${PUBLISH_BRANCH}" ]; then
echo "CONFIG_VERSION_REPO=\"https://repo.turris.cz/${TARGET_BOARD}-${PUBLISH_BRANCH}\"" >> .config
else
echo "CONFIG_VERSION_REPO=\"https://repo.turris.cz/${TARGET_BOARD}\"" >> .config
fi
echo "CONFIG_VERSION_NUMBER=\"$(_get_version)\"" >> .config
if [ "$EVERYTHING" = yes ]; then
echo "CONFIG_ALL_KMODS=y" >> .config
echo "CONFIG_ALL=y" >> .config
echo "CONFIG_IB=y" >> .config
echo "CONFIG_IB_STANDALONE=y" >> .config
echo "CONFIG_SDK=y" >> .config
else
echo "CONFIG_ALL=n" >> .config
echo "CONFIG_IB=n" >> .config
echo "CONFIG_IB_STANDALONE=n" >> .config
echo "CONFIG_SDK=n" >> .config
fi
OPENWRT_ARCH="$(sed -n 's|^CONFIG_TARGET_BOARD="\([^"]*\)"|\1|p' .config)"
EXTRA_KERNEL="$(cat "${SRC_DIR}"/configs/common/* "${SRC_DIR}/configs/${TARGET_BOARD}"/* | sed -n 's|^[[:blank:]]*CONFIG_KERNEL|CONFIG|p')"
TO_COMMIT=""
for config in target/linux/"${OPENWRT_ARCH}"/config-*; do
echo "${EXTRA_KERNEL}" >> "$config"
TO_COMMIT="$TO_COMMIT $config"
done
git commit -m 'kernel: Add customized kernel options' $TO_COMMIT
}
update_mirror_help="Updates all local mirrors"
update_mirror() {
[ -n "$GIT_MIRROR" ] || return 0
[ -z "$MIRROR_UPDATED" ] || return 0
_report "Updating local mirrors"
pushd "$GIT_MIRROR"
if [ \! -d openwrt ]; then
git clone --mirror "$OPENWRT_URL" openwrt
fi
OPENWRT_URL="$GIT_MIRROR/openwrt"
for mirror in ./*; do
cd "$GIT_MIRROR"
done
popd
MIRROR_UPDATED="yes"
}
sign_help=" Signs generated lists and resigns packages"
[ -x staging_dir/host/bin/usign ] || make package/usign/host/compile
[ -x staging_dir/host/bin/usign ] || _die "Don't have usign, can't sign"
[ -n "$KEY" ] || KEY=key-build
[ -f "$KEY" ] || _die "No key available"
for i in bin/packages/lists/*.lua bin/packages/*/*/Packages bin/targets/*/*/packages/Packages; do
[ -f "$i" ] || continue
rm -f "$i".sig
staging_dir/host/bin/usign -S -m "$i" -s "$KEY"
done
}
_report "Checking out clean OpenWRT repository"
update_mirror
rm -rf .git
git init
git remote add origin "$OPENWRT_URL"
}
_checkout_clean() {
BUILD_SETTINGS="$(cat turris-build.conf 2> /dev/null || true)"
[ -z "$BUILD_SETTINGS" ] || echo "$BUILD_SETTINGS" > turris-build.conf
echo "/turris-build.conf" >> ./.gitignore
echo "/version" >> ./.gitignore
git commit -m 'gitignore: Ignore some more files' ./.gitignore
rm -rf dl && ln -s "$DL_MIRROR" dl
fi
}
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
checkout_help="Start everything from scratch - all changes deleted and fresh copy of OpenWRT gets checked out"
checkout() {
_report "Starting out fresh!"
_checkout_init
HASH="$(echo "$OPENWRT_BRANCH" | sed -n 's|^#||p')"
if [ -n "$HASH" ]; then
git fetch origin
git checkout -f $HASH
else
git fetch $DEPTH origin "$OPENWRT_BRANCH"
git checkout -f "origin/$OPENWRT_BRANCH"
fi
_checkout_clean
cp "$SRC_DIR"/feeds.conf .
}
repo_checkout_help="Start everything from scratch relative to repository on repo.turris.cz - all changes deleted and copy of OpenWRT gets checked out with same hashes as repo.turris.cz"
repo_checkout() {
_report "Starting out $TARGET_BOARD-$PUBLISH_BRANCH!"
curl "https://repo.turris.cz/$TARGET_BOARD-$PUBLISH_BRANCH/git-hash" | tail -n +2 | grep -v '^$' | sed 's/^ \* //;s/: /:/' > repo-git-hashes
_checkout_init
git fetch
git checkout -f "$(awk -F : '/^openwrt:/{print $2}' repo-git-hashes)"
GIT_HASHES="$(cat repo-git-hashes)"
_checkout_clean
echo "$GIT_HASHES" > repo-git-hashes
rm -f feeds.conf
while read LINE; do
local HASH="$(awk -F : "/^feeds\/$(echo "$LINE" | awk '{print $2}')\:/{print \$2}" repo-git-hashes)"
if [ -z "$HASH" ]; then
echo "$LINE" >> feeds.conf
else
echo "$LINE" | sed "s/\^.*$//;s/$/\^$HASH/" >> feeds.conf
fi
done < "$SRC_DIR/feeds.conf"
}
patch_feeds_help="Apply patches to the feeds"
patch_feeds() {
_report "Patching feeds"
pushd "$SRC_DIR"/patches
for feed in *; do
[ -d "$BUILD_DIR"/feeds/$feed ] || continue
cd "$SRC_DIR"/patches/$feed
for patch in */*.patch; do
[ -f "$patch" ] || continue
cd "$BUILD_DIR"/feeds/$feed
git am --reject "$SRC_DIR"/patches/$feed/$patch
cd "$SRC_DIR"/patches/$feed
clean_ccache_help="Set persistent ccache paths"
clean_ccache() {
if [ "$(which ccache)" ]; then
_report "Cleaning ccache"
[ -z "$CCACHE_HOST_DIR" ] || CCACHE_DIR="$CCACHE_HOST_DIR" ccache -C
[ -z "$CCACHE_TARGET_DIR" ] || CCACHE_DIR="$CCACHE_TARGET_DIR/$TARGET_ARCH" ccache -C
else
_report "Not cleaning ccache as you don't have ccache installed"
fi
}
set_ccache_help="Set persistent ccache paths"
set_ccache() {
[ -z "$CCACHE_SET" ] || return 0
_report "Setting ccache paths"
CCACHE_SET=y
[ -z "$CCACHE_HOST_DIR" ] || sed -i 's|$(STAGING_DIR_HOST)/ccache|'"$CCACHE_HOST_DIR|" include/host-build.mk
[ -z "$CCACHE_TARGET_DIR" ] || [ -z "$TARGET_ARCH" ] || sed -i 's|$(STAGING_DIR)/ccache|'"$CCACHE_TARGET_DIR/$TARGET_ARCH|" include/package.mk
[ -z "$(git diff include/host-build.mk include/package.mk)" ] || git commit -m "include: ccache settings" include/host-build.mk include/package.mk
}
set_local_feeds_help="Change feed URL to their mirror counterparts"
set_local_feeds() {
[ -n "$GIT_MIRROR" ] || return 0
_report "Setting feeds to their local counterparts"
while read vcs name url rest; do
if [ "$vcs" = src-git ] && [ -d "$GIT_MIRROR"/$name ]; then
feed_url="$(echo "$url" | sed 's|[[:blank:]^].*||')"
sed -i "s|$feed_url|file://$GIT_MIRROR/$name|" feeds.conf
fi
done < feeds.conf
}
mirror_feeds_help="Creates initial mirrors of all configured feeds"
mirror_feeds() {
[ -n "$GIT_MIRROR" ] || return 0
pushd "$GIT_MIRROR"
cat "$SRC_DIR"/feeds.conf "$BUILD_DIR"/feeds.conf 2> /dev/null | while read vcs name url rest; do
if [ "$vcs" = src-git ] && [ \! -d "$GIT_MIRROR"/$name ]; then
feed_url="$(echo "$url" | sed 's|[[:blank:]^].*||')"
git clone --mirror "$feed_url" "$name"
fi
done
popd
update_mirror
}
patch_openwrt_help="Patch the main OpenWRT repository"
patch_openwrt() {
_report "Patching OpenWRT repository"
pushd "$SRC_DIR"/patches/openwrt
for patch in */*.patch; do
cd "$BUILD_DIR"
if [ -n "$(ls -1 "$SRC_DIR"/src/)" ]; then
cp -r "$SRC_DIR"/src/* .
find "$SRC_DIR"/src | sed 's|^'"$SRC_DIR"'/src/|git add |' | sh
git commit -m 'Add additional files'
fi
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
_git_remote_hash() {
pushd "$1" > /dev/null
br="$(LANG=C git status --long | sed -n 's|HEAD detached from \(.*\)|\1|p')"
[ -n "$br" ] || br="$(LANG=C git status --long | sed -n "s|Your branch is up to date with '\\([^']*\\)'.*|\1|p")"
[ -n "$br" ] || br="$(LANG=C git status --long | sed -n "s|Your branch is ahead of '\\([^']*\\)'.*|\1|p")"
[ -n "$br" ] || br=HEAD
git log -n1 --pretty=%H "$br"
popd > /dev/null
}
store_hash_help="Stores hashes of current build"
store_hash() {
_report "Storing hashes"
mkdir -p bin
{
echo "Project was build from following repositories:"
echo
echo " * turris-build: $(_git_remote_hash "${SRC_DIR}")"
echo " * openwrt: $(_git_remote_hash "${BUILD_DIR}")"
for feed in feeds/*; do
[ -d "$feed/.git" ] || continue
echo " * $feed: $(_git_remote_hash "${feed}")"
done
echo
} | tee bin/git-hash
}
get_feeds_help="Recreate configured feeds"
get_feeds() {
update_mirror
set_local_feeds
perl $PERL_DEBUG ./scripts/feeds clean -a
perl $PERL_DEBUG ./scripts/feeds update -a
perl $PERL_DEBUG ./scripts/feeds install -a
DISABLED_PACKAGES="$(cat "${SRC_DIR}"/disabled_packages/common "${SRC_DIR}"/disabled_packages/${TARGET_BOARD} 2> /dev/null)"
DISABLED_PACKAGES="$(echo "$DISABLED_PACKAGES" | sed 's|#.*||' | grep .)"
set -e
[ -z "$DISABLED_PACKAGES" ] || perl $PERL_DEBUG ./scripts/feeds uninstall $DISABLED_PACKAGES
repatch_feeds_help="Cleanup feeds, update them and patch them (Implies: get_feeds patch_feeds)"
repatch_feeds() {
get_feeds
patch_feeds
}
prefetch_help="Runs make download"
prefetch() {
make $MAKE_DEBUG $BUILD_ARGS $OWRT_DEBUG download
}
gen_junit_help="Generates junit output from build logs"
gen_junit() {
gen_lists_help="Generate updater package lists"
gen_lists() {
_report "Generating package lists for updater"
[ -n "${TARGET_BOARD}" ] || _die "You need to specify target board!"
mkdir -p bin/packages/lists
"${SRC_DIR}"/helpers/generate_userlists.sh --branch "${PUBLISH_BRANCH}" --src "${SRC_DIR}/lists" bin/packages/lists
build_help=" Builds everything"
build() {
_report "Starting real build"
set_ccache
make $MAKE_DEBUG IS_TTY=1 BUILD_LOG=1 $BUILD_ARGS $OWRT_DEBUG
JOBS="$(echo " $BUILD_ARGS " | sed -n 's|.*-j\([0-9]\+\)|\1\n|p' | head -n 1)"
[ -n "$JOBS" ] || JOBS=0
SUCCESS=""
while [ "$JOBS" -gt 0 ] && [ -z "$SUCCESS" ]; do
COUNTDOWN="$FORCE"
while [ $COUNTDOWN -gt 0 ] && [ -z "$SUCCESS" ]; do
if make $MAKE_DEBUG IGNORE_ERRORS=m IS_TTY=1 BUILD_LOG=1 $BUILD_ARGS $OWRT_DEBUG -j${JOBS}; then
COUNTDOWN="$(expr $COUNTDOWN - 1 || true)"
_report "Build job with -j$JOBS failed (try $(expr $FORCE - $COUNTDOWN || true)/$FORCE)..."
_report "Build job with -j$JOBS failed, decreasing parallelism..."
if [ "$JOBS" -le 1 ] && [ -z "$SUCCESS" ]; then
make $MAKE_DEBUG IGNORE_ERRORS=m IS_TTY=1 BUILD_LOG=1 $BUILD_ARGS $OWRT_DEBUG -j1 V=s
clean_help=" Clean current build directory"
clean() {
_report "Cleaning up current build directory"
rm -rf ./build_dir ./tmp ./staging_dir ./logs ./bin
mkdir -p tmp
}
prepare_help=" Prepare build but don't build it (Implies: checkout clean patch_openwrt repatch_feeds gen_version configure)"
prepare() {
checkout
clean
patch_openwrt
repatch_feeds
gen_version
configure
}
prepare_tools_help=" Prepare build and build tools"
prepare_tools() {
prepare
make $MAKE_DEBUG IS_TTY=1 BUILD_LOG=1 $BUILD_ARGS $OWRT_DEBUG tools/compile toolchain/compile target/compile
repo_prepare_help=" Same as prepare but instead of checkout uses repo_checkout"
repo_prepare() {
repo_checkout
clean
patch_openwrt
repatch_feeds
gen_version
configure
}
pkgauto_help=" Deploy pkgauto.mk that can be used for rolling software development"
pkgauto() {
cp "$SRC_DIR/helpers/pkgauto.mk" "$BUILD_DIR/include/pkgauto.mk"
}
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
# Sets various variables to match the specified target, just a helper
_set_target() {
_report "Setting target as $1"
case "$1" in
omnia)
TARGET_BOARD=omnia
TARGET_ARCH=armv7l
;;
turris)
TARGET_BOARD=turris
TARGET_ARCH=ppcspe
;;
mox)
TARGET_BOARD=mox
TARGET_ARCH=aarch64
;;
*)
echo "Invalid target board!!! Use -t [turris|omnia|mox]!!!"
exit 1
;;
esac
}
help_help=" Displays text with help"
help() {
echo "Usage: $CMD [options] [command] [command] ..."
echo
echo "Available options are:"
echo " -x Enable debug mode"
echo " -e Build everything, not just minimal set"
echo " -f[num] Try hard to get stuff to compile (optional number specifies number of tries)"
echo " -d Do not use shallow checkouts"
echo " -t board Set target board to _board_"
echo " -a \"arg1 arg2\" Set build arguments - passed directly to make"
echo " -b branch Branch to checkout"
echo " -p branch Name of the branch that is being build"
echo " -l Do not update local git mirrors"
echo
echo "Available commands are:"
declare -F | sed -n 's|declare -f \([a-z]\)|\1|p' | while read func; do
echo " $func $(eval echo \"\$${func}_help\")"
done
echo
echo "Default commands are: $DEFAULT_STEPS"
echo
}
[ -z "$BOARD" ] || _set_target "$BOARD"
while expr x$1 : x- > /dev/null; do
cmd="x$1"
shift
case $cmd in
x-x)
set -x
DEBUG="yes"
PERL_DEBUG="-d:Trace"
MAKE_DEBUG="-n"
x-e)
EVERYTHING="yes"
;;
FORCE="$(echo $cmd | sed -n 's|^x-f\([0-9]*\)$|\1|p')"
;;
x-t)
_set_target "$1"
shift
;;
x-a)
BUILD_ARGS="$1"
shift
;;
x-p)
PUBLISH_BRANCH="$1"
shift
;;