Unverified Commit 4672d939 authored by Michal Hrusecky's avatar Michal Hrusecky 🐭
Browse files

Initial version of build scripts



Commiting basic building script. Some functionality is already there, what is
missing is handling branches and freezing stuff, correct version setup (no need
in development for now), userlist generation and medkit creation (no need till
we build at least a big part of packages).
Signed-off-by: Michal Hrusecky's avatarMichal Hrusecky <Michal@Hrusecky.net>
parents
/README.html
Simple Turris OS building script
================================
Getting started
---------------
Create an empty directory you want to build Turris OS in, enter that directory
and from it call `compile_fw` script. In most basic use-case, this should be
enough. Be aware - *it will delete all previous content of the current
directory*. You can repeat individual steps of the compilation separately,
check `compile_fw help` for a list of available commands, generic options and
their brief descriptions.
Advanced usage
--------------
Patching
~~~~~~~~
If you need to patch OpenWRT or any of the feeds, put a git formatted patch
into respective subdirectory of patches directory. In patches directory, there
is openwrt directory and directory for each feed we are patching. Inside those
directories, patches are also divided into subdirectories like `to-upstream`,
`wip` or `hack` to somehow distinguish between quality of patches.
Configuration
~~~~~~~~~~~~~
Default config for OpenWRT is put in `configs` directory either in file
`common` or in a file named after the target board. Those two files gets merged
and will form the final `.config` file. Some parts of `compile_fw` might modify
it further.
#!/bin/bash -e
# OpenWRT compilation script
# (C) 2018 CZ.NIC, z.s.p.o.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
BUILD_DIR="$(pwd)"
SRC_DIR="$(dirname "$0")"
SRC_DIR="$(cd "$SRC_DIR"; pwd)"
CMD="${0}"
OPENWRT_URL="https://git.openwrt.org/openwrt/openwrt.git"
BRANCH=master
DEBUG=""
PERL_DEBUG=""
MAKE_DEBUG=""
DEPTH="--depth 1"
FORCE=""
DEFAULT_STEPS="checkout patch_openwrt get_feeds patch_feeds configure build"
export TMPDIR="${BUILD_DIR}/tmp"
mkdir -p "$TMPDIR"
[ \! -f "${SRC_DIR}"/turris-build.conf ] || . "${SRC_DIR}"/turris-build.conf
[ \! -f ~/.turris-build ] || . ~/.turris-build
[ \! -f ./turris-build.conf ] || . ./turris-build.conf
MIRROR_UPDATED=""
_report() {
echo -e '\033[0;34m'"$1"'\033[0m' >&2
}
_die() {
echo -e '\033[0;31m'"$1"'\033[0m' >&2
exit 1
}
configure_help="Recreates configuration for target boards"
configure() {
_report "Creating default configuration"
[ -n "${TARGET_BOARD}" ] || _die "No board selected!"
set -x
cat "${SRC_DIR}"/configs/common "${SRC_DIR}"/configs/${TARGET_BOARD} > ./.config
echo "CONFIG_CCACHE=y" >> .config
make $MAKE_DEBUG defconfig
}
update_mirror_help="Updates all local mirrors"
update_mirror() {
[ -n "$GIT_MIRROR" ] || return 0
[ -z "$MIRROR_UPDATED" ] || return 0
_report "Updating local mirrors"
pushd "$GIT_MIRROR"
if [ \! -d openwrt ]; then
mkdir openwrt
cd openwrt && git init --bare && git remote add --mirror=fetch origin "$OPENWRT_URL"
fi
OPENWRT_URL="$GIT_MIRROR/openwrt"
for mirror in ./*; do
cd "$mirror" && git fetch --all
cd "$GIT_MIRROR"
done
popd
MIRROR_UPDATED="yes"
}
checkout_help="Start everything from scratch - all changes deleted and fresh copy of OpenWRT gets checked out"
checkout() {
_report "Starting out fresh!"
_report "Checking out clean OpenWRT repository"
update_mirror
rm -rf .git
git init
git remote add origin "$OPENWRT_URL"
git fetch $DEPTH origin "$BRANCH"
git checkout -f "origin/$BRANCH"
BUILD_SETTINGS="$(cat turris-build.conf 2> /dev/null)"
git clean -dff
[ -z "$BUILD_SETTINGS" ] || echo "$BUILD_SETTINGS" > turris-build.conf
git config --local commit.gpgsign false
echo "/turris-build.conf" >> ./.gitignore
echo "/version" >> ./.gitignore
git commit -m 'gitignore: Ignore some more files' ./.gitignore
[ \! -f "$SRC_DIR"/feeds.conf ] || cp "$SRC_DIR"/feeds.conf .
if [ -n "$DL_MIRROR" ]; then
rm -rf dl && ln -s "$DL_MIRROR" dl
fi
git log -n 1 --format="%h" > version
}
patch_feeds_help="Apply patches to the feeds"
patch_feeds() {
_report "Patching feeds"
pushd "$SRC_DIR"/patches
for feed in *; do
[ -d "$BUILD_DIR"/feeds/$feed ] || continue
cd "$SRC_DIR"/patches/$feed
for patch in */*.patch; do
[ -f "$patch" ] || continue
cd "$BUILD_DIR"/feeds/$feed
git config --local commit.gpgsign false
git am "$SRC_DIR"/patches/$feed/$patch
done
done
popd
}
set_ccache_help="Set persistent ccache paths"
set_ccache() {
[ -z "$CCACHE_SET" ] || return 0
_report "Setting ccache paths"
CCACHE_SET=y
[ -z "$CCACHE_HOST_DIR" ] || sed -i 's|\(export CCACHE_DIR:=\).*|\1'"$CCACHE_HOST_DIR|" include/host-build.mk
[ -z "$CCACHE_TARGET_DIR" ] || [ -z "$TARGET_ARCH" ] || sed -i 's|\(export CCACHE_DIR:=\).*|\1'"$CCACHE_TARGET_DIR/$TARGET_ARCH|" include/package.mk
[ -z "$(git diff include/host-build.mk include/package.mk)" ] || git commit -m "include: ccache settings" include/host-build.mk include/package.mk
}
set_local_feeds_help="Change feed URL to their mirror counterparts"
set_local_feeds() {
[ -n "$GIT_MIRROR" ] || return 0
while read vcs name url rest; do
if [ "$vcs" = src-git ] && [ -d "$GIT_MIRROR"/$name ]; then
feed_url="$(echo "$url" | sed 's|[[:blank:]^].*||')"
sed -i "s|$feed_url|file://$GIT_MIRROR/$name|" feeds.conf
fi
done < feeds.conf
}
mirror_feeds_help="Creates initial mirrors of all configured feeds"
mirror_feeds() {
[ -n "$GIT_MIRROR" ] || return 0
pushd "$GIT_MIRROR"
cat "$SRC_DIR"/feeds.conf "$BUILD_DIR"/feeds.conf 2> /dev/null | while read vcs name url rest; do
if [ "$vcs" = src-git ] && [ \! -d "$GIT_MIRROR"/$name ]; then
feed_url="$(echo "$url" | sed 's|[[:blank:]^].*||')"
mkdir "$name"
cd "$name" && git init --bare && git remote add --mirror=fetch origin "$feed_url" && git fetch --all
cd "$GIT_MIRROR"
fi
done
popd
update_mirror
}
patch_openwrt_help="Patch the main OpenWRT repository"
patch_openwrt() {
_report "Patching OpenWRT repository"
pushd "$SRC_DIR"/patches/openwrt
for patch in */*.patch; do
cd "$BUILD_DIR"
git am "$SRC_DIR"/patches/openwrt/$patch
done
popd
set_ccache
}
get_feeds_help="Recreate configured feeds"
get_feeds() {
update_mirror
set_local_feeds
perl $PERL_DEBUG ./scripts/feeds clean -a
perl $PERL_DEBUG ./scripts/feeds update -a
perl $PERL_DEBUG ./scripts/feeds install -a
}
repatch_feeds_help="Cleanup feeds, update them and patch them"
repatch_feeds() {
get_feeds
patch_feeds
}
prefetch_help="Runs make download"
prefetch() {
make $MAKE_DEBUG $BUILD_ARGS $OWRT_DEBUG download
}
build_help=" Builds everything"
build() {
_report "Starting real build"
set_ccache
if [ -z "$FORCE" ]; then
make $MAKE_DEBUG IS_TTY=1 LOGFILE=1 BUILD_LOG=1 $BUILD_ARGS $OWRT_DEBUG
else
make $MAKE_DEBUG IS_TTY=1 LOGFILE=1 BUILD_LOG=1 $BUILD_ARGS $OWRT_DEBUG || \
IGNORE_ERRORS=m make $MAKE_DEBUG IS_TTY=1 LOGFILE=1 BUILD_LOG=1 $BUILD_ARGS $OWRT_DEBUG || \
IGNORE_ERRORS=m make $MAKE_DEBUG IS_TTY=1 LOGFILE=1 BUILD_LOG=1 $BUILD_ARGS $OWRT_DEBUG -j1 V=s
fi
}
# Sets various variables to match the specified target, just a helper
_set_target() {
_report "Setting target as $1"
case "$1" in
omnia)
TARGET_BOARD=omnia
TARGET_ARCH=armv7l
;;
turris)
TARGET_BOARD=turris
TARGET_ARCH=ppcspe
;;
mox)
TARGET_BOARD=mox
TARGET_ARCH=aarch64
;;
*)
echo "Invalid target board!!! Use -t [turris|omnia|mox]!!!"
exit 1
;;
esac
}
help_help=" Displays text with help"
help() {
echo "Usage: $CMD [options] [command] [command] ..."
echo
echo "Available options are:"
echo " -x Enable debug mode"
echo " -f Try hard to get stuff to compile"
echo " -d Do not use shallow checkouts"
echo " -t board Set target board to _board_"
echo " -a \"arg1 arg2\" Set build arguments - passed directly to make"
echo " -l Do not update local git mirrors"
echo
echo "Available commands are:"
declare -F | sed -n 's|declare -f \([a-z]\)|\1|p' | while read func; do
echo " $func $(eval echo \"\$${func}_help\")"
done
echo
echo "Default commands are: $DEFAULT_STEPS"
echo
}
[ -z "$BOARD" ] || _set_target "$BOARD"
while expr x$1 : x- > /dev/null; do
cmd="x$1"
shift
case $cmd in
x-x)
set -x
DEBUG="yes"
PERL_DEBUG="-d:Trace"
MAKE_DEBUG="-n"
OWRT_DEBUG="V=s"
;;
x-d)
DEPTH=""
;;
x-f)
FORCE="y"
;;
x-t)
_set_target "$1"
shift
;;
x-a)
BUILD_ARGS="$1"
shift
;;
x-b)
BRANCH="$1"
shift
;;
x-l)
MIRROR_UPDATED="override"
OPENWRT_URL="$GIT_MIRROR/openwrt"
;;
esac
done
[ -n "$1" ] || set $DEFAULT_STEPS
while [ -n "$1" ]; do
eval "$1"
shift
done
#
# Automatically generated file; DO NOT EDIT.
# OpenWrt Configuration
#
This diff is collapsed.
src-git turrispackages https://gitlab.labs.nic.cz/turris/turris-os-packages.git^origin/cleanup
src-git packages https://git.lede-project.org/feed/packages.git
src-git luci https://git.lede-project.org/project/luci.git
src-git routing https://git.lede-project.org/feed/routing.git
src-git telephony https://git.lede-project.org/feed/telephony.git
src-git printing https://github.com/miska/openwrt-printing-packages.git
src-git openwisp https://github.com/openwisp/openwisp-config.git
#src-git video https://github.com/openwrt/video.git
#src-git targets https://github.com/openwrt/targets.git
#src-git management https://github.com/openwrt-management/packages.git
#src-git oldpackages http://git.openwrt.org/packages.git
#src-link custom /usr/src/openwrt/custom-feed
From decaa09ed89d8685ff683526ea75dca8975039a0 Mon Sep 17 00:00:00 2001
From: Michal Hrusecky <michal.hrusecky@nic.cz>
Date: Mon, 23 Oct 2017 12:56:44 +0200
Subject: [PATCH] build: Allow to get just a subset of packages from feeds
Sometimes it is useful to get just some subset of packages from feed. This
commit changes scripts/feeds in a way that you can specify as the last optional
argument which packages should be used from the feed.
Signed-off-by: Michal Hrusecky <michal.hrusecky@nic.cz>
---
scripts/feeds | 25 ++++++++++++++++---------
1 file changed, 16 insertions(+), 9 deletions(-)
diff --git a/scripts/feeds b/scripts/feeds
index f49cdb3..cebe05a 100755
--- a/scripts/feeds
+++ b/scripts/feeds
@@ -52,7 +52,7 @@ sub parse_config() {
chomp;
s/#.+$//;
next unless /\S/;
- my @line = split /\s+/, $_, 3;
+ my @line = split /\s+/, $_, 4;
my @src;
$line++;
@@ -65,7 +65,7 @@ sub parse_config() {
$name{$line[1]} and die "Duplicate feed name '$line[1]', line: $line\n";
$name{$line[1]} = 1;
- push @feeds, [$line[0], $line[1], \@src];
+ push @feeds, [$line[0], $line[1], \@src, $line[3]];
}
close FEEDS;
}
@@ -165,12 +165,13 @@ my %update_method = (
# src-git: pull broken
# src-cpy: broken if `basename $src` != $name
-sub update_feed_via($$$$$) {
+sub update_feed_via($$$$$$) {
my $type = shift;
my $name = shift;
my $src = shift;
my $relocate = shift;
my $force = shift;
+ my $filter = shift;
my $m = $update_method{$type};
my $localpath = "./feeds/$name";
@@ -198,6 +199,11 @@ sub update_feed_via($$$$$) {
system("cd '$safepath'; $update_cmd") == 0 or return 1;
}
+ if($filter) {
+ system("cd '$safepath'; find ./[a-zA-Z0-9]* -name Makefile -not -regex '" . $filter . "' -exec dirname \{\} \\; | xargs -d '\n' rm -rf");
+ system("cd '$safepath'; find ./[a-zA-Z0-9]* -empty -exec rmdir -p \\{\\} \\; 2> /dev/null");
+ }
+
return 0;
}
@@ -715,13 +721,14 @@ sub uninstall {
return 0;
}
-sub update_feed($$$$$)
+sub update_feed($$$$$$)
{
my $type=shift;
my $name=shift;
my $src=shift;
my $perform_update=shift;
my $force_update=shift;
+ my $filter=shift;
my $force_relocate=update_location( $name, "@$src" );
my $rv=0;
@@ -736,7 +743,7 @@ sub update_feed($$$$$)
my $failed = 1;
foreach my $feedsrc (@$src) {
warn "Updating feed '$name' from '$feedsrc' ...\n";
- if (update_feed_via($type, $name, $feedsrc, $force_relocate, $force_update) != 0) {
+ if (update_feed_via($type, $name, $feedsrc, $force_relocate, $force_update, $filter) != 0) {
if ($force_update) {
$rv=1;
$failed=0;
@@ -788,17 +795,17 @@ sub update {
if ( ($#ARGV == -1) or $opts{a}) {
foreach my $feed (@feeds) {
- my ($type, $name, $src) = @$feed;
- update_feed($type, $name, $src, $perform_update, $opts{f}) == 0 or $failed=1;
+ my ($type, $name, $src, $filter) = @$feed;
+ update_feed($type, $name, $src, $perform_update, $opts{f}, $filter) == 0 or $failed=1;
}
} else {
while ($feed_name = shift @ARGV) {
foreach my $feed (@feeds) {
- my ($type, $name, $src) = @$feed;
+ my ($type, $name, $src, $filter) = @$feed;
if($feed_name ne $name) {
next;
}
- update_feed($type, $name, $src, $perform_update, $opts{f}) == 0 or $failed=1;
+ update_feed($type, $name, $src, $perform_update, $opts{f}, $filter) == 0 or $failed=1;
}
}
}
--
2.15.1
From c3868937dfff17c9e715563da23a394736ec1f14 Mon Sep 17 00:00:00 2001
From: Michal Hrusecky <Michal@Hrusecky.net>
Date: Wed, 17 Jan 2018 13:25:43 +0100
Subject: [PATCH] build: Optionally provide file checksums in package metadata
This may be useful if you don't entirely trust your flash and want to be able
to check for corruptions.
Signed-off-by: Michal Hrusecky <Michal@Hrusecky.net>
---
config/Config-build.in | 8 ++
include/package-ipkg.mk | 8 ++
package/base-files/Makefile | 2 +
package/base-files/files/sbin/pkg_check | 130 ++++++++++++++++++++++++++++++++
4 files changed, 148 insertions(+)
create mode 100755 package/base-files/files/sbin/pkg_check
diff --git a/config/Config-build.in b/config/Config-build.in
index f9987fc..38f725f 100644
--- a/config/Config-build.in
+++ b/config/Config-build.in
@@ -68,6 +68,14 @@ menu "Global build settings"
This removes all ipkg/opkg status data files from the target directory
before building the root filesystem.
+ config IPK_FILES_CHECKSUMS
+ bool
+ prompt "Record files checksums in package metadata"
+ default n
+ help
+ This makes file checksums part of package metadata. It increases size
+ but provides you with pkg_check command to check for flash coruptions.
+
config INCLUDE_CONFIG
bool "Include build configuration in firmware" if DEVEL
default n
diff --git a/include/package-ipkg.mk b/include/package-ipkg.mk
index 9e2717e..358aede 100644
--- a/include/package-ipkg.mk
+++ b/include/package-ipkg.mk
@@ -194,6 +194,15 @@ $(_endef)
$(CheckDependencies)
$(RSTRIP) $$(IDIR_$(1))
+
+ ifneq ($$(CONFIG_IPK_FILES_CHECKSUMS),)
+ (cd $$(IDIR_$(1)); \
+ ( \
+ find . -type f \! -path ./CONTROL/\* -exec sha256sum \{\} \; 2> /dev/null | \
+ sed 's|\([[:blank:]]\)\./|\1/|' > $$(IDIR_$(1))/CONTROL/files-sha256 \
+ ) || true \
+ )
+ endif
(cd $$(IDIR_$(1))/CONTROL; \
( \
echo "$$$$CONTROL"; \
diff --git a/package/base-files/Makefile b/package/base-files/Makefile
index 0055474..411ed8e 100644
--- a/package/base-files/Makefile
+++ b/package/base-files/Makefile
@@ -195,6 +195,8 @@ define Package/base-files/install
mkdir -p $(1)/etc/opkg; \
$(call FeedSourcesAppend,$(1)/etc/opkg/distfeeds.conf); \
$(VERSION_SED) $(1)/etc/opkg/distfeeds.conf)
+ $(if $(CONFIG_IPK_FILES_CHECKSUMS), \
+ rm -f $(1)/sbin/pkg_check,)
endef
ifneq ($(DUMP),1)
diff --git a/package/base-files/files/sbin/pkg_check b/package/base-files/files/sbin/pkg_check
new file mode 100755
index 0000000..9968c9b
--- /dev/null
+++ b/package/base-files/files/sbin/pkg_check
@@ -0,0 +1,130 @@
+#!/bin/sh
+#
+# Package checksums checking script
+# (C) 2018 CZ.NIC, z.s.p.o.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+ERRFATAL="no"
+QUIET="yes"
+MISSING=""
+SUMMARY=""
+NL="
+"
+
+# Arguments parsing
+while expr "x$1" : "x-" > /dev/null; do
+ if [ "x$1" = "x-s" ]; then
+ ERRFATAL="yes"
+ shift
+ elif [ "x$1" = "x-v" ]; then
+ QUIET=" no"
+ shift
+ else
+ echo "Usage: $(basename $0) [-s] [-v] [pkg1 pkg2 ...]"
+ echo
+ echo " -s Stop on first change"
+ echo " -v Verbose"
+ if [ "x$1" = "x-h" ]; then
+ exit 0
+ else
+ echo
+ echo "ERROR: Unknown option '$1'"
+ exit 1
+ fi
+ fi
+done
+
+# Check all packages by default
+if [ -z "$1" ]; then
+ set $(cd /usr/lib/opkg/info/; for i in *.files-sha256sum; do basename $i .files-sha256sum; done)
+fi
+
+# Iterate over packages
+while [ "$1" ]; do
+ if [ \! -f "/usr/lib/opkg/info/$1.files-sha256sum" ]; then
+ if [ "$ERRFATAL" = no ]; then
+ echo " * No checksums for $1 - skipping"
+ echo
+ else
+ echo " * No checksums for $1 - exiting"
+ exit 1
+ fi
+ if [ -z "$MISSING" ]; then
+ MISSING="$1"
+ else
+ MISSING="$MISSING, $1"
+ fi
+ shift
+ continue
+ fi
+ [ $QUIET = yes ] || echo " * Checking package $1:"
+ ERR=""
+ CHECK="`sha256sum -c /usr/lib/opkg/info/$1.files-sha256sum 2> /dev/null`"
+
+ # Are the changed files config files?
+ if [ $? -ne 0 ] && [ "`cat "/usr/lib/opkg/info/$1.files-sha256sum"`" ]; then