turris-maintain: split off turris-backup and rewrite

This makes backup scripts more accessible and also adds additional
features. The short list is:
* Inclusion of sysupgrade files (OpenWrt's way to mark files for backup)
* Optional inclusion of modified configuration files
* More strict inclusion for previously included files
* Protection against recovery from other major versions of OS

The last change is the most important as it is not just plain addition
but the incompatible change. This is intentional as the protection is
more desirable than the compatibility. The use of the scripts is same as
the previously but recovery of backup from different major OS version is
not allowed.

utils/turris-maintain/Makefile

@@ -8,41 +8,70 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=turris-maintain
-PKG_VERSION:=12
+PKG_VERSION:=13
 PKG_RELEASE:=1
 
 PKG_MAINTAINER:=CZ.NIC <packaging@turris.cz>
 
 include $(INCLUDE_DIR)/package.mk
 
-define Package/turris-maintain
+define Package/Common
   SECTION:=utils
   CATEGORY:=Utilities
-  TITLE:=Turris maintain scripts
+endef
+
+define Package/turris-backup
+  $(call Package/Common)
+  TITLE:=Turris backup scripts
+  DEPENDS:=+uci +coreutils-base64 +bzip2 +tar
+endef
+
+define Package/turris-restart
+  TITLE:=Turris restart utility scripts
   DEPENDS:=+uci +python3 +python3-uci +coreutils-base64 +bzip2 +crypto-wrapper
 endef
 
-define Package/turris-maintain/description
+define Package/turris-maintain
+  $(call Package/Common)
+  TITLE:=Turris maintain scripts
+  DEPENDS:=+turris-backup +turris-restart
+endef
+
+define Package/turris-backup/description
  Contains a couple of scripts which are resposible for storing/restoring backups.
 endef
 
-define Package/turris-maintain/conffiles
+define Package/turris-restart/description
+ Scripts that are used to distribute notice about reboot and specific services
+ restart to Turris components.
+endef
+
+define Package/turris-backup/conffiles
 /etc/config/backups
 endef
 
-define Package/turris-maintain/install
+define Package/turris-backup/install
 	$(INSTALL_DIR) $(1)/usr/bin
-	$(INSTALL_BIN) files/config-backup.sh $(1)/usr/bin/maintain-config-backup
-	$(INSTALL_BIN) files/config-restore.sh $(1)/usr/bin/maintain-config-restore
-	$(INSTALL_BIN) files/reboot-needed.sh $(1)/usr/bin/maintain-reboot-needed
-	$(INSTALL_BIN) files/reboot.py $(1)/usr/bin/maintain-reboot
-	$(INSTALL_BIN) files/network-restart.py $(1)/usr/bin/maintain-network-restart
-	$(INSTALL_BIN) files/lighttpd-restart.py $(1)/usr/bin/maintain-lighttpd-restart
+	$(INSTALL_BIN) ./files/config-backup.sh $(1)/usr/bin/turris-backup
+	$(INSTALL_BIN) ./files/config-restore.sh $(1)/usr/bin/turris-backup-restore
+	ln -sf ./turris-backup $(1)/usr/bin/maintain-config-backup
+	ln -sf ./turris-backup-restore $(1)/usr/bin/maintain-config-restore
 
 	$(INSTALL_DIR) $(1)/etc/config
-	$(INSTALL_CONF) ./files/backups.conf $(1)/etc/config/backups
+	$(INSTALL_CONF) ./files/backups.uci $(1)/etc/config/backups
+endef
+
+define Package/turris-restart/install
+	$(INSTALL_DIR) $(1)/usr/bin
+	$(INSTALL_BIN) ./files/reboot-needed.sh $(1)/usr/bin/maintain-reboot-needed
+	$(INSTALL_BIN) ./files/reboot.py $(1)/usr/bin/maintain-reboot
+	$(INSTALL_BIN) ./files/network-restart.py $(1)/usr/bin/maintain-network-restart
+	$(INSTALL_BIN) ./files/lighttpd-restart.py $(1)/usr/bin/maintain-lighttpd-restart
 endef
 
+Package/turris-maintain/install :=
 Build/Compile:=:
 
+$(eval $(call BuildPackage,turris-backup))
+$(eval $(call BuildPackage,turris-restart))
 $(eval $(call BuildPackage,turris-maintain))

utils/turris-maintain/files/README.md

+# Turris Backup scripts
+
+The `turris-backup` in an configuration backup tool. It integrates OpenWrt's
+backup as well as Turris specific files.
+
+The important feature and protection is that it checks major version of
+distribution in backup and compares it with current system. The default
+behavior, unless forced, is to not allow restore of backup if major version is
+not the same. The reason for this is because with major version of Turris OS the
+OpenWrt version is updated and there are commonly incompatible changes in the
+configuration. These changes regularly result in invalid the configuration for
+various software. In worst case scenario they can make the router inaccessible
+due to invalid network configuration.
+
+The backup script backups only files and links. It never dereferences the links
+and instead just includes links as they are. The backup is tar archive
+compressed with bzip2.
+
+## UCI configuration
+
+The backup script reads `/etc/config/backups` file and expects the following
+options in the `generate` section:
+
+* `include_sysupgrade`: boolean option if sysupgrade files should be included in
+  backup. This is OpenWrt's native list of files that should be backed up.
+  Various packages ship with additions to this list. The directories and files
+  to be backed up are collected from file `/etc/sysupgrade.conf` and files in
+  directory `/lib/upgrade/keep.d/`. Sysupgrade files are automatically included
+  in default unless explicitly disabled.
+* `include_modified_configs`: boolean option signaling that any modified
+  configuration file should be included. The packages can set some of their
+  files as configuration files and that way they are protected from updates if
+  they were modified. This includes any such modified configuration file in
+  backup. The modified files are NOT included automatically unless enabled
+  explicitly.
+* `include_default`: boolean option if default Turris files should be included.
+  This includes UCI files in `/etc/config/` and non-standard updater
+  configuration files from directory `/etc/updater/conf.d/`. The default is to
+  include these files unless explicitly disabled.
+* `dirs`: list with additional paths to files or directories that should be
+  included in backup. Feel free to add any additional directories you want to
+  include in the backup this way.

utils/turris-maintain/files/backups.conf

-package backups
-
-config generate generate
-	# custom firewall rules from LuCI config
-	list dirs "/etc/firewall.user"
-	## Some extra files and directories for inspiration:
-	#
-	# if you want to backup startup services
-	#list dirs "/etc/rc.d"
-	#
-	# to backup root ssh config, authorized and private keys
-	#list dirs "/root/.ssh"
-	#
-	# if you want to backup system users as well
-	#list dirs "/etc/passwd"
-	#list dirs "/etc/shadow"
-	#
-	# if you want to backup OpenVPN Foris plugin certificates
-	#list dirs "/etc/ssl/ca/openvpn"
-	#
-	# if you have some custom crontabs
-	#list dirs "/etc/cron.d/my_crontab"

utils/turris-maintain/files/backups.uci

+
+config generate 'generate'
+	# This controls if files from /etc/sysupdate.conf and /lib/upgrade/keep.d/*
+	# should be included in backup.
+	#option include_sysupgrade '1'
+	# This controls if any configuration file that was modified should be
+	# included as well.
+	#option include_modified_configs '0'
+	# This controls if Turris default file set should be included in the backup.
+	#option include_default '1'
+	# You can specify additonal files and directories to backup using:
+	#list dirs '/root/.ssh'
+

utils/turris-maintain/files/config-backup.sh

 #!/bin/sh
 # Turris OS configuration backup utility
 #
-# Copyright 2014-2018 CZ.NIC z.s.p.o. (http://www.nic.cz/)
+# Copyright 2014-2021 CZ.NIC z.s.p.o. (http://www.nic.cz/)
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,29 +15,135 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
 set -e
+. /lib/functions.sh
+turris_version="/etc/turris-version"
 
-DIR=/tmp/backup-$$
-SRC=/etc/config
-mkdir -p "$DIR"/etc
-trap 'rm -rf "$DIR"' EXIT INT QUIT TERM ABRT
-cd "$DIR"/etc
-cp -a "$SRC" config
-cd ..
-uci -q -c "$DIR"/etc/config delete foris.auth.password || true
-uci -c "$DIR"/etc/config commit
-if [ -d /etc/updater ] ; then
-	# Back up the updater options (mostly lists of packages)
-	cp -a /etc/updater "$DIR"/etc
-	# But exclude things coming from packages not marked as configs
-	rm -rf "$DIR"/etc/updater/keys
-	rm -rf "$DIR"/etc/updater/hook_*
-fi
-uci -q -d '
-' get backups.generate.dirs | while read dir ; do
-	DNAME=$(dirname "$dir")
-	mkdir -p "$DIR"/"$DNAME"
-	cp -a "$dir" "$DIR"/"$DNAME"/
+config_load "backups"
+
+# These are files requested to be backed up by OpenWrt packages
+sysupgrade_files() {
+	# Note: this is taken from sysupgrade script
+	# https://git.openwrt.org/?p=openwrt/openwrt.git;a=blob;f=package/base-files/files/sbin/sysupgrade;h=7e0a00e13b8ee4be7163936fd01a7beff0ce5c99;hb=HEAD#l116
+	sed -ne '/^[[:space:]]*$/d; /^#/d; p' /etc/sysupgrade.conf /lib/upgrade/keep.d/* 2>/dev/null \
+		| while read -r path; do
+			[ -e "$path" ] || continue
+			find -H "$path" -type f -o -type l
+		done
+}
+
+# These are configuration files that were modified
+modified_configs_files() {
+	# Note: this is taken from sysupgrade script as well (see link in sysupgrade_files)
+	awk '
+		BEGIN { conffiles = 0 }
+		/^Conffiles:/ { conffiles = 1; next }
+		!/^ / { conffiles = 0; next }
+		conffiles == 1 { print }
+	' /usr/lib/opkg/status \
+		| while read -r file csum; do
+			[ -r "$file" ] || continue
+			echo "${csum}  ${file}" | busybox sha256sum -sc - || echo "$file"
+		done
+}
+
+# These are files we want automatically include on Turris
+default_files() {
+	# UCI configuration
+	find -H /etc/config -type f \( -name "*-opkg" -o -name ".*" -o -print \)
+	# Updater configuration (the Turris ones contain warning that they should
+	# not be edited and that way we can filter them out)
+	find -H /etc/updater/conf.d -type f \( -exec grep -qF "Don't edit it." \{\} \; -o -print \)
+}
+
+# Files/directories configured in backups UCI
+uci_files() {
+	config_list_foreach "generate" "dirs" _uci_files_dirs
+}
+_uci_files_dirs() {
+	find -H "$1" -type f -o -type l
+}
+
+# This provides list of all files configured for backup and available in system
+files() {
+	local include_sysupgrade include_modified_configs include_default
+	config_get_bool include_sysupgrade "generate" "include_sysupgrade" "1"
+	config_get_bool include_modified_configs "generate" "include_modified_configs" "0"
+	config_get_bool include_default "generate" "include_default" "1"
+	{
+		[ -f "$turris_version" ] \
+			&& echo "$turris_version"
+		[ "$include_sysupgrade" = "1" ] \
+			&& sysupgrade_files
+		[ "$include_modified_configs" = "1" ] \
+			&& modified_configs_files
+		[ "$include_default" = "1" ] \
+			&& default_files
+		uci_files
+	} | sort -u
+}
+
+
+
+usage() {
+	echo "Usage: $0 [OPTION].. [OUTPUT]" >&2
+}
+help() {
+	usage
+	cat >&2 <<-"EOF"
+
+		In default the output is generated to stdout in base64 encoding.
+		Optionally you can specify OUTPUT where raw backup should be stored.
+		The backup is simply tar.bz2 archive with files.
+
+		Options:
+		  -l  List all files and links that would be backed up
+		  -h  Print this help text
+	EOF
+}
+
+while getopts "lh" opt; do
+	case "$opt" in
+		l)
+			files
+			exit 0
+			;;
+		h)
+			help
+			exit 0
+			;;
+		*)
+			usage
+			exit 1
+			;;
+	esac
 done
-tar c . | bzip2 -9c | base64
+shift $((OPTIND - 1))
+[ "$#" -le 1 ] || {
+	usage
+	exit 1
+}
+output="${1:-}"
+if [ -z "$output" ] || [ "$output" = "-" ]; then
+	archive="/tmp/backup-$$.tar"
+	trap 'rm -f "$archive"' INT QUIT TERM ABRT EXIT
+else
+	archive="${1%.bz2}" # This prevents bzip2 compain about suffix
+fi
+
+files | sed 's|^/||' | xargs /usr/bin/tar -rf "$archive" -C /
+
+bzip2 -z -f -9 "$archive"
+archive="$archive.bz2"
+
+case "$output" in
+	"")
+		base64 "$archive"
+		;;
+	"-")
+		cat "$archive"
+		;;
+	*)
+		mv "$archive" "$output"
+		;;
+esac

utils/turris-maintain/files/config-restore.sh

 #!/bin/sh
 # Turris OS configuration backup utility
 #
-# Copyright 2014-2018 CZ.NIC z.s.p.o. (http://www.nic.cz/)
+# Copyright 2014-2021 CZ.NIC z.s.p.o. (http://www.nic.cz/)
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,21 +15,126 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
 set -e
+. /lib/functions.sh
+
+
+usage() {
+	echo "Usage: $0 [OPTION].. [INPUT]" >&2
+}
+help() {
+	usage
+	cat >&2 <<-"EOF"
+
+		In default the input is expected on stdin and to be base64 encoded.
+		Optionally you can specify raw INPUT (not base64 encoded).
+
+		Options:
+		  -v  Print system version backup was created on instead of backup restoration
+		  -f  Force backup restoration even when it is from different major version of OS
+		  -p  Preserve the Turris password
+		  -n  Do not send notification about need to restart the router
+		  -h  Print this help text
+	EOF
+}
+
+force="n"
+only_version="n"
+preserve_passwords="n"
+send_notification="y"
+while getopts "vfpnh" opt; do
+	case "$opt" in
+		v)
+			only_version="y"
+			;;
+		f)
+			force="y"
+			;;
+		p)
+			preserve_passwords="y"
+			;;
+		n)
+			send_notification="n"
+			;;
+		h)
+			help
+			exit 0
+			;;
+		*)
+			usage
+			exit 1
+			;;
+	esac
+done
+shift $((OPTIND - 1))
+[ "$#" -le 1 ] || {
+	usage
+	exit 1
+}
+input="${1:-}"
+if [ -f "$input" ]; then
+	archive="$1"
+else
+	archive="/tmp/backup-$$.tar"
+	case "$input" in
+		"")
+			base64 -d >"$archive"
+			;;
+		"-")
+			cat >"$archive"
+			;;
+		*)
+			cat <"$input" >"$archive"
+			;;
+	esac
+	trap 'rm -f "$archive"' INT QUIT TERM ABRT EXIT
+fi
+
+version="$(tar -xOf backup.tar.bz2 etc/turris-version || echo "unknown")"
+if [ "$only_version" = "y" ]; then
+	echo "$version"
+	exit 0
+fi
+
+if [ "$force" = "n" ]; then
+	current="$(cat /etc/turris-version)"
+	if [ "${version%%.*}" != "${current%%.*}" ]; then
+		cat >&2 <<-EOF
+		You are trying to recover backup from a different major version of OS.
+		This is protection against possibly dangerous recoveries. The major OS
+		version commonly update system configuration in the incompatible way and
+		recovery of obsolete configuration can make device unaccessible.
+		The current versions is '$current' while backup was created on '$version'.
+		You can use -f option to force recovery anyway.
+		EOF
+		exit 1
+	fi
+fi
+
+if [ "$preserve_passwords" = "y" ]; then
+	foris_password="$(uci -q get foris.auth.password || true)"
+	auth_password="$(uci -q get turris-auth.auth.password || true)"
+	root_password="$(sed -n 's/^root:\([^:]\+\):.*/\1/p' /etc/shadow || true)"
+fi
+
+tar -xjf "$archive" -C /
+
+if [ "$preserve_passwords" = "y" ]; then
+	if [ -n "$foris_password" ]; then
+		uci set "foris.auth.password=$foris_password"
+		uci commit foris.auth.password
+	fi
+	if [ -n "$auth_password" ]; then
+		uci set "turris-auth.auth.password=$auth_password"
+		uci commit turris-auth.auth.password
+	fi
+	if [ -n "$auth_password" ]; then
+		sed -i "s/^root:\([^:]\+\):/root:$root_password:/" /etc/shadow
+	fi
+fi
+
 
-DIR=/tmp/restore-$$
-mkdir -p "$DIR"
-trap 'rm -rf "$DIR"' EXIT INT QUIT TERM ABRT
-cd "$DIR"
-base64 -d | bzip2 -cd | tar xp
-# Here we have a special-case/hack for foris password. It was requested NOT to restore the
-# password, as potentially confusing action. So we unpack the backed-up configuration,
-# extract the current password and implant it into the configuration. Then we just copy
-# the configs and overwrite the current ones.
-PASSWD="$(uci -q -c "/etc/config" get foris.auth.password || echo -n)"
-[ -z "$PASSWD" ] || uci -c "$DIR/etc/config" set foris.auth.password="$PASSWD"
-uci -c "$DIR/etc/config" commit
-cp -rf "$DIR/"* "/"
-cd /
-rm -rf "$DIR"
+if [ "$send_notification" = "y" ]; then
+	create_notification -t -s restart \
+		'Configuration backup was restored. To fully apply all configuration changes the router has to be restarted.'
+fi