compile_pkgs: split force-make to separate script

Running make multiple times for builds with last debug fallback run is
handy not even for calls available trough compile_pkgs. For example
building single selected package is not possible using compile_pkgs and
that script is not intended to even do it. The clean solution because of
that is to just simply extract logic for forced make run to separate
script that can be used without compile_pkgs,

compile_pkgs

@@ -125,10 +125,14 @@ _make() {
 
 # Make for OpenWRT
 _openwrt_make() {
+	local mk=("make" -j "$BUILD_JOBS")
+	[ -z "$FORCE" ] || mk=("$src_dir/helpers/force-make" "-j" "$BULD_JOBS" "-c" "$FORCE" "-f" "V=s" "--")
+
 	local args=( "IGNORE_ERRORS=m" "BUILD_LOG=1" )
 	[ -z "$owrt_debug" ] || args+=( "$owrt_debug" )
 	[ "${#BUILD_ARGS[@]}" -eq 0 ] || args+=( "${BUILD_ARGS[@]}" )
-	_make "${args[@]}" "$@"
+
+	"${mk[@]}" $make_debug "${args[@]}" "$@"
 }
 
 _enable_debug() {
@@ -416,7 +420,8 @@ repatch_feeds() {
 
 available_commands+=( ["prefetch"]="Runs make download" )
 prefetch() {
-	_openwrt_make -j"$BUILD_JOBS" download
+	# We do not want to use force here intentionally as we are not building here.
+	FORCE="" _openwrt_make download
 }
 
 available_commands+=( ["gen_junit"]="Generates junit output from build logs" )
@@ -435,51 +440,22 @@ pkgsrepo() {
 	mv bin/git-hash "$OUTPUT_DIR"
 }
 
-# Function calling make in OpenWRT tree that is designed to handle force option in
-# compact way.
-_compile() {
-	if [ -z "$FORCE" ]; then
-		_openwrt_make -j"$BUILD_JOBS" "$@"
-	else
-		local success=""
-		local force_jobs="$BUILD_JOBS"
-		while [ "$force_jobs" -gt 0 ] && [ -z "$success" ]; do
-			local countdown="$FORCE"
-			while [ $countdown -gt 0 ] && [ -z "$success" ]; do
-				if _openwrt_make -j"$force_jobs" "$@"; then
-					success="YES!!!"
-				else
-					countdown="$((countdown - 1))"
-					report "Build job with -j$force_jobs failed (trying $((FORCE - countdown))/$FORCE)..."
-				fi
-			done
-			if [ -z "$success" ]; then
-				report "Build job with -j$force_jobs failed, decreasing parallelism..."
-				force_jobs="$((force_jobs / 2))"
-			fi
-		done
-		if [ "$force_jobs" -le 1 ] && [ -z "$success" ]; then
-			_openwrt_make -j1 V=s "$@"
-		fi
-	fi
-}
-
 available_commands+=( ["compile_tools"]="Compile host tools" )
 compile_tools() {
 	report "Compiling tools"
-	_compile tools/compile toolchain/compile
+	_openwrt_make tools/compile toolchain/compile
 }
 
 available_commands+=( ["compile_target"]="Compile target specific software (Linux kernel)" )
 compile_target() {
 	report "Compiling target"
-	_compile target/compile
+	_openwrt_make target/compile
 }
 
 available_commands+=( ["compile_packages"]="Compile packages" )
 compile_packages() {
 	report "Compiling packages"
-	_compile package/compile
+	_openwrt_make package/compile
 }
 
 available_commands+=( ["compile"]="	Compile tools, target and packages" )
@@ -491,7 +467,7 @@ compile() {
 
 available_commands+=( ["sign"]="	Generate packages index and sign it" )
 sign() {
-	_compile package/index BUILD_KEY="$SIGN_KEY"
+	_openwrt_make package/index BUILD_KEY="$SIGN_KEY"
 }
 
 available_commands+=( ["clean"]="	Clean current build directory" )
@@ -534,7 +510,7 @@ available_commands+=( ["autopkg"]="	Deploy autopkg scripts that can be used for
 autopkg() {
 	report "Deploying autopkg scripts"
 	cp "$src_dir"/helpers/autopkg/* "$build_dir"/include/
-	git add include/autopkg-*.mk
+	_git add include/autopkg-*.mk
 	_git commit -m 'autopkg: Add autopkg scripts'
 }
 

helpers/force-make

+#!/bin/bash
+# Make wrapper with ability to decrease paralelism and try again.
+# (C) 2018-2020 CZ.NIC, z.s.p.o.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+set -eu
+
+usage() {
+	echo "Usage: $0 [OPTION].. [-- [ARGUMENTS]..]" >&2
+}
+
+help() {
+	usage
+	cat >&2 <<EOF
+This is make wrapper that tries to run build really hard. It runs build multiple
+times while it decreases paralelism. You can set amount of retries done given
+number of parallel jobs. You can also specify speed how fast number of parallel
+jobs are going to decay using -d option. There is last additional run on top of it
+all that appends all arguments to make call passed to -f option.
+
+Options:
+  -j NUM   Number of parallel jobs, modified and passed to make (in default 1)
+  -d OP    Divider used to decrease number of parallel jobs (in default 2)
+  -c NUM   Number of retries before paralelism is decreased (in default set to 1)
+  -f ARGS  Additional argument to be used for last single job attempt (can be
+           passed multiple times)
+EOF
+}
+
+jobs_count="1"
+jobs_divider="2"
+retry_count="1"
+fallback=()
+while getopts "j:d:c:f:h" opt; do
+	case "$opt" in
+		j)
+			jobs_count="$OPTARG"
+			;;
+		d)
+			jobs_divider="$OPTARG"
+			;;
+		c)
+			retry_count="$OPTARG"
+			;;
+		f)
+			fallback+=("$OPTARG")
+			;;
+		h)
+			help
+			exit 0
+			;;
+		*)
+			usage
+			exit 1
+			;;
+	esac
+done
+shift $((OPTIND - 1))
+
+
+while [ "$jobs_count" -gt 1 ]; do
+	while [ "$retry_count" -gt 0 ]; do
+		if make -j "$jobs_count" "$@"; then
+			exit 0
+		fi
+		retry_count="$((retry_count - 1))"
+	done
+	jobs_count="$((jobs_count / jobs_divider))"
+done
+
+make "$@" "${fallback[@]}"