pkg: Unpacking of the package files into a temp dir

Without any further parsing yet. With lot of TODOs.

src/lib/autoload/a_04_backend.lua

@@ -20,7 +20,13 @@ along with Updater.  If not, see <http://www.gnu.org/licenses/>.
 local error = error
 local type = type
 local pairs = pairs
+local pcall = pcall
+local unpack = unpack
 local io = io
+local table = table
+local mkdtemp = mkdtemp
+local run_command = run_command
+local events_wait = events_wait
 local DBG = DBG
 local WARN = WARN
 
@@ -34,6 +40,10 @@ needed) to modify these variables.
 status_file = "/usr/lib/opkg/status"
 -- The directory where unpacked control files of the packages live
 info_dir = "/usr/lib/opkg/info/"
+-- Time after which we SIGTERM external commands. Something incredibly long, just prevent them from being stuck.
+cmd_timeout = 600000
+-- Time after which we SIGKILL external commands
+cmd_kill_timeout = 900000
 
 --[[
 Parse a single block of mail-header-like records.
@@ -150,20 +160,31 @@ function package_postprocess(status)
 	return status
 end
 
--- Get pkg_name's file's content with given suffix. Nil on error.
-local function pkg_file(pkg_name, suffix, warn)
-	local fname = info_dir .. pkg_name .. "." .. suffix
-	local f, err = io.open(fname)
+--[[
+Read the whole content of given file. Return the content, or nil and error message.
+In case of errors during the reading (instead of when opening), it calls error()
+]]
+local function slurp(filename)
+	local f, err = io.open(filename)
 	if not f then
-		if warn then WARN("Could not read ." .. suffix .. "file of " .. pkg_name .. ": " .. err) end
-		return nil
+		return nil, err
 	end
 	local content = f:read("*a")
 	f:close()
-	if not content then error("Could not read content of " .. fname) end
+	if not content then error("Could not read content of " .. filename) end
 	return content
 end
 
+-- Get pkg_name's file's content with given suffix. Nil on error.
+local function pkg_file(pkg_name, suffix, warn)
+	local fname = info_dir .. pkg_name .. "." .. suffix
+	local content, err = slurp(fname)
+	if not content then
+		WARN("Could not read ." .. suffix .. " file of " .. pkg_name .. ": " .. err)
+	end
+	return content, err
+end
+
 -- Read pkg_name's .control file and return it as a parsed block
 local function pkg_control(pkg_name)
 	local content = pkg_file(pkg_name, "control", true)
@@ -216,4 +237,76 @@ function status_parse()
 	return result
 end
 
+--[[
+Take the .ipk package (passed as the data, not as a path to a file) and unpack it
+into a temporary location somewhere under tmp_dir. If you omit tmp_dir, /tmp is used.
+
+It returns a path to a subdirectory of tmp_dir, where the package is unpacked.
+There are two further subdirectories, control and data. Data are the files to be merged
+into the system, control are the control files for the package manager.
+
+TODO:
+• Sanity checking of the package.
+• Less calling of external commands.
+]]
+function pkg_unpack(package, tmp_dir)
+	-- The first unpack goes into the /tmp
+	-- We assume s1dir returs sane names of directories ‒ no spaces or strange chars in them
+	local s1dir = mkdtemp()
+	-- The results go into the provided dir, or to /tmp if none was provided
+	-- FIXME: Sanity-check provided tmp_dir ‒ it must not contain strange chars
+	local s2dir = mkdtemp(tmp_dir)
+	-- If anything goes wrong, this is where we find the error message
+	local err
+	-- Unpack the ipk into s1dir, getting control.tar.gz and data.tar.gz
+	local function stage1()
+		events_wait(run_command(function (ecode, killed, stdout, stderr)
+			if ecode ~= 0 then
+				err = "Stage 1 unpack failed: " .. stderr
+			end
+		end, nil, package, cmd_timeout, cmd_kill_timeout, "/bin/sh", "-c", "cd '" .. s1dir .. "' && /bin/gzip -dc | /bin/tar x"))
+		-- TODO: Sanity check debian-binary
+		return err == nil
+	end
+	-- Unpack the control.tar.gz and data.tar.gz under respective subdirs in s2dir
+	local function unpack_archive(what)
+		local archive = s1dir .. "/" .. what .. ".tar.gz"
+		local dir = s2dir .. "/" .. what
+		return run_command(function (ecode, killed, stdout, stderr)
+			if ecode ~= 0 then
+				err = "Stage 2 unpack of " .. what .. " failed: " .. stderr
+			end
+		end, nil, package, cmd_timeout, cmd_kill_timeout, "/bin/sh", "-c", "mkdir -p '" .. dir .. "' && cd '" .. dir .. "' && /bin/gzip -dc <'" .. archive .. "' | /bin/tar x")
+	end
+	local function stage2()
+		events_wait(unpack_archive("control"), unpack_archive("data"))
+		return err == nil
+	end
+	-- Try-finally like construct, make sure cleanup is called no matter what
+	local success, ok = pcall(function () return stage1() and stage2() end)
+	-- Do the cleanups
+	local events = {}
+	local function remove(dir)
+		-- TODO: Would it be better to remove from within our code, without calling rm?
+		table.insert(events, run_command(function (ecode, killed, stdout, stderr)
+			if ecode ~= 0 then
+				WARN("Failed to clean up work directory ", dir, ": ", stderr)
+			end
+		end, nil, nil, cmd_timeout, cmd_kill_timeout, "/bin/rm", "-rf", dir))
+	end
+	-- Intermediate work space, not needed by the caller
+	remove(s1dir)
+	if err then
+		-- Clean up the resulting directory in case of errors
+		remove(s2dir)
+	end
+	-- Run all the cleanup removes in parallel
+	events_wait(unpack(events))
+	-- Cleanup done, call error() if anything failed
+	if not success then error(ok) end
+	if not ok then error(err) end
+	-- Everything went well. So return path to the directory where the package is unpacked
+	return s2dir
+end
+
 return _M

src/lib/interpreter.c

@@ -291,7 +291,7 @@ static int lua_mkdtemp(lua_State *L) {
 	if (param_count > 1)
 		return luaL_error(L, "Too many parameters to mkdtemp: %d", param_count);
 	const char *base_dir = "/tmp";
-	if (param_count)
+	if (param_count && !lua_isnil(L, 1))
 		base_dir = luaL_checkstring(L, 1);
 	char *template = aprintf("%s/updater-XXXXXX", base_dir);
 	char *result = mkdtemp(template);

tests/backend.lua

@@ -231,6 +231,51 @@ end
 
 local orig_status_file = B.status_file
 local orig_info_dir = B.info_dir
+local tmp_dirs = {}
+
+function test_pkg_unpack()
+	local fname = (os.getenv("S") or ".") .. "/tests/data/updater.ipk"
+	local f = io.open(fname)
+	local input = f:read("*a")
+	f:close()
+	local path = B.pkg_unpack(input)
+	-- Make sure it is deleted on teardown
+	table.insert(tmp_dirs, path)
+	-- Check list of extracted files
+	events_wait(run_command(function (ecode, killed, stdout)
+		assert_equal(0, ecode, "Failed to check the list of files")
+		assert_equal([[.
+./control
+./control/conffiles
+./control/control
+./control/postinst
+./control/prerm
+./data
+./data/etc
+./data/etc/config
+./data/etc/config/updater
+./data/etc/cron.d
+./data/etc/init.d
+./data/etc/init.d/updater
+./data/etc/ssl
+./data/etc/ssl/updater.pem
+./data/usr
+./data/usr/bin
+./data/usr/bin/updater-resume.sh
+./data/usr/bin/updater.sh
+./data/usr/bin/updater-unstuck.sh
+./data/usr/bin/updater-utils.sh
+./data/usr/bin/updater-wipe.sh
+./data/usr/bin/updater-worker.sh
+./data/usr/share
+./data/usr/share/updater
+./data/usr/share/updater/hashes
+./data/usr/share/updater/keys
+./data/usr/share/updater/keys/release.pem
+./data/usr/share/updater/keys/standby.pem
+]], stdout)
+	end, nil, nil, -1, -1, '/bin/sh', '-c', "cd '" .. path .. "' && find | sort"))
+end
 
 function setup()
 	local sdir = os.getenv("S") or "."
@@ -243,4 +288,7 @@ function teardown()
 	-- Clean up, return the original file name
 	B.status_file = orig_status_file
 	B.info_dir = orig_info_dir
+	if next(tmp_dirs) then
+		events_wait(run_command(function (ecode) assert_equal(0, ecode) end, nil, nil, -1, -1, '/bin/rm', '-rf', unpack(tmp_dirs)))
+	end
 end