Verified Commit 3a649af2 authored by Karel Koci's avatar Karel Koci 🤘
Browse files

svupdater: add ability to set auto-approve window

The idea here is that user can set when exactly updater can update
system by setting start and end periodic times. We use cron syntax for
that.

There are few hacks that we had to implement to ensure that it is not
fragile. The first issue is combination with execution and random delay.
We want to always hit the allowed window so we can install any possible
update so we need to tweak the random delay to fix that. Also we had to
make sure that updater considers all hours of the day. We run updater by
cron every two hours now with still the same two hours random delay.
This way updater can hit every possible allowed window.

The combination of delayed approvals and approval window results in wait
for delay first and only after that installation in allowed window. Thus
both of them apply.
parent aecdf2c4
......@@ -55,6 +55,14 @@ that has to be approved.
setter for `updater.autorun.auto_approve_time`. This is number of hours before
approval is automatically granted. This implements update delay.
`auto_approve_window()` and `set_auto_approve_window(window)` is getter and
setter for `updater.autorun.auto_approve_start` and
`updater.autorun.auto_approve_end`. They serve to define the window when updates
are automatically approved. There can be multiple starts and ends set (it can
be a list in UCI). The values them self are crontab like period specifier. If
window is defined toggether with `auto_approve_time` then that time has to
elapse before auto approve window is considered.
### l10n
Updater in Turris OS support multiple languages. Supported languages are provided
by additional file provided by separate package but updater-supervisor serves as
......@@ -77,7 +85,8 @@ Approvals
---------
This is a feature that simulates otherwise normal package manager execution with
user approving changes to system explicitly. This feature can also be configured
to serve as delayed updates to just delay update by some amount of time.
to serve as delayed updates to just delay update by some amount of time or to
some time window.
The implementation expects updater to be run as usual in periodic runs but
supervisor automatically configures updater to not install update unless it was
......
MAILTO=""
0 0-23/4 * * * root /usr/bin/updater-supervisor -d --autorun --rand-sleep --no-network-fail
0 0-23/2 * * * root /usr/bin/updater-supervisor -d --autorun --rand-sleep --no-network-fail
......@@ -19,6 +19,7 @@ setup(
install_requires=[
"packaging",
"distro",
"crontab",
"pyuci @ git+https://gitlab.labs.nic.cz/turris/pyuci.git",
],
)
import sys
import argparse
import datetime
import sys
from . import autorun
from .prerun import random_sleep, wait_for_network
from ._supervisor import run
from .const import PKGUPDATE_TIMEOUT, PKGUPDATE_TIMEOUT_KILL
from .const import TURRIS_REPO_HEALTH_TIMEOUT
from .const import PKGUPDATE_TIMEOUT, PKGUPDATE_TIMEOUT_KILL, TURRIS_REPO_HEALTH_TIMEOUT
from .prerun import random_sleep, wait_for_network
from .utils import daemonize, report
HELP_DESCRIPTION = """
Updater-ng supervisor used for system updating.
"""
def parse_arguments():
"Parse script arguments"
prs = argparse.ArgumentParser(description=HELP_DESCRIPTION)
prs.add_argument('--daemon', '-d', action='store_true',
help="""
Run supervisor in background (detach from terminal).
""")
prs.add_argument('--autorun', '-a', action='store_true',
help="""
Use this option when this is automatic execution. It prevents run of updater when autorun is not
enabled.
""")
prs.add_argument('--rand-sleep', const=7200, nargs='?', type=int, default=0,
help="""
Sleep random amount of the time with maximum of given number of seconds. In default two hours are
used.
""")
prs.add_argument('--wait-for-network', const=TURRIS_REPO_HEALTH_TIMEOUT, type=int, default=10,
nargs='?', help="""
Check if Turris repository is accessible before running updater. You can specify timeout in seconds
as an argument. 10 seconds is used if no argument is specified. Specify zero to disable network
check.
""")
prs.add_argument('--no-network-fail', action='store_true',
help="""
Do not run pkgupdate when network connection is not detected.
""")
prs.add_argument('--ensure-run', action='store_true',
help="""
Make sure that updater runs at least once after current time. This can be used to ensure that
latest changes are applied as soon as possible even if another instance of updater is already
running.
""")
prs.add_argument('--quiet', '-q', action='store_true',
help="""
Don't print pkgupdate's output to console. But still print supervisor output.
""")
prs.add_argument('--timeout', default=PKGUPDATE_TIMEOUT,
help="""
Set time limit in seconds for updater execution. pkgupdate is gracefully exited when this timeout
runs out. This is protection for pkgupdate stall. In defaut one hour is set as timeout.
""")
prs.add_argument('--timeout-kill', default=PKGUPDATE_TIMEOUT_KILL,
help="""
Set time in seconds after which pkgupdate is killed. This is time from timeout. In default one
minute is used.
""")
"""Parse script arguments"""
prs = argparse.ArgumentParser(description="Updater-ng supervisor used for system updating.")
prs.add_argument(
"--daemon",
"-d",
action="store_true",
help="Run supervisor in background (detach from terminal).",
)
prs.add_argument(
"--autorun",
"-a",
action="store_true",
help="Use this option when this is automatic execution. It prevents run of updater when autorun is not enabled.",
)
prs.add_argument(
"--rand-sleep",
const=7200,
nargs="?",
type=int,
default=0,
help="Sleep random amount of the time with maximum of given number of seconds. In default two hours are used.",
)
prs.add_argument(
"--wait-for-network",
const=TURRIS_REPO_HEALTH_TIMEOUT,
type=int,
default=10,
nargs="?",
help="Check if Turris repository is accessible before running updater. You can specify timeout in seconds as an argument. 10 seconds is used if no argument is specified. Specify zero to disable network check.",
)
prs.add_argument(
"--no-network-fail",
action="store_true",
help="Do not run pkgupdate when network connection is not detected.",
)
prs.add_argument(
"--ensure-run",
action="store_true",
help="Make sure that updater runs at least once after current time. This can be used to ensure that latest changes are applied as soon as possible even if another instance of updater is already running.",
)
prs.add_argument(
"--quiet",
"-q",
action="store_true",
help="Don't print pkgupdate's output to console. But still print supervisor output.",
)
prs.add_argument(
"--timeout",
default=PKGUPDATE_TIMEOUT,
help="Set time limit in seconds for updater execution. pkgupdate is gracefully exited when this timeout runs out. This is protection for pkgupdate stall. In defaut one hour is set as timeout.",
)
prs.add_argument(
"--timeout-kill",
default=PKGUPDATE_TIMEOUT_KILL,
help="Set time in seconds after which pkgupdate is killed. This is time from timeout. In default one minute is used.",
)
return prs.parse_args()
......@@ -67,25 +74,39 @@ def main():
args = parse_arguments()
if args.autorun and not autorun.enabled():
print('Updater autorun disabled.')
print("Updater autorun disabled.")
sys.exit(0)
if args.daemon and daemonize():
return
fixednow = None
if args.rand_sleep > 0:
random_sleep(args.rand_sleep)
now = datetime.datetime.now()
# Note: the random sleep could skip the allowed window so if we detect one then we tweak sleep to hit it.
wstart, wend = autorun.auto_approve_window().next_window(now)
overlap_start = max(now, wstart)
overlap_end = min(now + datetime.timedelta(seconds=args.rand_sleep), wend)
if overlap_start >= overlap_end:
random_sleep(0, args.rand_sleep)
else:
random_sleep((overlap_start - now).total_seconds(), (overlap_end - now).total_seconds())
fixednow = min(datetime.datetime.now(), wend)
if not wait_for_network(args.wait_for_network) and args.no_network_fail:
report("There seems to be no network connection to Turris servers. Please try again later.")
sys.exit(1)
sys.exit(run(
ensure_run=args.ensure_run,
timeout=args.timeout,
timeout_kill=args.timeout_kill,
verbose=not args.quiet))
sys.exit(
run(
ensure_run=args.ensure_run,
timeout=args.timeout,
timeout_kill=args.timeout_kill,
verbose=not args.quiet,
now=fixednow,
)
)
if __name__ == '__main__':
if __name__ == "__main__":
main()
"""This module is core of udpdater-supervisor. It runs and supervise updater execution.
"""
import os
import sys
import subprocess
"""This module is core of udpdater-supervisor. It runs and supervise updater execution."""
import atexit
import datetime
import os
import signal
import errno
from threading import Thread, Lock
from . import autorun, approvals, notify, hook
from .utils import setup_alarm, report
from .const import PKGUPDATE_CMD, APPROVALS_ASK_FILE
import subprocess
import sys
import typing
from threading import Lock, Thread
from . import approvals, autorun, hook, notify
from ._pidlock import PidLock
from .const import APPROVALS_ASK_FILE, PKGUPDATE_CMD
from .utils import report, setup_alarm
class Supervisor:
"Supervisor itself."
"""Supervisor itself."""
def __init__(self, verbose):
def __init__(self, verbose: bool = False):
self.verbose = verbose
self.kill_timeout = 0
self.process = None
self.trace = None
self.trace = ""
self.trace_lock = Lock()
self._devnull = open(os.devnull, 'w')
self._stdout_thread = Thread(
target=self._stdout,
name="pkgupdate-stdout")
self._stderr_thread = Thread(
target=self._stderr,
name="pkgupdate-stderr")
self._devnull = open(os.devnull, "w")
self._stdout_thread = Thread(target=self._stdout, name="pkgupdate-stdout")
self._stderr_thread = Thread(target=self._stderr, name="pkgupdate-stderr")
atexit.register(self._at_exit)
def run(self):
"Run pkgupdate"
def run(self, now: typing.Optional[datetime.datetime] = None):
"""Run pkgupdate"""
if self.process is not None:
raise Exception("Only one call to Supervisor.run is allowed.")
self.trace = ""
# Prepare command to be run
cmd = list(PKGUPDATE_CMD)
if autorun.approvals():
cmd.append('--ask-approval=' + APPROVALS_ASK_FILE)
approved = approvals._approved()
if autorun.approvals() and (
autorun.auto_approve_time() is not None or autorun.auto_approve_window().in_window(now) is None
):
# Ask approval if approvals are enabled or when we are in approve window and there is no approve timeout
cmd.append("--ask-approval=" + str(APPROVALS_ASK_FILE))
approved = approvals._approved(now)
if approved is not None:
cmd.append('--approve=' + approved)
cmd.append("--approve=" + approved)
# Clear old dump files
notify.clear_logs()
# Open process
self.process = subprocess.Popen(
cmd,
stdin=self._devnull,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
self.process = subprocess.Popen(cmd, stdin=self._devnull, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
self._stdout_thread.start()
self._stderr_thread.start()
def join(self, timeout, killtimeout):
"Join pkgupdate execution and return exit code."
"""Join pkgupdate execution and return exit code."""
self.kill_timeout = killtimeout
# Wait for pkgupdate to exit (with timeout)
setup_alarm(self._timeout, timeout)
......@@ -79,7 +74,7 @@ class Supervisor:
if not line:
break
if self.verbose:
print(line, end='')
print(line, end="")
sys.stdout.flush()
def _stderr(self):
......@@ -91,7 +86,7 @@ class Supervisor:
if not line:
break
if self.verbose:
print(line, end='', file=sys.stderr)
print(line, end="", file=sys.stderr)
sys.stderr.flush()
def _at_exit(self):
......@@ -110,9 +105,15 @@ class Supervisor:
self.process.kill()
def run(ensure_run, timeout, timeout_kill, verbose, hooklist=None):
"""Run updater
"""
def run(
ensure_run: bool,
timeout: int,
timeout_kill: int,
verbose: bool,
hooklist=None,
now: typing.Optional[datetime.datetime] = None,
):
"""Run updater."""
pidlock = PidLock()
plown = pidlock.acquire(ensure_run)
hook.register_list(hooklist)
......@@ -124,7 +125,7 @@ def run(ensure_run, timeout, timeout_kill, verbose, hooklist=None):
pidlock.unblock()
supervisor = Supervisor(verbose=verbose)
report("Running pkgupdate")
supervisor.run()
supervisor.run(now)
exit_code = supervisor.join(timeout, timeout_kill)
if exit_code != 0:
report("pkgupdate exited with: " + str(exit_code))
......
"""Access and control functions of update approvals."""
import datetime
import os
import time
import typing
......@@ -126,7 +127,7 @@ def deny(hsh: str) -> None:
_set_stat("denied", hsh)
def _approved():
def _approved(now: typing.Optional[datetime.datetime] = None):
"""Return hash of approved plan.
If there is no approved plan then it returns None.
......@@ -135,11 +136,16 @@ def _approved():
if not const.APPROVALS_ASK_FILE.is_file() or not const.APPROVALS_STAT_FILE.is_file() or not autorun.approvals():
return None
now = now or datetime.datetime.now()
auto_grant_time = autorun.auto_approve_time()
auto_grant_window = autorun.auto_approve_window()
with const.APPROVALS_STAT_FILE.open("r") as file:
cols = file.readline().split(" ")
auto_grant_time = autorun.auto_approve_time()
if cols[1].strip() == "granted" or (
auto_grant_time and int(cols[2]) < (time.time() - (auto_grant_time * 3600))
(auto_grant_window is None or auto_grant_window.in_window(now))
and (
not auto_grant_time or auto_grant_time and (int(cols[2]) < (now.timestamp() - (auto_grant_time * 3600)))
)
):
return cols[0]
return None
......
"""Configuration of updater's automatic execution.
"""
"""Configuration of updater's automatic execution."""
import collections.abc
import datetime
import typing
import warnings
import crontab
import euci
def enabled() -> typing.Optional[bool]:
"""Returns True if updater can be automatically started by various system
utils. This includes automatic periodic execution, after-boot recovery and
other tools call to configuration aplication. This returns None if no
configuration was set so it is possible to catch no configuration case.
Relevant uci configuration is: updater.autorun.enable
"""Return True if updater can be automatically started by various system utils.
This includes automatic periodic execution, after-boot recovery and other tools call to configuration aplication.
This returns None if no configuration was set so it is possible to catch no configuration case. Relevant uci
configuration is: updater.autorun.enable
"""
with euci.EUci() as uci:
try:
return uci.get("updater", "autorun", "enable", dtype=bool)
except euci.UciExceptionNotFound:
# No option means disabled but instead of False we return None to
# allow to handle no setting situation.
return None
try:
return euci.EUci().get("updater", "autorun", "enable", dtype=bool)
except euci.UciExceptionNotFound:
# No option means disabled but instead of False we return None to
# allow to handle no setting situation.
return None
def set_enabled(enable: bool):
def set_enabled(enable: bool) -> None:
"""Set value that can be later received with enable function.
It sets uci configuration value: updater.autorun.enable
"""
with euci.EUci() as uci:
uci.set('updater', 'autorun', 'autorun')
uci.set('updater', 'autorun', 'enable', enable)
uci.set("updater", "autorun", "autorun")
uci.set("updater", "autorun", "enable", enable)
def approvals() -> bool:
"""Returns True if updater approvals are enabled.
"""Return True if updater approvals are enabled.
Relevant uci configuration is: updater.autorun.approvals
"""
with euci.EUci() as uci:
return uci.get("updater", "autorun", "approvals", dtype=bool, default=False)
return euci.EUci().get("updater", "autorun", "approvals", dtype=bool, default=False)
def set_approvals(enabled: bool):
def set_approvals(enabled: bool) -> None:
"""Set value that can later be received by enabled function.
This is relevant to uci config: updater.autorun.approvals
"""
with euci.EUci() as uci:
uci.set('updater', 'autorun', 'autorun')
uci.set('updater', 'autorun', 'approvals', enabled)
uci.set("updater", "autorun", "autorun")
uci.set("updater", "autorun", "approvals", enabled)
def auto_approve_time() -> typing.Optional[int]:
"""Returns number of hours before automatic approval is granted. If no
approval time is configured then this function returns None.
This is releavant to uci config: updater.autorun.auto_approve_time
"""Return number of hours before automatic approval is granted.
If no approval time is configured then this function returns None. This is releavant to uci config:
updater.autorun.auto_approve_time
"""
with euci.EUci() as uci:
value = uci.get("updater", "autorun", "auto_approve_time", dtype=int, default=0)
return value if value > 0 else None
value = euci.EUci().get("updater", "autorun", "auto_approve_time", dtype=int, default=0)
return value if value > 0 else None
def set_auto_approve_time(approve_time: typing.Optional[int]):
"""Sets time in hours after which approval is granted. You can provide None
or value that is less or equal to zero and in that case this feature is
disabled and if approvals are enabled only manual approve can be granted.
def set_auto_approve_time(approve_time: typing.Optional[int]) -> None:
"""Set time in hours after which approval is granted.
You can provide None or value that is less or equal to zero and in that case this feature is disabled and if
approvals are enabled only manual approve can be granted.
"""
with euci.EUci() as uci:
if approve_time and approve_time > 0:
uci.set('updater', 'autorun', 'autorun')
uci.set('updater', 'autorun', 'auto_approve_time', approve_time)
uci.set("updater", "autorun", "autorun")
uci.set("updater", "autorun", "auto_approve_time", approve_time)
else:
uci.delete("updater", "autorun", "auto_approve_time")
class ApproveWindow:
"""Description and abstraction of auto-approve window.
The window is specified by set of periodic points. The enable points open the window and disable points close it.
That is used to identify the appropriate window.
"""
def __init__(self, enables: collections.abc.Collection[str], disables: collections.abc.Collection[str]):
self.enables = set(enables)
self.disables = set(disables)
self._enables: dict[str, crontab.CronTab] = {}
self._disables: dict[str, crontab.CronTab] = {}
def _update_internal(self):
def update(srcset, resdict):
for cron in srcset - resdict.keys():
try:
resdict[cron] = crontab.CronTab(cron)
except ValueError as err:
warnings.warn(f"Invalid crontab format '{cron}': {err.args}")
for cron in resdict.keys() - srcset:
del resdict[cron]
update(self.enables, self._enables)
update(self.disables, self._disables)
def add_enable(self, entry: str):
"""Add given crontab-like entry as enable.
This is preffered way to add entry as it verifies it. It raises ValueError if entry is invalid.
"""
self._enables[entry] = crontab.CronTab(entry)
self.enables.add(entry)
def add_disable(self, entry: str):
"""Add given crontab-like entry as enable.
This is preffered way to add entry as it verifies it. It raises ValueError if entry is invalid.
"""
self._disables[entry] = crontab.CronTab(entry)
self.disables.add(entry)
def in_window(self, now: typing.Optional[datetime.datetime] = None) -> typing.Optional[datetime.datetime]:
"""Check if we are in auto-approve window.
It returns None if we are not or the end of the window if we are.
"""
self._update_internal()
now = datetime.datetime.now()
start = min(cron.previous(now) for cron in self._enables.values())
if any(cron.previous(now) < start for cron in self._disables.values()):
return None # There is disable that is closer to now than any enable so we are in disabled window
end = min(cron.next(now) for cron in self._disables.values())
return now + datetime.timedelta(seconds=end)
def next_window(
self, now: typing.Optional[datetime.datetime] = None
) -> tuple[datetime.datetime, datetime.datetime]:
"""Return closest window for auto-approve.
Two datetimes are returned. The first one is start of the window and the second one is end. If we are in window
then the first one is in the past.
"""
self._update_internal()
now = datetime.datetime.now()
start = -min(cron.previous(now) for cron in self._enables.values())
if any(cron.previous(now) <= -start for cron in self._disables.values()):
start = min(cron.next(now) for cron in self._enables.values())
startdate = now + datetime.timedelta(seconds=start)
end = min(cron.next(startdate) for cron in self._disables.values())
return startdate, startdate + datetime.timedelta(seconds=end)
def auto_approve_window() -> typing.Optional[ApproveWindow]:
"""Return configuration for approve window.
The approve window consists of periodic enables and disables. The discuite enables and disables are cron-like
periods. For any given time we can find if we are in auto-approve window by checking if closest time is for enable
and not for disable.
"""
uci = euci.EUci()
enables = uci.get("updater", "autorun", "auto_approve_start", dtype=str, list=True, default=())
disables = uci.get("updater", "autorun", "auto_approve_end", dtype=str, list=True, default=())
return ApproveWindow(enables, disables) if enables and disables else None
def set_auto_approve_window(window: typing.Optional[ApproveWindow]):
"""Set window when approvals are automatically granted.
You can provide None to disable approval window.
"""
with euci.EUci() as uci:
if window is not None and window.enables:
uci.set("updater", "autorun", "auto_approve_start", list(window.enables))
else:
uci.delete("updater", "autorun", "auto_approve_start")
if window is not None and window.disables:
uci.set("updater", "autorun", "auto_approve_end", list(window.disables))
else:
uci.delete('updater', 'autorun', 'auto_approve_time')
uci.delete("updater", "autorun", "auto_approve_end")
"""These are functions we use before we even take pid lock file. They allow
updater-supervisor to be suspended for random amount of time or it allows it to
wait for internet connection
"""These are functions we use before we even take pid lock file.
They allow updater-supervisor to be suspended for random amount of time or it allows it to wait for internet connection.
"""
import os
import time
import random
import typing
......@@ -11,33 +10,33 @@ import multiprocessing
from . import const, utils
def random_sleep(max_seconds: int):
"Sleep random amount of seconds with maximum of max_seconds"
if max_seconds is None or max_seconds <= 0: