Skip to content

GitLab

Verified Commit e84e0f04 authored by Štěpán Henek's avatar Štěpán Henek 🐻
Browse files

repository restructured + using netry points to deploy binaries

parent 55fcd1f3
Pipeline #42062 failed with stage
in 47 seconds
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
View file @ e84e0f04
......@@ -3,8 +3,8 @@ stages:
.run_script: &run_script
script:
- PYTHONPATH="$(pwd)/src" nose2 -c ci-cd/unittest.cfg
- PYTHONPATH="$(pwd)/src" pylint --rcfile=ci-cd/pylintrc src/ssbackups.py tests
- PYTHONPATH="$(pwd)/ssbackups_client" nose2 -c ci-cd/unittest.cfg
- PYTHONPATH="$(pwd)/ssbackups_client" pylint --rcfile=ci-cd/pylintrc ssbackups/*.py tests
test:python2:
image: python:2.7
......
setup.py
View file @ e84e0f04
from distutils.core import setup
from setuptools import setup
from os import path
here = path.abspath(path.dirname(__file__))
......@@ -33,6 +33,9 @@ setup(
],
keywords='server side backups',
packages=['ssbackups_client'],
package_dir={'ssbackups_client': 'src'},
scripts=['src/ssbackups.py']
entry_points={
"console_scripts": [
"ssbackups = ssbackups_client.__main__:main",
]
},
)
src/ssbackups.py → ssbackups_client/__init__.py 100755 → 100644
View file @ e84e0f04
#!/usr/bin/env python
"""
The client server-side-backups app is tool to manage your router's backups and thus it is a proxy to call remote API.
Copyright (C) 2017 CZ.NIC, z.s.p.o. (http://www.nic.cz/)
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
#
# The client server-side-backups app is tool to manage your router's backups and thus it is a proxy
# to call remote API.
#
# Copyright (C) 2017 CZ.NIC, z.s.p.o. (http://www.nic.cz/)
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
It provides these functions :
* authentication against server of server-side-backups app
......@@ -37,11 +40,9 @@ It always return additionaly data in JSON.
import os
import json
import getpass
import re
import sys
from sys import stdout, stdin
from argparse import ArgumentParser
from sys import stdin
from subprocess import PIPE, Popen, check_output, CalledProcessError
......@@ -74,68 +75,7 @@ ERR_CODE_MAX_FILE_SIZE = -8
ERR_CODE_UNKNOWN = -10
class SSBackupsException(Exception):
"""
General client exception for this module. It has 3 attributtes.
* detail -> verbal explanation of the exception,
* code -> a number expressing type of exception and a exit code for this script.
"""
def __init__(self, *args):
super(SSBackupsException, self).__init__(*args)
self.detail = args[0]
self.code = ERR_CODE_API_CALL if len(args) == 1 else args[1]
def type_filepath(value):
"""
Make sure that value is filepath or SSBackupsException is raised.
:param value: a file path
:return: a file path
"""
if os.path.isfile(value):
return value
raise SSBackupsException("Given path is not the existing path to any file!", ERR_CODE_ARGS)
def type_target_path(value):
"""
Make sure that value is file path and directory of that file exists or SSBackupsException is raised.
:param value: a directory path
:return: a directory path
"""
if not value.split(os.path.sep)[-1]:
raise SSBackupsException("Given file path needn't slash at the end of the path!", ERR_CODE_ARGS)
if not os.path.isdir(os.path.dirname(value)):
raise SSBackupsException("Given file path has no existing directory!", ERR_CODE_ARGS)
return value
def type_url(value):
"""
Make sure that value is URL in form or meaning of scheme://hostname:port
:param value: Remote URL
:return: Remote URL
"""
error = False
url_parts = value.split(':')
if url_parts[0] not in ('http', 'https'):
# URL scheme is not valid
error = True
elif 1 < len(url_parts) < 4:
if not url_parts[1].startswith('//'):
# before hostname must be double slash
error = True
if len(url_parts) == 3:
if not re.match(r"[0-9]{2,5}", url_parts[2]):
# port is not a number
error = True
else:
# count of parts of URL is not valid
error = True
if error:
raise SSBackupsException("Bad value of URL!", ERR_CODE_ARGS)
return value
DEFAULT_TIMEOUT = 10
# It maps method and URL path to required action
API_MAPPING = {
......@@ -161,7 +101,17 @@ API_MAPPING = {
)
}
CONNECTION_TIMEOUT = 10
class SSBackupsException(Exception):
"""
General client exception for this module. It has 3 attributtes.
* detail -> verbal explanation of the exception,
* code -> a number expressing type of exception and a exit code for this script.
"""
def __init__(self, *args):
super(SSBackupsException, self).__init__(*args)
self.detail = args[0]
self.code = ERR_CODE_API_CALL if len(args) == 1 else args[1]
def get_password(tty_recheck=False):
......@@ -214,8 +164,8 @@ def encrypt_backup(backup, password):
# --batch - Use batch mode. Never ask, do not allow interactive commands.
# Hide the prompt : "Reading passphrase from file descriptor 0..."
#
# --passphrase-fd 0 - Read the passphrase from file descriptor 0. Only the first line will be read.
# The passphrase will be read from STDIN.
# --passphrase-fd 0 - Read the passphrase from file descriptor 0. Only the first line will
# be read. The passphrase will be read from STDIN.
#
# --cipher-algo=AES256 - Use AES256 as cipher algorithm (default is AES128).
#
......@@ -243,7 +193,8 @@ def encrypt_backup(backup, password):
), ERR_CODE_ENCRYPT)
except CalledProcessError as exc:
raise SSBackupsException(
"Command << {cmd} >> ended with << return code = {return_code} >> and with output << {output} >>.".format(
"Command << {cmd} >> ended with << return code = {return_code} >> and with output "
"<< {output} >>.".format(
cmd=exc.cmd,
return_code=exc.returncode,
output=str(exc.output)
......@@ -270,8 +221,8 @@ def decrypt_backup(target, backup, password):
# --batch - Use batch mode. Never ask, do not allow interactive commands.
# Hide the prompt : "Reading passphrase from file descriptor 0..."
#
# --passphrase-fd 0 - Read the passphrase from file descriptor 0. Only the first line will be read.
# The passphrase will be read from STDIN.
# --passphrase-fd 0 - Read the passphrase from file descriptor 0. Only the first line will
# be read. The passphrase will be read from STDIN.
#
# -o - Write output to file.
#
......@@ -297,7 +248,8 @@ def decrypt_backup(target, backup, password):
), ERR_CODE_DECRYPT)
except CalledProcessError as exc:
raise SSBackupsException(
"Command << {cmd} >> ended with << return code = {return_code} >> and with output << {output} >>.".format(
"Command << {cmd} >> ended with << return code = {return_code} "
">> and with output << {output} >>.".format(
cmd=exc.cmd,
return_code=exc.returncode,
output=str(exc.output)
......@@ -320,7 +272,10 @@ def get_registration_code(reg_code_path):
# pylint: disable=too-many-arguments
def call_rest_api(reg_code, url, action, backup_id=None, backup=None, fail=True, content_type=False):
def call_rest_api(
reg_code, url, action, backup_id=None, backup=None, fail=True, content_type=False,
timeout=DEFAULT_TIMEOUT,
):
"""
Wrapper to call remote API
:param reg_code: a router registration code
......@@ -345,16 +300,18 @@ def call_rest_api(reg_code, url, action, backup_id=None, backup=None, fail=True,
###
# build command curl to able to call SSBackups REST API
# --fail triggers exit code of curl to appear when remote HTTP source returns 4xx or 5xx HTTP status codes.
# --fail triggers exit code of curl to appear when remote HTTP source
# returns 4xx or 5xx HTTP status codes.
###
cmd = ["/usr/bin/curl"]
if fail:
cmd.append("--fail")
if content_type:
cmd += ["-w", "\n%{http_code}\n%{content_type}"]
cmd += ["-X", method, "-m", str(CONNECTION_TIMEOUT),
"-H", "Accept:application/json", "-H", "Authorization:Token %s" % reg_code
] + form_data + [rest_api_url]
cmd += [
"-X", method, "-m", str(timeout),
"-H", "Accept:application/json", "-H", "Authorization:Token %s" % reg_code
] + form_data + [rest_api_url]
try:
###
# stderr=open('/dev/null','w') is here to hide progress meter
......@@ -367,14 +324,15 @@ def call_rest_api(reg_code, url, action, backup_id=None, backup=None, fail=True,
except CalledProcessError as exc:
raise SSBackupsException(
"Command << {cmd} >> ended with << return code = {return_code} >> and with output << {output} >>.".format(
"Command << {cmd} >> ended with << return code = {return_code} "
">> and with output << {output} >>.".format(
cmd=exc.cmd,
return_code=exc.returncode,
output=str(exc.output)
), ERR_CODE_API_CALL)
def backup_list(reg_code, url):
def backup_list(reg_code, url, timeout=DEFAULT_TIMEOUT):
"""
Wrapper to action list and its call_rest_api
:param reg_code: router registration code(16 bytes length)
......@@ -385,7 +343,8 @@ def backup_list(reg_code, url):
res = call_rest_api(
reg_code,
url,
'list'
'list',
timeout=timeout,
)
try:
return json.loads(res)
......@@ -398,7 +357,7 @@ def backup_list(reg_code, url):
)
def backup_delete(reg_code, url, backup_id):
def backup_delete(reg_code, url, backup_id, timeout=DEFAULT_TIMEOUT):
"""
Wrapper to action delete and its call_rest_api
:param reg_code: router registration code(16 bytes length)
......@@ -413,7 +372,8 @@ def backup_delete(reg_code, url, backup_id):
'delete',
backup_id=backup_id,
fail=False,
content_type=True
content_type=True,
timeout=timeout,
)
status_code = res.split('\n')[-2]
if int(status_code) // 100 != 2:
......@@ -423,7 +383,7 @@ def backup_delete(reg_code, url, backup_id):
)
def backup_ondemand(reg_code, url, backup_id):
def backup_ondemand(reg_code, url, backup_id, timeout=DEFAULT_TIMEOUT):
"""
Wrapper to action ondemand and its call_rest_api
:param reg_code: router registration code(16 bytes length)
......@@ -438,7 +398,8 @@ def backup_ondemand(reg_code, url, backup_id):
'ondemand',
backup_id=backup_id,
fail=False,
content_type=True
content_type=True,
timeout=timeout,
)
status_code = res.split('\n')[-2]
if int(status_code) // 100 != 2:
......@@ -448,7 +409,7 @@ def backup_ondemand(reg_code, url, backup_id):
)
def backup_create(reg_code, url, backup, password):
def backup_create(reg_code, url, backup, password, timeout=DEFAULT_TIMEOUT):
"""
Wrapper to action create and its call_rest_api
:param reg_code: router registration code(16 bytes length)
......@@ -467,7 +428,8 @@ def backup_create(reg_code, url, backup, password):
'create',
backup=backup,
fail=False,
content_type=True
content_type=True,
timeout=timeout,
)
status_code, content_type = res.split('\n')[-2:]
......@@ -510,7 +472,7 @@ def backup_create(reg_code, url, backup, password):
)
def backup_retrieve(reg_code, url, backup_id, target, password):
def backup_retrieve(reg_code, url, backup_id, target, password, timeout=DEFAULT_TIMEOUT):
"""
Wrapper to action retrieve and its call_rest_api
:param reg_code: router registration code(16 bytes length)
......@@ -525,169 +487,10 @@ def backup_retrieve(reg_code, url, backup_id, target, password):
reg_code,
url,
'retrieve',
backup_id=backup_id
backup_id=backup_id,
timeout=timeout,
)
decrypt_backup(target, res, password or get_password())
remove_encrypted_backup(target)
if __name__ == '__main__':
# exit code when it is sunny days
exit_code = 0
try:
parser = ArgumentParser(description=("It manages backups of Turris router. It allows to list, create, delete, "
"retrieve and mark stable router's backups. Behind the every each action "
"is remote API call. Password to encrypt or decrypt backup can be passed "
"via -w option or via prompt while command runs."))
parser.add_argument(
'-t', '--connection-timeout',
dest='connection_timeout',
default=10,
type=int,
metavar='SECONDS',
help="Timeout for http requests"
)
parser.add_argument(
'-u', '--url',
dest='url',
default='https://rb.turris.cz',
type=type_url,
metavar='URL',
help="URL of remote server-side-backups API - scheme://hostname:port"
)
parser.add_argument(
'-r', '--reg-code-path',
dest='reg_code_path',
default='/usr/share/server-uplink/registration_code',
type=type_filepath,
metavar='FILE',
help="File path to registration code"
)
parser.add_argument(
'-w', '--passwd',
dest='passwd',
default=None,
type=str,
metavar='PASSWORD',
help="Password to encrypt a backup"
)
# subparsers to dinstinguish between actions
subparsers = parser.add_subparsers(
description="Managing router backups on the server side",
help="Possible actions to do"
)
# action --list does not need additional arguments
parser_list = subparsers.add_parser('list', help="list all of the backups on the server-side")
parser_list.set_defaults(action='list')
# action --create requires backup argument
parser_create = subparsers.add_parser('create', help="encrypt and upload backup to server side")
parser_create.set_defaults(action='create')
parser_create.add_argument(
'backup',
type=type_filepath,
metavar='FILE',
help="Path to a backup file"
)
# action --delete requires backup_id argument
parser_delete = subparsers.add_parser('delete', help="delete backup forever on server side")
parser_delete.set_defaults(action='delete')
parser_delete.add_argument(
'backup_id',
type=int,
metavar='ID',
help="Id of a backup known to server-side-backups server",
)
# action --retrieve requires backup_id and target arguments
parser_retrieve = subparsers.add_parser('retrieve', help="download backup and decrypt from server side")
parser_retrieve.set_defaults(action='retrieve')
parser_retrieve.add_argument(
'backup_id',
type=int,
metavar='ID',
help="Id of a backup known to server-side-backups server",
)
parser_retrieve.add_argument(
'target',
type=type_target_path,
metavar='FILE',
help="Path to a file where backup will be saved"
)
# action --ondemand requires backup_id
parser_ondemand = subparsers.add_parser('ondemand', help="mark backup as stable on server side")
parser_ondemand.set_defaults(action='ondemand')
parser_ondemand.add_argument(
'backup_id',
type=int,
metavar='ID',
help="Id of a backup known to server-side-backups server"
)
# get arguments
cli_args = parser.parse_args()
CONNECTION_TIMEOUT = cli_args.connection_timeout
# get registration code first because everytime is needed
registration_code = get_registration_code(cli_args.reg_code_path)
response = None
if cli_args.action == 'create':
# returns JSON {"id": x}
response = backup_create(
registration_code,
cli_args.url,
cli_args.backup,
cli_args.passwd
)
elif cli_args.action == 'retrieve':
backup_retrieve(
registration_code,
cli_args.url,
cli_args.backup_id,
cli_args.target,
cli_args.passwd
)
elif cli_args.action == 'delete':
backup_delete(
registration_code,
cli_args.url,
cli_args.backup_id
)
elif cli_args.action == 'ondemand':
backup_ondemand(
registration_code,
cli_args.url,
cli_args.backup_id
)
elif cli_args.action == 'list':
response = backup_list(
registration_code,
cli_args.url
)
stdout.write(json.dumps(response))
except SSBackupsException as exc:
stdout.write(json.dumps({'detail': exc.detail, 'exit_code': exc.code}))
exit_code = exc.code
except Exception as exc: # pylint: disable=broad-except
# handle unhandled exceptions
stdout.write(json.dumps({'detail': repr(exc), 'exit_code': ERR_CODE_UNKNOWN}))
exit_code = ERR_CODE_UNKNOWN
stdout.flush()
exit(exit_code)
ssbackups_client/__main__.py 0 → 100644
View file @ e84e0f04
#
# The client server-side-backups app is tool to manage your router's backups and thus it is a proxy
# to call remote API.
#
# Copyright (C) 2017 CZ.NIC, z.s.p.o. (http://www.nic.cz/)
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
import json
import os
import re
from sys import stdout
from argparse import ArgumentParser
from . import (
SSBackupsException, ERR_CODE_ARGS, ERR_CODE_UNKNOWN, get_registration_code,
backup_create, backup_retrieve, backup_delete, backup_ondemand, backup_list,
)
def type_target_path(value):
"""
Make sure that value is file path and directory of that file exists or SSBackupsException
is raised.
:param value: a directory path
:return: a directory path
"""
if not value.split(os.path.sep)[-1]:
raise SSBackupsException(
"Given file path needn't slash at the end of the path!", ERR_CODE_ARGS)
if not os.path.isdir(os.path.dirname(value)):
raise SSBackupsException(
"Given file path has no existing directory!", ERR_CODE_ARGS)
return value
def type_filepath(value):
"""
Make sure that value is filepath or SSBackupsException is raised.
:param value: a file path
:return: a file path
"""
if os.path.isfile(value):
return value
raise SSBackupsException("Given path is not the existing path to any file!", ERR_CODE_ARGS)
def type_url(value):
"""
Make sure that value is URL in form or meaning of scheme://hostname:port
:param value: Remote URL
:return: Remote URL
"""
error = False
url_parts = value.split(':')
if url_parts[0] not in ('http', 'https'):
# URL scheme is not valid
error = True
elif 1 < len(url_parts) < 4:
if not url_parts[1].startswith('//'):
# before hostname must be double slash
error = True
if len(url_parts) == 3:
if not re.match(r"[0-9]{2,5}", url_parts[2]):
# port is not a number
error = True
else:
# count of parts of URL is not valid
error = True
if error:
raise SSBackupsException("Bad value of URL!", ERR_CODE_ARGS)
return value
def main():
# exit code when it is sunny days
exit_code = 0
try:
parser = ArgumentParser(description=(
"It manages backups of Turris router. It allows to list, create, delete, "
"retrieve and mark stable router's backups. Behind the every each action "
"is remote API call. Password to encrypt or decrypt backup can be passed "
"via -w option or via prompt while command runs.")
)
parser.add_argument(
'-t', '--connection-timeout',
dest='connection_timeout',
default=10,
type=int,
metavar='SECONDS',
help="Timeout for http requests"
)
parser.add_argument(
'-u', '--url',
dest='url',
default='https://rb.turris.cz',
type=type_url,
metavar='URL',
help="URL of remote server-side-backups API - scheme://hostname:port"
)
parser.add_argument(
'-r', '--reg-code-path',