Skip to content

GitLab

Verified Commit ba8520b8 authored by Michal Mladek's avatar Michal Mladek
Browse files

Merge branch 'backup-file-is-too-large'

parents 005d44d6 09dd20cd
Pipeline #36676 passed with stage
in 39 seconds
Hide whitespace changes
Inline Side-by-side
src/ssbackups.py
View file @ ba8520b8
......@@ -29,7 +29,7 @@ It always returns exit code :
When exit code is not zero than every time will appear JSON in stdout in the format:
{"detail": "lorem ipsum, lorem ipsum...", "code": -1, "type": "ArgumentError"}
{"detail": "lorem ipsum, lorem ipsum...", "code": -1}
It always return additionaly data in JSON.
"""
......@@ -43,9 +43,6 @@ from argparse import ArgumentParser
from subprocess import PIPE, Popen, check_output, CalledProcessError
ERROR_TYPES = ('ClientException', 'ArgumentTypeError', 'ArgumentError', 'CalledProcessError', 'UnknownException')
###
# error code meanings
# - Remote API error
......@@ -62,6 +59,8 @@ ERR_CODE_ENCRYPT = -5
ERR_CODE_DECRYPT = -6
# - Password from prompt mismatched
ERR_PASSWD_MISMATCHED = -7
# - Max file size reached
ERR_CODE_MAX_FILE_SIZE = -8
# - Unknown error
ERR_CODE_UNKNOWN = -10
......@@ -71,14 +70,11 @@ class SSBackupsException(Exception):
General client exception for this module. It has 3 attributtes.
* detail -> verbal explanation of the exception,
* code -> a number expressing type of exception and a exit code for this script.
* type -> real type of the exception -> builtin exception types are mapped to SSBackupsException,
it can be any value of ERROR_TYPES variable"""
"""
def __init__(self, *args):
super(SSBackupsException, self).__init__(*args)
self.detail = args[0]
self.code = ERR_CODE_API_CALL if len(args) == 1 else args[1]
self.type = ERROR_TYPES[0] if len(args) <= 2 else args[2]
def type_filepath(value):
......@@ -89,7 +85,7 @@ def type_filepath(value):
"""
if os.path.isfile(value):
return value
raise SSBackupsException("Given path is not the existing path to any file!", ERR_CODE_ARGS, ERROR_TYPES[1])
raise SSBackupsException("Given path is not the existing path to any file!", ERR_CODE_ARGS)
def type_target_path(value):
......@@ -99,9 +95,9 @@ def type_target_path(value):
:return: a directory path
"""
if not value.split(os.path.sep)[-1]:
raise SSBackupsException("Given file path needn't slash at the end of the path!", ERR_CODE_ARGS, ERROR_TYPES[1])
raise SSBackupsException("Given file path needn't slash at the end of the path!", ERR_CODE_ARGS)
if not os.path.isdir(os.path.dirname(value)):
raise SSBackupsException("Given file path has no existing directory!", ERR_CODE_ARGS, ERROR_TYPES[1])
raise SSBackupsException("Given file path has no existing directory!", ERR_CODE_ARGS)
return value
......@@ -128,7 +124,7 @@ def type_url(value):
# count of parts of URL is not valid
error = True
if error:
raise SSBackupsException("Bad value of URL!", ERR_CODE_ARGS, ERROR_TYPES[1])
raise SSBackupsException("Bad value of URL!", ERR_CODE_ARGS)
return value
......@@ -156,6 +152,8 @@ API_MAPPING = {
)
}
CONNECTION_TIMEOUT = 10
def get_password(tty_recheck=False):
"""
......@@ -174,8 +172,7 @@ def get_password(tty_recheck=False):
if tty_recheck and (passwd != getpass.getpass('Password for backup (again): ')):
raise SSBackupsException(
"Passwords inserted from prompt mismatched!",
ERR_PASSWD_MISMATCHED,
ERROR_TYPES[0],
ERR_PASSWD_MISMATCHED
)
else:
# Read stdin directly when the program is triggered using pipe
......@@ -234,14 +231,14 @@ def encrypt_backup(backup, password):
"Command << {cmd} >> failed having this on stderr << {error} >>.".format(
cmd=cmd,
error=error
), ERR_CODE_ENCRYPT, ERROR_TYPES[4])
), ERR_CODE_ENCRYPT)
except CalledProcessError as exc:
raise SSBackupsException(
"Command << {cmd} >> ended with << return code = {return_code} >> and with output << {output} >>.".format(
cmd=exc.cmd,
return_code=exc.returncode,
output=str(exc.output)
), ERR_CODE_ENCRYPT, ERROR_TYPES[3])
), ERR_CODE_ENCRYPT)
def decrypt_backup(target, backup, password):
......@@ -287,14 +284,14 @@ def decrypt_backup(target, backup, password):
"Command << {cmd} >> failed having this on stderr << {error} >>.".format(
cmd=cmd,
error=error
), ERR_CODE_DECRYPT, ERROR_TYPES[4])
), ERR_CODE_DECRYPT)
except CalledProcessError as exc:
raise SSBackupsException(
"Command << {cmd} >> ended with << return code = {return_code} >> and with output << {output} >>.".format(
cmd=exc.cmd,
return_code=exc.returncode,
output=str(exc.output)
), ERR_CODE_DECRYPT, ERROR_TYPES[3])
), ERR_CODE_DECRYPT)
def get_registration_code(reg_code_path):
......@@ -309,11 +306,11 @@ def get_registration_code(reg_code_path):
raise SSBackupsException(
"Attempt to get registration code failed with message << {message} >>".format(
message=exc.message
), ERR_CODE_REG_CODE, ERROR_TYPES[3])
), ERR_CODE_REG_CODE)
# pylint: disable=too-many-arguments
def call_rest_api(reg_code, url, action, backup_id=None, backup=None, connection_timeout=10):
def call_rest_api(reg_code, url, action, backup_id=None, backup=None, fail=True, content_type=False):
"""
Wrapper to call remote API
:param reg_code: a router registration code
......@@ -321,7 +318,8 @@ def call_rest_api(reg_code, url, action, backup_id=None, backup=None, connection
:param action: one of 'list', 'create', 'retrieve', 'delete', 'on-demand'
:param backup_id: id of backup stored in remote database
:param backup: a file path of backup where it is stored on local FS
:param connection_timeout: a network layer timeout in seconds
:param fail: flag says use option --fail to return exit code when HTTP 4xx and 5xx is returned
:param content_type: flag says use option -w '%{http_code}\n%{content_type}' in curl command
:return: response (in most cases JSON = action must be in ('list', 'create', 'ondemand'))
:raises: SSBackupsException: when anything went wrong
"""
......@@ -339,10 +337,14 @@ def call_rest_api(reg_code, url, action, backup_id=None, backup=None, connection
# build command curl to able to call SSBackups REST API
# --fail triggers exit code of curl to appear when remote HTTP source returns 4xx or 5xx HTTP status codes.
###
cmd = [
"/usr/bin/curl", "--fail", "-X", method, "-m", str(connection_timeout),
"-H", "Accept:application/json", "-H", "Authorization:Token %s" % reg_code
] + form_data + [rest_api_url]
cmd = ["/usr/bin/curl"]
if fail:
cmd.append("--fail")
if content_type:
cmd += ["-w", "\n%{http_code}\n%{content_type}"]
cmd += ["-X", method, "-m", str(CONNECTION_TIMEOUT),
"-H", "Accept:application/json", "-H", "Authorization:Token %s" % reg_code
] + form_data + [rest_api_url]
try:
###
# stderr=open('/dev/null','w') is here to hide progress meter
......@@ -354,17 +356,91 @@ def call_rest_api(reg_code, url, action, backup_id=None, backup=None, connection
cmd=exc.cmd,
return_code=exc.returncode,
output=str(exc.output)
), ERR_CODE_API_CALL, ERROR_TYPES[0])
), ERR_CODE_API_CALL)
def backup_list(reg_code, url):
"""
Wrapper to action list and its call_rest_api
:param reg_code: router registration code(16 bytes length)
:param url: url of remote REST API
:return: list of dictionaries parsed from JSON response [{},{},...{}]
:raises: SSBackupsException: when anything went wrong
"""
res = call_rest_api(
reg_code,
url,
'list'
)
try:
return json.loads(res)
except Exception as exc:
raise SSBackupsException(
"JSON decoding list of backups ended with this error << {msg} >>.".format(
msg=exc.message
),
ERR_CODE_API_CALL
)
def backup_delete(reg_code, url, backup_id):
"""
Wrapper to action delete and its call_rest_api
:param reg_code: router registration code(16 bytes length)
:param url: url of remote REST API
:param backup_id: id of backup stored in remote database
:return: None
:raises: SSBackupsException: when anything went wrong
"""
res = call_rest_api(
reg_code,
url,
'delete',
backup_id=backup_id,
fail=False,
content_type=True
)
status_code = res.split('\n')[-2]
if int(status_code) / 100 != 2:
raise SSBackupsException(
"Could not delete backup.",
ERR_CODE_API_CALL
)
def backup_ondemand(reg_code, url, backup_id):
"""
Wrapper to action ondemand and its call_rest_api
:param reg_code: router registration code(16 bytes length)
:param url: url of remote REST API
:param backup_id: id of backup stored in remote database
:return: None
:raises: SSBackupsException: when anything went wrong
"""
res = call_rest_api(
reg_code,
url,
'ondemand',
backup_id=backup_id,
fail=False,
content_type=True
)
status_code = res.split('\n')[-2]
if int(status_code) / 100 != 2:
raise SSBackupsException(
"Could not mark backup as on-demand.",
ERR_CODE_API_CALL
)
def backup_create(reg_code, url, backup, password, connection_timeout=10):
def backup_create(reg_code, url, backup, password):
"""
Wrapper to action create and its call_rest_api
:param reg_code: router registration code(16 bytes length)
:param url: url of remote REST API
:param backup: a file path to backup
:param password: passphrase for gpg to encrypt backup with AES256
:param connection_timeout: network layer timeout in seconds
:return: string: JSON response {'id': n}, where n is positive integer
:raises: SSBackupsException: when anything went wrong
"""
......@@ -376,15 +452,51 @@ def backup_create(reg_code, url, backup, password, connection_timeout=10):
url,
'create',
backup=backup,
connection_timeout=connection_timeout
fail=False,
content_type=True
)
status_code, content_type = res.split('\n')[-2:]
res = ''.join(res.split('\n')[0:-2])
if int(status_code) / 100 == 2:
try:
return json.loads(res)
except Exception as exc:
raise SSBackupsException(
"JSON decoding response on create call ended with this error << {msg} >>.".format(
msg=exc.message
),
ERR_CODE_API_CALL
)
remove_encrypted_backup(backup)
return res
if content_type == 'application/json':
if status_code == '400':
validation_error = json.loads(res)
if 'payload' in validation_error:
if [
err for err in validation_error['payload'] if
err == 'backup reaches or crosses over the limit max file size']:
raise SSBackupsException(
"Max file size reached.",
ERR_CODE_MAX_FILE_SIZE
)
if content_type == 'text/html':
if status_code == '413':
raise SSBackupsException(
"Max file size reached.",
ERR_CODE_MAX_FILE_SIZE
)
raise SSBackupsException(
"Connection error",
ERR_CODE_API_CALL
)
def backup_retrieve(reg_code, url, backup_id, target, password, connection_timeout=10):
def backup_retrieve(reg_code, url, backup_id, target, password):
"""
Wrapper to action retrieve and its call_rest_api
:param reg_code: router registration code(16 bytes length)
......@@ -392,7 +504,6 @@ def backup_retrieve(reg_code, url, backup_id, target, password, connection_timeo
:param backup_id: id of backup stored in remote database
:param target: a file path where backup should be stored on local FS
:param password: passphrase for gpg to decrypt backup
:param connection_timeout: network layer timeout in seconds
:return: None
:raises: SSBackupsException: when anything went wrong
"""
......@@ -400,8 +511,7 @@ def backup_retrieve(reg_code, url, backup_id, target, password, connection_timeo
reg_code,
url,
'retrieve',
backup_id=backup_id,
connection_timeout=connection_timeout
backup_id=backup_id
)
decrypt_backup(target, res, password or get_password())
......@@ -509,72 +619,59 @@ if __name__ == '__main__':
# get arguments
cli_args = parser.parse_args()
CONNECTION_TIMEOUT = cli_args.connection_timeout
# get registration code first because everytime is needed
registration_code = get_registration_code(cli_args.reg_code_path)
if cli_args.action == 'create':
response = None
if cli_args.action == 'create':
# returns JSON {"id": x}
response = backup_create(
registration_code,
cli_args.url,
cli_args.backup,
cli_args.passwd,
connection_timeout=cli_args.connection_timeout
cli_args.passwd
)
elif cli_args.action == 'retrieve':
backup_retrieve(
registration_code,
cli_args.url,
cli_args.backup_id,
cli_args.target,
cli_args.passwd,
connection_timeout=cli_args.connection_timeout
cli_args.passwd
)
# returned text from this script should be JSON and path to backup is important information
response = '{"path": "%(path)s"}' % {'path': cli_args.target}
elif cli_args.action == 'delete':
call_rest_api(
backup_delete(
registration_code,
cli_args.url,
cli_args.action,
backup_id=cli_args.backup_id,
connection_timeout=cli_args.connection_timeout
cli_args.backup_id
)
# returned text from this script should be JSON
response = '{"deleted": true}'
elif cli_args.action == 'ondemand':
response = call_rest_api(
backup_ondemand(
registration_code,
cli_args.url,
cli_args.action,
backup_id=cli_args.backup_id,
connection_timeout=cli_args.connection_timeout
cli_args.backup_id
)
elif cli_args.action == 'list':
# list
response = call_rest_api(
response = backup_list(
registration_code,
cli_args.url,
cli_args.action,
connection_timeout=cli_args.connection_timeout
cli_args.url
)
stdout.write(response)
stdout.write(json.dumps(response))
except SSBackupsException as exc:
stdout.write(json.dumps({'detail': exc.detail, 'exit_code': exc.code, 'type': exc.type}))
stdout.write(json.dumps({'detail': exc.detail, 'exit_code': exc.code}))
exit_code = exc.code
except Exception as exc: # pylint: disable=broad-except
# handle unhandled exceptions
stdout.write(json.dumps({'detail': exc.message, 'exit_code': ERR_CODE_UNKNOWN, 'type': exc.__class__.__name__}))
stdout.write(json.dumps({'detail': exc.message, 'exit_code': ERR_CODE_UNKNOWN}))
exit_code = ERR_CODE_UNKNOWN
stdout.flush()
......
tests/test_ssbackups.py
View file @ ba8520b8
......@@ -17,7 +17,8 @@ from subprocess import CalledProcessError
from mock import patch
from ssbackups import SSBackupsException, type_target_path, type_filepath, \
remove_encrypted_backup, encrypt_backup, decrypt_backup, get_registration_code, call_rest_api
remove_encrypted_backup, encrypt_backup, decrypt_backup, get_registration_code, call_rest_api, \
backup_create, ERR_CODE_MAX_FILE_SIZE, ERR_CODE_API_CALL
# test preparation things
......@@ -25,22 +26,6 @@ JSON_500_SERVER_ERROR = '{"status":500, "detail": "Internal Server Error", "exce
JSON_201_BACKUP_CREATED = '{"id": 1}'
# pylint: disable=too-many-instance-attributes, too-few-public-methods
class Options(object):
"""
Testing preparation - it is shortcut to say what could be in CLI arguments passed to the script
"""
def __init__(self, **kwargs):
self.url = kwargs.get('url', None)
self.connection_timeout = kwargs.get('connection_timeout', None)
self.challenge_url = kwargs.get('challenge_url', None)
self.action = kwargs.get('action', None)
self.target = kwargs.get('target', None)
self.backup = kwargs.get('backup', None)
self.backup_id = kwargs.get('backup_id', None)
self.passwd = kwargs.get('passwd', None)
class TestClientException(unittest.TestCase):
"""Some tests to test base exception for the project - ClientException
"""
......@@ -64,7 +49,6 @@ class TestClientException(unittest.TestCase):
except SSBackupsException as exc:
self.assertTrue(hasattr(exc, 'detail'))
self.assertTrue(hasattr(exc, 'code'))
self.assertTrue(hasattr(exc, 'type'))
class TestTypeTargetPath(unittest.TestCase):
......@@ -117,8 +101,6 @@ class TestRemoveEncryptedBackup(unittest.TestCase):
"""Some tests to assure that os.remove was called to remove backup from disk
"""
factory_options = Options
@patch('ssbackups.os.remove')
def test_remove_backup(self, mock_remove):
"""
......@@ -222,6 +204,22 @@ class TestCallRestApi(unittest.TestCase):
Some tests to test call_rest_api raising ClientException
"""
TEST_VALUE_1 = ('{"payload":["backup reaches or crosses over the limit max file size"],'
'"detail":"client error","status":400,"exception":"ValidationError"}'
'\n400\napplication/json')
TEST_VALUE_2 = ('<html>\n<head><title>413 Request Entity Too Large</title></head>\n'
'<body bgcolor="white">\n'
'<center><h1>413 Request Entity Too Large</h1></center>\n'
'<hr><center>nginx/1.10.3</center>\n'
'</body>\n'
'</html>\n'
'\n'
'413\n'
'text/html')
TEST_VALUE_3 = 'Some unexpected text\n\n500\ntext/html'
TEST_EXCEPTION_1 = SSBackupsException("Max file size reached.", ERR_CODE_MAX_FILE_SIZE)
TEST_EXCEPTION_2 = SSBackupsException("Connection error", ERR_CODE_API_CALL)
@patch('ssbackups.check_output', side_effect=CalledProcessError(-1, 'echo "hello world!"', 'something has failed'))
def test_raise_exception_on_bad_call(self, mock_check_output):
"""
......@@ -255,3 +253,66 @@ class TestCallRestApi(unittest.TestCase):
'https://rb.turris.cz',
'list'
)
@patch('ssbackups.call_rest_api', return_value=TEST_VALUE_1)
@patch('ssbackups.encrypt_backup', return_value=None)
@patch('ssbackups.remove_encrypted_backup', return_value=None)
def test_raise_max_file_size_error(self, mock_remove_encrypted_backup, mock_encrypt_backup,
mock_call_rest_api):
"""
What happend if user try to store backup larger than max file size set by Django?
SSBackupsException exception should be raised...
:return:
"""
try:
backup_create(
"registration-code", "https://rb.turris.cz", '/tmp/backup.tar.gz', 'password')
except SSBackupsException as exc:
self.assertEqual(exc.detail, self.TEST_EXCEPTION_1.detail)
self.assertEqual(exc.code, self.TEST_EXCEPTION_1.code)
self.assertTrue(mock_remove_encrypted_backup.called)
self.assertTrue(mock_encrypt_backup.called)
self.assertTrue(mock_call_rest_api.called)
@patch('ssbackups.call_rest_api', return_value=TEST_VALUE_2)
@patch('ssbackups.encrypt_backup', return_value=None)
@patch('ssbackups.remove_encrypted_backup', return_value=None)
def test_raise_max_file_size_error_text_html(self, mock_remove_encrypted_backup,
mock_encrypt_backup, mock_call_rest_api):
"""
What happend if user try to store backup larger than max file size set by Nginx?
SSBackupsException exception should be raised...
:return:
"""
try:
backup_create(
"registration-code", "https://rb.turris.cz", '/tmp/backup.tar.gz', 'password')
except SSBackupsException as exc:
self.assertEqual(exc.detail, self.TEST_EXCEPTION_1.detail)
self.assertEqual(exc.code, self.TEST_EXCEPTION_1.code)
self.assertTrue(mock_remove_encrypted_backup.called)
self.assertTrue(mock_encrypt_backup.called)
self.assertTrue(mock_call_rest_api.called)
@patch('ssbackups.call_rest_api', return_value=TEST_VALUE_3)
@patch('ssbackups.encrypt_backup', return_value=None)
@patch('ssbackups.remove_encrypted_backup', return_value=None)
def test_raise_error_text_html(self, mock_remove_encrypted_backup, mock_encrypt_backup,
mock_call_rest_api):
"""
What happend if user try to store backup and unexptected HTTP text/html response arrives?
SSBackupsException exception should be raised...
:return:
"""
try:
backup_create(
"registration-code", "https://rb.turris.cz", '/tmp/backup.tar.gz', 'password')
except SSBackupsException as exc:
self.assertEqual(exc.detail, self.TEST_EXCEPTION_2.detail)
self.assertEqual(exc.code, self.TEST_EXCEPTION_2.code)
self.assertTrue(mock_remove_encrypted_backup.called)
self.assertTrue(mock_encrypt_backup.called)
self.assertTrue(mock_call_rest_api.called)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment