commit 3eaf7cd3393ae66757c0e91636a5b44ac15cae57 Author: git Date: Wed Apr 27 21:51:59 2022 -0700 backup script (local only) diff --git a/backup/__init__.py b/backup/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backup/__pycache__/__init__.cpython-310.pyc b/backup/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..5eb2673 Binary files /dev/null and b/backup/__pycache__/__init__.cpython-310.pyc differ diff --git a/backup/__pycache__/exceptions.cpython-310.pyc b/backup/__pycache__/exceptions.cpython-310.pyc new file mode 100644 index 0000000..c61f430 Binary files /dev/null and b/backup/__pycache__/exceptions.cpython-310.pyc differ diff --git a/backup/__pycache__/repo.cpython-310.pyc b/backup/__pycache__/repo.cpython-310.pyc new file mode 100644 index 0000000..f4e55b5 Binary files /dev/null and b/backup/__pycache__/repo.cpython-310.pyc differ diff --git a/backup/__pycache__/server.cpython-310.pyc b/backup/__pycache__/server.cpython-310.pyc new file mode 100644 index 0000000..e4d73f7 Binary files /dev/null and b/backup/__pycache__/server.cpython-310.pyc differ diff --git a/backup/detail/__init__.py b/backup/detail/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backup/detail/__pycache__/__init__.cpython-310.pyc b/backup/detail/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..a56399a Binary files /dev/null and b/backup/detail/__pycache__/__init__.cpython-310.pyc differ diff --git a/backup/detail/__pycache__/server_local.cpython-310.pyc b/backup/detail/__pycache__/server_local.cpython-310.pyc new file mode 100644 index 0000000..32eb241 Binary files /dev/null and b/backup/detail/__pycache__/server_local.cpython-310.pyc differ diff --git a/backup/detail/server_local.py b/backup/detail/server_local.py new file mode 100644 index 0000000..0981c89 --- /dev/null +++ b/backup/detail/server_local.py @@ -0,0 +1,26 @@ +from pathlib import Path + +class LocalServer: + """ + Backup server on localhost + """ + + def __init__(self, params: dict): + """ + Constructor + """ + self.path = Path(params["path"]) + + def repo_path(self, repo_name: str) -> str: + """ + Returns a full path to a given repo + """ + return str( self.path / repo_name ) + + @staticmethod + def is_accessible() -> bool: + """ + Tests the connection to the server. + Since this is localhost, always return True + """ + return True diff --git a/backup/exceptions.py b/backup/exceptions.py new file mode 100644 index 0000000..7015db5 --- /dev/null +++ b/backup/exceptions.py @@ -0,0 +1,16 @@ +""" +Custom errors +""" + +from subprocess import CompletedProcess + +class BorgError(RuntimeError): + """ + Exception happened during borg command + """ + + def __init__( self, command: str, run: CompletedProcess ): + super().__init__(self) + self.command = command + self.stdout = str(run.stdout) + self.stderr = str(run.stderr) diff --git a/backup/repo.py b/backup/repo.py new file mode 100644 index 0000000..1cf4a40 --- /dev/null +++ b/backup/repo.py @@ -0,0 +1,162 @@ +""" +Declares a borg repository +""" + +import subprocess +from typing import List, Optional +from datetime import datetime + +import json +import yaml + +from backup.server import Server +from backup.exceptions import BorgError + +class Repo: + """ + Interface to a borg repository + """ + + def __init__(self, name: str, dirs: List[str]): + """ + Constructor. + + Parameters: + - name: name of the repo. No white spaces allowed. + - dirs: list of directories to backup + """ + self.name = name + self.dirs = dirs + + self.exclude_patterns: Optional[ List[str] ] = None + self.append_only = False + + self.prune = True + self.keep_daily = 7 + self.keep_weekly = 4 + self.keep_monthly = 6 + + def set_exclude_patterns(self, exclude_patterns: Optional[ List[str] ]): + """ + Sets exclude patterns + """ + self.exclude_patterns = exclude_patterns + + def set_append_only(self, append_only: bool): + """ + Sets (or remove) the "append_only" flag on the repo + """ + self.append_only = append_only + + def set_pruning(self, + do_prune: bool, + *, + keep_daily: int = 7, + keep_weekly: int = 4, + keep_monthly: int = 6): + """ + Sets up the pruning strategy. + + Parameters: + - do_prune: if True, prunes periodically the repo + - keep_daily: number of backups to keep daily (not used if do_prune is False) + - keep_weekly: number of backups to keep weekly (not used if do_prune is False) + - keep_monthly: number of backups to keep monthly (not used if do_prune is False) + """ + + def exists( self, server: Server ) -> bool: + """ + Checks if the repo exists on the server + """ + repo_path = server.repo_path(self.name) + + run = subprocess.run([ + "borg", + "check", + repo_path + ], check=False) + return run.returncode == 0 + + def init_or_config( self, server: Server ): + """ + Inits the repository (if not already done), + changes its configuration otherwise + """ + repo_path = server.repo_path(self.name) + + if not self.exists( server ): + run = subprocess.run([ + "borg", "init", + repo_path, + "--encryption", "none" + ], check=False, capture_output=True) + if run.returncode != 0: + raise BorgError("init", run) + + # configure append_only + run = subprocess.run([ + "borg", "config", + repo_path, + "append_only", ("1" if self.append_only else "0") + ], check=False, capture_output=True) + if run.returncode != 0: + raise BorgError("config (append_only)", run) + + def backup( self, server: Server ) -> dict: + """ + Creates a backup archive. + Returns a dict of stats about the archive. + """ + cli = ["borg", "create", + "--stats", "--json", + "--compression", "lz4", + "--exclude-caches"] + if self.exclude_patterns is not None: + for pattern in self.exclude_patterns: + cli.append("--exclude") + cli.append(pattern) + + repo_path = server.repo_path(self.name) + archive_name = datetime.now().isoformat() + cli.append(f"{ repo_path }::{ archive_name }") + + for directory in self.dirs: + cli.append(directory) + + run = subprocess.run(cli, check=False, capture_output=True) + if run.returncode != 0: + raise BorgError("create", run) + return json.loads(run.stdout) + + +def load_repo_from_dict( params: dict ) -> Repo: + """ + Creates a Repo from a parameters dict + """ + repo = Repo( params["name"], params["dirs"] ) + + # replace defaults + if "exclude_patterns" in params: + repo.set_exclude_patterns( params["exclude_patterns"] ) + + if "append_only" in params: + repo.set_append_only( params["append_only"] ) + + if "prune" in params: + if params["prune"] is dict: + repo.set_pruning(True, + keep_daily = params["prune"]["keep_daily"], + keep_weekly = params["prune"]["keep_weekly"], + keep_monthly = params["prune"]["keep_monthly"]) + else: + repo.set_pruning(False) + + return repo + +def load_repo_from_file( filename: str ) -> Repo: + """ + Creates a Repo from a yaml config file + """ + with open( filename, 'r', encoding="UTF-8" ) as file: + params = yaml.safe_load( file ) + return load_repo_from_dict( params ) diff --git a/backup/server.py b/backup/server.py new file mode 100644 index 0000000..75645e2 --- /dev/null +++ b/backup/server.py @@ -0,0 +1,62 @@ +""" +Declares the Borg backup server class, and functions to create servers from file +""" + +import yaml + +from backup.detail.server_local import LocalServer + +class Server: + """ + Interface for abstracting multiple types of Borg servers. + """ + + def __init__(self, server_type: str, params: dict): + """ + Constructor. + + Parameters: + type: server type + params: parameters + """ + if server_type == "local": + self.server = LocalServer(params) + else: + raise AssertionError("Unknown server type") + + def repo_path(self, repo_name: str) -> str: + """ + Returns a full path to a given repo + """ + return self.server.repo_path( repo_name ) + + def is_accessible(self) -> bool: + """ + Tests the connection to the server. + + Returns: + - True if the server is ready to be connected to + - False otherwise + """ + return self.server.is_accessible() + + +def load_server_from_dict( params: dict ) -> Server: + """ + Creates a Server from a parameters dict + """ + assert "type" in params + server_type = params["type"] + + if server_type == "local": + return Server("local", params) + + raise AssertionError("Unknown server type") + +def load_server_from_file( filename: str ) -> Server: + """ + Creates a Server from a yaml config file + """ + with open( filename, 'r', encoding="UTF-8" ) as file: + params = yaml.safe_load( file ) + return load_server_from_dict( params ) diff --git a/conf/backups/test.yml b/conf/backups/test.yml new file mode 100644 index 0000000..be68e42 --- /dev/null +++ b/conf/backups/test.yml @@ -0,0 +1,11 @@ +name: test +append_only: false +dirs: + - '/tmp/backup/src' +exclude_patterns: + - '/home/*/.cache/*' + - '/var/tmp/*' +prune: + keep_daily: 7 + keep_weekly: 4 + keep_monthly: 6 diff --git a/conf/servers/localhost.yml b/conf/servers/localhost.yml new file mode 100644 index 0000000..b6370f5 --- /dev/null +++ b/conf/servers/localhost.yml @@ -0,0 +1,3 @@ +name: "localhost" +type: local +path: "/tmp/backup/tgt" diff --git a/run_backup.py b/run_backup.py new file mode 100755 index 0000000..f765f0e --- /dev/null +++ b/run_backup.py @@ -0,0 +1,75 @@ +#!/bin/python3 +""" +Automated backup script. +""" + +import argparse +from pathlib import Path +import os + +from typing import ( + Callable, + List +) + +import yaml + +from backup.server import Server, load_server_from_file +from backup.repo import Repo, load_repo_from_file + +############# +# Arguments # +############# + +parser = argparse.ArgumentParser(description="Automated backup script") +parser.add_argument('--config_dir', + metavar="DIR", + default="./conf", + help="Directory for configuration files") + +args = parser.parse_args() + +############# +# Functions # +############# + +def list_yml_files_in_dir( directory: Path ) -> List[str]: + """ + Lists the YAML files in a given directory. + """ + files: List[str] = [] + with os.scandir( directory ) as iterator: + for entry in iterator: + if not entry.is_file(): + continue + if entry.name.startswith('.'): + continue + if not entry.name.endswith('.yml'): + continue + + files.append(entry.path) + return files + +########## +# Script # +########## + +config_dir = Path(args.config_dir) + +# load servers +server_dir = config_dir / "servers" +servers: List[Server] = [] +for server_file in list_yml_files_in_dir( server_dir ): + servers.append( load_server_from_file( server_file ) ) + +# load repos +repos_dir = config_dir / "backups" +repos: List[Repo] = [] +for repo_file in list_yml_files_in_dir( repos_dir ): + repos.append( load_repo_from_file( repo_file ) ) + +# init backups +for server in servers: + for repo in repos: + repo.init_or_config( server ) + print(repo.backup( server ))