backup script (local only)

backup/detail/server_local.py

@@ -0,0 +1,26 @@
+from pathlib import Path
+
+class LocalServer:
+    """
+    Backup server on localhost
+    """
+
+    def __init__(self, params: dict):
+        """
+        Constructor
+        """
+        self.path = Path(params["path"])
+
+    def repo_path(self, repo_name: str) -> str:
+        """
+        Returns a full path to a given repo
+        """
+        return str( self.path / repo_name )
+
+    @staticmethod
+    def is_accessible() -> bool:
+        """
+        Tests the connection to the server.
+        Since this is localhost, always return True
+        """
+        return True

backup/exceptions.py

@@ -0,0 +1,16 @@
+"""
+Custom errors
+"""
+
+from subprocess import CompletedProcess
+
+class BorgError(RuntimeError):
+    """
+    Exception happened during borg command
+    """
+
+    def __init__( self, command: str, run: CompletedProcess ):
+        super().__init__(self)
+        self.command = command
+        self.stdout = str(run.stdout)
+        self.stderr = str(run.stderr)

backup/repo.py

@@ -0,0 +1,162 @@
+"""
+Declares a borg repository
+"""
+
+import subprocess
+from typing import List, Optional
+from datetime import datetime
+
+import json
+import yaml
+
+from backup.server import Server
+from backup.exceptions import BorgError
+
+class Repo:
+    """
+    Interface to a borg repository
+    """
+
+    def __init__(self, name: str, dirs: List[str]):
+        """
+        Constructor.
+
+        Parameters:
+        - name: name of the repo. No white spaces allowed.
+        - dirs: list of directories to backup
+        """
+        self.name = name
+        self.dirs = dirs
+
+        self.exclude_patterns: Optional[ List[str] ] = None
+        self.append_only = False
+
+        self.prune = True
+        self.keep_daily   = 7
+        self.keep_weekly  = 4
+        self.keep_monthly = 6
+
+    def set_exclude_patterns(self, exclude_patterns: Optional[ List[str] ]):
+        """
+        Sets exclude patterns
+        """
+        self.exclude_patterns = exclude_patterns
+
+    def set_append_only(self, append_only: bool):
+        """
+        Sets (or remove) the "append_only" flag on the repo
+        """
+        self.append_only = append_only
+
+    def set_pruning(self,
+                    do_prune: bool,
+                    *,
+                    keep_daily:   int = 7,
+                    keep_weekly:  int = 4,
+                    keep_monthly: int = 6):
+        """
+        Sets up the pruning strategy.
+
+        Parameters:
+        - do_prune: if True, prunes periodically the repo
+        - keep_daily: number of backups to keep daily (not used if do_prune is False)
+        - keep_weekly: number of backups to keep weekly (not used if do_prune is False)
+        - keep_monthly: number of backups to keep monthly (not used if do_prune is False)
+        """
+
+    def exists( self, server: Server ) -> bool:
+        """
+        Checks if the repo exists on the server
+        """
+        repo_path = server.repo_path(self.name)
+
+        run = subprocess.run([
+            "borg",
+            "check",
+            repo_path
+            ], check=False)
+        return run.returncode == 0
+
+    def init_or_config( self, server: Server ):
+        """
+        Inits the repository (if not already done),
+        changes its configuration otherwise
+        """
+        repo_path = server.repo_path(self.name)
+
+        if not self.exists( server ):
+            run = subprocess.run([
+                "borg", "init",
+                repo_path,
+                "--encryption", "none"
+                ], check=False, capture_output=True)
+            if run.returncode != 0:
+                raise BorgError("init", run)
+
+        # configure append_only
+        run = subprocess.run([
+            "borg", "config",
+            repo_path,
+            "append_only", ("1" if self.append_only else "0")
+            ], check=False, capture_output=True)
+        if run.returncode != 0:
+            raise BorgError("config (append_only)", run)
+
+    def backup( self, server: Server ) -> dict:
+        """
+        Creates a backup archive.
+        Returns a dict of stats about the archive.
+        """
+        cli = ["borg", "create",
+               "--stats", "--json",
+               "--compression", "lz4",
+               "--exclude-caches"]
+        if self.exclude_patterns is not None:
+            for pattern in self.exclude_patterns:
+                cli.append("--exclude")
+                cli.append(pattern)
+
+        repo_path = server.repo_path(self.name)
+        archive_name = datetime.now().isoformat()
+        cli.append(f"{ repo_path }::{ archive_name }")
+
+        for directory in self.dirs:
+            cli.append(directory)
+
+        run = subprocess.run(cli, check=False, capture_output=True)
+        if run.returncode != 0:
+            raise BorgError("create", run)
+        return json.loads(run.stdout)
+
+
+def load_repo_from_dict( params: dict ) -> Repo:
+    """
+    Creates a Repo from a parameters dict
+    """
+    repo = Repo( params["name"], params["dirs"] )
+
+    # replace defaults
+    if "exclude_patterns" in params:
+        repo.set_exclude_patterns( params["exclude_patterns"] )
+
+    if "append_only" in params:
+        repo.set_append_only( params["append_only"] )
+
+    if "prune" in params:
+        if params["prune"] is dict:
+            repo.set_pruning(True,
+                             keep_daily = params["prune"]["keep_daily"],
+                             keep_weekly = params["prune"]["keep_weekly"],
+                             keep_monthly = params["prune"]["keep_monthly"])
+        else:
+            repo.set_pruning(False)
+
+    return repo
+
+def load_repo_from_file( filename: str ) -> Repo:
+    """
+    Creates a Repo from a yaml config file
+    """
+    with open( filename, 'r', encoding="UTF-8" ) as file:
+        params = yaml.safe_load( file )
+    return load_repo_from_dict( params )

backup/server.py

@@ -0,0 +1,62 @@
+"""
+Declares the Borg backup server class, and functions to create servers from file
+"""
+
+import yaml
+
+from backup.detail.server_local import LocalServer
+
+class Server:
+    """
+    Interface for abstracting multiple types of Borg servers.
+    """
+
+    def __init__(self, server_type: str, params: dict):
+        """
+        Constructor.
+
+        Parameters:
+        type: server type
+        params: parameters
+        """
+        if server_type == "local":
+            self.server = LocalServer(params)
+        else:
+            raise AssertionError("Unknown server type")
+
+    def repo_path(self, repo_name: str) -> str:
+        """
+        Returns a full path to a given repo
+        """
+        return self.server.repo_path( repo_name )
+
+    def is_accessible(self) -> bool:
+        """
+        Tests the connection to the server.
+
+        Returns:
+        - True if the server is ready to be connected to
+        - False otherwise
+        """
+        return self.server.is_accessible()
+
+
+def load_server_from_dict( params: dict ) -> Server:
+    """
+    Creates a Server from a parameters dict
+    """
+    assert "type" in params
+    server_type = params["type"]
+
+    if server_type == "local":
+        return Server("local", params)
+
+    raise AssertionError("Unknown server type")
+
+def load_server_from_file( filename: str ) -> Server:
+    """
+    Creates a Server from a yaml config file
+    """
+    with open( filename, 'r', encoding="UTF-8" ) as file:
+        params = yaml.safe_load( file )
+    return load_server_from_dict( params )

conf/backups/test.yml

@@ -0,0 +1,11 @@
+name: test
+append_only: false
+dirs:
+  - '/tmp/backup/src'
+exclude_patterns:
+  - '/home/*/.cache/*'
+  - '/var/tmp/*'
+prune:
+  keep_daily:   7
+  keep_weekly:  4
+  keep_monthly: 6

conf/servers/localhost.yml

@@ -0,0 +1,3 @@
+name: "localhost"
+type: local
+path: "/tmp/backup/tgt"

run_backup.py

@@ -0,0 +1,75 @@
+#!/bin/python3
+"""
+Automated backup script.
+"""
+
+import argparse
+from pathlib import Path
+import os
+
+from typing import (
+    Callable,
+    List
+)
+
+import yaml
+
+from backup.server import Server, load_server_from_file
+from backup.repo import Repo, load_repo_from_file
+
+#############
+# Arguments #
+#############
+
+parser = argparse.ArgumentParser(description="Automated backup script")
+parser.add_argument('--config_dir',
+                    metavar="DIR",
+                    default="./conf",
+                    help="Directory for configuration files")
+
+args = parser.parse_args()
+
+#############
+# Functions #
+#############
+
+def list_yml_files_in_dir( directory: Path ) -> List[str]:
+    """
+    Lists the YAML files in a given directory.
+    """
+    files: List[str] = []
+    with os.scandir( directory ) as iterator:
+        for entry in iterator:
+            if not entry.is_file():
+                continue
+            if entry.name.startswith('.'):
+                continue
+            if not entry.name.endswith('.yml'):
+                continue
+
+            files.append(entry.path)
+    return files
+
+##########
+# Script #
+##########
+
+config_dir = Path(args.config_dir)
+
+# load servers
+server_dir = config_dir / "servers"
+servers: List[Server] = []
+for server_file in list_yml_files_in_dir( server_dir ):
+    servers.append( load_server_from_file( server_file ) )
+
+# load repos
+repos_dir = config_dir / "backups"
+repos: List[Repo] = []
+for repo_file in list_yml_files_in_dir( repos_dir ):
+    repos.append( load_repo_from_file( repo_file ) )
+
+# init backups
+for server in servers:
+    for repo in repos:
+        repo.init_or_config( server )
+        print(repo.backup( server ))