From 3eaf7cd3393ae66757c0e91636a5b44ac15cae57 Mon Sep 17 00:00:00 2001 From: git Date: Wed, 27 Apr 2022 21:51:59 -0700 Subject: [PATCH] backup script (local only) --- backup/__init__.py | 0 backup/__pycache__/__init__.cpython-310.pyc | Bin 0 -> 140 bytes backup/__pycache__/exceptions.cpython-310.pyc | Bin 0 -> 736 bytes backup/__pycache__/repo.cpython-310.pyc | Bin 0 -> 4464 bytes backup/__pycache__/server.cpython-310.pyc | Bin 0 -> 2036 bytes backup/detail/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 147 bytes .../__pycache__/server_local.cpython-310.pyc | Bin 0 -> 1107 bytes backup/detail/server_local.py | 26 +++ backup/exceptions.py | 16 ++ backup/repo.py | 162 ++++++++++++++++++ backup/server.py | 62 +++++++ conf/backups/test.yml | 11 ++ conf/servers/localhost.yml | 3 + run_backup.py | 75 ++++++++ 15 files changed, 355 insertions(+) create mode 100644 backup/__init__.py create mode 100644 backup/__pycache__/__init__.cpython-310.pyc create mode 100644 backup/__pycache__/exceptions.cpython-310.pyc create mode 100644 backup/__pycache__/repo.cpython-310.pyc create mode 100644 backup/__pycache__/server.cpython-310.pyc create mode 100644 backup/detail/__init__.py create mode 100644 backup/detail/__pycache__/__init__.cpython-310.pyc create mode 100644 backup/detail/__pycache__/server_local.cpython-310.pyc create mode 100644 backup/detail/server_local.py create mode 100644 backup/exceptions.py create mode 100644 backup/repo.py create mode 100644 backup/server.py create mode 100644 conf/backups/test.yml create mode 100644 conf/servers/localhost.yml create mode 100755 run_backup.py diff --git a/backup/__init__.py b/backup/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backup/__pycache__/__init__.cpython-310.pyc b/backup/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5eb2673be012cb106a833bc1ea1a5c2cd8a1ea60 GIT binary patch literal 140 zcmd1j<>g`kf~9XVlR)%i5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!H%enx(7s(xO6 zZf3E5d45rLaY15os(w;pa&~C}jEaxX%*!l^kJl@xyv1RYo1apelWGStsF(>zurL4s D2Ol3S literal 0 HcmV?d00001 diff --git a/backup/__pycache__/exceptions.cpython-310.pyc b/backup/__pycache__/exceptions.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c61f4305985a78c607ff0a4bb4b39f47e1c63939 GIT binary patch literal 736 zcmYjPv2NQi5Istitq5`xw3!PHUDUyD_o65o1l@w7-Lw$0NGD1pQ6(ibHM}`Mexn`p zOI$nk7drKhwhJG?cXzzgdq;|THcJ3)b$+{)4B$5<$Hp=FfXlu@kWheRGH6&E6aj-z zAVbL?Kynps!-C7`E98^)X>u{zLARMQMjM;_!A~J)?(9OheWL~?KO0>sYbz=o8>x}m z*9Zyn^(~ z`(AZQW^y!jcbzSaqu2*D-RvFs#*CC6JgsE zb0(do9fi|^Xp*}kmQVp&#qV}Ee!aCSTIf0!xKq_}KM0h_Rt zow4=v{}(nt%$3J9V^_9H2$u-aYB@F(r$T%i%VuAh3L$lc#~?lv5RinM?dPmi(_r-( z6+Do*IVGU0q+UBgh=H@0-0xlGk$HhzF0tdX-#re`?9wk|H`J~2zt;C25qowbcJaI2 PuWSwpL5sksr^ zJzLS%dNur8ku$9K>T1u9++m~F(6wJ_yvCge8h7p+y(X`V7I#H+z<7flHC<~J~ zh&p=V@E{XeI24%Mw}reXBxdywz~QEpiR{zS`KOM>b&T{FKxj}~>ls|{ncNVJo5B)o zRfE8cTM*CYwy2djU=5`l>p4)Yj+M(BpmBMVw>Z>!Wb(GC^E0BsmoUzX#=zmr{EY`T zOtAt}yopncpW~~_6u^MTdHxnwi_`r!_Al_afjPt9fnhEd>=W1_UvoV?x8h95LC_ao zmb}7DL{hLg7>a`NP^NtoD$X978yGh*()R%-`b6`!i2(@L0~(y>CN}}0KcK}eK-d5f zE{8m|{E?D`UUegh(@c*0aM8M3VLuKe1c03BMuqEoB-Dn2UNZ2qU7_5&?rkOB!ETrd zFC9TG_&$n~1Hsqlj*=!DUIfR}QLF^LF$6J;upRXG$D@jRYxB<28gzR~LH^VR=s42B zpCq>zZSlu`H0Ht|1sNoa)1ncKMk3~Z5=V!HHIn046z;wdBcBH$Pz^Od5Mm#TrdkY> zINOa5J7&RB5e-h*Khfke-tzAsT-i;A;!2zh!}Q8QBKMVUS4vT5e-z#8qr<}S{V)zQ z->0K#8^CsK-8QB}clEqQ5wJd^lP%En$&<5(<}=TJLAhUgR{6`w->r;uTV$y>wSYIX zK()Ko(G}~CAVI!mh1>t?FZ{In{IB;*{sF43$4aDjNiv08dOwu99a6pvY>dLy51#=hQ}}GUcCzp<}2Vq!2BA6nqL~CB4#oJW^^A z$1vXUkY^}!JBQzxXEP}_yl~*%kz;Y$Q&OegNXRhZVILWG__E2m-bv6SPshV;At~L; zpjD|x1fI$YZ!JzTZ=4EFZ<)Q)3sY}1h*Ht1RGD%u)8}<;ON#%O?=%x%$9PKeKXcwv zaniBM9Jl}%o$}}DIBHNFP2Kb(kf)RhOv~F<6iqh^J5{$+VM~#XWn2XY=y9MNDloo} z#Y8{WCfc6Z9xkYTx;yRsrJF94T~*@>Hyv+}WP&oC7A69-aHpCi zTs&c=#onRwmk4}^02Q?osQO`-!YC~%YnqH1hOU0k3}%1jT6sg+t)zR&L{!k#;Xh$A zZ=#8IY)oLR^ioEKQN|)u?cfIEHm^N2mB}pb?3?mZX2Y1~edEvil)65_T@G{CWS2K$ zoi|^3?i$Xu%5!`HhF{FX%8<8E!;vM^d#!9+yfyfNG9`$l2>2l~B^I5_)tUZo5bxj` zqm%t49)vq%WH}leF99QY5T>G{l^!%M7+n>4v)dJMUmmI^sK;V(-MsU}P`>Le^n($? zOZdq+8;vvN%bs1bS4FYVuNL}y`ErT!*5-;_XA8Hba-O;}sj$m$6HvbVF3rdTijDjp z%})7uL0x5u^h@R+gCwPBZd$IvbOujjJY&k!uFg!tEX-q`=Vz7wU*z*b9FS;KM^zmB z1Ct$8(Z3^2;6IHUV_mM&>_hGMP59h6MpOG(*U{%3>k~D*+eG9LHC9I~W5?EG?bv>- z9a)NoeWs!Ju|CuOR-QvtfLHCvR=YOwy8FY-*`t`5#a3-n+jCWHeUvqLeNt1g)jVzi zzwns)CO7{><;xq0hOF#!RrlfrvcDVN6DLj2C#phFCyauv+MYBE5GBDjysDz?r6ZkM z86tLHanEbrt~!(3-R@qR#CfaR?I*(#8tfDy)-$90{k)AxDm$WXKR^vgxAJ<0%8iYU z%{#IK6RRkabVYU+Rbx=tW!TI`3&U<<#>qiZ57T6j$YGGFuIp7HR4|gMu%aa3DJo;e zp%{WsbskhhdS*Wgi`ukHQLpJWr%L;(!>!Lc!NXUVn)2of!7HUQ?pQ8!(M>h=rqR@w zbr-O0l$dGJDNk&5mZ&MCcldtc`u;HC-@iK!qUlb}_j%F>+Qj87sp`uW0w#fT z090F(NS>#u8OE7>n|3I4rOK?!iiR94e@w`eTv9VCn38(gMT~R}z|ok#!dA{M*hZad zru`-0bHg-HJB_al%l&exz0g`&Ri_GjM2&bVFXCjMD*EgpQL94w7c4T|G+bXLXsl}Y zjU%MYM7sxxkSO>t;O44EVNag~z+NoVhvPFzVeyFo>`l;Ft6FA`HFvAUVJf--jwmDOH;J6_qL{X+kBz8^~mcgiu3KC-7VNfT3BqZ|bo0#JCT8UDeX_2k;L4eupu`cdY!q*Vyqe7)H}u$0M;6X`##9 z`puoq?oa5;8kYsx$H`0kT!mDzkRIY;06|_0CLj zsBJ=Bl4*HwDX%MElRMxrq^Njo!aC6C0=*hztBUs_bAzMfMKIRihTU>-uLmnN7e6#1j@UO=cAxa$e%b^ zEer-yZlPqH*5S>?S*cPE& zYb0IK14$sfBNFvzPr`3yS|nN;UY$z*Sm_f!O476Ggd3^PrRFee%zd7ef@jk*t#Vbu zTE$Z>lS($L49~P0^DHl9_!rno4kK#2KdCe+j@&UR3RlY#)?<)L0&vLC6LjbcPcq@l z78s9^(D*h;R^cF&AoD~hx-j1nJ%nQ0wWisA$f5jDR#In4DtV?fPe!KF2^e3V@bR>$ z@<}0ibvXgpDyvb~aLvH&w%vLd;~LwZBsv+J)L#LBeGlqBRP!-3bGVQ*Pj@Ra_pa%j z#Be=Fr1AueUebuoPTk39+dfcO>1kRiJqVZ5pHYbftIgh0hU)hgH-UrYOl{I0cYn8Q zIb~Y~694wj)AFoT7v<{y!^qcYwr$q{?fN$cdU&hfX{~hRTV`aDX}llZg60pRyYPFv zvwNz>a<^3D-0WT`eP$-W?r!ZYi(h@{U~*}LIL^zwisNfUWd4DP@~?2<|C*Ejp?!3W`n_5j+;{o6N`d< zT<9#uGi!_A5NGf2OB;@*sp_yxRar_GT4*o^VngJOcQoK`Tw{kH>#5{wQ4;-KSE(xN`)#&Rqn5;x4M=WlwWf;I3LqV@W18h4B z(RAoztkqF>n`nEtP>G0ZhlnxF#(3m?SndG#SO8%aLe<+KvjHYyr1chH)PxSdkJ}EQ zfi&e_65;>q|F(HWEBe~Qn7ZhFN}6zYduHEY-8t)8m7kNAOAnvP_kL#C#Yr%hW`cmfK_prP9(QkO96 nyZQjvBhC6AHh9XvxugxgtR4L;%mq$6BLVAq>)zJq_j>;TH;d&y literal 0 HcmV?d00001 diff --git a/backup/detail/__init__.py b/backup/detail/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backup/detail/__pycache__/__init__.cpython-310.pyc b/backup/detail/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a56399ad6ddd17ac8cc7bfdc88e666feaefed6ba GIT binary patch literal 147 zcmd1j<>g`kf}bxllR)%i5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!HSenx(7s(xO6 zZf3E5d45rLaY15os(w;pa&~C}j7mu@NzBaAkB`sH%PfhH*DI*J#bJ}1pHiBWY6mi} Km8nu5P(UEmVaUgg^>!BB8e_ZC5VtM zd6~YF)=qtePQ8;8K`GjUdGe9RyZi2FCzBBYIe2lod>;|=6OFFKU>w8puR$rIXhw3n zpp<+fDpK(SQSp7Wh!i^~v)(W8Av0RDFCwfZ8eL&v9K-S_pfp)fMc^V`#46Hk1;G@1 zV2hsW=|m+uSuxdD1K8>70oX(E7^=|&yQhnh+JSWAdgrs1B0JaaTD$7aCulu^{0DJ;`20SauWhd9h0Tqh-&l9$H$v+9vMKxN zqjV4^n{Od_LvXmctp}W&!UWC-5XobnDXaE6Jve-FsE1B(EH6Z^>%{3$y5j%DyATM7 z$tAs`uYeh7Y4xrp``TRhsYq94mZgZ91`FmlbFGUM6Scm897Ag!@m&Iu9lo!IeXFqc z$MP@~CT~!7S@p7u(g2``1dnGz!D>c@eN|X)RGGGf3+;nX!&;}(7KN6E&$+S^wwW?AxFKd9qnd!K$3BQ)WUj;7 zs+ujW%^Y_2?^?0>E(V3mpq`w=JN KaT;SXjs5_PI{rog literal 0 HcmV?d00001 diff --git a/backup/detail/server_local.py b/backup/detail/server_local.py new file mode 100644 index 0000000..0981c89 --- /dev/null +++ b/backup/detail/server_local.py @@ -0,0 +1,26 @@ +from pathlib import Path + +class LocalServer: + """ + Backup server on localhost + """ + + def __init__(self, params: dict): + """ + Constructor + """ + self.path = Path(params["path"]) + + def repo_path(self, repo_name: str) -> str: + """ + Returns a full path to a given repo + """ + return str( self.path / repo_name ) + + @staticmethod + def is_accessible() -> bool: + """ + Tests the connection to the server. + Since this is localhost, always return True + """ + return True diff --git a/backup/exceptions.py b/backup/exceptions.py new file mode 100644 index 0000000..7015db5 --- /dev/null +++ b/backup/exceptions.py @@ -0,0 +1,16 @@ +""" +Custom errors +""" + +from subprocess import CompletedProcess + +class BorgError(RuntimeError): + """ + Exception happened during borg command + """ + + def __init__( self, command: str, run: CompletedProcess ): + super().__init__(self) + self.command = command + self.stdout = str(run.stdout) + self.stderr = str(run.stderr) diff --git a/backup/repo.py b/backup/repo.py new file mode 100644 index 0000000..1cf4a40 --- /dev/null +++ b/backup/repo.py @@ -0,0 +1,162 @@ +""" +Declares a borg repository +""" + +import subprocess +from typing import List, Optional +from datetime import datetime + +import json +import yaml + +from backup.server import Server +from backup.exceptions import BorgError + +class Repo: + """ + Interface to a borg repository + """ + + def __init__(self, name: str, dirs: List[str]): + """ + Constructor. + + Parameters: + - name: name of the repo. No white spaces allowed. + - dirs: list of directories to backup + """ + self.name = name + self.dirs = dirs + + self.exclude_patterns: Optional[ List[str] ] = None + self.append_only = False + + self.prune = True + self.keep_daily = 7 + self.keep_weekly = 4 + self.keep_monthly = 6 + + def set_exclude_patterns(self, exclude_patterns: Optional[ List[str] ]): + """ + Sets exclude patterns + """ + self.exclude_patterns = exclude_patterns + + def set_append_only(self, append_only: bool): + """ + Sets (or remove) the "append_only" flag on the repo + """ + self.append_only = append_only + + def set_pruning(self, + do_prune: bool, + *, + keep_daily: int = 7, + keep_weekly: int = 4, + keep_monthly: int = 6): + """ + Sets up the pruning strategy. + + Parameters: + - do_prune: if True, prunes periodically the repo + - keep_daily: number of backups to keep daily (not used if do_prune is False) + - keep_weekly: number of backups to keep weekly (not used if do_prune is False) + - keep_monthly: number of backups to keep monthly (not used if do_prune is False) + """ + + def exists( self, server: Server ) -> bool: + """ + Checks if the repo exists on the server + """ + repo_path = server.repo_path(self.name) + + run = subprocess.run([ + "borg", + "check", + repo_path + ], check=False) + return run.returncode == 0 + + def init_or_config( self, server: Server ): + """ + Inits the repository (if not already done), + changes its configuration otherwise + """ + repo_path = server.repo_path(self.name) + + if not self.exists( server ): + run = subprocess.run([ + "borg", "init", + repo_path, + "--encryption", "none" + ], check=False, capture_output=True) + if run.returncode != 0: + raise BorgError("init", run) + + # configure append_only + run = subprocess.run([ + "borg", "config", + repo_path, + "append_only", ("1" if self.append_only else "0") + ], check=False, capture_output=True) + if run.returncode != 0: + raise BorgError("config (append_only)", run) + + def backup( self, server: Server ) -> dict: + """ + Creates a backup archive. + Returns a dict of stats about the archive. + """ + cli = ["borg", "create", + "--stats", "--json", + "--compression", "lz4", + "--exclude-caches"] + if self.exclude_patterns is not None: + for pattern in self.exclude_patterns: + cli.append("--exclude") + cli.append(pattern) + + repo_path = server.repo_path(self.name) + archive_name = datetime.now().isoformat() + cli.append(f"{ repo_path }::{ archive_name }") + + for directory in self.dirs: + cli.append(directory) + + run = subprocess.run(cli, check=False, capture_output=True) + if run.returncode != 0: + raise BorgError("create", run) + return json.loads(run.stdout) + + +def load_repo_from_dict( params: dict ) -> Repo: + """ + Creates a Repo from a parameters dict + """ + repo = Repo( params["name"], params["dirs"] ) + + # replace defaults + if "exclude_patterns" in params: + repo.set_exclude_patterns( params["exclude_patterns"] ) + + if "append_only" in params: + repo.set_append_only( params["append_only"] ) + + if "prune" in params: + if params["prune"] is dict: + repo.set_pruning(True, + keep_daily = params["prune"]["keep_daily"], + keep_weekly = params["prune"]["keep_weekly"], + keep_monthly = params["prune"]["keep_monthly"]) + else: + repo.set_pruning(False) + + return repo + +def load_repo_from_file( filename: str ) -> Repo: + """ + Creates a Repo from a yaml config file + """ + with open( filename, 'r', encoding="UTF-8" ) as file: + params = yaml.safe_load( file ) + return load_repo_from_dict( params ) diff --git a/backup/server.py b/backup/server.py new file mode 100644 index 0000000..75645e2 --- /dev/null +++ b/backup/server.py @@ -0,0 +1,62 @@ +""" +Declares the Borg backup server class, and functions to create servers from file +""" + +import yaml + +from backup.detail.server_local import LocalServer + +class Server: + """ + Interface for abstracting multiple types of Borg servers. + """ + + def __init__(self, server_type: str, params: dict): + """ + Constructor. + + Parameters: + type: server type + params: parameters + """ + if server_type == "local": + self.server = LocalServer(params) + else: + raise AssertionError("Unknown server type") + + def repo_path(self, repo_name: str) -> str: + """ + Returns a full path to a given repo + """ + return self.server.repo_path( repo_name ) + + def is_accessible(self) -> bool: + """ + Tests the connection to the server. + + Returns: + - True if the server is ready to be connected to + - False otherwise + """ + return self.server.is_accessible() + + +def load_server_from_dict( params: dict ) -> Server: + """ + Creates a Server from a parameters dict + """ + assert "type" in params + server_type = params["type"] + + if server_type == "local": + return Server("local", params) + + raise AssertionError("Unknown server type") + +def load_server_from_file( filename: str ) -> Server: + """ + Creates a Server from a yaml config file + """ + with open( filename, 'r', encoding="UTF-8" ) as file: + params = yaml.safe_load( file ) + return load_server_from_dict( params ) diff --git a/conf/backups/test.yml b/conf/backups/test.yml new file mode 100644 index 0000000..be68e42 --- /dev/null +++ b/conf/backups/test.yml @@ -0,0 +1,11 @@ +name: test +append_only: false +dirs: + - '/tmp/backup/src' +exclude_patterns: + - '/home/*/.cache/*' + - '/var/tmp/*' +prune: + keep_daily: 7 + keep_weekly: 4 + keep_monthly: 6 diff --git a/conf/servers/localhost.yml b/conf/servers/localhost.yml new file mode 100644 index 0000000..b6370f5 --- /dev/null +++ b/conf/servers/localhost.yml @@ -0,0 +1,3 @@ +name: "localhost" +type: local +path: "/tmp/backup/tgt" diff --git a/run_backup.py b/run_backup.py new file mode 100755 index 0000000..f765f0e --- /dev/null +++ b/run_backup.py @@ -0,0 +1,75 @@ +#!/bin/python3 +""" +Automated backup script. +""" + +import argparse +from pathlib import Path +import os + +from typing import ( + Callable, + List +) + +import yaml + +from backup.server import Server, load_server_from_file +from backup.repo import Repo, load_repo_from_file + +############# +# Arguments # +############# + +parser = argparse.ArgumentParser(description="Automated backup script") +parser.add_argument('--config_dir', + metavar="DIR", + default="./conf", + help="Directory for configuration files") + +args = parser.parse_args() + +############# +# Functions # +############# + +def list_yml_files_in_dir( directory: Path ) -> List[str]: + """ + Lists the YAML files in a given directory. + """ + files: List[str] = [] + with os.scandir( directory ) as iterator: + for entry in iterator: + if not entry.is_file(): + continue + if entry.name.startswith('.'): + continue + if not entry.name.endswith('.yml'): + continue + + files.append(entry.path) + return files + +########## +# Script # +########## + +config_dir = Path(args.config_dir) + +# load servers +server_dir = config_dir / "servers" +servers: List[Server] = [] +for server_file in list_yml_files_in_dir( server_dir ): + servers.append( load_server_from_file( server_file ) ) + +# load repos +repos_dir = config_dir / "backups" +repos: List[Repo] = [] +for repo_file in list_yml_files_in_dir( repos_dir ): + repos.append( load_repo_from_file( repo_file ) ) + +# init backups +for server in servers: + for repo in repos: + repo.init_or_config( server ) + print(repo.backup( server ))