From 67605546c54b7c440b7793b22d5263abe6a0f7bb Mon Sep 17 00:00:00 2001 From: n0m1s Date: Sat, 4 Nov 2023 18:51:41 -0700 Subject: [PATCH] WIP --- .gitignore | 5 ++ config/example.yml | 2 + makefile | 18 ++++ ptaimport/__main__.py | 22 +++++ ptaimport/config.py | 65 ++++++++++++++ ptaimport/data_importers/__init__.py | 59 ++++++++++++ ptaimport/data_importers/mock.py | 39 ++++++++ ptaimport/data_importers/modo.py | 130 +++++++++++++++++++++++++++ ptaimport/passwords/__init__.py | 73 +++++++++++++++ ptaimport/passwords/gnupass.py | 66 ++++++++++++++ ptaimport/passwords/plaintext.py | 60 +++++++++++++ requirements.txt | 3 + tests/__init__.py | 0 tests/context.py | 5 ++ tests/test_data_importers.py | 112 +++++++++++++++++++++++ tests/test_passwords.py | 128 ++++++++++++++++++++++++++ 16 files changed, 787 insertions(+) create mode 100644 .gitignore create mode 100644 config/example.yml create mode 100644 makefile create mode 100644 ptaimport/__main__.py create mode 100644 ptaimport/config.py create mode 100644 ptaimport/data_importers/__init__.py create mode 100644 ptaimport/data_importers/mock.py create mode 100644 ptaimport/data_importers/modo.py create mode 100644 ptaimport/passwords/__init__.py create mode 100644 ptaimport/passwords/gnupass.py create mode 100644 ptaimport/passwords/plaintext.py create mode 100644 requirements.txt create mode 100644 tests/__init__.py create mode 100644 tests/context.py create mode 100644 tests/test_data_importers.py create mode 100644 tests/test_passwords.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..927a013 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +**/__pycache__ + +config/config.yml + +tests/data/* diff --git a/config/example.yml b/config/example.yml new file mode 100644 index 0000000..cd21505 --- /dev/null +++ b/config/example.yml @@ -0,0 +1,2 @@ +--- + diff --git a/makefile b/makefile new file mode 100644 index 0000000..cd52137 --- /dev/null +++ b/makefile @@ -0,0 +1,18 @@ +PY=python3 + +CONFIG=config/config.yml + +# avoid doing anything when typing "make" since there's nothing to build +dummy: + +run: ${CONFIG} + ${PY} -m ptaimport $^ + +test: + ${PY} -m unittest discover + +.PHONY: dummy run test + +# if running config is not already written, copy the example one +${CONFIG}: config/example.yml + cp $< $@ diff --git a/ptaimport/__main__.py b/ptaimport/__main__.py new file mode 100644 index 0000000..edbe9cc --- /dev/null +++ b/ptaimport/__main__.py @@ -0,0 +1,22 @@ +from .config import Config + +from .passwords import Manager + + +def main(): + config_file = "config/config.yml" #TODO: take as arg + + with open(config_file) as f: + config = Config( f ) + (ok, err) = config.is_correct() + if not ok: + print(f"Configuration error: {err}") + exit(255) + + for data_import in config.data_imports(): + print(data_import) + +#------------------------------------------------------------------------------- + +if __name__ == "__main__": + main() diff --git a/ptaimport/config.py b/ptaimport/config.py new file mode 100644 index 0000000..cb7185f --- /dev/null +++ b/ptaimport/config.py @@ -0,0 +1,65 @@ +import yaml +from typing import List, Tuple +import ptaimport.passwords as pwd +import ptaimport.data_importers as importers + +class Config: + + def __init__( self, yaml_data: str ): + self.data = yaml.safe_load(yaml_data) + + + def is_correct( self ) -> Tuple[bool, str]: + """ + Verifies that the configuration is correct + + Returns: + bool: true if the config is correct + str: error message to display to the user + """ + ok = True + errors = [] + + if "passwords" not in self.data: + errors.append('Missing "passwords" section') + else: + (pwd_ok, pwd_err) = pwd.verify( + self.data["passwords"], + "passwords" + ) + ok &= pwd_ok + errors = errors + pwd_err + + if "imports" not in self.data: + errors.append('missing "imports" section') + elif type(self.data["imports"]) is not list: + errors.append('"imports" section is not a list') + else: + for i, importer in enumerate(self.data["imports"]): + (imp_ok, imp_err) = importers.verify( + importer, + f"imports[{i}]" + ) + ok &= imp_ok + errors = errors + imp_err + + return (ok, "\n".join(errors)) + + + def passwords( self ) -> pwd.PasswordManager: + """ + Returns the password manager defined in the config + """ + return pwd.Manager( self.data["passwords"] ) + + + def data_imports( self ) -> List[importers.DataImporter]: + """ + Returns the list of data importers defined in the config + """ + ret = [] + for config in self.data["imports"]: + ret.append( importers.Importer( + config + ) ) + return ret diff --git a/ptaimport/data_importers/__init__.py b/ptaimport/data_importers/__init__.py new file mode 100644 index 0000000..63c5443 --- /dev/null +++ b/ptaimport/data_importers/__init__.py @@ -0,0 +1,59 @@ +from typing import List, Tuple + + +class DataImporter( object ): + """ + An abstract data importer. + + Instanciate concrete instances via the Importer() function + """ + + def retrieve( self ) -> List[str]: + """ + Retrieve a list of transactions + """ + return [] + +from .mock import MockDataImporter +from .modo import ModoDataImporter + +#------------------------------------------------------------------------------- + +def verify( config: dict, path: str = "" ) -> Tuple[bool, List[str]]: + """ + Verifies that the configuration is correct + + Args: + config: dict with password manager configuration + path: path to the "config" dict in the main configuration file + + Returns: + bool: true if the config is correct + List[str]: list of errors if the config is not correct + """ + if "import" not in config: + return (False, [f'Missing "{path}.import" value']) + if "name" not in config: + return (False, [f'Missing "{path}.name" value']) + + importer = config["import"] + for cls in DataImporter.__subclasses__(): + if importer == cls.name(): + return cls.verify(config, path) + return (False, [f'Unknown data importer "{importer}"']) + +#------------------------------------------------------------------------------- + +def Importer( config: dict ) -> DataImporter: + """ + Instanciate an Importer instance from a given configuration + """ + if "import" not in config: + raise ValueError + if "name" not in config: + raise ValueError + importer = config["import"] + for cls in DataImporter.__subclasses__(): + if importer == cls.name(): + return cls(config) + raise ValueError diff --git a/ptaimport/data_importers/mock.py b/ptaimport/data_importers/mock.py new file mode 100644 index 0000000..b9c021e --- /dev/null +++ b/ptaimport/data_importers/mock.py @@ -0,0 +1,39 @@ +from . import DataImporter + +from typing import List, Tuple + + +class MockDataImporter( DataImporter ): + """ + Mock data importer, does not retrieve any data + + Only useful for testing + """ + + @classmethod + def name( cls ): + return "mock" + + + @classmethod + def verify( cls, config: dict, path: str ) -> Tuple[bool, List[str]]: + """ + Verifies that the configuration is correct + + Args: + config: dict with password manager configuration + path: path to the "config" dict in the main configuration file + + Returns: + bool: true if the config is correct + List[str]: list of errors if the config is not correct + """ + return (True, []) + + + def __init__( self, config: dict, **kwargs ): + pass + + + def retrieve( self ) -> List[str]: + return [] diff --git a/ptaimport/data_importers/modo.py b/ptaimport/data_importers/modo.py new file mode 100644 index 0000000..7714a9d --- /dev/null +++ b/ptaimport/data_importers/modo.py @@ -0,0 +1,130 @@ +from . import DataImporter + +import re +import requests +from typing import Tuple, List +from bs4 import BeautifulSoup + +class ModoDataImporter( DataImporter ): + """ + Modo (https://modo.coop/) data importer. + + Automatically connects to your account and retrieve transactions + """ + + @classmethod + def name( cls ): + return "modo" + + + @classmethod + def verify( cls, config: dict, path: str ) -> Tuple[bool, List[str]]: + """ + Verifies that the configuration is correct + + Args: + config: dict with password manager configuration + path: path to the "config" dict in the main configuration file + + Returns: + bool: true if the config is correct + List[str]: list of errors if the config is not correct + """ + ok = True + errors = [] + + if "user_id" not in config: + ok = False + errors.append(f'Missing value "{path}.user_id"') + + if "password_id" not in config: + ok = False + errors.append(f'Missing value "{path}.password_id"') + + return (ok, errors) + + + def __init__( self, config: dict, **kwargs ): + self.name = str(config["name"]) + self.user_id = str(config["user_id"]) + self.pwd_id = str(config["password_id"]) + + self.regex_usage = re.compile("[A-Z][a-z]+ [0-9]{4} usage details") + self.regex_money = re.compile("-?\$[0-9]+\.[0-9]{2}") + self.regex_membership_fee = re.compile("annual membership fee") + self.regex_trip = re.compile("On .*, you drove ([0-9]+) km") + + + def retrieve( self ) -> List[str]: + """ + Retrieves the list of transactions from modo website + """ + raise NotImplementedError #FIXME: actually implement + + + def _parse_invoice( self, html: str ) -> List[str]: + """ + Parses an HTML modo invoice + + This might be quite brittle and require maintenance if the modo + website ever change + """ + data = BeautifulSoup(html, "html.parser") + trips = data.find( + class_="m_invoice" + ).find( + name="div", + class_="m_i_h", + string=self.regex_usage + ).find_parent( + name="div", + id=re.compile("m_i_tier_[0-9]+") + ).find_all( + class_="m_i_detail" + ) + + ret = [] + for trip in trips: + text_tag = trip.find(class_="m_id_text") + if text_tag is None: + continue + amounts = trip.find_all( + class_="m_id_amount", + string=self.regex_money + ) + if len(amounts) != 3: + continue + + text = "".join(text_tag.text.splitlines()) + + ret.append( + amounts[0].text + + "|" + + self._parse_text(text) + ) + + return ret + + + def _parse_text( self, text: str ) -> str: + if self.regex_membership_fee.search(text): + return "fee" + + trip = self.regex_trip.search(text) + if trip: + d = trip.group(1) + return f"trip|{d}km" + + return "unknown" + + +#payload = { +# 'fMemberNumber': '', +# 'fPassword': '' +#} +#with requests.Session() as s: +# p = s.post("https://bookit.modo.coop/home/login", data=payload) +# print(p.text) + +# r = s.get("https://bookit.modo.coop/account/invoices?invoice_id=#") +# print(r.text) diff --git a/ptaimport/passwords/__init__.py b/ptaimport/passwords/__init__.py new file mode 100644 index 0000000..ed6f3cd --- /dev/null +++ b/ptaimport/passwords/__init__.py @@ -0,0 +1,73 @@ +from dataclasses import dataclass +from typing import Tuple, List, Optional + + +@dataclass +class Password: + password: Optional[str] + login: Optional[str] = None + +#------------------------------------------------------------------------------- + +class PasswordManager( object ): + """ + An abstract password manager. + + Instanciate concrete instances via the Manager() function + """ + + def get( self, identifier: str ) -> Password: + """ + Gets a password from the manager. + + This function is overriden by the concrete PasswordManager instances. + + Args: + identifier: which password to get + + Returns: + None if the password is not found, + the password otherwise + """ + return Password( None ) + +from .plaintext import PlainTextPasswordManager +from .gnupass import GNUPassPasswordManager + +#------------------------------------------------------------------------------- + +def verify( config: dict, path: str = "" ) -> Tuple[bool, List[str]]: + """ + Verifies that the configuration is correct + + Args: + config: dict with password manager configuration + path: path to the "config" dict in the main configuration file + + Returns: + bool: true if the config is correct + List[str]: list of errors if the config is not correct + """ + if "manager" not in config: + return (False, [f'Missing "{path}.manager" value']) + + manager = config["manager"] + for cls in PasswordManager.__subclasses__(): + if manager == cls.name(): + return cls.verify(config, path) + return (False, [f'Unknown password manager "{manager}"']) + +#------------------------------------------------------------------------------- + +def Manager( config: dict ) -> PasswordManager: + """ + Instanciates a PasswordManager instance from a given configuration + """ + if "manager" not in config: + raise ValueError + manager = config["manager"] + for cls in PasswordManager.__subclasses__(): + if manager == cls.name(): + return cls(config) + raise ValueError + diff --git a/ptaimport/passwords/gnupass.py b/ptaimport/passwords/gnupass.py new file mode 100644 index 0000000..c6cddae --- /dev/null +++ b/ptaimport/passwords/gnupass.py @@ -0,0 +1,66 @@ +from . import Password, PasswordManager + +import subprocess +from typing import Tuple, List + +class GNUPassPasswordManager( PasswordManager ): + """ + GNU pass password manager + https://www.passwordstore.org/ + """ + + @classmethod + def name( cls ): + return "pass" + + + @classmethod + def verify( cls, config: dict, path: str ) -> Tuple[bool, List[str]]: + """ + Verifies that the configuration is correct + + Args: + config: dict with password manager configuration + path: path to the "config" dict in the main configuration file + + Returns: + bool: true if the config is correct + List[str]: list of errors if the config is not correct + """ + return (True, []) + + + def __init__( self, config: dict ): + # no parameters + pass + + def get( self, identifier: str ) -> Password: + """ + Gets a password from the manager. + + This function is overriden by the concrete PasswordManager instances. + + Args: + identifier: which password to get + + Returns: + None if the password is not found, + the password otherwise + """ + result = subprocess.run([ + "pass", + identifier + ], + capture_output = True + ) + lines = result.stdout.decode("utf-8").splitlines() + if len(lines) == 0: + return Password( None ) + ret = Password( lines[0] ) + for i in range(1, len(lines)): + fields = lines[i].split(': ', 1) + if len(fields) != 2: + continue + if fields[0] == "login": + ret.login = str(fields[1]) + return ret diff --git a/ptaimport/passwords/plaintext.py b/ptaimport/passwords/plaintext.py new file mode 100644 index 0000000..d88c577 --- /dev/null +++ b/ptaimport/passwords/plaintext.py @@ -0,0 +1,60 @@ +from . import Password, PasswordManager + +from typing import Tuple, List, Optional + +class PlainTextPasswordManager( PasswordManager ): + """ + Passwords provided as plain-text (bad idea, but always useful) + """ + + @classmethod + def name( cls ): + return "plaintext" + + + @classmethod + def verify( cls, config: dict, path: str ) -> Tuple[bool, List[str]]: + """ + Verifies that the configuration is correct + + Args: + config: dict with password manager configuration + path: path to the "config" dict in the main configuration file + + Returns: + bool: true if the config is correct + List[str]: list of errors if the config is not correct + """ + ok = True + errors = [] + + if "passwords" not in config: + ok = False + errors.append(f'Missing section "{path}.passwords"') + elif type(config["passwords"]) is not dict: + ok = False + errors.append(f'"{path}.passwords" is not dict') + + return (ok, errors) + + + def __init__( self, config: dict ): + self.passwords = config["passwords"] + + + def get( self, identifier: str ) -> Password: + """ + Gets a password from the manager. + + This function is overriden by the concrete PasswordManager instances. + + Args: + identifier: which password to get + + Returns: + None if the password is not found, + the password otherwise + """ + if identifier not in self.passwords: + return Password( None ) + return Password( self.passwords[identifier] ) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..48217ec --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +beautifulsoup4==4.8.2 +PyYAML==6.0.1 +Requests==2.31.0 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/context.py b/tests/context.py new file mode 100644 index 0000000..c901f50 --- /dev/null +++ b/tests/context.py @@ -0,0 +1,5 @@ +import os +import sys +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +import ptaimport diff --git a/tests/test_data_importers.py b/tests/test_data_importers.py new file mode 100644 index 0000000..956ec57 --- /dev/null +++ b/tests/test_data_importers.py @@ -0,0 +1,112 @@ +from os.path import exists +import unittest + +from .context import ptaimport +from ptaimport import data_importers as imp + +class TestDataImporters( unittest.TestCase ): + """ + Tests the data_importers modules functions + """ + + def setUp( self ): + # nothing provided + self.config_none = { + } + + # name provided, but not import + self.config_noImport = { + "name": "test importer" + } + + # import provided but no name + self.config_noName = { + "import": "mock" + } + + # wrong import provided + self.config_wrong = { + "name": "test importer", + "import": "unknown" + } + + # correct + self.config_mock = { + "name": "test importer", + "import": "mock" + } + + + def test_verify( self ): + """ + Tests the data_importers.verify() function + """ + (ok, err) = imp.verify( self.config_none ) + self.assertFalse(ok) + self.assertNotEqual( len(err), 0 ) + + (ok, err) = imp.verify( self.config_noImport ) + self.assertFalse(ok) + self.assertNotEqual( len(err), 0 ) + + (ok, err) = imp.verify( self.config_noName ) + self.assertFalse(ok) + self.assertNotEqual( len(err), 0 ) + + (ok, err) = imp.verify( self.config_wrong ) + self.assertFalse(ok) + self.assertNotEqual( len(err), 0 ) + + (ok, err) = imp.verify( self.config_mock ) + self.assertTrue(ok) + self.assertEqual( len(err), 0 ) + + + def test_factory( self ): + """ + Tests the data_importers.Importer() function + """ + self.assertRaises( ValueError, imp.Importer, self.config_none ) + self.assertRaises( ValueError, imp.Importer, self.config_noImport ) + self.assertRaises( ValueError, imp.Importer, self.config_noName ) + self.assertRaises( ValueError, imp.Importer, self.config_wrong ) + imp.Importer( self.config_mock ) # should not raise + + +#------------------------------------------------------------------------------- + +from ptaimport.data_importers.modo import ModoDataImporter + +class TestModoDataImporter( unittest.TestCase ): + + def setUp( self ): + pass + + # Note: to activate this test, save a modo invoice (HTML) from your account + # to the "tests/data/" directory (as "tests/data/modo_invoice.html"), and + # also create a "modo_invoice_expected.txt" file containing the expected + # transactions to test against + @unittest.skipUnless( exists("tests/data/modo_invoice.html") + and exists("tests/data/modo_invoice_expected.txt"), + "Missing test data" ) + def test_html_parse( self ): + modo = ModoDataImporter({ + "name": "test", + "user_id": 42, + "password_id": "none" + }) + + expected_transactions = [] + with open("tests/data/modo_invoice_expected.txt") as f: + expected_transactions = f.readlines() + + actual_transactions = [] + with open("tests/data/modo_invoice.html") as f: + actual_transactions = modo._parse_invoice( f.read() ) + + self.assertEqual( len(expected_transactions), len(actual_transactions) ) + for i in range(0, len(expected_transactions)): + self.assertEqual( + expected_transactions[i].strip(), + actual_transactions[i].strip() + ) diff --git a/tests/test_passwords.py b/tests/test_passwords.py new file mode 100644 index 0000000..2f21d5f --- /dev/null +++ b/tests/test_passwords.py @@ -0,0 +1,128 @@ +import unittest + +from .context import ptaimport +from ptaimport import passwords as pwd + + +class TestPasswords( unittest.TestCase ): + """ + Tests the passwords module functions + """ + + def setUp( self ): + # no manager name given + self.config_none = { + } + + # unknown manager name given + self.config_unknown = { + "manager": "unknown" + } + + + def test_verify( self ): + """ + Tests the passwords.verify() function + """ + (ok, err) = pwd.verify( self.config_none ) + self.assertFalse(ok) + self.assertNotEqual( len(err), 0 ) + + (ok, err) = pwd.verify( self.config_unknown ) + self.assertFalse(ok) + self.assertNotEqual( len(err), 0 ) + + + def test_factory( self ): + """ + Tests the passwords.Manager() function + """ + self.assertRaises(ValueError, pwd.Manager, self.config_none ) + self.assertRaises(ValueError, pwd.Manager, self.config_unknown ) + +#------------------------------------------------------------------------------- + +class TestPlainTextPasswords( unittest.TestCase ): + """ + Tests the PlainTextPasswordManager class + """ + + def setUp( self ): + # no passwords given + self.config_none = { + "manager": "plaintext" + } + + # not dict "passwords" section + self.config_wrong = { + "manager": "plaintext", + "passwords": "foo" + } + + # one password given + self.test_pass_name = "my_password" + self.test_pass_val = "qwerty1234" + self.config_pass = { + "manager": "plaintext", + "passwords": { + self.test_pass_name: self.test_pass_val + } + } + + + def test_verify(self): + """ + Tests the PlainTextPasswordManager.verify() method + """ + (ok, err) = pwd.verify( self.config_none ) + self.assertFalse(ok) + self.assertNotEqual( len(err), 0 ) + + (ok, err) = pwd.verify( self.config_wrong ) + self.assertFalse(ok) + self.assertNotEqual( len(err), 0 ) + + (ok, err) = pwd.verify( self.config_pass ) + self.assertTrue(ok) + self.assertEqual( len(err), 0 ) + + + def test_pass(self): + """ + Tests the PlainTextPasswordManager.get() method + """ + manager = pwd.Manager( self.config_pass ) + pwd_val = manager.get( self.test_pass_name ) + self.assertIsNotNone( pwd_val ) + self.assertEqual( pwd_val, pwd.Password(self.test_pass_val) ) + +#------------------------------------------------------------------------------- + +class TestGNUPassPasswords( unittest.TestCase ): + """ + Tests the GNUPassPasswordManager class + + Unfortunately, not much can be tested automatically, as any password + retrieval would trigger user input (and anyway, we cannot know the ground + truth for tests) + """ + + def setUp( self ): + self.config = { + "manager": "pass" + } + + def test_create( self ): + """ + Tests the creation of a GNU pass manager + """ + (ok, err) = pwd.verify( self.config ) + self.assertTrue(ok) + self.assertEqual( len(err), 0 ) + + pwd.Manager( self.config ) + +#------------------------------------------------------------------------------- + +if __name__ == "__main__": + unittest.main()