Merge pull request #121 from tarioch/feature/include_duplicate

Implement deduplication based on references for the importers that ha…

src/tariochbctools/importers/bitst/importer.py

@@ -10,6 +10,7 @@
 from dateutil.parser import parse
 from dateutil.relativedelta import relativedelta
 
+from tariochbctools.importers.general.deduplication import ReferenceDuplicatesComparator
 from tariochbctools.importers.general.priceLookup import PriceLookup
 
 
@@ -166,3 +167,5 @@ def fetchSingle(self, trx: dict[str, Any]) -> data.Transaction:
         return data.Transaction(
             meta, date, "*", "", narration, data.EMPTY_SET, data.EMPTY_SET, postings
         )
+
+    cmp = ReferenceDuplicatesComparator()

src/tariochbctools/importers/blockchain/importer.py

@@ -6,6 +6,7 @@
 from beancount.core import amount, data
 from beancount.core.number import D
 
+from tariochbctools.importers.general.deduplication import ReferenceDuplicatesComparator
 from tariochbctools.importers.general.priceLookup import PriceLookup
 
 
@@ -66,3 +67,5 @@ def extract(self, filepath: str, existing: data.Entries) -> data.Entries:
                 entries.append(entry)
 
         return entries
+
+    cmp = ReferenceDuplicatesComparator()

src/tariochbctools/importers/general/deduplication.py

@@ -0,0 +1,14 @@
+class ReferenceDuplicatesComparator:
+    def __init__(self, refs=["ref"]):
+        self.refs = refs
+
+    def __call__(self, entry1, entry2):
+        entry1Refs = set()
+        entry2Refs = set()
+        for ref in self.refs:
+            if ref in entry1.meta:
+                entry1Refs.add(entry1.meta[ref])
+            if ref in entry2.meta:
+                entry2Refs.add(entry2.meta[ref])
+
+        return entry1Refs & entry2Refs

src/tariochbctools/importers/general/mt940importer.py

@@ -6,6 +6,8 @@
 from beancount.core import amount, data
 from beancount.core.number import D
 
+from tariochbctools.importers.general.deduplication import ReferenceDuplicatesComparator
+
 
 class Importer(beangulp.Importer):
     """An importer for MT940 files."""
@@ -60,6 +62,8 @@ def extract(self, filepath: str, existing: data.Entries) -> data.Entries:
 
         return entries
 
+    cmp = ReferenceDuplicatesComparator()
+
     def prepare_payee(self, trxdata: dict[str, Any]) -> str:
         return ""
 

src/tariochbctools/importers/nordigen/importer.py

@@ -7,6 +7,8 @@
 from beancount.core import amount, data
 from beancount.core.number import D
 
+from tariochbctools.importers.general.deduplication import ReferenceDuplicatesComparator
+
 
 class HttpServiceException(Exception):
     pass
@@ -105,3 +107,5 @@ def extract(self, filepath: str, existing: data.Entries) -> data.Entries:
                 entries.append(entry)
 
         return entries
+
+    cmp = ReferenceDuplicatesComparator(["nordref"])

src/tariochbctools/importers/transferwise/importer.py

@@ -15,6 +15,8 @@
 from beancount.core.number import D
 from dateutil.relativedelta import relativedelta
 
+from tariochbctools.importers.general.deduplication import ReferenceDuplicatesComparator
+
 http = urllib3.PoolManager()
 
 
@@ -183,3 +185,5 @@ def extract(self, filepath, existing):
                 entries.append(entry)
 
         return entries
+
+    cmp = ReferenceDuplicatesComparator()

src/tariochbctools/importers/zak/importer.py

@@ -8,6 +8,8 @@
 from beancount.core.number import D
 from dateutil.parser import parse
 
+from tariochbctools.importers.general.deduplication import ReferenceDuplicatesComparator
+
 
 class Importer(beangulp.Importer):
     """An importer for Bank Cler ZAK PDF files files."""
@@ -127,3 +129,5 @@ def extract(self, filepath: str, existing: data.Entries) -> data.Entries:
                 entries.append(self.createBalanceEntry(filepath, date, saldo))
 
         return entries
+
+    cmp = ReferenceDuplicatesComparator(["zakref"])