Try find matching record in database automatically

reconcile.py

@@ -1,9 +1,86 @@
 #!/usr/bin/python3
-from datetime import datetime
+#
+# usage:  ./reconcile.py ~/gnucash/merged.gnucash 2021-11-01 2021-12-10 1
+#
+from datetime import datetime, timedelta
 from dateutil.parser import parse as parse_time
 import itertools
 import gnucash
 import sys
+import os
+import psycopg2
+from psycopg2.sql import (
+    Identifier,
+    SQL,
+    Literal,
+    Placeholder,
+    Composed,
+)
+user = os.getenv('USER')
+conn = psycopg2.connect(f"host=***REMOVED*** dbname=das user={user} password='***REMOVED***'")
+cur = conn.cursor()
+
+def get_record_from_database(date, store):
+    cur.execute(get_statement(date, store))
+    #print(cur.mogrify(get_statement(date, store)))
+    return sum([row['price'] for row in cursor_as_dict(cur)])
+
+def cursor_as_dict(cur):
+    _col_idx_map=dict(map(lambda col: (col[1].name, col[0]), enumerate(cur.description)))
+    for row in map(lambda row, _map=_col_idx_map: dict([
+        (name, row[i]) for name, i in _map.items()
+    ]), cur.fetchall()):
+        #print(row)
+        yield row
+
+def get_store_code(store, user_data=dict()):
+    if 'countdown' in store.lower():
+        return 'CD'
+    if 'pak n save' in store.lower():
+        return 'PnS'
+    
+    if store not in user_data:
+        user_data[store] = input(f"Enter code for {store}: ")
+    return user_data[store]
+
+select = {
+    'ts': SQL("""date_part('day',ts)||'/'||date_part('month',ts)||' '||date_part('hour',ts)::int%12"""),
+    'shop': Identifier('code'),
+    'description': SQL("""substr(description,1,32)"""),
+    'volume': Identifier('quantity'),
+    'unit': SQL("""substr(unit,1,4)"""),
+    'price': Identifier('price'),
+    '$/unit': SQL("""TRUNC(price/quantity,4)"""),
+    'total': SQL("""sum(transaction_view.price) OVER (PARTITION BY transaction_view.ts::date ORDER BY transaction_view.ts, transaction_view.description ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)"""),
+    'group': SQL("""substr(product,1,10) AS product, substr(category,1,8) AS category, substr("group",1,9)"""),
+    'og': Identifier('organic'),
+}
+
+def get_where(date, store):
+    where = [ ]
+    if store is not None:
+        where.append(SQL(' ').join([
+            Identifier('code'), SQL('='), Literal(store)
+        ]))
+    where.append(
+        SQL("{ts} BETWEEN {date}::date AND {date}::date + {interval}::interval").format(
+            ts=Identifier('ts'),
+            date=Literal(str(date)),
+            interval=Literal('23 hours 59 minutes 59 seconds'),
+        )
+    )
+    return SQL(" AND ").join(where)
+
+def get_statement(date, store):
+    statement = SQL("\n").join([
+        SQL("SELECT"),
+        SQL(',\n  ').join([
+            SQL(' ').join([v, SQL('AS'), Identifier(f'{k}')]) for k, v in select.items()
+        ]),
+        SQL("FROM transaction_view"),
+        SQL("WHERE"), get_where(date,store)
+    ])
+    return statement
 
 def _unwrap_list(root, blacklist):
   for i in map(lambda x: [
@@ -15,9 +92,10 @@ def _accounts(root, path, blacklist, whitelist):
   #print('blacklist1: {}'.format(blacklist))
   #print(f'whitelist1 ({root.name}: {whitelist}')
   
-  if root.name in [ w[0] for w in whitelist ] and \
-     root.name not in [ b[0] for b in blacklist ]:
-         yield path, root
+  if root.name in [ w[0] for w in whitelist ]:
+      if root.name not in [ b[0] for b in blacklist ]:
+          yield path, root
+
   blacklist = [b for b in itertools.chain(
     _unwrap_list(root, filter(lambda x: (x[0] == root.name), blacklist)),
     filter(lambda x: not (x[0] == root.name), blacklist)
@@ -47,8 +125,7 @@ def _accounts(root, path, blacklist, whitelist):
   ]
   #print('blacklist: {}'.format(blacklist))
   for a in root.get_children():
-    for p, c in _accounts(a, path, blacklist, whitelist):
-      yield p, c
+      yield from _accounts(a, path, blacklist, whitelist)
 
 def _path(account):
   if account.get_parent() is not None: return ':'.join([_path(account.get_parent()), account.name])
@@ -57,28 +134,18 @@ def _path(account):
 def _iter(root, blacklist, whitelist, _from, _to):
   for p, a in _accounts(root, [], blacklist, whitelist):
     apath = _path(a)
-    s = dict()
-
     for t in a.GetSplitList():
       tpath = _path(t.account)
-      if datetime.fromtimestamp(_to) > t.parent.GetDate() >= datetime.fromtimestamp(_from):
+      ts = t.parent.GetDate()
+      if datetime.fromtimestamp(_to) > ts >= datetime.fromtimestamp(_from):
         #if apath.startswith('Root Account:Trading'):
         #  if tpath.startswith('Root Account:Trading:CURRENCY'):
         #    s+=t.GetValue().num()
         #elif tpath == apath:
         if tpath == apath:
           #print(t.parent.GetDescription())
-          desc = t.parent.GetDescription()
-          if desc not in s:
-              s[desc] = 0
-          #print(t.parent.GetNotes())
-          s[desc]+=t.GetValue().num()
-        #if apath.startswith('Assets'): s+= t.GetValue().num()
-        #elif apath.startswith('Liabilities'): s-= t.GetValue().num()
-        #if apath.startswith('Income'): s+= t.GetValue().num()
-    yield p, a.name, s
-    #for t in a.GetSplitList():
-    #  print(t.parent.GetDate(), type(t.parent.GetDate())) 
+          ts = ts.replace(hour=0, minute=0)
+          yield p, a.name, t.GetValue().num(), t.parent.GetDescription(), ts
 
 if __name__ == '__main__':
   args = sys.argv
@@ -98,16 +165,31 @@ if __name__ == '__main__':
     _from = int(parse_time(args[2]).timestamp())# 1554030000 # 1 April 2019 as seconds since epoch
     _to = int(parse_time(args[3]).timestamp())
     _interval = int(args[4])*60*60*24
-    for f, t in [ (i, i+_interval) for i in range(_from, _to, _interval)]:
-      ts = datetime.fromtimestamp(f)
-      to = datetime.fromtimestamp(t)
-      tot = 0
-      for p,n,s in _iter(session.book.get_root_account(), blacklist, whitelist, f, t):
+    
+    ranges = dict([
+        [
+            el for el in map(lambda x: datetime.fromtimestamp(x), (i, i+_interval))
+        ] for i in range(_from, _to, _interval)
+    ])
+    tot = dict()
+    for p,n,value,desc,d in sorted(_iter(session.book.get_root_account(), blacklist, whitelist, _from, _to)):
         account_name = ':'.join([*p, n])
-        for k in sorted(s.keys()):
-            print(f'{ts} {k:35s}: {s[k]/100:>06.2f}')
-            tot += s[k]
-      if tot > 0:
-          print(f'{to} {str("TOTAL"):35s}: {tot/100:>06.2f}')
+        f = next(( f for (f, t) in ranges.items() if f <= d < t ))
+        tot.update({
+            f: value if f not in tot else value+tot[f]
+        })
+        ts = d
+        store = get_store_code(desc)
+        book = None
+        while ts > d - timedelta(days=3):
+            book = get_record_from_database(ts, store)
+            book = int(book*100)
+            if book == int(value):
+                break;
+            ts = ts - timedelta(days=1)
+            book = get_record_from_database(ts, store)
+        dbentry = f'{book/100:>06.2f} : {store:5s}' if book == int(value) else ''
+        print(f'{d} {desc:35s} : {value/100:>06.2f} | {dbentry}')
+        
   finally:
     session.end()