add display and filter by tags

app/activities/PriceCheck.py

@@ -37,7 +37,7 @@ def get_historic_prices(df):
     return df.drop(labels=[
         'id', 'last', 'avg', 'min', 'max', 'price', 'quantity', 'ts_raw', 'product', 'category', 'group'
     ], axis=1).to_string(header=[
-        'Date', 'Store', '$/unit', 'Org',
+        'Date', 'Store', '$/unit', 'Org', 'Tags'
     ], justify='justify-all', max_colwidth=16, index=False)
 
 class PriceCheck(FocusWidget):

app/data/PriceView.py

@@ -18,7 +18,8 @@ from .util import(
     get_include_exclude,
     get_select,
     get_from,
-    get_where_include_exclude
+    get_where_include_exclude,
+    get_groupby,
 )
 
 def get_window(unit, organic):
@@ -43,25 +44,10 @@ def get_selectors(
         ('$/unit', SQL("""TRUNC(
     price / quantity / convert_unit(units.name, {unit}, products.name), 4
     )""").format(unit=Literal(unit))),
-        ('last', SQL("""TRUNC(last_value(
-    price / quantity / convert_unit(units.name, {unit}, products.name)
-) OVER {window}, 4)
-""").format(unit=Literal(unit), window=window)),
-        ('avg', SQL("""TRUNC(sum(CASE
-    WHEN convert_unit(units.name, {unit}, products.name) IS NOT NULL THEN price
-    ELSE NULL
-END) OVER {window} / sum(
-    quantity * convert_unit(units.name, {unit}, products.name)
-) OVER {window}, 4)
-""").format(unit=Literal(unit), window=window)),
-        ('min', SQL("""TRUNC(min(
-    price / quantity / convert_unit(units.name, {unit}, products.name)
-) OVER {window}, 4)
-""").format(unit=Literal(unit), window=window)),
-        ('max', SQL("""TRUNC(max(
-    price / quantity / convert_unit(units.name, {unit}, products.name)
-) OVER {window}, 4)
-""").format(unit=Literal(unit), window=window)),
+        *[(
+            k, v.format(unit=Literal(unit), window=window)
+        ) for k, v in WINDOW.items()
+        ],
         ('price', SQL("""TRUNC(price, 4)""")),
         ('quantity', SQL("""TRUNC(
     quantity * convert_unit(units.name, {unit}, products.name), 4
@@ -70,6 +56,11 @@ END) OVER {window} / sum(
         ('category', Identifier('categories', 'name')),
         ('group', Identifier('groups', 'name')),
         ('organic', Identifier('organic')),
+        ('tags', SQL(
+"""array_agg({tag_name}) FILTER (WHERE {tag_name} IS NOT NULL)"""
+        ).format(
+            tag_name=Identifier('tags','name'),
+        ))
     ])
 
 JOINS = OrderedDict([
@@ -78,17 +69,63 @@ JOINS = OrderedDict([
     ('products', ('id', 'product_id')),
     ('categories', ('id', 'category_id')),
     ('groups', ('id', 'group_id')),
+    ('tags_map', ('transaction_id', ('transactions','id'))),
+    ('tags', ('id', ('tags_map','tag_id'))),
+])
+
+WINDOW = OrderedDict([
+    ('last', SQL("""TRUNC(last_value(
+    price / quantity / convert_unit(units.name, {unit}, products.name)
+) OVER {window}, 4)
+""")),
+    ('avg', SQL("""TRUNC(sum(CASE
+    WHEN convert_unit(units.name, {unit}, products.name) IS NOT NULL THEN price
+    ELSE NULL
+END) OVER {window} / sum(
+    quantity * convert_unit(units.name, {unit}, products.name)
+) OVER {window}, 4)
+""")),
+    ('min', SQL("""TRUNC(min(
+    price / quantity / convert_unit(units.name, {unit}, products.name)
+) OVER {window}, 4)
+""")),
+    ('max', SQL("""TRUNC(max(
+    price / quantity / convert_unit(units.name, {unit}, products.name)
+) OVER {window}, 4)
+""")),
+])
+
+GROUPBY = OrderedDict([
+    ('id', Identifier('transactions', 'id')),
+    ('ts_raw', SQL("""(transactions.ts AT TIME ZONE 'UTC')::timestamp without time zone""")),
+    ('%d/%m/%y %_I%P', SQL("""(transactions.ts AT TIME ZONE 'UTC')::timestamp without time zone""")),
+    ('code', Identifier('stores', 'code')),
+    ('$/unit', SQL("""
+TRUNC(
+    price / quantity / convert_unit(units.name, {unit}, products.name), 4
+)""")),
+    ('price', SQL("""TRUNC(price, 4)""")),
+    ('quantity', SQL("""
+TRUNC(
+    quantity * convert_unit(units.name, {unit}, products.name), 4
+)""")),
+    ('group', Identifier('groups', 'name')),
+    ('category', Identifier('categories', 'name')),
+    ('product', Identifier('products', 'name')),
+    ('organic', Identifier('organic')),
+    ('unit', Identifier('units', 'name')),
 ])
 
 
-def get_where(product=None, category=None, group=None, organic=None, limit='90 days'):
+def get_where(product=None, category=None, group=None, tag=None, organic=None, limit='90 days'):
     where = [
         get_where_include_exclude(
             k, 'name', *get_include_exclude(v)
         ) for k, v in {
             'products': product,
             'categories': category,
-            'groups': group
+            'groups': group,
+            'tags': tag,
         }.items()
     ]
     if organic is not None:
@@ -111,12 +148,13 @@ def get_where(product=None, category=None, group=None, organic=None, limit='90 d
     ]) if where else SQL('')
 
 
-def get_historic_prices_statement(unit, sort=None, product=None, category=None, group=None, organic=None, limit='90 days'):
+def get_historic_prices_statement(unit, sort=None, product=None, category=None, group=None, tag=None, organic=None, limit='90 days'):
     
     return SQL('\n').join([
         get_select(get_selectors(unit, organic)),
         get_from("transactions", JOINS),
-        get_where(product=product, category=category, group=group, organic=organic, limit=limit),
+        get_where(product=product, category=category, group=group, tag=tag, organic=organic, limit=limit),
+        get_groupby(GROUPBY, lambda x: x.format(unit=Literal(unit))),
         get_sort(sort, organic),
     ])
 

app/data/QueryManager.py

@@ -129,8 +129,8 @@ class QueryManager(object):
         self.display = display
         self.cursor = cursor
 
-    def get_historic_prices_data(self, unit, sort=None, product=None, category=None, group=None, organic=None, limit=None):
-        statement = get_historic_prices_statement(unit, sort=sort, product=product, category=category, group=group, organic=organic, limit=limit)
+    def get_historic_prices_data(self, unit, sort=None, product=None, category=None, group=None, tag=None, organic=None, limit=None):
+        statement = get_historic_prices_statement(unit, sort=sort, product=product, category=category, group=group, tag=tag, organic=organic, limit=limit)
         data = get_data(self.cursor, statement)
         return pd.DataFrame(map(lambda x: dict(
             map(lambda k: (k, self.display(x[k], k)), x)

app/data/util.py

@@ -30,7 +30,7 @@ def get_where_include_exclude(
     return SQL("""
   ({identifier} = ANY({include}) OR ARRAY[]::text[] @> {include}::text[])
 AND
-  NOT {identifier} = ANY({exclude})
+  (NOT {identifier} = ANY({exclude}) OR {identifier} IS NULL)
 """).format(
         identifier=Identifier(table, col),
         include=Literal(include),
@@ -65,9 +65,13 @@ LEFT JOIN """).format(base=Identifier(base)),
 LEFT JOIN """).join(joins)])
 
 
-def get_groupby(alias_to_sql: dict[str, Composable]) -> Composable:
+def get_groupby(alias_to_sql: dict[str, Composable], formatter=None) -> Composable:
     groupby = SQL(""",
-    """).join([ v for k, v in alias_to_sql.items() if k != 'tags'])
+""").join([
+        formatter(v) if formatter is not None and isinstance(
+            v, SQL
+        ) else v for k, v in alias_to_sql.items()
+    ])
     return SQL("""
-    """).join([SQL("GROUP BY"), *groupby])
+""").join([SQL("GROUP BY"), *groupby])
 

app/rest/data-item.tpl

@@ -2,5 +2,4 @@
     <td>{{ product }}</td>
     <td>{{ category }}</td>
     <td>{{ group }}</td>
-    <td></td>
 </tr>

app/rest/filter-heading.tpl

@@ -1,4 +1 @@
-<th>
-    <span> {{fname.title()}} </span>
-    <button style="position: absolute; right: 0.2em" type="submit" {{ "hidden" if not first else "" }}>Apply</button>
-</th>
+<th>{{fname.title()}}</th>

app/rest/filter.tpl

@@ -1,16 +0,0 @@
-<form id="filter" style="display: inline-block"  method="{{ method }}" action="{{ action }}">
-    <table class="pure-table pure-table-bordered pure-table-striped">
-        <thead style="position: sticky; top: 0">
-            <tr>
-{{! ''.join( header )}}
-            </tr>
-            <tr>
-{{! ''.join( items )}}
-            </tr>
-        </thead>
-        <tbody>       
-{{! ''.join( data )}}
-            </details>
-        </tbody>
-    </table>
-</form>

app/rest/form.tpl

@@ -0,0 +1,59 @@
+<form id="filter" style="display: inline-flex"  method="{{ method }}" action="{{ action }}">
+    <div>
+    <table class="pure-table pure-table-bordered pure-table-striped" style="width: 60vw">
+        <thead style="position: sticky; top: 0">
+            <tr>
+{{! ''.join( header )}}
+            </tr>
+            <tr>
+{{! ''.join( items )}}
+            </tr>
+        </thead>
+        <tbody>       
+{{! ''.join( data )}}
+        </tbody>
+    </table>
+    </div>
+    <div>
+    <table class="pure-table pure-table-bordered pure-table-striped" style="width: 20vw; position: sticky; top: 0">
+        <thead>
+            <tr>
+                <th>Tags</th>
+                <th>
+                    Unit
+                    <button style="position: absolute; right: 0.2em" type="submit">
+                        Apply
+                    </button>
+                </th>
+            </tr>
+            <tr>
+                <td>
+                    <input
+                        type="text"
+                        id="tag"
+                        name="tag"
+                        size="26"
+                        pattern="((.*\\|)*(.*))?(!(.*\\|)*(.*))?"
+                        title="Must be of the form include!exclude where include and exclude are | separated lists"
+                        value="{{tag}}"
+                    />
+                </td>
+                <td>
+                    <input
+                        type="text"
+                        id="unit"
+                        name="unit"
+                        size="10"
+                        pattern="{{pattern}}"
+                        title="Must be one of: {{units}}"
+                        value="{{unit}}"
+                    />
+                </td>
+            </tr>
+        </thead>
+        <tbody>
+{{! ''.join( tags )}}
+        </tbody>
+    </table>
+    </div>
+</form>

app/rest/pyapi.py

@@ -3,6 +3,7 @@
 # All rights reserved
 #
 # THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+from itertools import chain
 from time import time
 from typing import Iterable
 from io import StringIO
@@ -37,8 +38,8 @@ def line(pivot, ylabel=None, xlabel=None):
     ax.set_xlabel(xlabel)
     ax.set_ylabel(ylabel)
 
-ALL_UNITS = {'g','kg','mL','L','Pieces','Bunches','Bags'}
-PARAMS = { 'group', 'category', 'product', 'unit' }
+ALL_UNITS = {'g', 'kg', 'mL', 'L', 'Pieces', 'Bunches', 'Bags'}
+PARAMS = { 'group', 'category', 'product', 'unit', 'tag' }
 CACHE = dict()
 
 host = f"host={os.getenv('HOST')}"
@@ -72,39 +73,50 @@ def normalize_query(query: DictProperty, allow: Iterable[str] = None):
     return get_query(**get_filter(query, allow=allow))
 
 def get_form(action, method, filter_data, data):
-    keys = sorted(filter_data, key=lambda x: {
+    keys = sorted(filter(lambda x: x not in ('unit', 'tag'), filter_data), key=lambda x: {
         'product': 0,
         'category': 1,
         'group': 2,
-        'unit': 3
     }[x])
-    data = data[data['$/unit'].apply(lambda x: (x or False) and True)].groupby([
-        k for k in keys if k != 'unit'
+    in_chart = data['$/unit'].apply(lambda x: (x or False) and True)
+    group = data[in_chart].groupby([
+        k for k in keys
     ]).size().reset_index()[[
-        k for k in keys if k != 'unit'
+        k for k in keys
     ]]
+    tags = set(chain(*data[in_chart]['tags']))
     return template(
-        'app/rest/filter.tpl',
+        'app/rest/form',
         action=action,
         method=method,
-        header=[ template(
-            'app/rest/filter-heading',
-            fname=k,
-            first=(idx == 0),
-        ) for idx, k in enumerate(keys) ],
+        unit=get_query_param(*filter_data['unit']),
+        pattern='|'.join(ALL_UNITS),
+        units=', '.join(ALL_UNITS),
+        header=[
+            template(
+                'app/rest/filter-heading',
+                fname=k,
+                first=(idx == 0),
+            ) for idx, k in enumerate(keys)
+        ],
         items=[ template(
-            'app/rest/filter-item.tpl',
+            'app/rest/filter-item',
             fname=k,
             fvalue=get_query_param(*filter_data[k]),
             size=36,
         ) for k in keys ],
         data=[ template(
-            'app/rest/data-item.tpl',
+            'app/rest/data-item',
             product=row['product'],
             category=row['category'],
             group=row['group'],
             size=36,
-        ) for _, row in data.iterrows() ]
+        ) for _, row in group.iterrows() ],
+        tag=get_query_param(*filter_data['tag']),
+        tags=[
+            template('app/rest/tag-item', tag=x) for x in tags
+        ],
+        size=36,
     )
 
 
@@ -205,11 +217,13 @@ style = """
       </head>
       <body>
         <table class="pure-table pure-table-bordered pure-table-striped">
+          <thead>
           <tr>
             <xsl:for-each select="$schema/xsd:complexType[@name=$rowtypename]/xsd:sequence/xsd:element/@name">
               <th><xsl:value-of select="."/></th>
             </xsl:for-each>
           </tr>
+          </thead>
 
           <xsl:for-each select="row">
             <tr>
@@ -309,4 +323,24 @@ SELECT query_to_xml_and_xmlschema({inner}, false, false, ''::text)
     response.content_type = 'application/xhtml+xml; charset=utf-8'
     return f"{heading}" + '\n'.join(xml)
 
+@route('/grocery/tags')
+def tags():
+    try:
+        with conn.cursor() as cur:
+            inner = SQL('\n').join([SQL("""
+SELECT count(DISTINCT txn.id) AS "Uses", tg.name AS "Name"
+FROM tags tg
+JOIN tags_map tm ON tg.id = tm.tag_id
+JOIN transactions txn ON txn.id = tm.transaction_id
+GROUP BY tg.name
+ORDER BY 1 DESC, 2
+""")]).as_string(cur)
+            xml = cur.execute(SQL("""
+SELECT query_to_xml_and_xmlschema({inner}, false, false, ''::text)
+""").format(inner=Literal(inner))).fetchone()[0].splitlines()
+    finally:
+        conn.commit()
+    response.content_type = 'application/xhtml+xml; charset=utf-8'
+    return f"{heading}" + '\n'.join(xml)
+
 run(host='0.0.0.0', port=6772)

app/rest/tag-item.tpl

@@ -0,0 +1,4 @@
+<tr>
+    <td>{{ tag }}</td>
+    <td></td>
+</tr>