Merge branch 'limit-url-length' of gogsadmin/grocery-manager into master

Cache by hash and limit URL length by referencing hash instead when parameters are too many

app/activities/ActivityManager.py

@@ -5,8 +5,8 @@
 #
 # THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
 from typing import Iterator
-import urwid
 from urwid.display_common import BaseScreen
+from urwid import ExitMainLoop
 
 def show_or_exit(key,
     screen: BaseScreen = None, palettes: Iterator = None
@@ -25,7 +25,7 @@ def show_or_exit(key,
         screen.clear()
 
     if key in ('esc',):
-        raise urwid.ExitMainLoop()
+        raise ExitMainLoop()
 
 class ActivityManager():
 
@@ -58,4 +58,3 @@ class ActivityManager():
         if self.app is None:
             return None
         return self.app.original_widget
-

app/activities/Banner.py

@@ -0,0 +1,16 @@
+#
+# Copyright (c) Daniel Sheffield 2023
+#
+# All rights reserved
+#
+# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+from urwid import AttrMap, Padding, Pile, Text, AttrMap
+from .. import COPYRIGHT
+
+def banner(title: str) -> AttrMap:
+    header = Text(title, 'center')
+    _copyright = Text(COPYRIGHT, 'center')
+    return AttrMap(Pile([
+            Padding(header, 'center', width=('relative', 100)),
+            Padding(_copyright, 'center', width=('relative', 100)),
+        ]), 'banner')

app/activities/PriceCheck.py

@@ -4,7 +4,6 @@
 # All rights reserved
 #
 # THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from decimal import Decimal, InvalidOperation
 from itertools import chain
 from urwid import (
     connect_signal,
@@ -14,13 +13,13 @@ from urwid import (
     Divider,
     Filler,
     LineBox,
-    Padding,
     Pile,
     RadioButton,
     Text,
 )
 
-from .. import COPYRIGHT
+from ..data.decimal_util import decimal_or_none
+from ..data.dataframe_util import get_caption, get_time_range, stats
 from ..widgets import (
     AutoCompleteEdit,
     AutoCompleteFloatEdit,
@@ -32,6 +31,8 @@ from ..widgets import (
 from ..data.QueryManager import QueryManager
 from .ActivityManager import ActivityManager, show_or_exit
 from .Rating import Rating
+from .Banner import banner
+
 
 def get_historic_prices(df):
     return df.drop(labels=[
@@ -118,46 +119,23 @@ class PriceCheck(FocusWidget):
         ).truncate(
             before=max(0, len(df.index)-self.graph._canvas_width)
         )
-        data = df[['$/unit','quantity']].apply(
-            lambda x: (float(x['$/unit']), float(x['quantity'])),
+        data = df[['price', '$/unit','quantity']].apply(
+            lambda x: (float(x['price']), float(x['$/unit']), float(x['quantity'])),
             axis=1, result_type='broadcast'
         )
-        data['avg'] = (data['$/unit']*data['quantity']).sum()/data['quantity'].sum()
-        data_max = data.max()['$/unit'] #.max()
-        assert len(data['avg'].unique()) == 1
         norm = [ (x,) for x in data['$/unit'] ] #.to_records(index=False)
-        self.graph.set_data(norm, data_max,
-            vscale=list(map(float, [
-                data['$/unit'].min(),
-                data['$/unit'].median(),
-                data['avg'].iloc[0],
-                data_max
-            ]))
-        )
-        #self.graph.set_bar_width(1)
-        # canvas_width = 10 + pad + pad + 10
-        date_strlen = (self.graph.canvas_width - 20)
-        ex = "─" if date_strlen % 2 else ""
-        plen = date_strlen//2
-        caption = f"{df['ts_raw'].min():%d/%m/%Y}"
-        caption += f"{{p:>{plen}}}{ex}{{p:<{plen}}}".format(
-            p="─")
-        caption += f"{df['ts_raw'].max():%d/%m/%Y}"
+        scale = stats(data, 'price', 'quantity', '$/unit')
+        self.graph.set_data(norm, scale[-1], vscale=scale)
+        time_range = get_time_range(df, 'ts_raw')
+        caption = get_caption(time_range, self.graph.canvas_width)
         self.graph.set_caption(caption)
 
     def update_historic_prices(self, data):
         organic = None if data['organic'] == 'mixed' else data['organic']
         sort = '$/unit' if self.buttons['sort_price'].state else 'ts'
         product, unit = data['product'] or None, data['unit'] or None
-        try:
-            price = Decimal(data['price'])
-        except InvalidOperation:
-            price = None
-
-        try:
-            quantity = Decimal(data['quantity'])
-        except InvalidOperation:
-            quantity = None
+        price = decimal_or_none(data['price'])
+        quantity = decimal_or_none(data['quantity'])
 
         if None in (sort, product, unit):
             self.text_fields['dbview'].set_text('')
@@ -177,10 +155,11 @@ class PriceCheck(FocusWidget):
         ]
         self.rating.update_rating(_avg, _min, _max, unit, price=price, quantity=quantity)
 
+        self.update_graph(df)
         self.text_fields['dbview'].set_text(
             get_historic_prices(df)
         )
-        self.update_graph(df)
+
 
     def __init__(self,
         activity_manager: ActivityManager,
@@ -210,21 +189,6 @@ class PriceCheck(FocusWidget):
           self.text_fields.items()
         )))
 
-        left_pane = [
-            'product',
-            'organic',
-        ]
-        badge = [
-            'rating',
-            'spread',
-            'marker',
-        ]
-        right_pane = [
-            'unit',
-            'quantity',
-            'price',
-        ]
-
         self.query_manager = query_manager
         self.organic_checkbox = self.checkboxes['organic']
         connect_signal(self.organic_checkbox, 'postchange', lambda _,v: self.update())
@@ -248,15 +212,6 @@ class PriceCheck(FocusWidget):
 
         self.clear()
 
-        header = Text(u'Price Check', 'center')
-        _copyright = Text(COPYRIGHT, 'center')
-
-        banner = Pile([
-            Padding(header, 'center', width=('relative', 100)),
-            Padding(_copyright, 'center', width=('relative', 100)),
-        ])
-        banner = AttrMap(banner, 'banner')
-
         _widgets = dict(chain(*list(map(lambda x: x.items(), [
                 self.edit_fields, self.text_fields, self.checkboxes
             ])
@@ -280,35 +235,32 @@ class PriceCheck(FocusWidget):
             ),
         })
         components = {
-          'top_pane': Columns([
-              (9, Pile([
-                 Divider(),
-                 AttrMap(self.buttons['clear'], 'streak'),
-                 Divider(),
-              ])),
-              LineBox(
-                Columns([ v for k,v in self.buttons.items() if 'sort' in k]),
-                title="Sort price by",
-                title_align='left',
-              ),
-              (9, Pile([
-                Divider(),
-                AttrMap(self.buttons['exit'], 'streak'),
-                Divider(),
-              ]))
+            'top_pane': Columns([
+                (9, Pile([
+                    Divider(),
+                    AttrMap(self.buttons['clear'], 'streak'),
+                    Divider(),
+                ])),
+                LineBox(
+                    Columns([ v for k,v in self.buttons.items() if 'sort' in k]),
+                    title="Sort price by",
+                    title_align='left',
+                ),
+                (9, Pile([
+                    Divider(),
+                    AttrMap(self.buttons['exit'], 'streak'),
+                    Divider(),
+                ]))
             ], dividechars=1),
-          'right_pane': (16, Pile(map(
-            lambda x: _widgets[x] if x is not None else Divider,
-            right_pane
-          ))),
-          'left_pane': Pile(map(
-            lambda x: _widgets[x] if x is not None else Divider,
-            left_pane
-          )),
-          'badge': Pile(map(
-            lambda x: _widgets[x] if x is not None else Divider,
-            badge
-          )),
+            'right_pane': (16, Pile([
+                _widgets[x] for x in ['unit', 'quantity', 'price']
+            ])),
+            'left_pane': Pile([
+                _widgets[x] for x in ['product', 'organic']
+            ]),
+            'badge': Pile([
+                _widgets[x] for x in ['rating', 'spread', 'marker']
+            ]),
         }
         _widgets.update({
             'graph': LineBox(
@@ -329,7 +281,7 @@ class PriceCheck(FocusWidget):
         ])})
 
         widget = Pile([
-            banner,
+            banner(u'Price Check'),
             Divider(),
             components['top_pane'],
             Columns((components['left_pane'], components['right_pane']),

app/activities/RecipeEditor.py

@@ -29,7 +29,13 @@ from urwid.numedit import FloatEdit
 import yaml
 from yaml.representer import SafeRepresenter
 
-from .. import COPYRIGHT
+from .grouped_widget_util import (
+    in_same_row,
+    to_numbered_field,
+    to_unnumbered_field,
+    to_named_value,
+)
+
 from ..widgets import (
     AutoCompleteEdit,
     FocusWidget,
@@ -38,6 +44,7 @@ from ..widgets import (
 )
 from ..data.QueryManager import QueryManager
 from .ActivityManager import ActivityManager, show_or_exit
+from .Banner import banner
 
 def change_style(style, representer):
     def new_representer(dumper, data):
@@ -76,28 +83,13 @@ f"""<root>
     for e in filter(lambda x: x.tag == 'strong', depth_first_elements(xhtml)):
         yield e.text
 
-def to_numbered_field(x):
-    if len(x[0].split('#', 1)) > 1:
-        name, idx = x[0].split('#', 1)
-        idx = int(idx)
-    else:
-        name, idx = x[0], 0
-
-    return (name, int(idx)), x[1]
-
-def to_unnumbered_field(x):
-    return x[0][0], x[1]
-
-def in_same_row(name):
-    if len(name.split('#', 1)) > 1:
-        _, row = name.split('#', 1)
-    return lambda x: x[0][1] == int(row)
 
 def unzip(_iter: List[Tuple[AutoCompleteEdit, FloatEdit, AutoCompleteEdit]]) -> Tuple[
     List[AutoCompleteEdit], List[FloatEdit], List[AutoCompleteEdit]
 ]:
     return zip(*_iter)
 
+
 def extract_values(x: Union[List[AutoCompleteEdit], List[FloatEdit]]) -> Iterable[str]:
     if isinstance(x, (list, tuple)):
         if len(x) == 0:
@@ -105,8 +97,6 @@ def extract_values(x: Union[List[AutoCompleteEdit], List[FloatEdit]]) -> Iterabl
         return ( v.get_edit_text() for v in x )
     raise Exception(f"Unsupported type: {type(x)}")
 
-def to_named_value(name: str) -> Callable[[str], Tuple[str,str]]:
-    return lambda e: (f'{name}#{e[0]}', e[1])
 
 def blank_ingredients_row(idx: int) -> Tuple[AutoCompleteEdit, FloatEdit, AutoCompleteEdit]:
     return (
@@ -470,15 +460,6 @@ class RecipeEditor(FocusWidget):
         connect_signal(self.buttons['exit'], 'click', lambda _: show_or_exit('esc'))
         connect_signal(self.instructions, 'postchange', lambda w,_: self.update(w))
 
-        header = Text(u'Recipe Editor', 'center')
-        _copyright = Text(COPYRIGHT, 'center')
-
-        banner = Pile([
-            Padding(header, 'center', width=('relative', 100)),
-            Padding(_copyright, 'center', width=('relative', 100)),
-        ])
-        banner = AttrMap(banner, 'banner')
-        
         left_pane, middle_pane, right_pane, gutter = self.init_ingredients()
 
         self.components = {
@@ -507,7 +488,7 @@ class RecipeEditor(FocusWidget):
         }
 
         widget = Pile([
-            banner,
+            banner(u'Recipe Editor'),
             Divider(),
             self.components['top_pane'],
             Columns([

app/activities/TransactionEditor.py

@@ -4,10 +4,8 @@
 # All rights reserved
 #
 # THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from decimal import Decimal, InvalidOperation
 from itertools import chain
 from typing import (
-    Callable,
     Union,
     Tuple,
     List,
@@ -25,13 +23,18 @@ from urwid import (
     Edit,
     Filler,
     LineBox,
-    Padding,
     Pile,
     Text,
 )
 
-from .. import COPYRIGHT
+from ..data.decimal_util import decimal_or_none
+from ..data.dataframe_util import get_caption, get_time_range, stats
 from ..data.QueryManager import QueryManager
+from .grouped_widget_util import (
+    to_numbered_field,
+    to_unnumbered_field,
+    to_named_value,
+)
 from ..widgets import (
     AutoCompleteEdit,
     AutoCompleteFloatEdit,
@@ -43,31 +46,14 @@ from ..widgets import (
 from . import ActivityManager
 from .Rating import Rating
 from .NewProduct import NewProduct
-
-def to_numbered_field(x):
-    if len(x[0].split('#', 1)) > 1:
-        name, idx = x[0].split('#', 1)
-        idx = int(idx)
-    else:
-        name, idx = x[0], 0
-
-    return (name, int(idx)), x[1]
-
-def to_unnumbered_field(x):
-    return x[0][0], x[1]
-
-def in_same_row(name):
-    if len(name.split('#', 1)) > 1:
-        _, row = name.split('#', 1)
-    else:
-        row = 0
-    return lambda x: x[0][1] == int(row)
+from .Banner import banner
 
 def unzip(_iter: List[Tuple[AutoCompleteEdit, Edit]]) -> Tuple[
     List[AutoCompleteEdit], List[Edit]
 ]:
     return zip(*_iter)
 
+
 def extract_values(x: Union[List[AutoCompleteEdit], List[Edit]]) -> Iterable[str]:
     if isinstance(x, (list, tuple)):
         if len(x) == 0:
@@ -75,8 +61,6 @@ def extract_values(x: Union[List[AutoCompleteEdit], List[Edit]]) -> Iterable[str
         return ( v.get_edit_text() for v in x )
     raise Exception(f"Unsupported type: {type(x)}")
 
-def to_named_value(name: str) -> Callable[[str], Tuple[str,str]]:
-    return lambda e: (f'{name}#{e[0]}', e[1])
 
 def blank_tags_row(idx: int) -> Tuple[AutoCompleteEdit, Edit]:
     return (
@@ -110,10 +94,11 @@ class TransactionEditor(FocusWidget):
 
     def apply_choice(self, name, value):
         self.apply_changes(name, value)
-        data = dict(#filter(
-        #    in_same_row(name),
-            map(to_numbered_field, self.data.items())
-        )#)
+        data = {
+            (field, idx): v for field, idx, v in map(
+                to_numbered_field, self.data.items()
+            )
+        }
         for k,v in data.items():
             if f'{k[0]}#{k[1]}' == name or v:
                 continue
@@ -172,16 +157,12 @@ class TransactionEditor(FocusWidget):
 
 
     def init_tags(self):
-        #_tags = LineBox(Pile([AttrMap(
         _tags = Pile([AttrMap(
             AutoCompletePopUp(
                 tag[0],
                 self.apply_choice,
                 lambda: self.activity_manager.show(self.update())
             ), 'streak') for tag in self._tags])
-        #    title=f'Tags',
-        #    title_align='left'
-        #)
         gutter = Pile([
             *[ Divider() for _ in self._tags[:-1] ],
             Divider(),
@@ -196,7 +177,6 @@ class TransactionEditor(FocusWidget):
             blank_tags_row(len(self._tags))
         )
         _tags, gutter = self.init_tags()
-        #self.components['tags'][1].original_widget.contents = list(_tags.original_widget.contents)
         self.components['tags'].contents = list(_tags.contents)
         self.components['gutter'][1].contents = list(gutter.contents)
         for widget in self._tags:
@@ -204,16 +184,13 @@ class TransactionEditor(FocusWidget):
             connect_signal(widget[0], 'apply', lambda w, name: self.autocomplete_callback(
                 w, name, self.autocomplete_options(name, dict(map(
                     to_unnumbered_field,
-                    #filter(
-                    #    in_same_row(name),
-                        map(to_numbered_field, self.data.items()
-                    #)
+                    map(to_numbered_field, self.data.items()
                 ))))
             ))
 
 
     def clear(self):
-        self._tags = []
+        self._tags: List[Tuple[AutoCompleteEdit, Edit]] = []
         self.add_tag()
         for (k, ef) in self.edit_fields.items():
             if k in ('ts', 'store',):
@@ -249,46 +226,23 @@ class TransactionEditor(FocusWidget):
         ).truncate(
             before=max(0, len(df.index)-self.graph._canvas_width)
         )
-        data = df[['$/unit','quantity']].apply(
-            lambda x: (float(x['$/unit']), float(x['quantity'])),
+        data = df[['price', '$/unit','quantity']].apply(
+            lambda x: (float(x['price']), float(x['$/unit']), float(x['quantity'])),
             axis=1, result_type='broadcast'
         )
-        data['avg'] = (data['$/unit']*data['quantity']).sum()/data['quantity'].sum()
-        data_max = data.max()['$/unit'] #.max()
-        assert len(data['avg'].unique()) == 1
         norm = [ (x,) for x in data['$/unit'] ] #.to_records(index=False)
-        self.graph.set_data(norm, data_max,
-            vscale=list(map(float, [
-                data['$/unit'].min(),
-                data['$/unit'].median(),
-                data['avg'].iloc[0],
-                data_max
-            ]))
-        )
-        #self.graph.set_bar_width(1)
-        # canvas_width = 10 + pad + pad + 10
-        date_strlen = (self.graph.canvas_width - 20)
-        ex = "─" if date_strlen % 2 else ""
-        plen = date_strlen//2
-        caption = f"{df['ts_raw'].min():%d/%m/%Y}"
-        caption += f"{{p:>{plen}}}{ex}{{p:<{plen}}}".format(
-            p="─")
-        caption += f"{df['ts_raw'].max():%d/%m/%Y}"
+        scale = stats(data, 'price', 'quantity', '$/unit')
+        self.graph.set_data(norm, scale[-1], vscale=scale)
+        time_range = get_time_range(df, 'ts_raw')
+        caption = get_caption(time_range, self.graph.canvas_width)
         self.graph.set_caption(caption)
 
     def update_historic_prices(self, data):
         organic = None if data['organic'] == 'mixed' else data['organic']
         #sort = '$/unit' if self.buttons['sort_price'].state else 'ts'
         product, unit = data['product'] or None, data['unit'] or None
-        try:
-            price = Decimal(data['price'])
-        except InvalidOperation:
-            price = None
-
-        try:
-            quantity = Decimal(data['quantity'])
-        except InvalidOperation:
-            quantity = None
+        price = decimal_or_none(data['price'])
+        quantity = decimal_or_none(data['quantity'])
 
         if None in (product, unit):
             self.rating.update_rating(None, None, None, unit)
@@ -341,7 +295,7 @@ class TransactionEditor(FocusWidget):
         query_manager: QueryManager,
     ):
         self.autocomplete_options = lambda name, data: self.query_manager.unique_suggestions(name.split('#', 1)[0], **data)
-        self.activity_manager = activity_manager
+        self.activity_manager: ActivityManager = activity_manager
         self.query_manager = query_manager
         self.buttons = {
             'done': Button(('streak', u'Done')),
@@ -385,26 +339,7 @@ class TransactionEditor(FocusWidget):
         )))
         self.organic_checkbox = self.checkboxes['organic']
         connect_signal(self.organic_checkbox, 'postchange', lambda _,v: self.update())
-        layout = [
-            [ 'ts',          'store',     ],
-            [ 'organic',     'product',   ],
-            [ 'category',    'group',     ],
-        ]
-        side_pane = [
-            'unit',
-            'quantity',
-            'price',
-        ]
-        bottom_pane = [
-            'description',
-            'dbview',
-        ]
-        badge = [
-            'rating',
-            'spread',
-            'marker',
-        ]
-        #self.clear()
+
         _widgets = dict(chain(*list(map(lambda x: x.items(), [
                 self.edit_fields, self.text_fields, self.checkboxes
             ])
@@ -421,10 +356,7 @@ class TransactionEditor(FocusWidget):
             connect_signal(ef, 'apply', lambda w, name: self.autocomplete_callback(
                 w, name, self.autocomplete_options(name, dict(map(
                     to_unnumbered_field,
-                    #filter(
-                    #    in_same_row(name),
                         map(to_numbered_field, self.data.items()
-                    #)
                 ))))
             ))
 
@@ -438,24 +370,20 @@ class TransactionEditor(FocusWidget):
                 title=k.title(), title_align='left'
             ) for k in self.edit_fields if k != 'product'
         })
-        header = Text(u'Fill Transaction', 'center')
-        _copyright = Text(COPYRIGHT, 'center')
 
         self.components.update({
             'bottom_button_bar': Columns(
                 [(8, self.buttons['done']), Divider(), (9, self.buttons['clear'])]
             ),
-            'badge': Pile(map(
-                lambda x: _widgets[x] if x is not None else Divider,
-                badge
-            )),
+            'badge': Pile([
+                _widgets[x] for x in ['rating', 'spread', 'marker']
+            ]),
         })
         self.components.update({
             'bottom_pane': Columns([
-                Pile(map(
-                    lambda x: _widgets[x] if x is not None else Divider(),
-                    bottom_pane
-                )),
+                Pile([
+                    _widgets[x] for x in ['description', 'dbview']
+                ]),
                 (self.graph.total_width+2, Pile([
                     LineBox(
                         AttrMap(self.components['badge'], 'badge'),
@@ -472,11 +400,6 @@ class TransactionEditor(FocusWidget):
         connect_signal(self.buttons['clear'], 'click', lambda _: self.clear())
         connect_signal(self.buttons['add'], 'click', lambda _: self.add_tag())
 
-        banner = Pile([
-            Padding(header, 'center', width=('relative', 100)),
-            Padding(_copyright, 'center', width=('relative', 100)),
-        ])
-        banner = AttrMap(banner, 'banner')
         _widgets.update({
             'product': LineBox(Columns([
                 AttrMap(AutoCompletePopUp(
@@ -489,25 +412,23 @@ class TransactionEditor(FocusWidget):
         })
 
         self.components['side_pane'] = (12, Pile([
-            _widgets[r] if r is not None else Divider() for r in side_pane
+            _widgets[r] for r in ['unit', 'quantity', 'price']
         ]))
-        self.components['main_pane'] = []
-        for _, r in enumerate(layout):
-            col = []
-            for c in r:
-                if c is not None:
-                    if c == 'organic':
-                        continue
-                    col.append(_widgets[c])
-                else:
-                    col.append(Divider())
-            self.components['main_pane'].append(Columns(col))
-
-        self.components['main_pane'] = Pile(self.components['main_pane'])
+
+        self.components['main_pane'] = Pile([
+            Columns([
+                _widgets[c] for c in ['ts', 'store']
+            ]),
+            _widgets['product'],
+            Columns([
+                _widgets[c] for c in ['category', 'group']
+            ])
+        ])
+
         self.add_tag()
 
         widget = Pile([
-            banner,
+            banner(u'Fill Transaction'),
             Divider(),
             Columns([
                 self.components['main_pane'],

app/activities/grouped_widget_util.py

@@ -0,0 +1,29 @@
+#
+# Copyright (c) Daniel Sheffield 2023
+#
+# All rights reserved
+#
+# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+from typing import Callable, Tuple
+
+def to_numbered_field(x: Tuple[str, str]) -> Tuple[str, int, str]:
+    if len(x[0].split('#', 1)) > 1:
+        name, idx = x[0].split('#', 1)
+        idx = int(idx)
+    else:
+        name, idx = x[0], 0
+
+    return name, int(idx), x[1]
+
+def to_unnumbered_field(x: Tuple[str, int, str]) -> Tuple[str, str]:
+    return x[0], x[2]
+
+def in_same_row(name: str) -> Callable[[Tuple[str, int, str]], bool]:
+    if len(name.split('#', 1)) > 1:
+        _, row = name.split('#', 1)
+    else:
+        row = 0
+    return lambda x: x[1] == int(row)
+
+def to_named_value(name: str) -> Callable[[Tuple[int, str]], Tuple[str, str]]:
+    return lambda e: (f'{name}#{e[0]}', e[1])

app/data/dataframe_util.py

@@ -0,0 +1,34 @@
+#
+# Copyright (c) Daniel Sheffield 2023
+#
+# All rights reserved
+#
+# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+from typing import List, Tuple
+from pandas import DataFrame
+
+def stats(data: DataFrame, num: str, den: str, frac: str) -> List[float]:
+    return list(map(float, [
+        data[frac].min(),
+        data[frac].median(),
+        data[num].sum()/data[den].sum(), # mean
+        data[frac].max()
+    ]))
+
+def get_time_range(data: DataFrame, ts_col: str) -> Tuple[str, str]:
+    return f"{data[ts_col].min():%d/%m/%Y}" , f"{data[ts_col].max():%d/%m/%Y}"
+
+def get_divider(width: int, marker:str = "─") -> str:
+    if width <= 3:
+        return " ─ "
+
+    plen = width//2 - 1
+    ex = marker if (width % 2) and plen > 0 else ""
+    return f" {{p:>{plen}}}{ex}{{p:<{plen}}} ".format(p=marker)
+
+def get_caption(time_range: Tuple[str, str], width: int) -> str:
+    left, right = time_range
+    pad = width - len(left) - len(right)
+    return ''.join([
+        left, get_divider(pad, marker="─"), right
+    ])

app/data/decimal_util.py

@@ -0,0 +1,15 @@
+#
+# Copyright (c) Daniel Sheffield 2023
+#
+# All rights reserved
+#
+# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+from decimal import Decimal, InvalidOperation
+from typing import Any
+
+
+def decimal_or_none(maybe_decimal: Any) -> Decimal:
+    try:
+        return Decimal(maybe_decimal)
+    except InvalidOperation:
+        return None

app/palette/__init__.py

@@ -1 +1,12 @@
-
+#
+# Copyright (c) Daniel Sheffield 2023
+#
+# All rights reserved
+#
+# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+def iter_palettes(theme):
+    palettes = [ v for k,v in theme.items() ]
+    while True:
+        p = palettes.pop(0)
+        palettes.append(p)
+        yield p

app/rest/Cache.py

@@ -1,44 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from typing import Dict
-
-from .CachedLoadingPage import CachedLoadingPage
-
-class Cache:
-    def __init__(self, limit) -> None:
-        self._cache: Dict[str, CachedLoadingPage] = dict()
-        self._limit = limit
-
-    def get(self, key: str) -> str:
-        if key not in self._cache:
-            return None
-        
-        page = self._cache[key]
-        if page.stale:
-            del self._cache[key]
-            return None
-        return page.value if page.loaded else page.update()
-    
-    def _enforce_limit(self, limit):
-        for idx, (_, k) in enumerate(sorted([
-                (v.age, k) for k, v in self._cache.items()
-            ])):
-            if idx >= limit: del self._cache[k]
-
-    def _clear_stale(self):
-        for k in [k for k, v in self._cache.items() if v.stale]:
-            del self._cache[k]
-    
-    def add(self, key: str, page: CachedLoadingPage) -> str:
-        self._clear_stale()
-        self._enforce_limit(self._limit)
-        self._cache[key] = page
-        return page.value
-    
-    def remove(self, key: str):
-        if key in self._cache:
-            del self._cache[key]
-

app/rest/CachedLoadingPage.py

@@ -6,19 +6,22 @@
 from queue import Queue, Empty
 from time import time
 from threading import Lock
-from typing import Callable
+from typing import Callable, Union
+
+STALE = 7*24*60*60
 
 class CachedLoadingPage():
     
     value: str
 
-    def __init__(self, initial_value: str, provider: Callable[[Queue], None]):
+    def __init__(self, initial_value: Union[str, list], provider: Callable[[Queue], None], incremental: bool = True):
         self._created = time()
         self._queue = Queue()
         self._loaded = False
         self.value = initial_value
         self._lock = Lock()
         self.provider = provider
+        self.incremental = incremental
 
     @property
     def age(self) -> float:
@@ -38,16 +41,16 @@ class CachedLoadingPage():
     
     @property
     def stale(self) -> bool:
-        return self.age > 10*60
+        return self.age > STALE
     
     def _start(self) -> None:
         if not self.provider:
             return
-        
+
         self.provider(self.queue)
         self.provider = None
 
-    def update(self) -> str:
+    def update(self) -> Union[str, list]:
         if not self._lock.acquire(blocking=True, timeout=0.5):
             return self.value
         try:
@@ -57,9 +60,13 @@ class CachedLoadingPage():
                 self._queue.task_done()
                 self._set_loaded(True)
             else:
-                self.value = item
+                if self.incremental:
+                    self.value.append(item)
+                else:
+                    self.value = item
                 self.queue.task_done()
         except Empty:
             pass
-        self._lock.release()
+        finally:
+            self._lock.release()
         return self.value

app/rest/PageCache.py

@@ -0,0 +1,151 @@
+#
+# Copyright (c) Daniel Sheffield 2023
+# All rights reserved
+#
+# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+
+import os
+from time import time
+from typing import Dict
+import cherrypy
+
+from .hash_util import blake, bytes_to_hash, hash_to_base32
+from .CachedLoadingPage import STALE, CachedLoadingPage
+
+def delete_page(name: str, root: str = 'app/rest/static/files'):
+    directory = f'{root}/{name}'
+    try:
+        os.remove(f'{directory}/{name}.file')
+    except FileNotFoundError:
+        pass
+
+
+def save_page(name: str, content: bytes, tool: str, root='app/rest/static/files') -> str:
+    directory = f'{root}/{name}'
+    try:
+        os.mkdir(directory, mode=0o700, dir_fd=None)
+    except FileExistsError:
+        pass
+
+    fd = os.open(f'{directory}/{name}.file', os.O_WRONLY | os.O_TRUNC | os.O_CREAT, 0o600)
+    with open(fd, "wb") as f:
+        f.write(content)
+
+
+def get_page(name: str, root: str = 'app/rest/static/files') -> str:
+    directory = f'{root}/{name}'
+
+    try:
+        mtime = os.stat(f'{directory}/{name}.file').st_mtime
+    except:
+        mtime = None
+    
+    if mtime is None:
+        return None
+    
+    if mtime and time() - mtime > STALE:
+        delete_page(name)
+        return None
+
+    # todo: store file hash and validate it
+    fd = os.open(f'{directory}/{name}.file', os.O_RDONLY, 0o600)
+    with open(fd, "rb") as f:
+        f.seek(0)
+        page = f.read()
+    return page.decode('utf-8')
+
+
+def key_to_hash(key):
+
+    if isinstance(key, tuple):
+        orig, _hash = key
+    else:
+        if isinstance(key, int):
+            orig, _hash = None, key
+        else:
+            orig, _hash = key, None
+
+    if None not in (orig, _hash):
+        if get_hash(orig) != _hash:
+            raise KeyError(f"Invalid key: {key}")
+
+        return _hash
+
+    if (_hash, orig) is (None, None):
+        raise KeyError(f"Invalid key: {key}")
+    
+    return get_hash(orig) if _hash is None else _hash
+
+
+def get_hash(key):
+    _bytes = blake(key.encode('utf-8'), person='grocery'.encode('utf-8'))
+    return bytes_to_hash(_bytes)
+
+
+class PageCache:
+    def __init__(self, limit) -> None:
+        self._cache: Dict[str, CachedLoadingPage] = dict()
+        self._limit = limit
+
+    def __delitem__(self, key):
+        key = key_to_hash(key)
+        self._cache.pop(key, None)
+        delete_page(hash_to_base32(key))
+
+    def __getitem__(self, key):
+        return self.get(key)
+
+    def __setitem__(self, key, value):
+        self._cache[key_to_hash(key)] = value
+
+    def get(self, key: str) -> CachedLoadingPage:
+        key = key_to_hash(key)
+
+        if key not in self._cache:
+            try:
+                existing = get_page(hash_to_base32(key))
+            except:
+                existing = None
+            if existing is None:
+                return None
+            
+            return self.add(key, CachedLoadingPage(existing, lambda q: q.put(None), incremental=True))
+        
+        page = self._cache[key]
+        if page.stale:
+            del self._cache[key]
+            delete_page(hash_to_base32(key))
+            return None
+
+        if not page.loaded:
+            page.update()
+
+        if page.loaded:
+            try:
+                existing = get_page(hash_to_base32(key))
+            except:
+                existing = None
+            if existing is None:
+                content = ''.join(page.value) if isinstance(page.value, list) else page.value
+                save_page(hash_to_base32(key), content.encode('utf-8'), tool='grocery')
+
+        return page
+
+    def _enforce_limit(self, limit):
+        for idx, (_, k) in enumerate(sorted([
+                (v.age, k) for k, v in self._cache.items()
+            ])):
+            if idx >= limit: del self[k]
+
+    def _clear_stale(self):
+        for k in [k for k, v in self._cache.items() if v.stale]:
+            del self[k]
+    
+    def add(self, key: str, page: CachedLoadingPage) -> CachedLoadingPage:
+        self._clear_stale()
+        self._enforce_limit(self._limit)
+        self._cache[key_to_hash(key)] = page
+        return page
+    
+    def remove(self, key: str):
+        del self[key]

app/rest/QueryCache.py

@@ -0,0 +1,162 @@
+#
+# Copyright (c) Daniel Sheffield 2023
+# All rights reserved
+#
+# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+import os
+from typing import Dict, Iterable, Tuple
+from urllib.parse import urlencode
+
+from bottle import FormsDict
+from ..data.filter import get_filter, get_query_param
+
+from . import BOOLEAN, PARAMS
+from .hash_util import base32_to_hash, blake, bytes_to_hash, hash_to_base32, normalize_base32
+
+def delete_query(name: str, root: str = 'app/rest/static/files'):
+    directory = f'{root}/{name}'
+    try:
+        os.remove(f'{directory}/{name}.query')
+    except FileNotFoundError:
+        pass
+
+def save_query(name: str, content: bytes, tool: str, root='app/rest/static/files') -> str:
+    directory = f'{root}/{name}'
+    try:
+        os.mkdir(directory, mode=0o700, dir_fd=None)
+    except FileExistsError:
+        pass
+
+    fd = os.open(f'{directory}/{name}.query', os.O_WRONLY | os.O_TRUNC | os.O_CREAT, 0o600)
+    with open(fd, "wb") as f:
+        f.write(content)
+
+def get_query(name: str, root: str = 'app/rest/static/files') -> str:
+    directory = f'{root}/{name}'
+
+    try:
+        mtime = os.stat(f'{directory}/{name}.query').st_mtime
+    except:
+        mtime = None
+    
+    # if mtime and time() - mtime > STALE:
+    #     delete_query(name)
+    #     return None
+
+    # todo: store query hash and validate it
+    fd = os.open(f'{directory}/{name}.query', os.O_RDONLY, 0o600)
+    with open(fd, "rb") as f:
+        f.seek(0)
+        page = f.read()
+    return page.decode('utf-8')
+
+
+def norm(key):
+
+    if isinstance(key, tuple):
+        query, _hash = key
+    else:
+        if isinstance(key, int):
+            query, _hash = None, key
+        else:
+            query, _hash = key, None
+
+    if _hash and not isinstance(_hash, int):
+        _hash = base32_to_hash(normalize_base32(query.hash))
+        # TODO: normalize should be implicit
+        #_hash = base32_to_hash(query.hash)
+
+    if None not in (query, _hash):
+        if get_hash(query) != _hash:
+            raise KeyError(f"Invalid key: {key}")
+
+        return query, _hash
+
+    if (_hash, query) is (None, None):
+        raise KeyError(f"Invalid key: {key}")
+    
+    return query, _hash if _hash else get_hash(query)
+
+def get_hash(key):
+    _bytes = blake(key.encode('utf-8'), person='grocery'.encode('utf-8'))
+    return bytes_to_hash(_bytes)
+
+
+def normalize_query(query: FormsDict, allow: Iterable[str] = None) -> Tuple[str, str]:
+    _hash = query.hash
+    allow = allow or PARAMS
+    param = get_filter(query, allow=allow)
+    norm = urlencode(map(
+        lambda k: (
+            k, get_query_param(*param[k]) if k != 'organic' else BOOLEAN[
+                BOOLEAN.get(query.organic, None)
+            ]
+        ),
+        sorted(filter(bool, param))
+    ))
+    return norm, _hash
+
+
+class QueryCache:
+    def __init__(self, limit) -> None:
+        self._cache: Dict[int, str] = dict()
+        self._limit = limit
+
+    def __delitem__(self, key):
+        return self.remove(key)
+
+    def __getitem__(self, key):
+        return self.get(key)
+
+    def __setitem__(self, key, value):
+        return self.add(key, value)
+
+    def get(self, key: str) -> str:
+        query, _hash = norm(key)
+        if _hash not in self._cache:
+            if query:
+                return self.add(_hash, query)
+            
+            try:
+                existing = get_query(hash_to_base32(_hash))
+            except:
+                existing = None
+            
+            if existing:
+                return self.add(_hash, existing)
+            
+            return self.add(_hash, query)
+        
+        value = self._cache[_hash]
+        # if value.stale:
+        #     del self._cache[key]
+        #     delete_query(hash_to_base32(key))
+        #     return None
+        return value
+
+    # def _enforce_limit(self, limit):
+    #     for idx, (_, k) in enumerate(sorted([
+    #             (v.age, k) for k, v in self._cache.items()
+    #         ])):
+    #         if idx >= limit: del self[k]
+
+    # def _clear_stale(self):
+    #     for k in [k for k, v in self._cache.items() if v.stale]:
+    #         del self[k]
+    
+    def add(self, key: str, value: str) -> str:
+        #self._clear_stale()
+        #self._enforce_limit(self._limit)
+        query, _hash = norm(key)
+        value = value or query
+        
+        if not value:
+            raise ValueError("Invalid query string: {value}")
+        self._cache[_hash] = value
+        save_query(hash_to_base32(_hash), value.encode("utf-8"), 'query')
+        return value
+    
+    def remove(self, key: str):
+        key = norm(key)
+        self._cache.pop(key, None)
+        delete_query(hash_to_base32(key))

app/rest/__init__.py

@@ -7,5 +7,12 @@
 from bottle import TEMPLATE_PATH
 
 TEMPLATE_PATH.append("app/rest/templates")
-ALL_UNITS = {'g', 'kg', 'mL', 'L', 'Pieces', 'Bunches', 'Bags'}
-PARAMS = { 'group', 'category', 'product', 'unit', 'tag' }
+ALL_UNITS = { 'g', 'kg', 'mL', 'L', 'Pieces', 'Bunches', 'Bags' }
+PARAMS = { 'group', 'category', 'product', 'unit', 'tag', 'organic' }
+BOOLEAN = {
+    "1": True,
+    True: "1",
+    "0": False,
+    False: "0",
+    None: "0.5",
+}

app/rest/cherrypy.py

@@ -1,13 +1,17 @@
 import cherrypy
-import wsgigzip
 import bottle
 from .pyapi import *
 
-application = wsgigzip.GzipMiddleware(bottle.default_app())
+application = bottle.default_app()
 
+cherrypy.config.update({'environment' : 'staging'})
 cherrypy.config.update({
     'server.socket_host': "0.0.0.0",
     'server.socket_port': 6772,
+    'engine.autoreload.on': True,
+    'request.show_tracebacks': True,
+    'request.show_mismatched_params': True,
+    'log.screen': True,
 })
 
 cherrypy.tree.graft(application, "/")

app/rest/form.py

@@ -8,14 +8,18 @@ from typing import Dict, Tuple
 from bottle import template
 from pandas import DataFrame
 from itertools import chain
-from ..rest import ALL_UNITS
+from . import ALL_UNITS, BOOLEAN
 
 def get_option_groups(
     data: DataFrame, filter_data: Dict[str, Tuple[set, set]],
     k: str, g: str, _type: str
 ):
-    in_chart = data['$/unit'].apply(lambda x: (x or False) and True)
-    groups = sorted(set(data[g] if g is not None else []))
+    if k in data or (k == 'tag' and 'tags' in data):
+        k_data_in_chart = chain(*data["tags"]) if k == "tag" else data[k]
+    else:
+        k_data_in_chart = []
+    
+    groups = sorted(set(data[g] if g is not None and g in data else []))
     groups.append(None)
     if _type == "exclude":
         prefix = "!"
@@ -28,47 +32,43 @@ def get_option_groups(
         if group is None:
             if _type == "include":
                 selected.extend(filter_data[k][0] - (
-                    set((chain(*data[in_chart]["tags"]) if k == "tag" else data[in_chart][k]) if g is not None else set())
+                    set(k_data_in_chart if g is not None else set())
                 ))
                 unselected.extend(((
-                    set((chain(*data[in_chart]["tags"]) if k == "tag" else data[in_chart][k]) if g is None else set())
+                    set(k_data_in_chart if g is None else set())
                 ) | filter_data[k][1]) - filter_data[k][0])
             else:
                 selected.extend(filter_data[k][1])
                 unselected.extend((
-                    set((chain(*data[in_chart]["tags"]) if k == "tag" else data[in_chart][k]) if g is None else set())
+                    set(k_data_in_chart if g is None else set())
                 ) | (
-                    filter_data[k][0] - set(chain(*data[in_chart]["tags"]) if k == "tag" else data[in_chart][k]) - filter_data[k][1]
+                    filter_data[k][0] - set(k_data_in_chart) - filter_data[k][1]
                 ))
         else:
+            k_grouped_data_in_chart = set(data[data[g].apply(lambda x,axis=None: x == group)][k]) if k in data else set()
             if _type == "include":
-                selected.extend(filter_data[k][0] & set(
-                    data[in_chart & (data[g].apply(lambda x,axis=None: x == group))][k]
-                ))
-                unselected.extend(set(
-                    data[in_chart & (data[g].apply(lambda x,axis=None: x == group))][k]
-                ) - filter_data[k][0])
+                selected.extend(filter_data[k][0] & set(k_grouped_data_in_chart))
+                unselected.extend(k_grouped_data_in_chart - filter_data[k][0])
             else:
-                unselected.extend(set(
-                    data[in_chart & (data[g].apply(lambda x,axis=None: x == group))][k]
-                ))
+                unselected.extend(set(k_grouped_data_in_chart))
         assert set(selected) - set(unselected) == set(selected), f"{set(selected)} {set(unselected)}"
-        
-        yield {
-            "optgroup": group,
-            "options": sorted(map(lambda x: {
-                "selected": x[0],
-                "value": f"{prefix}{x[1]}",
-                "display": x[1]  
-            }, chain(
-                map(lambda x: (True, x), set(selected)),
-                map(lambda x: (False, x), set(unselected)),
-            )), key=lambda x: x["display"] if "display" in x else x["value"])
-        }
+        options = sorted(map(lambda x: {
+            "selected": x[0],
+            "value": f"{prefix}{x[1]}",
+            "display": x[1]  
+        }, chain(
+            map(lambda x: (True, x), set(selected)),
+            map(lambda x: (False, x), set(unselected)),
+        )), key=lambda x: x["display"] if "display" in x else x["value"])
+        if len(options) > 0:
+            yield {
+                "optgroup": group,
+                "options": options,
+            }
 
 
-def get_form(action: str, method: str, filter_data, data: DataFrame):
-    keys = sorted(filter(lambda x: x not in ('unit', 'tag'), filter_data), key=lambda x: {
+def get_form(action: str, method: str, filter_data: Dict[str, Tuple[set, set]], organic: bool, data: DataFrame):
+    keys = sorted(filter(lambda x: x not in ('unit', 'tag', 'organic'), filter_data), key=lambda x: {
         'product': 0,
         'category': 1,
         'group': 2,
@@ -104,5 +104,6 @@ def get_form(action: str, method: str, filter_data, data: DataFrame):
                 map(lambda x: (True, x), filter_data['unit'][0]),
                 map(lambda x: (False, x), ALL_UNITS - filter_data['unit'][0])
             )), key=lambda x: x["display"] if "display" in x else x["value"])
-        }
+        },
+        organic=BOOLEAN[organic],
     )

app/rest/hash_util.py

@@ -0,0 +1,180 @@
+from hashlib import blake2b, shake_128, md5, sha256, sha1
+import os
+from base64 import b64encode, b64decode, b85encode, b85decode
+import base32_lib as b32
+
+DIGEST_SIZE_BYTES = 3
+DIGEST_SIZE_BITMASK = 0xffffff
+DIGEST_SIZE_SIGNED_TO_UNSIGNED_BITMASK = 0x1ffffff
+DIGEST_SIZE_SIGNED_TO_UNSIGNED_BIT = 0x1000000
+DIGEST_SIZE_NIBBLES = DIGEST_SIZE_BYTES * 2
+B64ALTCHARS = b'.-'
+
+def sha1hash(data: str):
+    return sha1(data.encode("utf-8"), usedforsecurity=False).hexdigest()[:DIGEST_SIZE_BYTES]
+
+def sha256hash(data: str):
+    return sha256(data.encode("utf-8"), usedforsecurity=False).hexdigest()[:DIGEST_SIZE_BYTES]
+
+def md5hash(data: str):
+    return md5(data.encode("utf-8"), usedforsecurity=False).hexdigest()[:DIGEST_SIZE_BYTES]
+
+def shake(data: str):
+    return shake_128(data.encode("utf-8"), usedforsecurity=False).hexdigest(DIGEST_SIZE_BYTES)
+
+def blake(data: bytes, person: bytes = None) -> bytes:
+    return blake2b(
+        data,
+        usedforsecurity=False,
+        digest_size=DIGEST_SIZE_BYTES,
+        person=person
+    ).digest()
+
+def blake_file(path: str, person: bytes = None, root: str ='rest/static') -> bytes:
+    fd = os.open(f'{root}/{path}', os.O_RDONLY, 0o600)
+    with open(fd, "rb") as f:
+        f.seek(0)
+        _blake = blake2b(usedforsecurity=False, digest_size=DIGEST_SIZE_BYTES, person=person)
+        while f.peek(1):
+            _blake.update(f.read(1024))
+    return _blake.digest()
+        
+
+def python(data: str):
+    return hash(data)
+
+def normalize_hash(_hash: int) -> int:
+    #hex = hash_to_hex(_hash)
+    #return int(hex, 16)
+    #_bytes = _hash.to_bytes(8, byteorder='big', signed=True)
+    #return bytes_to_hash(_bytes)
+    return _hash & DIGEST_SIZE_BITMASK
+
+def normalize_bytes(_bytes: bytes) -> bytes:
+    return (b'\x00' * DIGEST_SIZE_BYTES + _bytes)[-DIGEST_SIZE_BYTES:]
+
+def normalize_hex(_hex: str) -> str:
+    #_bytes = hex_to_bytes(hex)
+    #return _bytes.hex()
+    return _hex.zfill(DIGEST_SIZE_NIBBLES)[-DIGEST_SIZE_NIBBLES:]
+
+def hex_to_bytes(_hex: str) -> bytes:
+    _bytes = bytes.fromhex(_hex.zfill(DIGEST_SIZE_NIBBLES))
+    return normalize_bytes(_bytes)
+
+def bytes_to_hex(_bytes: bytes) -> str:
+    return normalize_bytes(_bytes).hex()
+
+def hash_to_bytes(_hash: int) -> bytes:
+    _bytes = _hash.to_bytes(8, byteorder='big', signed=True)
+    return normalize_bytes(_bytes)
+
+def bytes_to_hash(_bytes: bytes) -> int:
+    norm = normalize_bytes(_bytes)
+    return int.from_bytes(norm, byteorder='big', signed=False)
+
+def hash_to_hex(_hash: int) -> str:
+    #return hash_to_bytes(_hash).hex()
+    #return normalize_hex(
+    #return f"{_hash + (1 << 64):x}"[-4:]
+    #return hex(_hash + (1<<64))[2:][-4:]
+    #return f"{_hash & 0xffff:04x}"
+    return hex((_hash|DIGEST_SIZE_SIGNED_TO_UNSIGNED_BIT) & DIGEST_SIZE_SIGNED_TO_UNSIGNED_BITMASK)[3:]
+
+def hex_to_hash(_hex: str) -> int:
+    #_bytes = bytes.fromhex(hex.zfill(4))
+    #return bytes_to_hash(_bytes)
+    #return int(normalize_hex(hex), 16)
+    return int(_hex, 16) & DIGEST_SIZE_BITMASK
+
+def remove_padding(f):
+    def wrap(*args, **kwargs):
+        return f(*args, **kwargs).split('=')[0]
+    return wrap
+
+def fix_padding(f):
+    def wrap(_b64, *args, **kwargs):
+        pad = (4 - len(_b64)) % 4
+        fixed = _b64 + '='*pad
+        return f(fixed, *args, **kwargs)
+    return wrap
+
+def normalize_base32(_b32: str):
+    return b32.encode(b32.decode(_b32)).upper().zfill(DIGEST_SIZE_BYTES*8//5+1)
+
+def add_padding_base32(f):
+    def wrap(*args, **kwargs):
+        return normalize_base32(f(*args, **kwargs))
+    return wrap
+
+@remove_padding
+def hash_to_base64(_hash: int) -> str:
+    return b64encode(hash_to_bytes(_hash), altchars=B64ALTCHARS).decode("utf-8")
+
+@remove_padding
+def hex_to_base64(_hex: str) -> str:
+    return b64encode(hex_to_bytes(_hex), altchars=B64ALTCHARS).decode("utf-8")
+
+@remove_padding
+def bytes_to_base64(_bytes: str) -> str:
+    return b64encode(normalize_bytes(_bytes), altchars=B64ALTCHARS).decode("utf-8")
+
+@fix_padding
+def base64_to_hash(_b64: str) -> str:
+    return bytes_to_hash(b64decode(_b64, altchars=B64ALTCHARS))
+
+@fix_padding
+def base64_to_hex(_b64: str) -> str:
+    return bytes_to_hex(b64decode(_b64, altchars=B64ALTCHARS))
+
+@fix_padding
+def base64_to_bytes(_b64: str) -> str:
+    return normalize_bytes(b64decode(_b64, altchars=B64ALTCHARS))
+
+#@remove_padding
+def hash_to_base85(_hash: int) -> str:
+    return b85encode(hash_to_bytes(_hash)).decode("utf-8")
+
+#@remove_padding
+def hex_to_base85(_hex: str) -> str:
+    return b85encode(hex_to_bytes(_hex)).decode("utf-8")
+
+#@remove_padding
+def bytes_to_base85(_bytes: str) -> str:
+    return b85encode(normalize_bytes(_bytes)).decode("utf-8")
+
+#@fix_padding
+def base85_to_hash(_b64: str) -> str:
+    return bytes_to_hash(b85decode(_b64))
+
+#@fix_padding
+def base85_to_hex(_b64: str) -> str:
+    return bytes_to_hex(b85decode(_b64))
+
+#@fix_padding
+def base85_to_bytes(_b64: str) -> str:
+    return normalize_bytes(b85decode(_b64))
+
+@add_padding_base32
+def hash_to_base32(_hash: int) -> str:
+    return b32.encode(_hash & DIGEST_SIZE_BITMASK)
+
+@add_padding_base32
+def hex_to_base32(_hex: str) -> str:
+    return b32.encode(hex_to_hash(_hex))
+
+@add_padding_base32
+def bytes_to_base32(_bytes: bytes) -> str:
+    return b32.encode(bytes_to_hash(_bytes))
+
+#@fix_padding
+def base32_to_hash(_b64: str) -> str:
+    return b32.decode(_b64)
+
+#@fix_padding
+def base32_to_hex(_b64: str) -> str:
+    return hash_to_hex(base32_to_hash(_b64))
+
+#@fix_padding
+def base32_to_bytes(_b64: str) -> str:
+    return hash_to_bytes(base32_to_hash(_b64))

app/rest/pyapi.py

@@ -3,34 +3,24 @@
 # All rights reserved
 #
 # THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from typing import Iterable, Dict
 import os
-from urllib.parse import urlencode
+from threading import Thread
+from typing import Tuple
 from bottle import (
-    route,
-    request,
-    response,
-    FormsDict,
-    redirect,
-    template,
+    route, request, response,
     static_file,
+    FormsDict,
 )
-from psycopg import connect
-from psycopg.sql import SQL, Literal
-from threading import Thread
-
-from ..data.filter import(
-    get_filter,
-    get_query_param,
-)
+from psycopg import Cursor, connect
+from psycopg.rows import TupleRow
+from urllib.parse import parse_qs
 
-from ..data.util import(
-    get_where_include_exclude
-)
-from . import trend as worker
-from . import PARAMS
+from .QueryCache import QueryCache
+from .route_decorators import cache, cursor
+from .query_to_xml import get_categories, get_groups, get_products, get_tags
 from .CachedLoadingPage import CachedLoadingPage
-from .Cache import Cache
+from .PageCache import PageCache
+from . import trend as worker
 
 host = f"host={os.getenv('HOST')}"
 db = f"dbname={os.getenv('DB', 'grocery')}"
@@ -40,194 +30,96 @@ if not password.split('=',1)[1]:
     password = ''
 conn = connect(f"{host} {db} {user} {password}")
 
-CACHE = Cache(10)
-
-def get_product_rollup_statement(filters, having=None):
-    where = [ get_where_include_exclude(
-        k[0], "name", list(include), list(exclude)
-    ) for k, (include, exclude) in filters.items() ]
-    return SQL('\n').join([
-        SQL("""
-SELECT
-  count(DISTINCT p.id) AS "Products",
-  count(DISTINCT c.id) AS "Categories",
-  count(DISTINCT g.id) AS "Groups",
-  p.name AS "Product",
-  c.name AS "Category",
-  g.name AS "Group"
-FROM products p
-JOIN categories c ON p.category_id = c.id
-JOIN groups g ON c.group_id = g.id
-"""),
-        SQL("""
-WHERE {where}
-""").format(where=SQL("\nAND").join(where)) if where else SQL(''),
-        SQL("""
-GROUP BY ROLLUP (g.name, c.name, p.name)
-"""),
-        SQL("""
-HAVING {having}
-ORDER BY "Product", "Category", "Group"
-""").format(having=having) if having else SQL('')
-    ])
-
-
-def normalize_query(query: FormsDict, allow: Iterable[str] = None) -> str:
-    param = get_filter(query, allow=allow)
-    return urlencode([
-        (k, get_query_param(*param[k])) for k in sorted(param) if param[k]
-    ])
-
-def _normalize_decorator(func, poison_on_reload=False):
-    def wrap(*args, **kwargs):
-        _, _, path, *_ = request.urlparts
-        normalized = normalize_query(request.query, allow=PARAMS)
-        if poison_on_reload and request.params.get('reload') == 'true':
-            CACHE.remove(normalized)
-        if request.query_string != normalized:
-            return redirect(f'{path}?{normalized}')
-        return func(*args, **kwargs)
-    return wrap
-
-def normalize(*args, **kwargs):
-    if not len(args):
-        return lambda f: _normalize_decorator(f, **kwargs)
-    
-    return _normalize_decorator(*args)
-
 @route('/grocery/static/<filename:path>')
 def send_static(filename):
     return static_file(filename, root='app/rest/static')
 
-@route('/grocery/trend')
-@normalize(poison_on_reload=True)
-def trend():
+def trend_thread(conn, path, forms):
+    def cb(queue):
+        return Thread(target=worker.trend, args=(
+            queue, conn, path, forms
+        )).start()
+    return cb
+
+PAGE_CACHE = PageCache(100)
+QUERY_CACHE = QueryCache(None)
+
+@route('/grocery/trend', method=['GET', 'POST'])
+@cache(query_cache=QUERY_CACHE, page_cache=PAGE_CACHE)
+def trend(key: Tuple[str, int], cache: PageCache):
     _, _, path, *_ = request.urlparts
-    normalized = normalize_query(request.query, allow=PARAMS)
-    if request.params.get('reload') == 'true':
-        CACHE.remove(normalized)
 
-    if request.query_string != normalized:
-        return redirect(f'{path}?{normalized}')
+    page = cache[key]
+    if page is None:
+        form = key_to_form(key)
+        page = cache.add(key, CachedLoadingPage([], trend_thread(conn, path, form)))
     
-    page = CACHE.get(normalized)
+    for i in iter_page(page):
+        yield i
 
-    return page if page else CACHE.add(normalized, CachedLoadingPage(
-        template("loading", progress=[]),
-        lambda queue: Thread(target=worker.trend, args=(
-            queue, conn, path, request.query
-        )).start()
-    ))
-
-@route('/grocery/groups')
-@normalize
-def groups():
-    filters = get_filter(request.query, allow=('group', 'category', 'product'))
-    form = template('form-nav', action='groups', method='get', params=[
-        {'name': k, 'value': request.params[k]} for k in request.params if k in PARAMS
-    ])
-    try:
-        with conn.cursor() as cur:
-            inner = get_product_rollup_statement(
-                filters,
-                having=SQL("c.name IS NULL")
-            )
-            xml = cur.execute(SQL("""
-SELECT query_to_xml_and_xmlschema({q}, false, false, ''::text)
-""").format(q=Literal(SQL("""SELECT
-    "Products",
-    "Categories",
-    COALESCE("Group", "Groups"||'') "Group"
-FROM (
-{inner}
-) q""").format(inner=inner).as_string(cur)))).fetchone()[0]
-    finally:
-        conn.commit()
-    response.content_type = 'application/xhtml+xml; charset=utf-8'
-    return template("query-to-xml", title="Groups", xml=xml, form=form)
-
-@route('/grocery/categories')
-@normalize
-def categories():
-    filters = get_filter(request.query, allow=('group', 'category', 'product'))
-    form = template('form-nav', action='categories', method='get', params=[
-        {'name': k, 'value': request.params[k]} for k in request.params if k in PARAMS
-    ])
-    try:
-        with conn.cursor() as cur:
-            inner = get_product_rollup_statement(
-                filters,
-                having=SQL("p.name IS NULL AND (c.name IS NOT NULL OR g.name IS NULL)")
-            )
-            xml = cur.execute(SQL("""
-SELECT query_to_xml_and_xmlschema({q}, false, false, ''::text)
-""").format(q=Literal(SQL("""SELECT
-    "Products",
-    COALESCE("Category", "Categories"||'') "Category",
-    COALESCE("Group", "Groups"||'') "Group"
-FROM (
-{inner}
-) q""").format(inner=inner).as_string(cur)))).fetchone()[0]
-    finally:
-        conn.commit()
+
+def query_to_form(query):
+    form = FormsDict()
+    for k, v in parse_qs(query).items():
+        for item in v:
+            form.append(k, item)
+    return form
+
+
+def key_to_form(key):
+    query, _ = key
+    return query_to_form(query)
+
+
+def iter_page(page):
+    # copy first to avoid races
+    resp = list(page.value)
+    pos = len(resp)
+    yield ''.join(resp)
+    
+    while not page.loaded:
+        page.update()
+        # all changes since last yield
+        resp = list(page.value[pos:])
+        pos = pos + len(resp)
+        yield ''.join(resp)
+    
+    # possibly have not yielded the entire page
+    if pos < len(page.value):
+        yield ''.join(page.value[pos:])
+
+
+@route('/grocery/groups', method=['GET', 'POST'])
+@cache(query_cache=QUERY_CACHE, page_cache=PAGE_CACHE)
+@cursor(connection=conn)
+def groups(cur: Cursor[TupleRow], key: Tuple[str, int], cache: PageCache):
+    form = key_to_form(key)
     response.content_type = 'application/xhtml+xml; charset=utf-8'
-    return template("query-to-xml", title="Categories", xml=xml, form=form)
-
-@route('/grocery/products')
-@normalize
-def products():
-    filters = get_filter(request.query, allow=('group', 'category', 'product'))
-    form = template('form-nav', action='products', method='get', params=[
-        {'name': k, 'value': request.params[k]} for k in request.params if k in PARAMS
-    ])
-    try:
-        with conn.cursor() as cur:
-            inner = get_product_rollup_statement(
-                filters,
-                having=SQL("p.name IS NOT NULL OR g.name IS NULL")
-            )
-            xml = cur.execute(SQL("""
-SELECT query_to_xml_and_xmlschema({q}, false, false, ''::text)
-""").format(q=Literal(SQL("""SELECT
-    --"Transactions",
-    COALESCE("Product", "Products"||'') "Product",
-    COALESCE("Category", "Categories"||'') "Category",
-    COALESCE("Group", "Groups"||'') "Group"
-FROM (
-{inner}
-) q""").format(inner=inner).as_string(cur)))).fetchone()[0]
-    finally:
-        conn.commit()
+    return get_groups(cur, form)
+
+
+@route('/grocery/categories', method=['GET', 'POST'])
+@cache(query_cache=QUERY_CACHE, page_cache=PAGE_CACHE)
+@cursor(connection=conn)
+def categories(cur: Cursor[TupleRow], key: Tuple[str, int], cache: PageCache):
+    form = key_to_form(key)
     response.content_type = 'application/xhtml+xml; charset=utf-8'
-    return template("query-to-xml", title="Products", xml=xml, form=form)
-
-@route('/grocery/tags')
-@normalize
-def tags():
-    form = template('form-nav', action='tags', method='get', params=[
-        {'name': k, 'value': request.params[k]} for k in request.params if k in PARAMS
-    ])
-    try:
-        with conn.cursor() as cur:
-            inner = SQL('\n').join([SQL("""
-SELECT * FROM (SELECT count(DISTINCT txn.id) AS "Uses", tg.name AS "Name"
-FROM tags tg
-JOIN tags_map tm ON tg.id = tm.tag_id
-JOIN transactions txn ON txn.id = tm.transaction_id
-GROUP BY tg.name
-ORDER BY 1 DESC, 2) q
-UNION ALL
-SELECT count(DISTINCT txn.id) AS "Uses", count(DISTINCT tg.name)||'' AS "Name"
-FROM tags tg
-JOIN tags_map tm ON tg.id = tm.tag_id
-JOIN transactions txn ON txn.id = tm.transaction_id
-""")]).as_string(cur)
-            xml = cur.execute(SQL("""
-SELECT query_to_xml_and_xmlschema({inner}, false, false, ''::text)
-""").format(inner=Literal(inner))).fetchone()[0]
-    finally:
-        conn.commit()
+    return get_categories(cur, form)
+
+
+@route('/grocery/products', method=['GET', 'POST'])
+@cache(query_cache=QUERY_CACHE, page_cache=PAGE_CACHE)
+@cursor(connection=conn)
+def products(cur: Cursor[TupleRow], key: Tuple[str, int], cache: PageCache):
+    form = key_to_form(key)
     response.content_type = 'application/xhtml+xml; charset=utf-8'
-    return template("query-to-xml", title="Tags", xml=xml, form=form)
+    return get_products(cur, form)
 
 
+@route('/grocery/tags', method=['GET', 'POST'])
+@cache(query_cache=QUERY_CACHE, page_cache=PAGE_CACHE)
+@cursor(connection=conn)
+def tags(cur: Cursor[TupleRow], key: Tuple[str, int], cache: PageCache):
+    form = key_to_form(key)
+    response.content_type = 'application/xhtml+xml; charset=utf-8'
+    return get_tags(cur, form)

app/rest/query_to_xml.py

@@ -0,0 +1,143 @@
+#
+# Copyright (c) Daniel Sheffield 2023
+# All rights reserved
+#
+# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+from bottle import request, template, FormsDict
+from pandas import DataFrame
+from psycopg import Cursor
+from psycopg.sql import SQL, Literal, Identifier
+from typing import List
+
+from ..data.filter import get_filter
+from ..data.util import get_where_include_exclude
+from ..data.QueryManager import get_data
+from .form import get_form
+from . import BOOLEAN, PARAMS
+
+
+def get_product_rollup_statement(filters, orderby: List[str]) -> SQL:
+    _map = { k: k[0] for k in ('product', 'category', 'group') }
+    where = [ get_where_include_exclude(
+        _map[k], "name", list(include), list(exclude)
+    ) for k, (include, exclude) in filters.items() ]
+    return SQL("""
+SELECT
+  count(DISTINCT p.id) AS "Products",
+  count(DISTINCT c.id) AS "Categories",
+  count(DISTINCT g.id) AS "Groups",
+  p.name AS "product",
+  c.name AS "category",
+  g.name AS "group"
+FROM products p
+JOIN categories c ON p.category_id = c.id
+JOIN groups g ON c.group_id = g.id
+WHERE {where}
+GROUP BY ROLLUP (g.name, c.name, p.name)
+ORDER BY {orderby}
+""").format(
+        where=SQL("\nAND").join(where),
+        orderby=SQL(", ").join(map(Identifier,orderby)),
+    )
+
+
+def get_inner_query(query: FormsDict, orderby: List[str]) -> SQL:
+    filters = get_filter(query, allow=('group', 'category', 'product'))
+    inner = get_product_rollup_statement(filters, orderby)
+    return inner
+
+
+def render_form(cur: Cursor, inner: str, query: FormsDict):
+    _filter = get_filter(query, allow=PARAMS)
+    data = DataFrame(get_data(cur, inner)).dropna()
+    action = request.path.split('/')[-1]
+    organic = BOOLEAN.get(query.organic, None)
+    return get_form(action, 'post', _filter, organic, data)
+
+
+def get_xml(cur: Cursor, sql: str):
+    return cur.execute(SQL(
+        "SELECT query_to_xml_and_xmlschema({q}, false, false, ''::text)"
+    ).format(q=Literal(sql))).fetchone()[0]
+
+
+def get_products(cur: Cursor, query: FormsDict):
+    inner = get_inner_query(query, ['product', 'category', 'group'])
+    form = render_form(cur, inner, query)
+    sql = SQL("""
+SELECT
+    --"Transactions",
+    COALESCE("product", "Products"||'') "Product",
+    COALESCE("category", "Categories"||'') "Category",
+    COALESCE("group", "Groups"||'') "Group"
+FROM ({inner}) q
+WHERE q.product IS NOT NULL OR q.group IS NULL
+""").format(inner=inner).as_string(cur)
+    xml = get_xml(cur, sql)
+    return template("query-to-xml", title="Products", xml=xml, form=form)
+
+
+def get_categories(cur: Cursor, query: FormsDict):
+    inner = get_inner_query(query, ['category', 'group'])
+    form = render_form(cur, inner, query)
+    sql = SQL("""
+SELECT
+    "Products",
+    COALESCE("category", "Categories"||'') "Category",
+    COALESCE("group", "Groups"||'') "Group"
+FROM ({inner}) q
+WHERE q.product IS NULL AND (q.category IS NOT NULL OR q.group IS NULL)
+""").format(inner=inner).as_string(cur)
+    xml = get_xml(cur, sql)
+    return template("query-to-xml", title="Categories", xml=xml, form=form)
+
+
+def get_groups(cur: Cursor, query: FormsDict):
+    inner = get_inner_query(query, ['group',])
+    form = render_form(cur, inner, query)
+    sql = SQL("""
+SELECT
+    "Products",
+    "Categories",
+    COALESCE("group", "Groups"||'') "Group"
+FROM ({inner}) q
+WHERE q.category IS NULL
+""").format(inner=inner).as_string(cur)
+    xml = get_xml(cur, sql)
+    return template("query-to-xml", title="Groups", xml=xml, form=form)
+
+def get_tags_statement(filters) -> SQL:
+    _map = {
+        k: k[0] for k in ('product', 'category', 'group')
+    }
+    _map.update({ 'tag': 'tg' })
+    where = [ get_where_include_exclude(
+        _map[k], "name", list(include), list(exclude)
+    ) for k, (include, exclude) in filters.items() ]
+    return SQL("""
+SELECT * FROM (SELECT count(DISTINCT txn.id) AS "Uses", tg.name AS "Name"
+FROM tags tg
+JOIN tags_map tm ON tg.id = tm.tag_id
+JOIN transactions txn ON txn.id = tm.transaction_id
+WHERE {where}
+GROUP BY tg.name
+ORDER BY 1 DESC, 2) q
+UNION ALL
+SELECT count(DISTINCT txn.id) AS "Uses", count(DISTINCT tg.name)||'' AS "Name"
+FROM tags tg
+JOIN tags_map tm ON tg.id = tm.tag_id
+JOIN transactions txn ON txn.id = tm.transaction_id
+WHERE {where}
+""").format(where=SQL("\nAND").join(where))
+
+def get_inner_tags_query(query: FormsDict) -> SQL:
+    filters = get_filter(query, allow=('tag',))
+    inner = get_tags_statement(filters)
+    return inner
+
+def get_tags(cur: Cursor, query: FormsDict):
+    inner = get_inner_tags_query(query)
+    form = render_form(cur, inner, query)
+    sql = inner.as_string(cur)
+    xml = get_xml(cur, sql)
+    return template("query-to-xml", title="Tags", xml=xml, form=form)

app/rest/requirements.txt

@@ -1,5 +1,5 @@
 seaborn
 psycopg[binary]
 bottle
-wsgigzip
 cherrypy
+base32-lib

app/rest/route_decorators.py

@@ -0,0 +1,90 @@
+#
+# Copyright (c) Daniel Sheffield 2023
+# All rights reserved
+#
+# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+from typing import Callable, Iterable, Tuple
+from urllib.parse import urlencode
+from bottle import request, FormsDict, redirect, HTTPError
+from psycopg import Connection
+from psycopg.connection import TupleRow
+
+from .QueryCache import QueryCache, get_hash
+from ..data.filter import get_filter, get_query_param
+from . import BOOLEAN, PARAMS
+from .PageCache import PageCache
+from .hash_util import base32_to_hash, hash_to_base32, normalize_base32
+
+def normalize_query(query: FormsDict, allow: Iterable[str] = None) -> Tuple[str, str]:
+    if query.hash:
+        _hash = normalize_base32(query.hash)
+    else:
+        _hash = None
+
+    allow = allow or PARAMS
+    param = get_filter(query, allow=allow)
+    norm = urlencode([
+        (
+            k, get_query_param(*param[k])
+        ) if k != 'organic' else (
+            "organic", BOOLEAN[BOOLEAN.get(query.organic, None)]
+        ) for k in sorted(param) if param[k]
+    ])
+    return norm if not _hash or len(query.keys()) > 1 else None, _hash
+
+
+def _cache_decorator(func: Callable, query_cache: QueryCache = None, page_cache: PageCache = None):
+    def wrap(*args, **kwargs):
+        _, _, path, *_ = request.urlparts
+        
+        query, _hash = normalize_query(request.params)
+        if not _hash:
+            _hashInt = get_hash(query)
+            _hash = hash_to_base32(_hashInt)
+            key = (query, _hashInt)
+        else:
+            key = (query, base32_to_hash(_hash))
+        
+        if request.params.reload == "true":
+            page_cache.remove(key)
+
+        # key with tuple to avoid ambiguity
+        cached = query_cache.get(key)
+        if not cached:
+            if query:
+                cached = query_cache.add(key, None)
+            else:
+                return HTTPError(404, f"No query found for hash: {_hash}")
+
+        if not request.params.hash:
+            if cached and len(cached) > 2000:
+                return redirect(f"{path}?hash={_hash}")
+
+            if cached and request.query_string != cached:
+                return redirect(f"{path}?{cached}")
+
+        return func((cached, key[1]), page_cache, *args, **kwargs)
+    return wrap
+
+
+def cache(*args, **kwargs):
+    if not len(args):
+        return lambda f: _cache_decorator(f, **kwargs)
+    
+    raise Exception("decorator argument required")
+
+
+def _cursor_decorator(func: Callable, connection: Connection[TupleRow] = None):
+    def wrap(*args, **kwargs):
+        try:
+            with connection.cursor() as cur:
+                return func(cur, *args, **kwargs)
+        finally:
+            connection.commit()
+    return wrap
+
+
+def cursor(*args, **kwargs):
+    if not len(args):
+        return lambda f: _cursor_decorator(f, **kwargs)
+    raise Exception("decorator argument required")

app/rest/static/manifest.json

@@ -0,0 +1,22 @@
+{
+  "id": "/grocery",
+  "name": "Grocery Manager Web Application",
+  "short_name": "Grocery",
+  "description": "View trending price data and tracked product info",
+  "start_url": "/grocery/trend",
+  "theme_color": "firebrick",
+  "background_color": "black",
+  "display": "standalone",
+  "icons": [
+    {
+      "src": "/grocery/static/favicon.svg",
+      "sizes": "any",
+      "purpose": "any"
+    },
+    {
+      "src": "/grocery/static/favicon_square.svg",
+      "sizes": "any",
+      "purpose": "any"
+    }
+  ]
+}

app/rest/templates/done.tpl

@@ -0,0 +1 @@
+<div class="done"></div>

app/rest/templates/error-500.tpl

@@ -0,0 +1 @@
+<span style="font-size: 3em">{{ error }}</span>

app/rest/templates/filter-set.tpl

@@ -6,6 +6,12 @@
 %>
 <%
   include('include-exclude', **tags)
-  include('select-one', **units)
+
 %>
+  <div class="pure-u-1-3 pure-u-lg-1-12">
+    <div class="pure-g">
+    <% include('select-one', **units) %>
+    <% include('range-organic') %>
+    </div>
+  </div>
 <div class="pure-u-lg-1-8"></div>

app/rest/templates/form-filter.tpl

@@ -1,5 +1,24 @@
 % from app.data.filter import get_query_param
 <form id="filter" method="{{ method }}" action="{{ action }}">
+  <style>
+  select::-webkit-scrollbar {
+  width: 11px;
+}
+select {
+  color: #cccccc;
+  background-color: #080808;
+  scrollbar-width: thin;
+  scrollbar-color: var(--thumbBG) var(--scrollbarBG);
+}
+select::-webkit-scrollbar-track {
+  background: var(--scrollbarBG);
+}
+select::-webkit-scrollbar-thumb {
+  background-color: var(--thumbBG) ;
+  border-radius: 6px;
+  border: 3px solid var(--scrollbarBG);
+}
+  </style>
   % include('button-style')
   % include('button-nav', action=action)
   <details style="padding: 1em 0">

app/rest/templates/loading.tpl

@@ -1,27 +0,0 @@
-<html>
-  <head>
-    <link rel="stylesheet" href="/grocery/static/cloud-gears.css"/>
-    <style>
-body {
-  background-color: #080808;
-  color: #cccccc;
-}
-.loader-container {
-  position: absolute;
-  left: 45vw;
-  top: 45vh;
-}
-    </style>
-  </head>
-  <body>
-    <div class="loader-container">
-      <span class="loader"></span>
-      <%
-      for indicator in progress:
-          include('progress', **indicator)
-      end
-      %>
-    </div>
-    <meta http-equiv="Refresh" content="0;" />
-  </body>
-</html>

app/rest/templates/progress.tpl

@@ -1,3 +1,5 @@
-<p>
-{{name}}... {{status}}
-</p>
+<div class="progress">
+  <progress id="loading-{{stage}}" value="{{percent}}" max="100"></progress>
+  <br/>
+  <label for="loading-{{stage}}">{{stage}}</label>
+</div>

app/rest/templates/query-to-xml.tpl

@@ -8,6 +8,8 @@
     <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/purecss@2.1.0/build/pure-min.css" integrity="sha384-yHIFVG6ClnONEA5yB5DJXfW2/KC173DIQrYoZMEtBvGzmf0PKiGyNEqe9N6BNDBH" crossorigin="anonymous"/>
     <link rel="stylesheet" href="https://shandan.one/css/grids-responsive-min.css"/>
     <link rel="stylesheet" href="https://shandan.one/css/responsive-visibility-collapse.css"/>
+    <link rel="manifest" href="/grocery/static/manifest.json"/>
+    <link rel="icon" type="image/png" href="/grocery/static/favicon.png"/>
     <style>
 body {
   background-color: #080808;

app/rest/templates/range-organic.tpl

@@ -0,0 +1,14 @@
+<div class="pure-u-1">
+  <div class="l-box">
+    <h3>Organic</h3>
+  </div>
+  </div>
+  <label for="organic-state" hidden="true">Organic</label>
+  <div class="pure-g">
+  <div class="pure-u-1-3">No</div>
+  <div class="pure-u-1-3">Any</div>
+  <div class="pure-u-1-3">Yes</div>
+  <div class="pure-u-1">
+    <input type="range" id="organic-state" name="organic" min="0" max="1" step="0.5" value="{{organic}}" />
+  </div>
+</div>

app/rest/templates/select-one.tpl

@@ -1,13 +1,10 @@
-<div class="pure-u-1-3 pure-u-lg-1-12">
-<div class="pure-g">
-  <div class="pure-u-1">
-    <div class="l-box">
-      <h3>{{name.title()}}</h3>
-    </div>
+<div class="pure-u-1">
+  <div class="l-box">
+    <h3>{{name.title()}}</h3>
   </div>
-  <% include('select', id=f"{name}-select-one", name=name,
-     children=[{
-        "options": options
-     }]) %>
 </div>
-</div>
+<% include('select', id=f"{name}-select-one", name=name,
+   children=[{
+      "options": options
+   }])
+%>

app/rest/templates/select.tpl

@@ -1,11 +1,11 @@
 % from bottle import template
-% multiple = (get("multiple", False) and "multiple") or ""
+% multiple = (get("multiple", False) and 'multiple="true"') or ""
 <div class="pure-u-1">
 %  if defined("label"):
 %    include('label', id=id, label=label)
 %  end
 
-<select id="{{id}}" name="{{name}}" size=10 {{multiple}} style="width: calc(100% - 1em); margin: 0 1em 1em">
+<select id="{{id}}" name="{{name}}" size="10" {{!multiple}} style="width: calc(100% - 1em); margin: 0 1em 1em">
 %  if defined("hint"):
 %    include('option', value=hint, disabled=True)
 %  end

app/rest/templates/trend.tpl

@@ -1,46 +1,77 @@
-<!DOCTYPE html>
+% setdefault("start", False)
+% setdefault("end", False)
+% setdefault("error", '')
+% if start:
 <html>
-    <head>
-        <style>
+  <head>
+    <title>Trend</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1"/>
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/purecss@2.1.0/build/pure-min.css" integrity="sha384-yHIFVG6ClnONEA5yB5DJXfW2/KC173DIQrYoZMEtBvGzmf0PKiGyNEqe9N6BNDBH" crossorigin="anonymous"/>
+    <link rel="stylesheet" href="https://shandan.one/css/grids-responsive-min.css"/>
+    <link rel="stylesheet" href="https://shandan.one/css/responsive-visibility-collapse.css"/>
+    <link rel="stylesheet" href="/grocery/static/cloud-gears.css"/>
+    <link rel="manifest" href="/grocery/static/manifest.json"/>
+    <link rel="icon" type="image/png" href="/grocery/static/favicon.png"/>
+    <style>
 html {
   --scrollbarBG: #333333;
   --thumbBG: #080808;
 }
+svg {
+  max-height: min(100vh, calc(100vw * 9 / 16));
+  max-width: calc(100vw - 2em);
+}
 body {
   background-color: #080808;
   color: #cccccc;
+  text-align: center;
 }
-select::-webkit-scrollbar {
-  width: 11px;
+div.loader-container {
+  position: absolute;
+  left: 50vw;
+  top: 50vh;
+  margin-top: -5.5em;
+  margin-left: -87.5px;
+  padding-bottom: 2em;
+  height: 9em;
+  width: 175px;
 }
-select {
-  color: #cccccc;
-  background-color: #080808;
-  scrollbar-width: thin;
-  scrollbar-color: var(--thumbBG) var(--scrollbarBG);
+div.loader-container:not(:has(+ .done)) {
+  display: block;
 }
-select::-webkit-scrollbar-track {
-  background: var(--scrollbarBG);
+.loader-container:not(:last-child) {
+  display: none;
 }
-select::-webkit-scrollbar-thumb {
-  background-color: var(--thumbBG) ;
-  border-radius: 6px;
-  border: 3px solid var(--scrollbarBG);
+div.progress {
+  margin: 1em 0 1em;
 }
-
-svg {
-  max-height: min(100vh, calc(100vw * 9 / 16));
-  max-width: calc(100vw - 2em);
+div.progress:not(:has(+ .done)) {
+  display: block;
+}
+.progress label {
+  text-align:left;
+}
+.progress label:after {
+  content: "...";
+}
+.progress:not(:last-child) {
+  display: none;
 }
-        </style>
-        <title>Trend</title>
-        <meta name="viewport" content="width=device-width, initial-scale=1"/>
-        <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/purecss@2.1.0/build/pure-min.css" integrity="sha384-yHIFVG6ClnONEA5yB5DJXfW2/KC173DIQrYoZMEtBvGzmf0PKiGyNEqe9N6BNDBH" crossorigin="anonymous"/>
-        <link rel="stylesheet" href="https://shandan.one/css/grids-responsive-min.css"/>
-        <link rel="stylesheet" href="https://shandan.one/css/responsive-visibility-collapse.css"/>
-    </head>
-    <body align="center" style="text-align: center">
+    </style>
+  </head>
+  <body>
+    <div class="loader-container">
+    <span class="loader"></span>
+% end
+% if end:
+    </div>
+    <div class="done"></div>
 {{!form}}
+    % if error:
+    % include('error-500', error=error)
+    % else:
 {{!svg}}
+    % end
     </body>
 </html>
+% end

app/rest/trend.py

@@ -4,25 +4,28 @@
 # All rights reserved
 #
 # THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+from io import StringIO
+from queue import Queue
 from bottle import (
-    DictProperty,
+    FormsDict,
     HTTPError,
     template,
 )
-from io import StringIO
 import matplotlib.pyplot as plt
 import matplotlib
+from pandas import DataFrame
 import seaborn as sns
 from psycopg import Connection
 from psycopg.connection import TupleRow
-from queue import Queue
-from . import ALL_UNITS
+
+from . import ALL_UNITS, BOOLEAN, PARAMS
 from ..data.QueryManager import (
     display_mapper,
     QueryManager,
 )
 from ..data.filter import (
     get_filter,
+    get_query_param,
 )
 from ..activities.Plot import (
     get_data,
@@ -30,42 +33,50 @@ from ..activities.Plot import (
 from .form import(
     get_form,
 )
-from . import PARAMS
 
 matplotlib.use('agg')
 
 def abort(code, text):
-    return HTTPError(code, text)
+    raise HTTPError(code, text)
 
-def trend(queue: Queue, conn: Connection[TupleRow], path: str, query: DictProperty):
+def trend(queue: Queue, conn: Connection[TupleRow], path: str, query: FormsDict):
     for item in trend_internal(conn, path, query):
         queue.put(item, block=True)
     queue.put(None)
 
-def trend_internal(conn: Connection[TupleRow], path: str, query: DictProperty):
-    progress = []
+def trend_internal(conn: Connection[TupleRow], path: str, query: FormsDict):
+    progress = {
+        'stage': None,
+        'percent': None,
+    }
+    action = path.split('/')[-1]
+    organic = BOOLEAN.get(query.organic, None)
+    _filter = get_filter(query, allow=PARAMS)
+    yield template("trend", start=True)
     try:
         with conn.cursor() as cur:
             query_manager = QueryManager(cur, display_mapper)
-            fields = { k: query[k] or None for k in query.keys() if k in PARAMS }
-            unit = fields['unit'] = fields['unit'] or 'kg' if 'unit' in fields else 'kg'
+            fields = {
+                k: get_query_param(*_filter[k])
+                for k in sorted(_filter) if k not in ('organic', 'unit') and _filter[k]
+            }
+            unit = fields['unit'] = query.unit or 'kg'
+            fields['organic'] = BOOLEAN.get(query.organic, None)
             if unit and unit not in ALL_UNITS:
-                yield abort(400, f"Unsupported unit {unit}")
-                return
+                abort(400, f"Unsupported unit: {unit}")
 
-            progress.append({ "name": "Loading data", "status": ""})
-            yield template("loading", progress=progress)
+            progress.update({ "stage": "Querying database", "percent": "10"})
+            yield template("done") + template("progress", **progress)
             data = get_data(query_manager, **fields)
-
-            progress[-1]["status"] = "done"
-            yield template("loading", progress=progress)
-
+            
             if data.empty:
-                yield abort(404, f"No data for {fields}")
-                return
+                abort(404, f"No data.")
+            
+            progress.update({ "stage": "Preparing data", "percent": "30"})
+            yield template("done") + template("progress", **progress)
             
-            progress.append({ "name": "Loading chart", "status": ""})
-            yield template("loading", progress=progress)
+            in_chart = data['$/unit'].apply(lambda x: (x or False) and True)
+            data = data[in_chart]
             
             pivot = data.pivot_table(index=['ts_raw',], columns=['product',], values=['$/unit'], aggfunc='mean')
             pivot.columns = pivot.columns.droplevel()
@@ -107,21 +118,24 @@ def trend_internal(conn: Connection[TupleRow], path: str, query: DictProperty):
             for _, spine in ax.spines.items():
                 spine.set_color('#ffffff')
             
-            progress[-1]["status"] = "done"
-            yield template("loading", progress=progress)
+            progress.update({ "stage": "Rendering chart", "percent": "50"})
+            yield template("done") + template("progress", **progress)
             
-            progress.append({ "name": "Rendering chart", "status": ""})
-            yield template("loading", progress=progress)
-
             f = StringIO()
             plt.savefig(f, format='svg')
-            _filter = get_filter(query, allow=PARAMS)
-            form = get_form(path.split('/')[-1], 'get', _filter, data)
+            progress.update({ "stage": "Done", "percent": "100" })
+            yield template("done") + template("progress", **progress)
             
-            progress[-1]["status"] = "done"
-            yield template("loading", progress=progress)
+            form = get_form(action, 'post', _filter, organic, data)
             
-            yield template("trend", form=form, svg=f.getvalue())
+            yield template("trend", end=True, form=form, svg=f.getvalue())
+
+    except HTTPError as e:
+        if 'data' not in locals():
+            data = DataFrame()
+        if 'form' not in locals():
+            form = get_form(action, 'post', _filter, organic, data)
+        yield template("done") + template("trend", end=True, form=form, error=e.body)
 
     finally:
         conn.commit()

grocery_transactions.py

@@ -11,7 +11,7 @@ from urwid import raw_display, WidgetPlaceholder, SolidFill, MainLoop
 from app.activities.ActivityManager import ActivityManager, show_or_exit
 from app.activities.TransactionEditor import TransactionEditor
 from app.data.QueryManager import QueryManager, display_mapper
-from app.palette import solarized
+from app.palette import iter_palettes, solarized
 
 try:
     from db_credentials import HOST, PASSWORD
@@ -114,14 +114,8 @@ activity_manager.create(TransactionEditor, 'transaction',
 )
 
 app = None
-def iter_palettes():
-    palettes = [ v for k,v in solarized.theme.items() ]
-    while True:
-        p = palettes.pop(0)
-        palettes.append(p)
-        yield p
-
-palettes = iter_palettes()
+
+palettes = iter_palettes(solarized.theme)
 
 try:
     app = GroceryTransactionEditor(activity_manager, cur, log)

requirements.txt

@@ -9,4 +9,3 @@ pandas
 pyyaml
 numpy
 seaborn
-urllib

test/activities/test_Rating.py

@@ -1,10 +1,9 @@
 #
-# Copyright (c) Daniel Sheffield 2021 - 2023
+# Copyright (c) Daniel Sheffield 2023
 #
 # All rights reserved
 #
 # THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-import numpy as np
 from app.activities.Rating import Rating
 from pytest import mark, fixture
 from urwid import Text

test/activities/test_activity_manager.py

@@ -0,0 +1,83 @@
+#
+# Copyright (c) Daniel Sheffield 2023
+#
+# All rights reserved
+#
+# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+from itertools import chain
+from pytest import mark, fixture, raises
+from urwid.raw_display import Screen
+from urwid import ExitMainLoop
+from urwid.widget import Widget, Text
+from app.activities import ActivityManager
+from app.activities.ActivityManager import show_or_exit, ActivityManager
+
+from app.palette import high_contrast, iter_palettes, solarized
+
+@fixture
+def palettes():
+    return chain(iter_palettes(solarized.theme), iter_palettes(high_contrast.theme))
+
+@fixture
+def activity_manager():
+    return ActivityManager()
+
+@fixture
+def widget():
+    return Widget()
+
+@mark.parametrize('screen', [
+    None,
+    Screen(),
+])
+@mark.parametrize('key, expected', [
+  ((1,), None),
+  ('ctrl home', None),
+  ('esc', (ExitMainLoop, ""))
+])
+def test_show_or_exit(key, screen, palettes, expected):
+    if not isinstance(expected, tuple):
+        assert expected == show_or_exit(key, screen=screen, palettes=palettes)
+        return
+    
+    with raises(expected[0]):
+        show_or_exit(key, screen=screen, palettes=palettes)
+
+def test_add(activity_manager, widget):
+    assert widget == activity_manager.add(widget, 'myActivity')
+    assert 'myActivity' in activity_manager.widgets
+    assert widget == activity_manager.add(widget, 'myOtherActivity')
+    assert 'myOtherActivity' in activity_manager.widgets
+    assert 'myActivity' in activity_manager.widgets
+
+def test_get(activity_manager, widget):
+    assert activity_manager.get('myActivity') is None
+    activity_manager.add(widget, 'myActivity')
+    assert widget == activity_manager.get('myActivity')
+    activity_manager.add(widget, 'myOtherActivity')
+    assert widget == activity_manager.get('myOtherActivity')
+    assert widget == activity_manager.get('myActivity')
+
+def test_create(activity_manager):
+    widget = activity_manager.create(Text, 'widget_name', 'abc', align='right')
+    assert isinstance(widget, Text)
+    assert activity_manager.get('widget_name') is widget
+    activity_manager.show(widget)
+    assert activity_manager.app is widget
+
+def test_show(activity_manager, widget):
+    assert activity_manager.app is None
+    activity_manager.add(widget, 'myActivity')
+    assert activity_manager.show(widget) is None
+    assert activity_manager.app is widget
+    assert getattr(activity_manager.app, 'original_widget', None) is None
+    assert activity_manager.show(widget) is None
+    assert activity_manager.app.original_widget is widget
+
+def test_current(activity_manager, widget):
+    assert activity_manager.current() is None
+    activity_manager.add(widget, 'myActivity')
+    activity_manager.show(widget)
+    # fix requiring call twice if not a container widget
+    activity_manager.show(widget)
+    assert activity_manager.current() is widget

test/activities/test_banner.py

@@ -0,0 +1,24 @@
+#
+# Copyright (c) Daniel Sheffield 2023
+#
+# All rights reserved
+#
+# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+from app import COPYRIGHT
+from app.activities.Banner import banner
+from urwid import Text, Pile, AttrMap, Padding
+
+def test_banner():
+    myBanner = banner('test-banner')
+    assert isinstance(myBanner.original_widget, Pile)
+    assert isinstance(myBanner, AttrMap)
+    contents = myBanner.original_widget.contents
+    assert len(contents) == 2
+    for (w,_), expected in zip(contents, [
+        'test-banner',
+        COPYRIGHT
+    ]):
+        assert isinstance(w, Padding)
+        original = w.original_widget
+        assert isinstance(original, Text)
+        assert original.get_text()[0] == expected

test/activities/test_grouped_widget_util.py

@@ -0,0 +1,77 @@
+#
+# Copyright (c) Daniel Sheffield 2023
+#
+# All rights reserved
+#
+# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+from pytest import mark, fixture
+from itertools import chain
+from app.activities.grouped_widget_util import (
+    to_named_value,
+    to_numbered_field,
+    to_unnumbered_field,
+    in_same_row,
+)
+
+@mark.parametrize("idx", chain(range(2), [None, 'any']))
+@mark.parametrize("key, value", [
+    ('label', 'value'),
+    ('label2', 3),
+    ('checkbox', True),
+    ('checkbox', 'mixed'),
+])
+def test_to_unnumbered_field(key, idx, value):
+    assert (key, value) == to_unnumbered_field((key, idx, value))
+
+@mark.parametrize("key, value, expected", [
+    ('label#1', 'value', ('label', 1, 'value')),
+    ('label2#0', 3, ('label2', 0, 3)),
+    ('checkbox', True, ('checkbox', 0, True)),
+    ('checkbox', 'mixed', ('checkbox', 0, 'mixed')),
+])
+def test_to_numbered_field(key, value, expected):
+    assert expected == to_numbered_field((key, value))
+
+@mark.parametrize("key, values, expected", [
+    ('label#1', [
+        ('label', 1, 'value'),
+        ('label2', 0, 3),
+        ('checkbox', 0, True),
+        ('checkbox', 0, 'mixed'),
+        ('label2', 1, 0),
+    ], [
+        ('label', 1, 'value'),
+        ('label2', 1, 0),
+    ]),
+    ('label', [
+        ('label', 1, 'value'),
+        ('label2', 0, 3),
+        ('checkbox', 0, True),
+        ('checkbox', 0, 'mixed'),
+        ('label2', 1, 0),
+    ], [
+        ('label2', 0, 3),
+        ('checkbox', 0, True),
+        ('checkbox', 0, 'mixed'),
+    ]),
+])
+def test_in_same_row(key, values, expected):
+    assert expected == list(filter(in_same_row(key), values))
+
+@mark.parametrize("key, values, expected", [
+    ('label2', [
+        (1, 'value'),
+        (0, 3),
+        (0, True),
+        (0, 'mixed'),
+        (1, 0),
+    ], [
+        ('label2#1', 'value'),
+        ('label2#0', 3),
+        ('label2#0', True),
+        ('label2#0', 'mixed'),
+        ('label2#1', 0)
+    ]),
+])
+def test_to_named_value(key, values, expected):
+    assert expected == list(map(to_named_value(key), values))

test/data/test_dataframe_util.py

@@ -0,0 +1,67 @@
+#
+# Copyright (c) Daniel Sheffield 2023
+#
+# All rights reserved
+#
+# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+from datetime import datetime
+from pytest import mark, raises
+from pandas import DataFrame
+
+from app.data.dataframe_util import(
+    get_caption,
+    get_divider,
+    get_time_range,
+    stats,
+)
+
+@mark.parametrize('data', [
+    DataFrame([
+        {'ts': datetime(1970, 1, 1)}, {'ts': datetime(2000, 2, 28)}
+    ])
+])
+def test_get_timerange(data):
+    assert ("01/01/1970", "28/02/2000") == get_time_range(data, 'ts')
+
+@mark.parametrize('width, marker, expected', [
+    #(0, '─', (AssertionError, "")),
+    #(1, '─', (AssertionError, "")),
+    #(2, '─', (AssertionError, "")),
+    (0, '─', ' ─ '),
+    (1, '─', ' ─ '),
+    (2, '─', ' ─ '),
+    (3, '─', ' ─ '),
+    (4, '─', ' ── '),
+    (5, '─', ' ─── '),
+    (6, '─', '  ──  '),
+    (7, '─', '  ───  '),
+    (8, '─', '   ──   '),
+])
+def test_get_divider(width, marker, expected):
+    if not isinstance(expected, tuple):
+        assert expected == get_divider(width, marker=marker)
+        return
+
+    with raises(expected[0]):
+        get_divider(width, marker=marker)
+
+@mark.parametrize('data', [
+    DataFrame([
+        {'ts': 0, 'price': 3.0, 'amount': 10, '$/unit': 0.3 },
+        {'ts': datetime(2000, 2, 28), 'price': 11, 'amount': 2, '$/unit': 5.5 },
+    ])
+])
+def test_stats(data):
+    assert [0.3, 2.9, 14/12, 5.5] == stats(data, 'price', 'amount', '$/unit')
+
+@mark.parametrize('data', [
+    DataFrame([
+        {'ts': datetime(1970, 1, 1)}, {'ts': datetime(2000, 2, 28)}
+    ])
+])
+@mark.parametrize('width, div', [
+    (26, ' ── '),
+    (27, ' ─── '),
+])
+def test_get_caption(data, width, div):
+    assert f"01/01/1970 {div} 28/02/2000" == get_caption(get_time_range(data, 'ts'), width)

test/data/test_decimal.py

@@ -0,0 +1,20 @@
+#
+# Copyright (c) Daniel Sheffield 2023
+#
+# All rights reserved
+#
+# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+from decimal import Decimal
+from pytest import mark
+
+from app.data.decimal_util import(
+    decimal_or_none
+)
+
+@mark.parametrize('maybe_decimal, expected', [
+    ('1', Decimal('1')),
+    ('1.0', Decimal('1.0')),
+    ('', None),
+])
+def test_decimal_or_none(maybe_decimal, expected):
+    assert decimal_or_none(maybe_decimal) == expected

test/data/test_util.py

@@ -4,7 +4,7 @@
 # All rights reserved
 #
 # THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from psycopg.sql import SQL, Identifier, Literal, Composable
+from psycopg.sql import SQL, Literal, Composable
 from typing import Callable, Iterable, Union
 from pytest import mark, raises
 from app.data.util import (

test/rest/templates/test_include-exclude.py

@@ -12,7 +12,7 @@ from bottle import template
     </div>
 <div class="pure-u-1">
 
-<select id="item-include" name="item" size=10 multiple style="width: calc(100% - 1em); margin: 0 1em 1em">
+<select id="item-include" name="item" size="10" multiple="true" style="width: calc(100% - 1em); margin: 0 1em 1em">
   <option value="Include" disabled="true" >Include</option>
 
   <option value="val1-to-backend"  >val1</option>
@@ -21,7 +21,7 @@ from bottle import template
 </div>
 <div class="pure-u-1">
 
-<select id="item-exclude" name="item" size=10 multiple style="width: calc(100% - 1em); margin: 0 1em 1em">
+<select id="item-exclude" name="item" size="10" multiple="true" style="width: calc(100% - 1em); margin: 0 1em 1em">
   <option value="Exclude" disabled="true" >Exclude</option>
 
   <option value="val1-to-backend"  >val1</option>

test/rest/templates/test_select-one.py

@@ -3,16 +3,14 @@ from pytest import mark, raises
 from bottle import template
 
 @mark.parametrize('expected, params', [
-    ("""<div class="pure-u-1-3 pure-u-lg-1-12">
-<div class="pure-g">
-  <div class="pure-u-1">
-    <div class="l-box">
-      <h3>Unit</h3>
-    </div>
+    ("""<div class="pure-u-1">
+  <div class="l-box">
+    <h3>Unit</h3>
   </div>
+</div>
 <div class="pure-u-1">
 
-<select id="unit-select-one" name="unit" size=10  style="width: calc(100% - 1em); margin: 0 1em 1em">
+<select id="unit-select-one" name="unit" size="10"  style="width: calc(100% - 1em); margin: 0 1em 1em">
 
 
   <option value="Bags"  >Bags</option>
@@ -23,7 +21,6 @@ from bottle import template
   <option value="kg"  selected="true">kg</option>
   <option value="mL"  >mL</option>
 </select>
-</div></div>
 </div>""", {
     "name": "unit", "options": [{
         "value": "Bags",

test/rest/templates/test_select.py

@@ -5,7 +5,7 @@ from bottle import template
 @mark.parametrize('expected, params', [
     ("""<div class="pure-u-1">
 <label for="select-id">Choose: </label>
-<select id="select-id" name="select-name" size=10 multiple style="width: calc(100% - 1em); margin: 0 1em 1em">
+<select id="select-id" name="select-name" size="10" multiple="true" style="width: calc(100% - 1em); margin: 0 1em 1em">
   <option value="hint" disabled="true" >hint</option>
 <optgroup label="Group">
   <option value="val1-to-backend"  >val1</option>
@@ -25,7 +25,7 @@ from bottle import template
     }, ]}, ]}),
     ("""<div class="pure-u-1">
 
-<select id="select-unit-id" name="unit" size=10  style="width: calc(100% - 1em); margin: 0 1em 1em">
+<select id="select-unit-id" name="unit" size="10"  style="width: calc(100% - 1em); margin: 0 1em 1em">
 
 
   <option value="val1"  >val1</option>
@@ -42,7 +42,7 @@ from bottle import template
     }, ]}, ]}),
     ("""<div class="pure-u-1">
 <label for="select-id">Choose: </label>
-<select id="select-id" name="select-name" size=10 multiple style="width: calc(100% - 1em); margin: 0 1em 1em">
+<select id="select-id" name="select-name" size="10" multiple="true" style="width: calc(100% - 1em); margin: 0 1em 1em">
   <option value="hint" disabled="true" >hint</option>
 <optgroup label="Group">
   <option value="val1-to-backend"  >val1</option>

test/rest/test_Cache.py

@@ -6,67 +6,68 @@
 # THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
 from pytest import fixture
 from time import time
-from app.rest.Cache import Cache
+from app.rest.PageCache import PageCache
 from app.rest.CachedLoadingPage import (
     CachedLoadingPage,
+    STALE,
 )
 
 @fixture
 def cache():
-    return Cache(0)
+    return PageCache(0)
 
-def test_add(cache: Cache):
+def test_add(cache: PageCache):
     val = 'test-cached-value'
     key = 'test-key'
-    assert cache.add(key, CachedLoadingPage(val, lambda _: None)) == val
+    assert cache.add(key, CachedLoadingPage(val, lambda _: None, incremental=False)).value == val
 
-def test_get(cache: Cache):
+def test_get(cache: PageCache):
     val = 'test-cached-value'
     key = 'test-key'
 
     assert cache.get(key) is None
 
-    assert cache.add(key, CachedLoadingPage(val, lambda q: q.put('next-val'))) == val
-    assert cache.get(key) == 'next-val'
+    assert cache.add(key, CachedLoadingPage(val, lambda q: q.put('next-val'), incremental=False)).value == val
+    assert cache.get(key).value == 'next-val'
 
-def test_remove(cache: Cache):
+def test_remove(cache: PageCache):
     val = 'test-cached-value'
     key = 'test-key'
 
     assert cache.get(key) is None
-    assert cache.add(key, CachedLoadingPage(val, lambda q: q.put('next-val'))) == val
+    assert cache.add(key, CachedLoadingPage(val, lambda q: q.put('next-val'), incremental=False)).value == val
     cache.remove(key)
     assert cache.get(key) is None
 
-def test_enforce_limit(cache: Cache):
+def test_enforce_limit(cache: PageCache):
     val = 'test-cached-value'
     key = 'test-key'
-    assert cache.add(key, CachedLoadingPage(val, lambda _: None)) == val
+    assert cache.add(key, CachedLoadingPage(val, lambda _: None, incremental=False)).value == val
     # adding more exceeds limit
-    assert cache.add('other', CachedLoadingPage(val, lambda _: None)) == val
+    assert cache.add('other', CachedLoadingPage(val, lambda _: None, incremental=False)).value == val
     assert cache.get(key) is None
-    assert cache.get('other') == val
+    assert cache.get('other').value == val
 
-def test_clean_stale(cache: Cache):
+def test_clean_stale(cache: PageCache):
     val = 'test-cached-value'
     key = 'test-key'
-    page = CachedLoadingPage(val, lambda _: None)
-    page._created = time() - 10*60
+    page = CachedLoadingPage(val, lambda _: None, incremental=False)
+    page._created = time() - STALE
     # add stale page
-    assert cache.add(key, page) == val
+    assert cache.add(key, page).value == val
     assert cache.get(key) is None
-    
-    page = CachedLoadingPage(val, lambda _: None)
-    assert cache.add(key, page) == val
+
+    page = CachedLoadingPage(val, lambda _: None, incremental=False)
+    assert cache.add(key, page).value == val
     # make page stale
-    page._created = time() - 10*60
+    page._created = time() - STALE
     assert cache.get(key) is None
 
-    page = CachedLoadingPage(val, lambda _: None)
-    assert cache.add(key, page) == val
+    page = CachedLoadingPage(val, lambda _: None, incremental=False)
+    assert cache.add(key, page).value == val
     # make page stale
-    page._created = time() - 10*60
+    page._created = time() - STALE
     # stale page is rotated out on addition
-    assert cache.add('other', CachedLoadingPage(val, lambda _: None)) == val
+    assert cache.add('other', CachedLoadingPage(val, lambda _: None, incremental=False)).value == val
     assert cache.get(key) is None
-    assert cache.get('other') == val
+    assert cache.get('other').value == val

test/rest/test_CachedLoadingPage.py

@@ -10,12 +10,13 @@ from pytest import mark, fixture
 from time import sleep, time
 from app.rest.CachedLoadingPage import (
     CachedLoadingPage,
+    STALE,
 )
 
 
 @fixture
 def cache():
-    return CachedLoadingPage("start", lambda _: None)
+    return CachedLoadingPage("start", lambda _: None, incremental=False)
 
 
 def test_get_age(cache: CachedLoadingPage):
@@ -52,7 +53,7 @@ def test_update(cache: CachedLoadingPage):
     assert cache.update() == "final value"
 
 def test_stale(cache: CachedLoadingPage):
-    cache._created = time() - 10*60
+    cache._created = time() - STALE
     assert cache.stale
 
 def test_lock(cache: CachedLoadingPage):

test/rest/test_route_decorators.py

@@ -0,0 +1,76 @@
+#
+# Copyright (c) Daniel Sheffield 2023
+#
+# All rights reserved
+#
+# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+from bottle import FormsDict
+from pytest import fixture, mark, raises
+from app.rest import PARAMS
+from app.rest.route_decorators import (
+    cache,
+    cursor,
+    normalize_query,
+)
+
+def incr(_dict, key):
+    _dict[key] = _dict[key] + 1
+
+@fixture
+def counter():
+    return dict()
+
+@fixture
+def tracker():
+    return dict()
+
+@fixture
+def method(counter, tracker):
+    counter['method'] = 0
+    tracker['method'] = []
+    return lambda *args, **kwargs: incr(counter, 'method') or \
+        tracker['method'].push({'args': args, 'kwargs': kwargs})
+
+@mark.parametrize('form_data, expected', [
+    ([
+        ('product', 'kiwi'),
+        ('product', 'apple'),
+        ('organic', ''),
+    ], ('category=&group=&organic=0.5&product=apple%7Ckiwi&tag=&unit=', None)),
+    ([
+        ('product', 'kiwi'),
+        ('category', '!baking'),
+        ('organic', '1'),
+    ], ('category=%21baking&group=&organic=1&product=kiwi&tag=&unit=', None)),
+])
+def test_normalise_query(form_data, expected):
+    form = FormsDict()
+    for k,v in form_data:
+        form.append(k, v)
+    
+    assert normalize_query(form, allow=PARAMS) == expected
+
+
+# need to mock bottle request.params
+def test_cache(counter, tracker, method):
+    decorated = cache(query_cache=None, page_cache=None)
+    assert callable(decorated)
+    #decorated()
+    #assert counter['method'] == 1
+    #assert tracker['method'][0]['args'] == []
+    #assert tracker['method'][0]['kwargs'] == {'allow': None}
+    with raises(Exception) as ex:
+        cursor(method)
+    assert ex.value.args[0] == "decorator argument required"
+
+
+def test_cursor(counter, tracker, method):
+    decorated = cursor(cache=None)
+    assert callable(decorated)
+    #decorated()
+    #assert counter['method'] == 1
+    #assert tracker['method'][0]['args'] == []
+    #assert tracker['method'][0]['kwargs'] == {'allow': None}
+    with raises(Exception) as ex:
+        cursor(method)
+    assert ex.value.args[0] == "decorator argument required"