refactor

app/data/PriceView.py

@@ -4,9 +4,7 @@
 # All rights reserved
 #
 # THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from collections import (
-    OrderedDict,
-)
+from collections import OrderedDict
 from typing import Tuple
 from psycopg.sql import (
     Identifier,
@@ -14,12 +12,14 @@ from psycopg.sql import (
     Literal,
     Composable,
 )
-from .util import(
+from .filter import (
     get_include_exclude,
+)
+from .util import(
     get_select,
     get_from,
-    get_where_include_exclude,
     get_groupby,
+    get_where_include_exclude,
 )
 
 def get_window(unit, organic):
@@ -120,7 +120,7 @@ TRUNC(
 def get_where(product=None, category=None, group=None, tag=None, organic=None, limit='90 days'):
     where = [
         get_where_include_exclude(
-            k, 'name', *list(map(set, get_include_exclude(v)))
+            k, 'name', *list(map(list, get_include_exclude(v)))
         ) for k, v in {
             'products': product,
             'categories': category,

app/data/filter.py

@@ -0,0 +1,37 @@
+#
+# Copyright (c) Daniel Sheffield 2023
+#
+# All rights reserved
+#
+# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+from bottle import FormsDict
+from typing import Iterable, List, Tuple, Union
+
+
+def get_include_exclude(
+        value: Union[str, List, Tuple]
+    ) -> Tuple[set[str], set[str]]:
+    value = value or ''
+    include, exclude = [], []
+    if isinstance(value, (list, tuple)):
+        for v in value:
+            inc, exc = get_include_exclude(v)
+            include.extend(inc)
+            exclude.extend(exc)
+    else:
+        inc, exc, *_ = [
+            *map(lambda x: x.split('|') if x else [], value.split('!')), []
+        ]
+        include.extend(inc)
+        exclude.extend(exc)
+    return set(include), set(exclude)
+
+
+def get_filter(
+    query: FormsDict, allow: Iterable[str] = None
+) -> dict[str, Tuple[List[str], List[str]]]:
+    return {
+        k: get_include_exclude(
+            (query[k] or 'kg' if k == 'unit' else query.getall(k)) if k in query else None
+        ) for k in sorted(set(query.keys()) | set(allow)) if allow is None or k in allow
+    }

app/data/util.py

@@ -13,23 +13,6 @@ from psycopg.sql import (
 )
 
 
-def get_include_exclude(value) -> Tuple[List[str], List[str]]:
-    value = value or ''
-    include, exclude = [], []
-    if isinstance(value, (list, tuple)):
-        for v in value:
-            inc, exc = get_include_exclude(v)
-            include.extend(inc)
-            exclude.extend(exc)
-    else:
-        inc, exc, *_ = [
-            *map(lambda x: x.split('|') if x else [], value.split('!')), []
-        ]
-        include.extend(inc)
-        exclude.extend(exc)
-    return list(set(include)), list(set(exclude))
-
-
 def get_where_include_exclude(
         table: str,
         col: str,
@@ -46,6 +29,7 @@ AND
         exclude=Literal(exclude)
     )
 
+
 def get_select(alias_to_sql: dict[str,Composable]) -> Composable:
     select = SQL(""",
     """).join([

app/rest/__init__.py

@@ -4,3 +4,8 @@
 # All rights reserved
 #
 # THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+from bottle import TEMPLATE_PATH
+
+TEMPLATE_PATH.append("app/rest/templates")
+ALL_UNITS = {'g', 'kg', 'mL', 'L', 'Pieces', 'Bunches', 'Bags'}
+PARAMS = { 'group', 'category', 'product', 'unit', 'tag' }

app/rest/form.py

@@ -0,0 +1,103 @@
+#
+# Copyright (c) Daniel Sheffield 2023
+#
+# All rights reserved
+#
+# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+from bottle import template
+from pandas import DataFrame
+from itertools import chain
+from ..rest import ALL_UNITS
+
+def get_option_groups(data: DataFrame, filter_data, k, g, _type):
+    in_chart = data['$/unit'].apply(lambda x: (x or False) and True)
+    groups = sorted(set(data[g] if g is not None else []))
+    groups.append(None)
+    if _type == "exclude":
+        prefix = "!"
+    else:
+        prefix = ""
+    
+    for group in groups:
+        selected = []
+        unselected = []
+        if group is None:
+            if _type == "include":
+                selected.extend(filter_data[k][0] - (
+                    set((chain(*data[in_chart]["tags"]) if k == "tag" else data[in_chart][k]) if g is not None else set())
+                ))
+                unselected.extend(((
+                    set((chain(*data[in_chart]["tags"]) if k == "tag" else data[in_chart][k]) if g is None else set())
+                ) | filter_data[k][1]) - filter_data[k][0])
+            else:
+                selected.extend(filter_data[k][1])
+                unselected.extend((
+                    set((chain(*data[in_chart]["tags"]) if k == "tag" else data[in_chart][k]) if g is None else set())
+                ) | (
+                    filter_data[k][0] - set(chain(*data[in_chart]["tags"]) if k == "tag" else data[in_chart][k]) - filter_data[k][1]
+                ))
+        else:
+            if _type == "include":
+                selected.extend(filter_data[k][0] & set(
+                    data[in_chart & (data[g].apply(lambda x,axis=None: x == group))][k]
+                ))
+                unselected.extend(set(
+                    data[in_chart & (data[g].apply(lambda x,axis=None: x == group))][k]
+                ) - filter_data[k][0])
+            else:
+                unselected.extend(set(
+                    data[in_chart & (data[g].apply(lambda x,axis=None: x == group))][k]
+                ))
+        assert set(selected) - set(unselected) == set(selected), f"{set(selected)} {set(unselected)}"
+        
+        yield {
+            "optgroup": group,
+            "options": sorted(map(lambda x: {
+                "selected": x[0],
+                "value": f"{prefix}{x[1]}",
+                "display": x[1]  
+            }, chain(
+                map(lambda x: (True, x), set(selected)),
+                map(lambda x: (False, x), set(unselected)),
+            )), key=lambda x: x["display"] if "display" in x else x["value"])
+        } 
+
+
+def get_form(action: str, method: str, filter_data, data: DataFrame):
+    keys = sorted(filter(lambda x: x not in ('unit', 'tag'), filter_data), key=lambda x: {
+        'product': 0,
+        'category': 1,
+        'group': 2,
+    }[x])
+    return template('form', action=action, method=method,
+        **{
+            k: {
+                "name": k,
+                "_include": {
+                    "option_groups": get_option_groups(data, filter_data, k, g, "include"),
+                },
+                "_exclude": {
+                    "option_groups": get_option_groups(data, filter_data, k, g, "exclude"),
+                }
+            } for k, g in zip(keys, [*keys[1:], None])
+        },
+        tags={
+            "name": "tag",
+            "_include": {
+                "option_groups": get_option_groups(data, filter_data, "tag", None, "include")
+            },
+            "_exclude": {
+                "option_groups": get_option_groups(data, filter_data, "tag", None, "exclude")
+            },
+        },
+        units={
+            "name": "unit",
+            "options": sorted(map(lambda x: {
+                "selected": x[0],
+                "value": x[1],
+            },chain(
+                map(lambda x: (True, x), filter_data['unit'][0]),
+                map(lambda x: (False, x), ALL_UNITS - filter_data['unit'][0])
+            )), key=lambda x: x["display"] if "display" in x else x["value"])
+        }
+    )

app/rest/pyapi.py

@@ -3,10 +3,7 @@
 # All rights reserved
 #
 # THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from itertools import chain
-from time import time
 from typing import Iterable
-from io import StringIO
 import os
 from urllib.parse import urlencode
 from bottle import (
@@ -14,35 +11,30 @@ from bottle import (
     request,
     run,
     response,
-    abort,
     FormsDict,
     redirect,
     template,
-    HTTPError,
     static_file,
     TEMPLATE_PATH,
 )
 from matplotlib.axes import Axes
-TEMPLATE_PATH.append("app/rest/templates")
 from psycopg import connect
 from psycopg.sql import SQL, Literal
-from pandas import DataFrame
-import matplotlib.pyplot as plt
 import seaborn as sns
 from multiprocessing import Lock
-from ..activities.Plot import (
-    get_data,
+
+from ..data.filter import(
+    get_filter,
 )
-from ..data.QueryManager import QueryManager, display_mapper
+
 from ..data.util import(
-    get_include_exclude,
     get_where_include_exclude
 )
+from .trend import trend_internal
+from . import PARAMS
 import matplotlib
 matplotlib.use('agg')
 
-ALL_UNITS = {'g', 'kg', 'mL', 'L', 'Pieces', 'Bunches', 'Bags'}
-PARAMS = { 'group', 'category', 'product', 'unit', 'tag' }
 CACHE = dict()
 
 host = f"host={os.getenv('HOST')}"
@@ -83,12 +75,6 @@ ORDER BY "Product", "Category", "Group"
 """).format(having=having) if having else SQL('')
     ])
 
-def get_filter(query: FormsDict, allow: Iterable[str] = None):
-    return {
-        k: list(get_include_exclude(
-            (query[k] or 'kg' if k == 'unit' else query.getall(k)) if k in query else None
-        )) for k in sorted(set(query.keys()) | set(allow)) if allow is None or k in allow
-    }
 
 def get_query_param(include, exclude):
     return '!'.join([
@@ -96,112 +82,19 @@ def get_query_param(include, exclude):
         *([ '|'.join(sorted(exclude)) ] if exclude else [ ]),
     ])
 
-def get_query(**param):
+def normalize_query(query: FormsDict, allow: Iterable[str] = None) -> str:
+    param = get_filter(query, allow=allow)
     return urlencode([
         (k, get_query_param(*param[k])) for k in sorted(param) if param[k]
     ])
 
-def normalize_query(query: FormsDict, allow: Iterable[str] = None):
-    return get_query(**get_filter(query, allow=allow))
-
-def get_option_groups(data, filter_data, k, g, _type):
-    in_chart = data['$/unit'].apply(lambda x: (x or False) and True)
-    groups = sorted(set(data[g] if g is not None else []))
-    groups.append(None)
-    if _type == "exclude":
-        prefix = "!"
-    else:
-        prefix = ""
-    
-    for group in groups:
-        selected = []
-        unselected = []
-        if group is None:
-            if _type == "include":
-                selected.extend(filter_data[k][0] - (
-                    set((chain(*data[in_chart]["tags"]) if k == "tag" else data[in_chart][k]) if g is not None else set())
-                ))
-                unselected.extend(((
-                    set((chain(*data[in_chart]["tags"]) if k == "tag" else data[in_chart][k]) if g is None else set())
-                ) | filter_data[k][1]) - filter_data[k][0])
-            else:
-                selected.extend(filter_data[k][1])
-                unselected.extend((
-                    set((chain(*data[in_chart]["tags"]) if k == "tag" else data[in_chart][k]) if g is None else set())
-                ) | (
-                    filter_data[k][0] - set(chain(*data[in_chart]["tags"]) if k == "tag" else data[in_chart][k]) - filter_data[k][1]
-                ))
-        else:
-            if _type == "include":
-                selected.extend(filter_data[k][0] & set(
-                    data[in_chart & (data[g].apply(lambda x,axis=None: x == group))][k]
-                ))
-                unselected.extend(set(
-                    data[in_chart & (data[g].apply(lambda x,axis=None: x == group))][k]
-                ) - filter_data[k][0])
-            else:
-                unselected.extend(set(
-                    data[in_chart & (data[g].apply(lambda x,axis=None: x == group))][k]
-                ))
-        assert set(selected) - set(unselected) == set(selected), f"{set(selected)} {set(unselected)}"
-        
-        yield {
-            "optgroup": group,
-            "options": sorted(map(lambda x: {
-                "selected": x[0],
-                "value": f"{prefix}{x[1]}",
-                "display": x[1]  
-            }, chain(
-                map(lambda x: (True, x), set(selected)),
-                map(lambda x: (False, x), set(unselected)),
-            )), key=lambda x: x["display"] if "display" in x else x["value"])
-        } 
 
-def get_form(action, method, filter_data, data):
-    keys = sorted(filter(lambda x: x not in ('unit', 'tag'), filter_data), key=lambda x: {
-        'product': 0,
-        'category': 1,
-        'group': 2,
-    }[x])
-    return template('form', action=action, method=method,
-        **{
-            k: {
-                "name": k,
-                "_include": {
-                    "option_groups": get_option_groups(data, filter_data, k, g, "include"),
-                },
-                "_exclude": {
-                    "option_groups": get_option_groups(data, filter_data, k, g, "exclude"),
-                }
-            } for k, g in zip(keys, [*keys[1:], None])
-        },
-        tags={
-            "name": "tag",
-            "_include": {
-                "option_groups": get_option_groups(data, filter_data, "tag", None, "include")
-            },
-            "_exclude": {
-                "option_groups": get_option_groups(data, filter_data, "tag", None, "exclude")
-            },
-        },
-        units={
-            "name": "unit",
-            "options": sorted(map(lambda x: {
-                "selected": x[0],
-                "value": x[1],
-            },chain(
-                map(lambda x: (True, x), filter_data['unit'][0]),
-                map(lambda x: (False, x), ALL_UNITS - filter_data['unit'][0])
-            )), key=lambda x: x["display"] if "display" in x else x["value"])
-        }
-    )
 
 @route('/grocery/static/<filename:path>')
 def send_static(filename):
     return static_file(filename, root='app/rest/static')
 
 
-
 global LOCK
 LOCK = Lock()
 
@@ -234,86 +127,13 @@ def trend():
             return CACHE[request.query_string]["state"]
         try:
             CACHE[request.query_string] = {
-                "iter": trend_internal(path, request.query),
+                "iter": trend_internal(conn, path, request.query),
                 "state": loading,
             }
         finally:
             LOCK.release()
     return loading
 
-def trend_internal(path, query):
-    progress = []
-    try:
-        with conn.cursor() as cur:
-            query_manager = QueryManager(cur, display_mapper)
-            fields = { k: query[k] or None for k in query.keys() if k in PARAMS }
-            unit = fields['unit'] = fields['unit'] or 'kg' if 'unit' in fields else 'kg'
-            if unit and unit not in ALL_UNITS:
-                raise abort(400, f"Unsupported unit {unit}")
-
-            progress.append({ "name": "Loading data", "status": ""})
-            yield template("loading", progress=progress)
-            data = get_data(query_manager, **fields)
-
-            progress[-1]["status"] = "done"
-            yield template("loading", progress=progress)
-
-            if data.empty:
-                raise abort(404, f"No data for {fields}")
-            
-            progress.append({ "name": "Loading chart", "status": ""})
-            yield template("loading", progress=progress)
-            
-            pivot = data.pivot_table(index=['ts_raw',], columns=['product',], values=['$/unit'], aggfunc='mean')
-            pivot.columns = pivot.columns.droplevel()
-            
-            sns.set(style="darkgrid", palette='pastel', context="talk")
-            plt.style.use("dark_background")
-            plt.rcParams.update({"grid.linewidth":0.1, "grid.alpha":0.5})
-            plt.figure(figsize=[16, 9], layout="tight")
-            xlabel='Time'
-            ylabel=f'$ / {unit}'
-            if pivot.columns.size > 50:
-                ax = sns.scatterplot(data=pivot, markers=True)
-            else:
-                ax = sns.lineplot(data=pivot, markers=True)
-                legend = plt.figlegend(
-                    loc='upper center', ncol=6,
-                    title_fontsize="14", fontsize="12", labelcolor='#a0a0a0',
-                    framealpha=0.5
-                )
-                legend.set_title(title="Products")
-            ax.legend().set_visible(False)
-
-            ax.set_xlabel(xlabel, color='#a0a0a0', fontsize="14")
-            ax.set_ylabel(ylabel, color='#a0a0a0', fontsize="14")
-            ax.axes.tick_params(colors='#a0a0a0', labelsize="12", which='both')
-            for _, spine in ax.spines.items():
-                spine.set_color('#a0a0a0')
-            
-            progress[-1]["status"] = "done"
-            yield template("loading", progress=progress)
-            
-            progress.append({ "name": "Rendering chart", "status": ""})
-            yield template("loading", progress=progress)
-
-            f = StringIO()
-            plt.savefig(f, format='svg')
-            form = get_form(path.split('/')[-1], 'get', get_filter(request.query, allow=PARAMS), data)
-            
-            progress[-1]["status"] = "done"
-            yield template("loading", progress=progress)
-            
-            resp = lambda: template("trend", form=form, svg=f.getvalue())
-
-    except HTTPError as e:
-        resp = lambda exception=e: exception
-
-    finally:
-        conn.commit()
-     
-    yield from iter(resp, lambda started=time(): time() - started > 600)
-
 
 @route('/grocery/groups')
 def groups():

app/rest/trend.py

@@ -0,0 +1,110 @@
+#
+# Copyright (c) Daniel Sheffield 2023
+#
+# All rights reserved
+#
+# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+from bottle import (
+    DictProperty,
+    HTTPError,
+    template,
+    request,
+)
+from io import StringIO
+import matplotlib.pyplot as plt
+import seaborn as sns
+from psycopg import Connection
+from psycopg.connection import TupleRow
+from time import time
+from . import ALL_UNITS
+from ..data.QueryManager import (
+    display_mapper,
+    QueryManager,
+)
+from ..data.filter import (
+    get_filter,
+)
+from ..activities.Plot import (
+    get_data,
+)
+from .form import(
+    get_form,
+)
+from . import PARAMS
+
+def abort(code, text):
+    return HTTPError(code, text)
+
+def trend_internal(conn: Connection[TupleRow], path: str, query: DictProperty):
+    progress = []
+    try:
+        with conn.cursor() as cur:
+            query_manager = QueryManager(cur, display_mapper)
+            fields = { k: query[k] or None for k in query.keys() if k in PARAMS }
+            unit = fields['unit'] = fields['unit'] or 'kg' if 'unit' in fields else 'kg'
+            if unit and unit not in ALL_UNITS:
+                raise abort(400, f"Unsupported unit {unit}")
+
+            progress.append({ "name": "Loading data", "status": ""})
+            yield template("loading", progress=progress)
+            data = get_data(query_manager, **fields)
+
+            progress[-1]["status"] = "done"
+            yield template("loading", progress=progress)
+
+            if data.empty:
+                raise abort(404, f"No data for {fields}")
+            
+            progress.append({ "name": "Loading chart", "status": ""})
+            yield template("loading", progress=progress)
+            
+            pivot = data.pivot_table(index=['ts_raw',], columns=['product',], values=['$/unit'], aggfunc='mean')
+            pivot.columns = pivot.columns.droplevel()
+            
+            sns.set(style="darkgrid", palette='pastel', context="talk")
+            plt.style.use("dark_background")
+            plt.rcParams.update({"grid.linewidth":0.1, "grid.alpha":0.5})
+            plt.figure(figsize=[16, 9], layout="tight")
+            xlabel='Time'
+            ylabel=f'$ / {unit}'
+            if pivot.columns.size > 50:
+                ax = sns.scatterplot(data=pivot, markers=True)
+            else:
+                ax = sns.lineplot(data=pivot, markers=True)
+                legend = plt.figlegend(
+                    loc='upper center', ncol=6,
+                    title_fontsize="14", fontsize="12", labelcolor='#a0a0a0',
+                    framealpha=0.5
+                )
+                legend.set_title(title="Products")
+            ax.legend().set_visible(False)
+
+            ax.set_xlabel(xlabel, color='#a0a0a0', fontsize="14")
+            ax.set_ylabel(ylabel, color='#a0a0a0', fontsize="14")
+            ax.axes.tick_params(colors='#a0a0a0', labelsize="12", which='both')
+            for _, spine in ax.spines.items():
+                spine.set_color('#a0a0a0')
+            
+            progress[-1]["status"] = "done"
+            yield template("loading", progress=progress)
+            
+            progress.append({ "name": "Rendering chart", "status": ""})
+            yield template("loading", progress=progress)
+
+            f = StringIO()
+            plt.savefig(f, format='svg')
+            _filter = get_filter(request.query, allow=PARAMS)
+            form = get_form(path.split('/')[-1], 'get', _filter, data)
+            
+            progress[-1]["status"] = "done"
+            yield template("loading", progress=progress)
+            
+            resp = lambda: template("trend", form=form, svg=f.getvalue())
+
+    except HTTPError as e:
+        resp = lambda exception=e: exception
+
+    finally:
+        conn.commit()
+     
+    yield from iter(resp, lambda started=time(): time() - started > 600)