move xslt generated pages into separate file

app/activities/TransactionEditor.py

@@ -341,7 +341,7 @@ class TransactionEditor(FocusWidget):
         query_manager: QueryManager,
     ):
         self.autocomplete_options = lambda name, data: self.query_manager.unique_suggestions(name.split('#', 1)[0], **data)
-        self.activity_manager = activity_manager
+        self.activity_manager: ActivityManager = activity_manager
         self.query_manager = query_manager
         self.buttons = {
             'done': Button(('streak', u'Done')),

app/rest/form.py

@@ -14,9 +14,8 @@ def get_option_groups(
     data: DataFrame, filter_data: Dict[str, Tuple[set, set]],
     k: str, g: str, _type: str
 ):
-    in_chart = data['$/unit'].apply(lambda x: (x or False) and True)
     if k in data or (k == 'tag' and 'tags' in data):
-        k_data_in_chart = chain(*data[in_chart]["tags"]) if k == "tag" else data[in_chart][k]  
+        k_data_in_chart = chain(*data["tags"]) if k == "tag" else data[k]
     else:
         k_data_in_chart = []
     
@@ -46,7 +45,7 @@ def get_option_groups(
                     filter_data[k][0] - set(k_data_in_chart) - filter_data[k][1]
                 ))
         else:
-            k_grouped_data_in_chart = set(data[in_chart & (data[g].apply(lambda x,axis=None: x == group))][k]) if k in data else set()
+            k_grouped_data_in_chart = set(data[data[g].apply(lambda x,axis=None: x == group)][k]) if k in data else set()
             if _type == "include":
                 selected.extend(filter_data[k][0] & set(k_grouped_data_in_chart))
                 unselected.extend(k_grouped_data_in_chart - filter_data[k][0])

app/rest/pyapi.py

@@ -3,7 +3,7 @@
 # All rights reserved
 #
 # THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from typing import Iterable, Dict
+from typing import Iterable
 import os
 from urllib.parse import urlencode
 from bottle import (
@@ -15,23 +15,15 @@ from bottle import (
     template,
     static_file,
 )
-from pandas import DataFrame
 from psycopg import connect
-from psycopg.sql import SQL, Literal
 from threading import Thread
 
-from app.data.QueryManager import QueryManager, get_data
+from .query_to_xml import get_categories, get_groups, get_products, get_tags
 
 from ..data.filter import(
     get_filter,
     get_query_param,
 )
-from .form import (
-    get_form,
-)
-from ..data.util import(
-    get_where_include_exclude
-)
 from . import trend as worker
 from . import PARAMS
 from .CachedLoadingPage import CachedLoadingPage
@@ -47,31 +39,6 @@ conn = connect(f"{host} {db} {user} {password}")
 
 CACHE = Cache(10)
 
-def get_product_rollup_statement(filters):
-    where = [ get_where_include_exclude(
-        k[0], "name", list(include), list(exclude)
-    ) for k, (include, exclude) in filters.items() ]
-    return SQL('\n').join([
-        SQL("""
-SELECT
-  count(DISTINCT p.id) AS "Products",
-  count(DISTINCT c.id) AS "Categories",
-  count(DISTINCT g.id) AS "Groups",
-  p.name AS "product",
-  c.name AS "category",
-  g.name AS "group"
-FROM products p
-JOIN categories c ON p.category_id = c.id
-JOIN groups g ON c.group_id = g.id
-"""),
-        SQL("""
-WHERE {where}
-""").format(where=SQL("\nAND").join(where)) if where else SQL(''),
-        SQL("""
-GROUP BY ROLLUP (g.name, c.name, p.name)
-"""),
-    ])
-
 def normalize_query(query: FormsDict, allow: Iterable[str] = None) -> str:
     param = get_filter(query, allow=allow)
     return urlencode([
@@ -122,124 +89,46 @@ def trend():
 @route('/grocery/groups')
 @normalize
 def groups():
-    filters = get_filter(request.query, allow=('group', 'category', 'product'))
-    _filter = get_filter(request.query, allow=PARAMS)
     try:
         with conn.cursor() as cur:
-            inner = get_product_rollup_statement(filters)
-            query = SQL("""
-SELECT
-    "Products",
-    "Categories",
-    COALESCE("group", "Groups"||'') "Group"
-FROM (
-{inner}
-) q
-WHERE q.category IS NULL
-""").format(inner=inner)
-            data = DataFrame(get_data(cur, inner)).dropna()
-            data['$/unit'] = 1
-            form = get_form(request.path.split('/')[-1], 'get', _filter, data)
-            xml = cur.execute(SQL("""
-SELECT query_to_xml_and_xmlschema({q}, false, false, ''::text)
-""").format(q=Literal(query.as_string(cur)))).fetchone()[0]
+            page = get_groups(cur, request.query)
     finally:
         conn.commit()
     response.content_type = 'application/xhtml+xml; charset=utf-8'
-    return template("query-to-xml", title="Groups", xml=xml, form=form)
+    return page
+
 
 @route('/grocery/categories')
 @normalize
 def categories():
-    filters = get_filter(request.query, allow=('group', 'category', 'product'))
-    _filter = get_filter(request.query, allow=PARAMS)
-    form = template('form-nav', action='categories', method='get', params=[
-        {'name': k, 'value': request.params[k]} for k in request.params if k in PARAMS
-    ])
     try:
         with conn.cursor() as cur:
-            inner = get_product_rollup_statement(filters)
-            query = SQL("""
-SELECT
-    "Products" "Product",
-    COALESCE("category", "Categories"||'') "Category",
-    COALESCE("group", "Groups"||'') "Group"
-FROM (
-{inner}
-) q
-WHERE q.product IS NULL AND (q.category IS NOT NULL OR q.group IS NULL)
-""").format(inner=inner)
-            data = DataFrame(get_data(cur, inner)).dropna()
-            data['$/unit'] = 1
-            form = get_form(request.path.split('/')[-1], 'get', _filter, data)
-            xml = cur.execute(SQL("""
-SELECT query_to_xml_and_xmlschema({q}, false, false, ''::text)
-""").format(q=Literal(query.as_string(cur)))).fetchone()[0]
+            page = get_categories(cur, request.query)
     finally:
         conn.commit()
     response.content_type = 'application/xhtml+xml; charset=utf-8'
-    return template("query-to-xml", title="Categories", xml=xml, form=form)
+    return page
 
 @route('/grocery/products')
 @normalize
 def products():
-    filters = get_filter(request.query, allow=('group', 'category', 'product'))
-    _filter = get_filter(request.query, allow=PARAMS)
-    form = template('form-nav', action='products', method='get', params=[
-        {'name': k, 'value': request.params[k]} for k in request.params if k in PARAMS
-    ])
     try:
         with conn.cursor() as cur:
-            inner = get_product_rollup_statement(filters)
-            query = SQL("""
-SELECT
-    --"Transactions",
-    COALESCE("product", "Products"||'') "Product",
-    COALESCE("category", "Categories"||'') "Category",
-    COALESCE("group", "Groups"||'') "Group"
-FROM (
-{inner}
-) q
-WHERE q.product IS NOT NULL OR q.group IS NULL
-""").format(inner=inner)
-            data = DataFrame(get_data(cur, inner)).dropna()
-            data['$/unit'] = 1
-            form = get_form(request.path.split('/')[-1], 'get', _filter, data)
-            xml = cur.execute(SQL("""
-SELECT query_to_xml_and_xmlschema({q}, false, false, ''::text)
-""").format(q=Literal(query.as_string(cur)))).fetchone()[0]
+            page = get_products(cur, request.query)
     finally:
         conn.commit()
     response.content_type = 'application/xhtml+xml; charset=utf-8'
-    return template("query-to-xml", title="Products", xml=xml, form=form)
+    return page
+
 
 @route('/grocery/tags')
 @normalize
 def tags():
-    form = template('form-nav', action='tags', method='get', params=[
-        {'name': k, 'value': request.params[k]} for k in request.params if k in PARAMS
-    ])
     try:
         with conn.cursor() as cur:
-            inner = SQL('\n').join([SQL("""
-SELECT * FROM (SELECT count(DISTINCT txn.id) AS "Uses", tg.name AS "Name"
-FROM tags tg
-JOIN tags_map tm ON tg.id = tm.tag_id
-JOIN transactions txn ON txn.id = tm.transaction_id
-GROUP BY tg.name
-ORDER BY 1 DESC, 2) q
-UNION ALL
-SELECT count(DISTINCT txn.id) AS "Uses", count(DISTINCT tg.name)||'' AS "Name"
-FROM tags tg
-JOIN tags_map tm ON tg.id = tm.tag_id
-JOIN transactions txn ON txn.id = tm.transaction_id
-""")]).as_string(cur)
-            xml = cur.execute(SQL("""
-SELECT query_to_xml_and_xmlschema({inner}, false, false, ''::text)
-""").format(inner=Literal(inner))).fetchone()[0]
+            page = get_tags(cur, request.query)
     finally:
         conn.commit()
     response.content_type = 'application/xhtml+xml; charset=utf-8'
-    return template("query-to-xml", title="Tags", xml=xml, form=form)
-
+    return page
 

app/rest/query_to_xml.py

@@ -0,0 +1,120 @@
+#
+# Copyright (c) Daniel Sheffield 2023
+# All rights reserved
+#
+# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+from bottle import request, template, FormsDict
+from pandas import DataFrame
+from psycopg import Cursor
+from psycopg.sql import SQL, Literal
+
+from ..data.filter import get_filter
+from ..data.util import get_where_include_exclude
+from ..data.QueryManager import get_data
+from .form import get_form
+from . import PARAMS
+
+
+def get_product_rollup_statement(filters) -> SQL:
+    where = [ get_where_include_exclude(
+        k[0], "name", list(include), list(exclude)
+    ) for k, (include, exclude) in filters.items() ]
+    return SQL("""
+SELECT
+  count(DISTINCT p.id) AS "Products",
+  count(DISTINCT c.id) AS "Categories",
+  count(DISTINCT g.id) AS "Groups",
+  p.name AS "product",
+  c.name AS "category",
+  g.name AS "group"
+FROM products p
+JOIN categories c ON p.category_id = c.id
+JOIN groups g ON c.group_id = g.id
+WHERE {where}
+GROUP BY ROLLUP (g.name, c.name, p.name)
+""").format(where=SQL("\nAND").join(where))
+
+
+def get_inner_query(query: FormsDict) -> SQL:
+    filters = get_filter(query, allow=('group', 'category', 'product'))
+    inner = get_product_rollup_statement(filters)
+    return inner
+
+
+def render_form(cur: Cursor, inner: str, query: FormsDict):
+    _filter = get_filter(query, allow=PARAMS)
+    data = DataFrame(get_data(cur, inner)).dropna()
+    return get_form(request.path.split('/')[-1], 'get', _filter, data)
+
+
+def get_xml(cur: Cursor, sql: str):
+    return cur.execute(SQL(
+        "SELECT query_to_xml_and_xmlschema({q}, false, false, ''::text)"
+    ).format(q=Literal(sql))).fetchone()[0]
+
+
+def get_products(cur: Cursor, query: FormsDict):
+    inner = get_inner_query(query)
+    form = render_form(cur, inner, query)
+    sql = SQL("""
+SELECT
+    --"Transactions",
+    COALESCE("product", "Products"||'') "Product",
+    COALESCE("category", "Categories"||'') "Category",
+    COALESCE("group", "Groups"||'') "Group"
+FROM ({inner}) q
+WHERE q.product IS NOT NULL OR q.group IS NULL
+""").format(inner=inner).as_string(cur)
+    xml = get_xml(cur, sql)
+    return template("query-to-xml", title="Products", xml=xml, form=form)
+
+
+def get_categories(cur: Cursor, query: FormsDict):
+    inner = get_inner_query(query)
+    form = render_form(cur, inner, query)
+    sql = SQL("""
+SELECT
+    "Products" "Product",
+    COALESCE("category", "Categories"||'') "Category",
+    COALESCE("group", "Groups"||'') "Group"
+FROM ({inner}) q
+WHERE q.product IS NULL AND (q.category IS NOT NULL OR q.group IS NULL)
+""").format(inner=inner).as_string(cur)
+    xml = get_xml(cur, sql)
+    return template("query-to-xml", title="Categories", xml=xml, form=form)
+
+
+def get_groups(cur: Cursor, query: FormsDict):
+    inner = get_inner_query(query)
+    form = render_form(cur, inner, query)
+    sql = SQL("""
+SELECT
+    "Products",
+    "Categories",
+    COALESCE("group", "Groups"||'') "Group"
+FROM ({inner}) q
+WHERE q.category IS NULL
+""").format(inner=inner).as_string(cur)
+    xml = get_xml(cur, sql)
+    return template("query-to-xml", title="Groups", xml=xml, form=form)
+
+def get_tags(cur: Cursor, query: FormsDict):
+    form = template('form-nav', action='tags', method='get', params=[
+        {'name': k, 'value': request.params[k]} for k in request.params if k in PARAMS
+    ])
+    sql = SQL("""
+SELECT * FROM (SELECT count(DISTINCT txn.id) AS "Uses", tg.name AS "Name"
+FROM tags tg
+JOIN tags_map tm ON tg.id = tm.tag_id
+JOIN transactions txn ON txn.id = tm.transaction_id
+GROUP BY tg.name
+ORDER BY 1 DESC, 2) q
+UNION ALL
+SELECT count(DISTINCT txn.id) AS "Uses", count(DISTINCT tg.name)||'' AS "Name"
+FROM tags tg
+JOIN tags_map tm ON tg.id = tm.tag_id
+JOIN transactions txn ON txn.id = tm.transaction_id
+""").as_string(cur)
+    xml = get_xml(cur, sql)
+    return template("query-to-xml", title="Tags", xml=xml, form=form)
+

app/rest/trend.py

@@ -56,7 +56,8 @@ def trend_internal(conn: Connection[TupleRow], path: str, query: DictProperty):
             progress.append({ "name": "Loading data", "status": ""})
             yield template("loading", progress=progress)
             data = get_data(query_manager, **fields)
-
+            in_chart = data['$/unit'].apply(lambda x: (x or False) and True)
+            data = data[in_chart]
             progress[-1]["status"] = "done"
             yield template("loading", progress=progress)
 

test/data/test_util.py

@@ -4,7 +4,7 @@
 # All rights reserved
 #
 # THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from psycopg.sql import SQL, Identifier, Literal, Composable
+from psycopg.sql import SQL, Literal, Composable
 from typing import Callable, Iterable, Union
 from pytest import mark, raises
 from app.data.util import (