Merge branch 'sqlpage' of gogsadmin/grocery-manager into master

LICENSE

@@ -0,0 +1,13 @@
+This software is licensed under the MIT license.
+
+The database is licensed under the CC0 license. See db/LICENSE
+
+The full software license wording follows:
+
+Copyright 2024 Daniel Sheffield
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

app/activities/PriceCheck.py

@@ -68,8 +68,8 @@ class PriceCheck(FocusWidget):
             )
             if k == 'unit':
                 options = list(filter(lambda x: x, [
-                self.query_manager.get_preferred_unit(self.data['product'])
-            ])) or options
+                    self.query_manager.get_preferred_unit(self.data['product'])
+                ])) or options
             if len(options) == 1:
                 self.apply_changes(k, list(options)[0])
 

app/activities/Rating.py

@@ -17,10 +17,11 @@ class Rating():
             self.text_fields['marker'].set_text('')
             return
         current = None if None in (price, quantity or None) else float(price/quantity)
-        
+
         chars = ['|', *['-']*(self.size - 2), '|' ]
         rating = [' ']*len(chars)
         _min, _max = min(_min, current or _min), max(_max, current or _max)
+        _avg = max(_avg, 0.0001)
         ls = np.linspace(_min, _max, len(chars))
         if current is not None and _avg is not None:
             if current <= _avg:

app/activities/TransactionEditor.py

@@ -240,14 +240,18 @@ class TransactionEditor(FocusWidget):
     def update_historic_prices(self, data):
         organic = None if data['organic'] == 'mixed' else data['organic']
         #sort = '$/unit' if self.buttons['sort_price'].state else 'ts'
-        product, unit = data['product'] or None, data['unit'] or None
+        product = data['product'] or None
+        unit = self.query_manager.get_preferred_unit(data['product']) or data['unit'] or None
         price = decimal_or_none(data['price'])
         quantity = decimal_or_none(data['quantity'])
-
         if None in (product, unit):
             self.rating.update_rating(None, None, None, unit)
             return
 
+        if quantity is not None:
+            factor = self.query_manager.convert_unit(product, data['unit'], unit)
+            quantity = quantity * factor if factor is not None else None
+
         df = self.query_manager.get_historic_prices_data(unit, product=product, organic=organic, sort='ts').dropna()
         if df.empty:
             self.rating.update_rating(None, None, None, unit)

app/data/QueryManager.py

@@ -149,6 +149,18 @@ class QueryManager(object):
     def insert_new_product(self, product, category, group, unit):
         self.cursor.execute(get_insert_product_statement(product, category, group, unit))
 
+    def convert_unit(self, product, _from, _to):
+        self.cursor.execute(SQL(
+            "SELECT convert_unit({_from}, {_to}, {product}) AS factor"
+        ).format(
+            product=Literal(product), _from=Literal(_from), _to=Literal(_to)
+        ))
+        res = list(cursor_as_dict(self.cursor))
+        if len(res) == 0:
+            return None
+        assert len(res) == 1
+        return res[0]['factor']
+
     def get_preferred_unit(self, product):
         self.cursor.execute(SQL("""SELECT
     units.name AS preferred_unit
@@ -159,4 +171,4 @@ WHERE products.name = {product}""").format(product=Literal(product)))
         if len(res) == 0:
             return None
         assert len(res) == 1
-        return res[0]['preferred_unit']
+        return res[0]['preferred_unit']

app/rest/CachedLoadingPage.py

@@ -1,72 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from queue import Queue, Empty
-from time import time
-from threading import Lock
-from typing import Callable, Union
-
-STALE = 7*24*60*60
-
-class CachedLoadingPage():
-    
-    value: str
-
-    def __init__(self, initial_value: Union[str, list], provider: Callable[[Queue], None], incremental: bool = True):
-        self._created = time()
-        self._queue = Queue()
-        self._loaded = False
-        self.value = initial_value
-        self._lock = Lock()
-        self.provider = provider
-        self.incremental = incremental
-
-    @property
-    def age(self) -> float:
-        return time() - self._created
-
-    @property
-    def queue(self) -> Queue:
-        return self._queue
-
-    @property
-    def loaded(self) -> bool:
-        return self._loaded
-
-    def _set_loaded(self, value: bool) -> bool:
-        self._loaded = value
-        return self._loaded
-    
-    @property
-    def stale(self) -> bool:
-        return self.age > STALE
-    
-    def _start(self) -> None:
-        if not self.provider:
-            return
-
-        self.provider(self.queue)
-        self.provider = None
-
-    def update(self) -> Union[str, list]:
-        if not self._lock.acquire(blocking=True, timeout=0.5):
-            return self.value
-        try:
-            self._start()
-            item = self._queue.get(block=True, timeout=0.5)
-            if item is None:
-                self._queue.task_done()
-                self._set_loaded(True)
-            else:
-                if self.incremental:
-                    self.value.append(item)
-                else:
-                    self.value = item
-                self.queue.task_done()
-        except Empty:
-            pass
-        finally:
-            self._lock.release()
-        return self.value

app/rest/Dockerfile

@@ -1,7 +0,0 @@
-FROM python:3-slim
-WORKDIR /usr/src/app
-COPY app/rest/requirements.txt ./
-RUN python3 -m pip install --upgrade pip && \
-    python3 -m pip install --no-cache-dir -r requirements.txt
-COPY app ./app
-CMD [ "python", "-m", "app.rest.cherrypy" ]

app/rest/PageCache.py

@@ -1,151 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-
-import os
-from time import time
-from typing import Dict
-import cherrypy
-
-from .hash_util import blake, bytes_to_hash, hash_to_base32
-from .CachedLoadingPage import STALE, CachedLoadingPage
-
-def delete_page(name: str, root: str = 'app/rest/static/files'):
-    directory = f'{root}/{name}'
-    try:
-        os.remove(f'{directory}/{name}.file')
-    except FileNotFoundError:
-        pass
-
-
-def save_page(name: str, content: bytes, tool: str, root='app/rest/static/files') -> str:
-    directory = f'{root}/{name}'
-    try:
-        os.mkdir(directory, mode=0o700, dir_fd=None)
-    except FileExistsError:
-        pass
-
-    fd = os.open(f'{directory}/{name}.file', os.O_WRONLY | os.O_TRUNC | os.O_CREAT, 0o600)
-    with open(fd, "wb") as f:
-        f.write(content)
-
-
-def get_page(name: str, root: str = 'app/rest/static/files') -> str:
-    directory = f'{root}/{name}'
-
-    try:
-        mtime = os.stat(f'{directory}/{name}.file').st_mtime
-    except:
-        mtime = None
-    
-    if mtime is None:
-        return None
-    
-    if mtime and time() - mtime > STALE:
-        delete_page(name)
-        return None
-
-    # todo: store file hash and validate it
-    fd = os.open(f'{directory}/{name}.file', os.O_RDONLY, 0o600)
-    with open(fd, "rb") as f:
-        f.seek(0)
-        page = f.read()
-    return page.decode('utf-8')
-
-
-def key_to_hash(key):
-
-    if isinstance(key, tuple):
-        orig, _hash = key
-    else:
-        if isinstance(key, int):
-            orig, _hash = None, key
-        else:
-            orig, _hash = key, None
-
-    if None not in (orig, _hash):
-        if get_hash(orig) != _hash:
-            raise KeyError(f"Invalid key: {key}")
-
-        return _hash
-
-    if (_hash, orig) is (None, None):
-        raise KeyError(f"Invalid key: {key}")
-    
-    return get_hash(orig) if _hash is None else _hash
-
-
-def get_hash(key):
-    _bytes = blake(key.encode('utf-8'), person='grocery'.encode('utf-8'))
-    return bytes_to_hash(_bytes)
-
-
-class PageCache:
-    def __init__(self, limit) -> None:
-        self._cache: Dict[str, CachedLoadingPage] = dict()
-        self._limit = limit
-
-    def __delitem__(self, key):
-        key = key_to_hash(key)
-        self._cache.pop(key, None)
-        delete_page(hash_to_base32(key))
-
-    def __getitem__(self, key):
-        return self.get(key)
-
-    def __setitem__(self, key, value):
-        self._cache[key_to_hash(key)] = value
-
-    def get(self, key: str) -> CachedLoadingPage:
-        key = key_to_hash(key)
-
-        if key not in self._cache:
-            try:
-                existing = get_page(hash_to_base32(key))
-            except:
-                existing = None
-            if existing is None:
-                return None
-            
-            return self.add(key, CachedLoadingPage(existing, lambda q: q.put(None), incremental=True))
-        
-        page = self._cache[key]
-        if page.stale:
-            del self._cache[key]
-            delete_page(hash_to_base32(key))
-            return None
-
-        if not page.loaded:
-            page.update()
-
-        if page.loaded:
-            try:
-                existing = get_page(hash_to_base32(key))
-            except:
-                existing = None
-            if existing is None:
-                content = ''.join(page.value) if isinstance(page.value, list) else page.value
-                save_page(hash_to_base32(key), content.encode('utf-8'), tool='grocery')
-
-        return page
-
-    def _enforce_limit(self, limit):
-        for idx, (_, k) in enumerate(sorted([
-                (v.age, k) for k, v in self._cache.items()
-            ])):
-            if idx >= limit: del self[k]
-
-    def _clear_stale(self):
-        for k in [k for k, v in self._cache.items() if v.stale]:
-            del self[k]
-    
-    def add(self, key: str, page: CachedLoadingPage) -> CachedLoadingPage:
-        self._clear_stale()
-        self._enforce_limit(self._limit)
-        self._cache[key_to_hash(key)] = page
-        return page
-    
-    def remove(self, key: str):
-        del self[key]

app/rest/QueryCache.py

@@ -1,162 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-import os
-from typing import Dict, Iterable, Tuple
-from urllib.parse import urlencode
-
-from bottle import FormsDict
-from ..data.filter import get_filter, get_query_param
-
-from . import BOOLEAN, PARAMS
-from .hash_util import base32_to_hash, blake, bytes_to_hash, hash_to_base32, normalize_base32
-
-def delete_query(name: str, root: str = 'app/rest/static/files'):
-    directory = f'{root}/{name}'
-    try:
-        os.remove(f'{directory}/{name}.query')
-    except FileNotFoundError:
-        pass
-
-def save_query(name: str, content: bytes, tool: str, root='app/rest/static/files') -> str:
-    directory = f'{root}/{name}'
-    try:
-        os.mkdir(directory, mode=0o700, dir_fd=None)
-    except FileExistsError:
-        pass
-
-    fd = os.open(f'{directory}/{name}.query', os.O_WRONLY | os.O_TRUNC | os.O_CREAT, 0o600)
-    with open(fd, "wb") as f:
-        f.write(content)
-
-def get_query(name: str, root: str = 'app/rest/static/files') -> str:
-    directory = f'{root}/{name}'
-
-    try:
-        mtime = os.stat(f'{directory}/{name}.query').st_mtime
-    except:
-        mtime = None
-    
-    # if mtime and time() - mtime > STALE:
-    #     delete_query(name)
-    #     return None
-
-    # todo: store query hash and validate it
-    fd = os.open(f'{directory}/{name}.query', os.O_RDONLY, 0o600)
-    with open(fd, "rb") as f:
-        f.seek(0)
-        page = f.read()
-    return page.decode('utf-8')
-
-
-def norm(key):
-
-    if isinstance(key, tuple):
-        query, _hash = key
-    else:
-        if isinstance(key, int):
-            query, _hash = None, key
-        else:
-            query, _hash = key, None
-
-    if _hash and not isinstance(_hash, int):
-        _hash = base32_to_hash(normalize_base32(query.hash))
-        # TODO: normalize should be implicit
-        #_hash = base32_to_hash(query.hash)
-
-    if None not in (query, _hash):
-        if get_hash(query) != _hash:
-            raise KeyError(f"Invalid key: {key}")
-
-        return query, _hash
-
-    if (_hash, query) is (None, None):
-        raise KeyError(f"Invalid key: {key}")
-    
-    return query, _hash if _hash else get_hash(query)
-
-def get_hash(key):
-    _bytes = blake(key.encode('utf-8'), person='grocery'.encode('utf-8'))
-    return bytes_to_hash(_bytes)
-
-
-def normalize_query(query: FormsDict, allow: Iterable[str] = None) -> Tuple[str, str]:
-    _hash = query.hash
-    allow = allow or PARAMS
-    param = get_filter(query, allow=allow)
-    norm = urlencode(map(
-        lambda k: (
-            k, get_query_param(*param[k]) if k != 'organic' else BOOLEAN[
-                BOOLEAN.get(query.organic, None)
-            ]
-        ),
-        sorted(filter(bool, param))
-    ))
-    return norm, _hash
-
-
-class QueryCache:
-    def __init__(self, limit) -> None:
-        self._cache: Dict[int, str] = dict()
-        self._limit = limit
-
-    def __delitem__(self, key):
-        return self.remove(key)
-
-    def __getitem__(self, key):
-        return self.get(key)
-
-    def __setitem__(self, key, value):
-        return self.add(key, value)
-
-    def get(self, key: str) -> str:
-        query, _hash = norm(key)
-        if _hash not in self._cache:
-            if query:
-                return self.add(_hash, query)
-            
-            try:
-                existing = get_query(hash_to_base32(_hash))
-            except:
-                existing = None
-            
-            if existing:
-                return self.add(_hash, existing)
-            
-            return self.add(_hash, query)
-        
-        value = self._cache[_hash]
-        # if value.stale:
-        #     del self._cache[key]
-        #     delete_query(hash_to_base32(key))
-        #     return None
-        return value
-
-    # def _enforce_limit(self, limit):
-    #     for idx, (_, k) in enumerate(sorted([
-    #             (v.age, k) for k, v in self._cache.items()
-    #         ])):
-    #         if idx >= limit: del self[k]
-
-    # def _clear_stale(self):
-    #     for k in [k for k, v in self._cache.items() if v.stale]:
-    #         del self[k]
-    
-    def add(self, key: str, value: str) -> str:
-        #self._clear_stale()
-        #self._enforce_limit(self._limit)
-        query, _hash = norm(key)
-        value = value or query
-        
-        if not value:
-            raise ValueError("Invalid query string: {value}")
-        self._cache[_hash] = value
-        save_query(hash_to_base32(_hash), value.encode("utf-8"), 'query')
-        return value
-    
-    def remove(self, key: str):
-        key = norm(key)
-        self._cache.pop(key, None)
-        delete_query(hash_to_base32(key))

app/rest/__init__.py

@@ -1,18 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-#
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from bottle import TEMPLATE_PATH
-
-TEMPLATE_PATH.append("app/rest/templates")
-ALL_UNITS = { 'g', 'kg', 'mL', 'L', 'Pieces', 'Bunches', 'Bags' }
-PARAMS = { 'group', 'category', 'product', 'unit', 'tag', 'organic' }
-BOOLEAN = {
-    "1": True,
-    True: "1",
-    "0": False,
-    False: "0",
-    None: "0.5",
-}

app/rest/cherrypy.py

@@ -1,19 +0,0 @@
-import cherrypy
-import bottle
-from .pyapi import *
-
-application = bottle.default_app()
-
-cherrypy.config.update({'environment' : 'staging'})
-cherrypy.config.update({
-    'server.socket_host': "0.0.0.0",
-    'server.socket_port': 6772,
-    'engine.autoreload.on': True,
-    'request.show_tracebacks': True,
-    'request.show_mismatched_params': True,
-    'log.screen': True,
-})
-
-cherrypy.tree.graft(application, "/")
-cherrypy.engine.start()
-cherrypy.engine.block()

app/rest/dev-requirements.txt

@@ -1 +0,0 @@
-pytest

app/rest/form.py

@@ -1,109 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-#
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from typing import Dict, Tuple
-from bottle import template
-from pandas import DataFrame
-from itertools import chain
-from . import ALL_UNITS, BOOLEAN
-
-def get_option_groups(
-    data: DataFrame, filter_data: Dict[str, Tuple[set, set]],
-    k: str, g: str, _type: str
-):
-    if k in data or (k == 'tag' and 'tags' in data):
-        k_data_in_chart = chain(*data["tags"]) if k == "tag" else data[k]
-    else:
-        k_data_in_chart = []
-    
-    groups = sorted(set(data[g] if g is not None and g in data else []))
-    groups.append(None)
-    if _type == "exclude":
-        prefix = "!"
-    else:
-        prefix = ""
-    
-    for group in groups:
-        selected = []
-        unselected = []
-        if group is None:
-            if _type == "include":
-                selected.extend(filter_data[k][0] - (
-                    set(k_data_in_chart if g is not None else set())
-                ))
-                unselected.extend(((
-                    set(k_data_in_chart if g is None else set())
-                ) | filter_data[k][1]) - filter_data[k][0])
-            else:
-                selected.extend(filter_data[k][1])
-                unselected.extend((
-                    set(k_data_in_chart if g is None else set())
-                ) | (
-                    filter_data[k][0] - set(k_data_in_chart) - filter_data[k][1]
-                ))
-        else:
-            k_grouped_data_in_chart = set(data[data[g].apply(lambda x,axis=None: x == group)][k]) if k in data else set()
-            if _type == "include":
-                selected.extend(filter_data[k][0] & set(k_grouped_data_in_chart))
-                unselected.extend(k_grouped_data_in_chart - filter_data[k][0])
-            else:
-                unselected.extend(set(k_grouped_data_in_chart))
-        assert set(selected) - set(unselected) == set(selected), f"{set(selected)} {set(unselected)}"
-        options = sorted(map(lambda x: {
-            "selected": x[0],
-            "value": f"{prefix}{x[1]}",
-            "display": x[1]  
-        }, chain(
-            map(lambda x: (True, x), set(selected)),
-            map(lambda x: (False, x), set(unselected)),
-        )), key=lambda x: x["display"] if "display" in x else x["value"])
-        if len(options) > 0:
-            yield {
-                "optgroup": group,
-                "options": options,
-            }
-
-
-def get_form(action: str, method: str, filter_data: Dict[str, Tuple[set, set]], organic: bool, data: DataFrame):
-    keys = sorted(filter(lambda x: x not in ('unit', 'tag', 'organic'), filter_data), key=lambda x: {
-        'product': 0,
-        'category': 1,
-        'group': 2,
-    }[x])
-    return template('form-filter', action=action, method=method,
-        params=filter_data,
-        **{
-            k: {
-                "name": k,
-                "_include": {
-                    "option_groups": get_option_groups(data, filter_data, k, g, "include"),
-                },
-                "_exclude": {
-                    "option_groups": get_option_groups(data, filter_data, k, g, "exclude"),
-                }
-            } for k, g in zip(keys, [*keys[1:], None])
-        },
-        tags={
-            "name": "tag",
-            "_include": {
-                "option_groups": get_option_groups(data, filter_data, "tag", None, "include")
-            },
-            "_exclude": {
-                "option_groups": get_option_groups(data, filter_data, "tag", None, "exclude")
-            },
-        },
-        units={
-            "name": "unit",
-            "options": sorted(map(lambda x: {
-                "selected": x[0],
-                "value": x[1],
-            },chain(
-                map(lambda x: (True, x), filter_data['unit'][0]),
-                map(lambda x: (False, x), ALL_UNITS - filter_data['unit'][0])
-            )), key=lambda x: x["display"] if "display" in x else x["value"])
-        },
-        organic=BOOLEAN[organic],
-    )

app/rest/hash_util.py

@@ -1,180 +0,0 @@
-from hashlib import blake2b, shake_128, md5, sha256, sha1
-import os
-from base64 import b64encode, b64decode, b85encode, b85decode
-import base32_lib as b32
-
-DIGEST_SIZE_BYTES = 3
-DIGEST_SIZE_BITMASK = 0xffffff
-DIGEST_SIZE_SIGNED_TO_UNSIGNED_BITMASK = 0x1ffffff
-DIGEST_SIZE_SIGNED_TO_UNSIGNED_BIT = 0x1000000
-DIGEST_SIZE_NIBBLES = DIGEST_SIZE_BYTES * 2
-B64ALTCHARS = b'.-'
-
-def sha1hash(data: str):
-    return sha1(data.encode("utf-8"), usedforsecurity=False).hexdigest()[:DIGEST_SIZE_BYTES]
-
-def sha256hash(data: str):
-    return sha256(data.encode("utf-8"), usedforsecurity=False).hexdigest()[:DIGEST_SIZE_BYTES]
-
-def md5hash(data: str):
-    return md5(data.encode("utf-8"), usedforsecurity=False).hexdigest()[:DIGEST_SIZE_BYTES]
-
-def shake(data: str):
-    return shake_128(data.encode("utf-8"), usedforsecurity=False).hexdigest(DIGEST_SIZE_BYTES)
-
-def blake(data: bytes, person: bytes = None) -> bytes:
-    return blake2b(
-        data,
-        usedforsecurity=False,
-        digest_size=DIGEST_SIZE_BYTES,
-        person=person
-    ).digest()
-
-def blake_file(path: str, person: bytes = None, root: str ='rest/static') -> bytes:
-    fd = os.open(f'{root}/{path}', os.O_RDONLY, 0o600)
-    with open(fd, "rb") as f:
-        f.seek(0)
-        _blake = blake2b(usedforsecurity=False, digest_size=DIGEST_SIZE_BYTES, person=person)
-        while f.peek(1):
-            _blake.update(f.read(1024))
-    return _blake.digest()
-        
-
-def python(data: str):
-    return hash(data)
-
-def normalize_hash(_hash: int) -> int:
-    #hex = hash_to_hex(_hash)
-    #return int(hex, 16)
-    #_bytes = _hash.to_bytes(8, byteorder='big', signed=True)
-    #return bytes_to_hash(_bytes)
-    return _hash & DIGEST_SIZE_BITMASK
-
-def normalize_bytes(_bytes: bytes) -> bytes:
-    return (b'\x00' * DIGEST_SIZE_BYTES + _bytes)[-DIGEST_SIZE_BYTES:]
-
-def normalize_hex(_hex: str) -> str:
-    #_bytes = hex_to_bytes(hex)
-    #return _bytes.hex()
-    return _hex.zfill(DIGEST_SIZE_NIBBLES)[-DIGEST_SIZE_NIBBLES:]
-
-def hex_to_bytes(_hex: str) -> bytes:
-    _bytes = bytes.fromhex(_hex.zfill(DIGEST_SIZE_NIBBLES))
-    return normalize_bytes(_bytes)
-
-def bytes_to_hex(_bytes: bytes) -> str:
-    return normalize_bytes(_bytes).hex()
-
-def hash_to_bytes(_hash: int) -> bytes:
-    _bytes = _hash.to_bytes(8, byteorder='big', signed=True)
-    return normalize_bytes(_bytes)
-
-def bytes_to_hash(_bytes: bytes) -> int:
-    norm = normalize_bytes(_bytes)
-    return int.from_bytes(norm, byteorder='big', signed=False)
-
-def hash_to_hex(_hash: int) -> str:
-    #return hash_to_bytes(_hash).hex()
-    #return normalize_hex(
-    #return f"{_hash + (1 << 64):x}"[-4:]
-    #return hex(_hash + (1<<64))[2:][-4:]
-    #return f"{_hash & 0xffff:04x}"
-    return hex((_hash|DIGEST_SIZE_SIGNED_TO_UNSIGNED_BIT) & DIGEST_SIZE_SIGNED_TO_UNSIGNED_BITMASK)[3:]
-
-def hex_to_hash(_hex: str) -> int:
-    #_bytes = bytes.fromhex(hex.zfill(4))
-    #return bytes_to_hash(_bytes)
-    #return int(normalize_hex(hex), 16)
-    return int(_hex, 16) & DIGEST_SIZE_BITMASK
-
-def remove_padding(f):
-    def wrap(*args, **kwargs):
-        return f(*args, **kwargs).split('=')[0]
-    return wrap
-
-def fix_padding(f):
-    def wrap(_b64, *args, **kwargs):
-        pad = (4 - len(_b64)) % 4
-        fixed = _b64 + '='*pad
-        return f(fixed, *args, **kwargs)
-    return wrap
-
-def normalize_base32(_b32: str):
-    return b32.encode(b32.decode(_b32)).upper().zfill(DIGEST_SIZE_BYTES*8//5+1)
-
-def add_padding_base32(f):
-    def wrap(*args, **kwargs):
-        return normalize_base32(f(*args, **kwargs))
-    return wrap
-
-@remove_padding
-def hash_to_base64(_hash: int) -> str:
-    return b64encode(hash_to_bytes(_hash), altchars=B64ALTCHARS).decode("utf-8")
-
-@remove_padding
-def hex_to_base64(_hex: str) -> str:
-    return b64encode(hex_to_bytes(_hex), altchars=B64ALTCHARS).decode("utf-8")
-
-@remove_padding
-def bytes_to_base64(_bytes: str) -> str:
-    return b64encode(normalize_bytes(_bytes), altchars=B64ALTCHARS).decode("utf-8")
-
-@fix_padding
-def base64_to_hash(_b64: str) -> str:
-    return bytes_to_hash(b64decode(_b64, altchars=B64ALTCHARS))
-
-@fix_padding
-def base64_to_hex(_b64: str) -> str:
-    return bytes_to_hex(b64decode(_b64, altchars=B64ALTCHARS))
-
-@fix_padding
-def base64_to_bytes(_b64: str) -> str:
-    return normalize_bytes(b64decode(_b64, altchars=B64ALTCHARS))
-
-#@remove_padding
-def hash_to_base85(_hash: int) -> str:
-    return b85encode(hash_to_bytes(_hash)).decode("utf-8")
-
-#@remove_padding
-def hex_to_base85(_hex: str) -> str:
-    return b85encode(hex_to_bytes(_hex)).decode("utf-8")
-
-#@remove_padding
-def bytes_to_base85(_bytes: str) -> str:
-    return b85encode(normalize_bytes(_bytes)).decode("utf-8")
-
-#@fix_padding
-def base85_to_hash(_b64: str) -> str:
-    return bytes_to_hash(b85decode(_b64))
-
-#@fix_padding
-def base85_to_hex(_b64: str) -> str:
-    return bytes_to_hex(b85decode(_b64))
-
-#@fix_padding
-def base85_to_bytes(_b64: str) -> str:
-    return normalize_bytes(b85decode(_b64))
-
-@add_padding_base32
-def hash_to_base32(_hash: int) -> str:
-    return b32.encode(_hash & DIGEST_SIZE_BITMASK)
-
-@add_padding_base32
-def hex_to_base32(_hex: str) -> str:
-    return b32.encode(hex_to_hash(_hex))
-
-@add_padding_base32
-def bytes_to_base32(_bytes: bytes) -> str:
-    return b32.encode(bytes_to_hash(_bytes))
-
-#@fix_padding
-def base32_to_hash(_b64: str) -> str:
-    return b32.decode(_b64)
-
-#@fix_padding
-def base32_to_hex(_b64: str) -> str:
-    return hash_to_hex(base32_to_hash(_b64))
-
-#@fix_padding
-def base32_to_bytes(_b64: str) -> str:
-    return hash_to_bytes(base32_to_hash(_b64))

app/rest/pyapi.py

@@ -1,139 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-import os
-from threading import Thread
-from typing import Tuple
-from bottle import (
-    route, request, response,
-    static_file,
-    FormsDict,
-)
-from psycopg import Cursor, connect
-from psycopg.rows import TupleRow
-from urllib.parse import parse_qs
-
-from .QueryCache import QueryCache
-from .route_decorators import cache, cursor
-from .query_to_xml import get_categories, get_groups, get_products, get_tags
-from .CachedLoadingPage import CachedLoadingPage
-from .PageCache import PageCache
-from . import trend as worker
-
-host = f"host={os.getenv('HOST')}"
-db = f"dbname={os.getenv('DB', 'grocery')}"
-user = f"user={os.getenv('USER', 'das')}"
-password = f"password={os.getenv('PASSWORD','')}"
-if not password.split('=',1)[1]:
-    password = ''
-conn = connect(f"{host} {db} {user} {password}")
-
-@route('/grocery/static/<filename:path>')
-def send_static(filename):
-    return static_file(filename, root='app/rest/static')
-
-def new_thread(target, conn, path, forms):
-    def cb(queue):
-        return Thread(target=target, args=(
-            queue, conn, path, forms
-        )).start()
-    return cb
-
-PAGE_CACHE = PageCache(100)
-QUERY_CACHE = QueryCache(None)
-
-@route('/grocery/volume', method=['GET', 'POST'])
-@cache(query_cache=QUERY_CACHE, page_cache=PAGE_CACHE)
-def volume(key: Tuple[str, int], cache: PageCache):
-    _, _, path, *_ = request.urlparts
-
-    page = cache[key]
-    if page is None:
-        form = key_to_form(key)
-        page = cache.add(key, CachedLoadingPage([], new_thread(worker.volume, conn, path, form)))
-    
-    for i in iter_page(page):
-        yield i
-
-
-@route('/grocery/trend', method=['GET', 'POST'])
-@cache(query_cache=QUERY_CACHE, page_cache=PAGE_CACHE)
-def trend(key: Tuple[str, int], cache: PageCache):
-    _, _, path, *_ = request.urlparts
-
-    page = cache[key]
-    if page is None:
-        form = key_to_form(key)
-        page = cache.add(key, CachedLoadingPage([], new_thread(worker.trend, conn, path, form)))
-    
-    for i in iter_page(page):
-        yield i
-
-
-def query_to_form(query):
-    form = FormsDict()
-    for k, v in parse_qs(query).items():
-        for item in v:
-            form.append(k, item)
-    return form
-
-
-def key_to_form(key):
-    query, _ = key
-    return query_to_form(query.split('?', 1)[-1])
-
-
-def iter_page(page):
-    # copy first to avoid races
-    resp = list(page.value)
-    pos = len(resp)
-    yield ''.join(resp)
-    
-    while not page.loaded:
-        page.update()
-        # all changes since last yield
-        resp = list(page.value[pos:])
-        pos = pos + len(resp)
-        yield ''.join(resp)
-    
-    # possibly have not yielded the entire page
-    if pos < len(page.value):
-        yield ''.join(page.value[pos:])
-
-
-@route('/grocery/groups', method=['GET', 'POST'])
-@cache(query_cache=QUERY_CACHE, page_cache=PAGE_CACHE)
-@cursor(connection=conn)
-def groups(cur: Cursor[TupleRow], key: Tuple[str, int], cache: PageCache):
-    form = key_to_form(key)
-    response.content_type = 'application/xhtml+xml; charset=utf-8'
-    return get_groups(cur, form)
-
-
-@route('/grocery/categories', method=['GET', 'POST'])
-@cache(query_cache=QUERY_CACHE, page_cache=PAGE_CACHE)
-@cursor(connection=conn)
-def categories(cur: Cursor[TupleRow], key: Tuple[str, int], cache: PageCache):
-    form = key_to_form(key)
-    response.content_type = 'application/xhtml+xml; charset=utf-8'
-    return get_categories(cur, form)
-
-
-@route('/grocery/products', method=['GET', 'POST'])
-@cache(query_cache=QUERY_CACHE, page_cache=PAGE_CACHE)
-@cursor(connection=conn)
-def products(cur: Cursor[TupleRow], key: Tuple[str, int], cache: PageCache):
-    form = key_to_form(key)
-    response.content_type = 'application/xhtml+xml; charset=utf-8'
-    return get_products(cur, form)
-
-
-@route('/grocery/tags', method=['GET', 'POST'])
-@cache(query_cache=QUERY_CACHE, page_cache=PAGE_CACHE)
-@cursor(connection=conn)
-def tags(cur: Cursor[TupleRow], key: Tuple[str, int], cache: PageCache):
-    form = key_to_form(key)
-    response.content_type = 'application/xhtml+xml; charset=utf-8'
-    return get_tags(cur, form)

app/rest/query_to_xml.py

@@ -1,143 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from bottle import request, template, FormsDict
-from pandas import DataFrame
-from psycopg import Cursor
-from psycopg.sql import SQL, Literal, Identifier
-from typing import List
-
-from ..data.filter import get_filter
-from ..data.util import get_where_include_exclude
-from ..data.QueryManager import get_data
-from .form import get_form
-from . import BOOLEAN, PARAMS
-
-
-def get_product_rollup_statement(filters, orderby: List[str]) -> SQL:
-    _map = { k: k[0] for k in ('product', 'category', 'group') }
-    where = [ get_where_include_exclude(
-        _map[k], "name", list(include), list(exclude)
-    ) for k, (include, exclude) in filters.items() ]
-    return SQL("""
-SELECT
-  count(DISTINCT p.id) AS "Products",
-  count(DISTINCT c.id) AS "Categories",
-  count(DISTINCT g.id) AS "Groups",
-  p.name AS "product",
-  c.name AS "category",
-  g.name AS "group"
-FROM products p
-JOIN categories c ON p.category_id = c.id
-JOIN groups g ON c.group_id = g.id
-WHERE {where}
-GROUP BY ROLLUP (g.name, c.name, p.name)
-ORDER BY {orderby}
-""").format(
-        where=SQL("\nAND").join(where),
-        orderby=SQL(", ").join(map(Identifier,orderby)),
-    )
-
-
-def get_inner_query(query: FormsDict, orderby: List[str]) -> SQL:
-    filters = get_filter(query, allow=('group', 'category', 'product'))
-    inner = get_product_rollup_statement(filters, orderby)
-    return inner
-
-
-def render_form(cur: Cursor, inner: str, query: FormsDict):
-    _filter = get_filter(query, allow=PARAMS)
-    data = DataFrame(get_data(cur, inner)).dropna()
-    action = request.path.split('/')[-1]
-    organic = BOOLEAN.get(query.organic, None)
-    return get_form(action, 'post', _filter, organic, data)
-
-
-def get_xml(cur: Cursor, sql: str):
-    return cur.execute(SQL(
-        "SELECT query_to_xml_and_xmlschema({q}, false, false, ''::text)"
-    ).format(q=Literal(sql))).fetchone()[0]
-
-
-def get_products(cur: Cursor, query: FormsDict):
-    inner = get_inner_query(query, ['product', 'category', 'group'])
-    form = render_form(cur, inner, query)
-    sql = SQL("""
-SELECT
-    --"Transactions",
-    COALESCE("product", "Products"||'') "Product",
-    COALESCE("category", "Categories"||'') "Category",
-    COALESCE("group", "Groups"||'') "Group"
-FROM ({inner}) q
-WHERE q.product IS NOT NULL OR q.group IS NULL
-""").format(inner=inner).as_string(cur)
-    xml = get_xml(cur, sql)
-    return template("query-to-xml", title="Products", xml=xml, form=form)
-
-
-def get_categories(cur: Cursor, query: FormsDict):
-    inner = get_inner_query(query, ['category', 'group'])
-    form = render_form(cur, inner, query)
-    sql = SQL("""
-SELECT
-    "Products",
-    COALESCE("category", "Categories"||'') "Category",
-    COALESCE("group", "Groups"||'') "Group"
-FROM ({inner}) q
-WHERE q.product IS NULL AND (q.category IS NOT NULL OR q.group IS NULL)
-""").format(inner=inner).as_string(cur)
-    xml = get_xml(cur, sql)
-    return template("query-to-xml", title="Categories", xml=xml, form=form)
-
-
-def get_groups(cur: Cursor, query: FormsDict):
-    inner = get_inner_query(query, ['group',])
-    form = render_form(cur, inner, query)
-    sql = SQL("""
-SELECT
-    "Products",
-    "Categories",
-    COALESCE("group", "Groups"||'') "Group"
-FROM ({inner}) q
-WHERE q.category IS NULL
-""").format(inner=inner).as_string(cur)
-    xml = get_xml(cur, sql)
-    return template("query-to-xml", title="Groups", xml=xml, form=form)
-
-def get_tags_statement(filters) -> SQL:
-    _map = {
-        k: k[0] for k in ('product', 'category', 'group')
-    }
-    _map.update({ 'tag': 'tg' })
-    where = [ get_where_include_exclude(
-        _map[k], "name", list(include), list(exclude)
-    ) for k, (include, exclude) in filters.items() ]
-    return SQL("""
-SELECT * FROM (SELECT count(DISTINCT txn.id) AS "Uses", tg.name AS "Name"
-FROM tags tg
-JOIN tags_map tm ON tg.id = tm.tag_id
-JOIN transactions txn ON txn.id = tm.transaction_id
-WHERE {where}
-GROUP BY tg.name
-ORDER BY 1 DESC, 2) q
-UNION ALL
-SELECT count(DISTINCT txn.id) AS "Uses", count(DISTINCT tg.name)||'' AS "Name"
-FROM tags tg
-JOIN tags_map tm ON tg.id = tm.tag_id
-JOIN transactions txn ON txn.id = tm.transaction_id
-WHERE {where}
-""").format(where=SQL("\nAND").join(where))
-
-def get_inner_tags_query(query: FormsDict) -> SQL:
-    filters = get_filter(query, allow=('tag',))
-    inner = get_tags_statement(filters)
-    return inner
-
-def get_tags(cur: Cursor, query: FormsDict):
-    inner = get_inner_tags_query(query)
-    form = render_form(cur, inner, query)
-    sql = inner.as_string(cur)
-    xml = get_xml(cur, sql)
-    return template("query-to-xml", title="Tags", xml=xml, form=form)

app/rest/requirements.txt

@@ -1,5 +0,0 @@
-seaborn
-psycopg[binary]
-bottle
-cherrypy
-base32-lib

app/rest/route_decorators.py

@@ -1,93 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from typing import Callable, Iterable, Tuple
-from urllib.parse import urlencode
-from bottle import request, FormsDict, redirect, HTTPError
-from psycopg import Connection
-from psycopg.connection import TupleRow
-
-from .QueryCache import QueryCache, get_hash
-from ..data.filter import get_filter, get_query_param
-from . import BOOLEAN, PARAMS
-from .PageCache import PageCache
-from .hash_util import base32_to_hash, hash_to_base32, normalize_base32
-
-def normalize_query(query: FormsDict, allow: Iterable[str] = None) -> Tuple[str, str]:
-    if query.hash:
-        _hash = normalize_base32(query.hash)
-    else:
-        _hash = None
-
-    allow = allow or PARAMS
-    param = get_filter(query, allow=allow)
-    norm = urlencode([
-        (
-            k, get_query_param(*param[k])
-        ) if k != 'organic' else (
-            "organic", BOOLEAN[BOOLEAN.get(query.organic, None)]
-        ) for k in sorted(param) if param[k]
-    ])
-    return norm if not _hash or len(query.keys()) > 1 else None, _hash
-
-
-def _cache_decorator(func: Callable, query_cache: QueryCache = None, page_cache: PageCache = None):
-    def wrap(*args, **kwargs):
-        _, _, path, *_ = request.urlparts
-        
-        endpoint = path.split('/', 2)[-1]
-
-        query, _hash = normalize_query(request.params)
-        query = f"{endpoint}?{query}"
-        if not _hash:
-            _hashInt = get_hash(query)
-            _hash = hash_to_base32(_hashInt)
-            key = (query, _hashInt)
-        else:
-            key = (query, base32_to_hash(_hash))
-        
-        if request.params.reload == "true":
-            page_cache.remove(key)
-
-        # key with tuple to avoid ambiguity
-        cached = query_cache.get(key)
-        if not cached:
-            if query:
-                cached = query_cache.add(key, None)
-            else:
-                return HTTPError(404, f"No query found for hash: {_hash}")
-
-        if not request.params.hash:
-            if cached and len(cached) > 2000:
-                return redirect(f"{path}?hash={_hash}")
-
-            if cached and f"{endpoint}?{request.query_string}" != cached:
-                return redirect(cached)
-
-        return func((cached, key[1]), page_cache, *args, **kwargs)
-    return wrap
-
-
-def cache(*args, **kwargs):
-    if not len(args):
-        return lambda f: _cache_decorator(f, **kwargs)
-    
-    raise Exception("decorator argument required")
-
-
-def _cursor_decorator(func: Callable, connection: Connection[TupleRow] = None):
-    def wrap(*args, **kwargs):
-        try:
-            with connection.cursor() as cur:
-                return func(cur, *args, **kwargs)
-        finally:
-            connection.commit()
-    return wrap
-
-
-def cursor(*args, **kwargs):
-    if not len(args):
-        return lambda f: _cursor_decorator(f, **kwargs)
-    raise Exception("decorator argument required")

app/rest/static/cloud-gears.css

@@ -1,49 +0,0 @@
-.loader {
-  width: 175px;
-  height: 80px;
-  display: block;
-  margin:auto;
-  background-image: radial-gradient(circle 25px at 25px 25px, #FFF 100%, transparent 0), radial-gradient(circle 50px at 50px 50px, #FFF 100%, transparent 0), radial-gradient(circle 25px at 25px 25px, #FFF 100%, transparent 0), linear-gradient(#FFF 50px, transparent 0);
-  background-size: 50px 50px, 100px 76px, 50px 50px, 120px 40px;
-  background-position: 0px 30px, 37px 0px, 122px 30px, 25px 40px;
-  background-repeat: no-repeat;
-  position: relative;
-  box-sizing: border-box;
-}
-.loader::before {
-  content: '';  
-  left: 60px;
-  bottom: 18px;
-  position: absolute;
-  width: 36px;
-  height: 36px;
-  border-radius: 50%;
-  background-color: #555555;
-  background-image: radial-gradient(circle 8px at 18px 18px, #FFF 100%, transparent 0), radial-gradient(circle 4px at 18px 0px, #FFF 100%, transparent 0), radial-gradient(circle 4px at 0px 18px, #FFF 100%, transparent 0), radial-gradient(circle 4px at 36px 18px, #FFF 100%, transparent 0), radial-gradient(circle 4px at 18px 36px, #FFF 100%, transparent 0), radial-gradient(circle 4px at 30px 5px, #FFF 100%, transparent 0), radial-gradient(circle 4px at 30px 5px, #FFF 100%, transparent 0), radial-gradient(circle 4px at 30px 30px, #FFF 100%, transparent 0), radial-gradient(circle 4px at 5px 30px, #FFF 100%, transparent 0), radial-gradient(circle 4px at 5px 5px, #FFF 100%, transparent 0);
-  background-repeat: no-repeat;
-  box-sizing: border-box;
-  animation: rotationBack 3s linear infinite;
-}
-.loader::after {
-  content: '';  
-  left: 94px;
-  bottom: 15px;
-  position: absolute;
-  width: 24px;
-  height: 24px;
-  border-radius: 50%;
-  background-color: #555555;
-  background-image: radial-gradient(circle 5px at 12px 12px, #FFF 100%, transparent 0), radial-gradient(circle 2.5px at 12px 0px, #FFF 100%, transparent 0), radial-gradient(circle 2.5px at 0px 12px, #FFF 100%, transparent 0), radial-gradient(circle 2.5px at 24px 12px, #FFF 100%, transparent 0), radial-gradient(circle 2.5px at 12px 24px, #FFF 100%, transparent 0), radial-gradient(circle 2.5px at 20px 3px, #FFF 100%, transparent 0), radial-gradient(circle 2.5px at 20px 3px, #FFF 100%, transparent 0), radial-gradient(circle 2.5px at 20px 20px, #FFF 100%, transparent 0), radial-gradient(circle 2.5px at 3px 20px, #FFF 100%, transparent 0), radial-gradient(circle 2.5px at 3px 3px, #FFF 100%, transparent 0);
-  background-repeat: no-repeat;
-  box-sizing: border-box;
-  animation: rotationBack 4s linear infinite reverse;
-}
-
-@keyframes rotationBack {
-  0% {
-    transform: rotate(0deg);
-  }
-  100% {
-    transform: rotate(-360deg);
-  }
-}

app/rest/static/query-to-xml-xslt.xml

@@ -1,91 +0,0 @@
-<xsl:stylesheet version="1.1"
-  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-  xmlns:xsd="http://www.w3.org/2001/XMLSchema"
-  xmlns:xhtml="http://www.w3.org/1999/xhtml"
->
-
-  <xsl:output method="xml"
-    doctype-system="http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"
-    doctype-public="-//W3C/DTD XHTML 1.0 Strict//EN"
-    indent="yes"/>
-  
-  <!-- This is an identity template - it copies everything
-        that doesn't match another template -->
-  <xsl:template match="@* | node()">
-    <xsl:copy>
-      <xsl:apply-templates select="@* | node()"/>
-    </xsl:copy>
-  </xsl:template>
-  <xsl:template match="processing-instruction()">
-  </xsl:template>
-
-  <xsl:template match="//xhtml:div[@id='xmldata']/*">
-    <xsl:variable name="schema" select="//xsd:schema"/>
-    <xsl:variable name="tabletypename"
-                  select="$schema/xsd:element[@name=name(current())]/@type"/>
-    <xsl:variable name="rowtypename"
-                  select="$schema/xsd:complexType[@name=$tabletypename]/xsd:sequence/xsd:element[@name='row']/@type"/>
-    <xsl:variable name="fieldname"
-                  select="$schema/xsd:complexType[@name=$rowtypename]/xsd:sequence/xsd:element[@name='Product' or @name='Category' or @name='Group' or @name='Name'][1]/@name"/>
-    
-    <table xmlns="http://www.w3.org/1999/xhtml"
-      class="pure-table pure-table-bordered pure-table-striped" style="text-align: left; width: 100%;">
-    
-      <thead style="text-align: center">
-      <tr>
-        <th></th>
-        <xsl:for-each select="$schema/xsd:complexType[@name=$rowtypename]/xsd:sequence/xsd:element/@name">
-          <th><xsl:value-of select="."/></th>
-        </xsl:for-each>
-      </tr>
-      </thead>
-
-      <tbody>
-      <xsl:for-each select="xhtml:row">
-        <xsl:choose>
-          <xsl:when test="position() != last()">
-            <tr>
-              <td>
-                <input type="checkbox">
-                  <xsl:attribute name="name">
-                    <xsl:choose>
-                      <xsl:when test="$fieldname = 'Name'">tag</xsl:when>
-                      <xsl:otherwise>
-                        <xsl:value-of select="translate($fieldname,'PCG', 'pcg')"/>
-                      </xsl:otherwise>
-                    </xsl:choose>
-                  </xsl:attribute>
-                  <xsl:attribute name="value">
-                    <xsl:value-of select="*[name() = $fieldname][1]"/>
-                  </xsl:attribute>
-                  <xsl:attribute name="form">filter</xsl:attribute>
-                </input>
-              </td>
-              <xsl:for-each select="*">
-                <td><xsl:value-of select="."/></td>
-              </xsl:for-each>
-            </tr>
-          </xsl:when>
-        </xsl:choose>
-      </xsl:for-each>
-      </tbody>
-      <tfoot>
-      <xsl:for-each select="xhtml:row">
-        <xsl:choose>
-          <xsl:when test="position() = last()">
-              <tr>
-                <th></th>
-                <xsl:for-each select="*">
-                  <th style="text-align: center"><xsl:value-of select="."/></th>
-                </xsl:for-each>
-              </tr>
-          </xsl:when>
-        </xsl:choose>
-      </xsl:for-each>
-      </tfoot>
-
-    </table>
-
-  </xsl:template>
-
-</xsl:stylesheet>

app/rest/templates/button-action.tpl

@@ -1,5 +0,0 @@
-<div class="pure-button-group vertical-button-group" role="action" style="padding: 1em 0.25em 0;">
-  <button class="pure-button" type="submit"> Apply </button>
-  <button form="clear" class="pure-button" type="submit"> Clear </button>
-  <button form="reload" class="pure-button" type="submit"> Reload </button>
-</div>

app/rest/templates/button-style.tpl

@@ -1,16 +0,0 @@
-<style>
-.vertical-button-group .pure-button:first-child {
-    border-top-right-radius: 2px;
-    border-bottom-left-radius: 0px;
-}
-.vertical-button-group .pure-button:last-child {
-    border-top-right-radius: 0px;
-    border-bottom-left-radius: 2px;
-}
-.vertical-button-group .pure-button {
-    width: 100%;
-}
-.vertical-button-group .pure-button-hover, .pure-button:focus, .pure-button:hover {
-    background-image: linear-gradient(.25turn, rgba(0, 0, 0, .1), rgba(0, 0, 0, .05) 20%, rgba(0, 0, 0, .05) 80%, rgba(0, 0, 0, 0.1));
-}
-</style>

app/rest/templates/buttongroup-nav.tpl

@@ -1,13 +0,0 @@
-% setdefault('style', '')
-<div class="pure-button-group vertical-button-group" role="{{role}}" style="padding: 1em 0.25em 0;">
-% for target in targets:
-%   active = 'pure-button-active' if target == action else ''
-%   label = target.title()
-  <button
-    class="pure-button {{active}}"
-    type="submit"
-    formaction="{{target}}"
-    style="{{style}}"
-  > {{label}} </button>
-% end
-</div>

app/rest/templates/done.tpl

@@ -1 +0,0 @@
-<div class="done"></div>

app/rest/templates/error-500.tpl

@@ -1 +0,0 @@
-<span style="font-size: 3em">{{ error }}</span>

app/rest/templates/filter-set.tpl

@@ -1,17 +0,0 @@
-<div class="pure-u-lg-1-8"></div>
-<%
-  for filter in (product, category, group):
-    include('include-exclude', **filter)
-  end
-%>
-<%
-  include('include-exclude', **tags)
-
-%>
-  <div class="pure-u-1-3 pure-u-lg-1-12">
-    <div class="pure-g">
-    <% include('select-one', **units) %>
-    <% include('range-organic') %>
-    </div>
-  </div>
-<div class="pure-u-lg-1-8"></div>

app/rest/templates/form-clear.tpl

@@ -1,8 +0,0 @@
-<form id="clear" method="{{ method }}" action="{{ action }}">
-  <div style="width: 0; height: 0">
-  % for param in params:
-  %   include('hidden-input', name=param['name'])
-
-  % end
-  </div>
-</form>

app/rest/templates/form-filter.tpl

@@ -1,47 +0,0 @@
-% from app.data.filter import get_query_param
-<form id="filter" method="{{ method }}" action="{{ action }}">
-  <style>
-  select::-webkit-scrollbar {
-  width: 11px;
-}
-select {
-  color: #cccccc;
-  background-color: #080808;
-  scrollbar-width: thin;
-  scrollbar-color: var(--thumbBG) var(--scrollbarBG);
-}
-select::-webkit-scrollbar-track {
-  background: var(--scrollbarBG);
-}
-select::-webkit-scrollbar-thumb {
-  background-color: var(--thumbBG) ;
-  border-radius: 6px;
-  border: 3px solid var(--scrollbarBG);
-}
-  </style>
-  <div class="pure-g">
-    <div class="pure-u-1-24 pure-u-lg-1-5"></div>  
-    <div class="pure-u-11-12 pure-u-lg-3-5">
-    % include('button-style')
-    % include('menu', action=action)
-
-    </div>
-    <div class="pure-u-1-24 pure-u-lg-1-5"></div>    
-  </div>
-  <details style="padding: 1em 0">
-    <summary>Click to expand filter...</summary>
-    <div class="pure-g">
-      <%
-      include('filter-set',
-        product=product, category=category, group=group,
-        tags=tags, units=units)
-      %>
-    </div>
-  </details>
-</form>
-% params = [{
-%   'name': k,
-%   'value': get_query_param(inc, ex),
-% } for k, (inc, ex) in params.items()]
-% include('form-clear', params=params)
-% include('form-reload', params=params)

app/rest/templates/form-reload.tpl

@@ -1,10 +0,0 @@
-<form id="reload" method="{{ method }}" action="{{ action }}">
-  <div style="width: 0; height: 0">
-  % for param in params:
-  %   include('hidden-input', **param)
-
-  % end
-  % include('hidden-input', name='reload', value='true')
-
-  </div>
-</form>

app/rest/templates/hidden-input.tpl

@@ -1,2 +0,0 @@
-% setdefault("value", "")
-<input type="text" name="{{name}}" value="{{value}}" hidden="true"/>

app/rest/templates/include-exclude.tpl

@@ -1,26 +0,0 @@
-<div class="pure-u-1-3 pure-u-lg-{{"1-6" if name != "tag" else "1-8"}}">
-  <div class="pure-g">
-    <div class="pure-u-1">
-      <div class="l-box">
-        <h3>{{name.title()}}</h3>
-      </div>
-    </div>
-    <%
-      include('select', id=f"{name}-include", name=name,
-        children=_include["option_groups"] if "option_groups" in _include else [{
-          "options": _include["options"]
-        }],
-        hint="Include", multiple=True)
-    %>
-
-    <%
-    if defined("_exclude"):
-      include('select', id=f"{name}-exclude", name=name,
-        children=_exclude["option_groups"] if "option_groups" in _exclude else [{
-          "options": _exclude["options"]
-        }], hint="Exclude", multiple=True)
-    end
-    %>
-
-  </div>
-</div>

app/rest/templates/label.tpl

@@ -1 +0,0 @@
-<label for="{{id}}">{{label}}</label>

app/rest/templates/menu.tpl

@@ -1,19 +0,0 @@
-<div class="pure-g">
-  <div class="pure-u-1-3">
-  % include('buttongroup-nav', role='chart', style='background-color: firebrick', targets=[
-  %   "trend",
-  %   "volume",
-  % ])
-
-  </div>
-  <div class="pure-u-1-3">
-  % include('buttongroup-nav', role='data', targets=[
-  %   "products", "categories", "groups", "tags"
-  % ])
-
-  </div>
-  <div class="pure-u-1-3">
-    % include('button-action')
-
-  </div>
-</div>

app/rest/templates/optgroup.tpl

@@ -1,2 +0,0 @@
-<optgroup label="{{name}}">{{!base}}
-</optgroup>

app/rest/templates/option.tpl

@@ -1,5 +0,0 @@
-% disabled = (get("disabled", False) and 'disabled="true"') or ""
-% selected = (get("selected", False) and 'selected="true"') or ""
-% display = get("display", value)
-% setdefault("indent", "  ")
-{{indent}}<option value="{{value}}" {{!disabled}} {{!selected}}>{{display}}</option>

app/rest/templates/options.tpl

@@ -1,5 +0,0 @@
-% defined("optgroup") and rebase('optgroup', name=optgroup)
-% for opt in options:
-
-%   include('option', **opt)
-% end

app/rest/templates/progress.tpl

@@ -1,5 +0,0 @@
-<div class="progress">
-  <progress id="loading-{{stage}}" value="{{percent}}" max="100"></progress>
-  <br/>
-  <label for="loading-{{stage}}">{{stage}}</label>
-</div>

app/rest/templates/query-to-xml.tpl

@@ -1,47 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<?xml-stylesheet type="text/xsl" href="/grocery/static/query-to-xml-xslt.xml"?>
-<html xmlns="http://www.w3.org/1999/xhtml">
-  <head>
-    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
-    <title>{{ title }}</title>
-    <meta name="viewport" content="width=device-width, initial-scale=1"/>
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/purecss@2.1.0/build/pure-min.css" integrity="sha384-yHIFVG6ClnONEA5yB5DJXfW2/KC173DIQrYoZMEtBvGzmf0PKiGyNEqe9N6BNDBH" crossorigin="anonymous"/>
-    <link rel="stylesheet" href="https://shandan.one/css/grids-responsive-min.css"/>
-    <link rel="stylesheet" href="https://shandan.one/css/responsive-visibility-collapse.css"/>
-    <link rel="manifest" href="/grocery/static/manifest.json"/>
-    <link rel="icon" type="image/png" href="/grocery/static/favicon.png"/>
-    <style>
-body {
-  background-color: #080808;
-  color: #cccccc;
-  text-align: center;
-}
-.pure-table thead{
-  background-color:#a0a0a0;
-  color:#000;
-  text-align:left;
-  vertical-align:bottom;
-}
-.pure-table td{
-  background-color:transparent;
-}
-.pure-table-odd td{
-  background-color:#181818;
-}
-.pure-table-striped tr:nth-child(2n-1) td{
-  background-color:#181818;
-}
-    </style>
-  </head>
-  <body align="center">
-{{!form}}
-    <div class="pure-g">
-      <div class="pure-u-lg-1-3"> </div>
-      <div id="xmldata" class="pure-u-1 pure-u-lg-1-3">
-{{!xml}}
-      </div>
-      <div class="pure-u-lg-1-3"> </div>
-    </div>
-  </body>
-</html>
-

app/rest/templates/range-organic.tpl

@@ -1,14 +0,0 @@
-<div class="pure-u-1">
-  <div class="l-box">
-    <h3>Organic</h3>
-  </div>
-  </div>
-  <label for="organic-state" hidden="true">Organic</label>
-  <div class="pure-g">
-  <div class="pure-u-1-3">No</div>
-  <div class="pure-u-1-3">Any</div>
-  <div class="pure-u-1-3">Yes</div>
-  <div class="pure-u-1">
-    <input type="range" id="organic-state" name="organic" min="0" max="1" step="0.5" value="{{organic}}" />
-  </div>
-</div>

app/rest/templates/select-one.tpl

@@ -1,10 +0,0 @@
-<div class="pure-u-1">
-  <div class="l-box">
-    <h3>{{name.title()}}</h3>
-  </div>
-</div>
-<% include('select', id=f"{name}-select-one", name=name,
-   children=[{
-      "options": options
-   }])
-%>

app/rest/templates/select.tpl

@@ -1,20 +0,0 @@
-% from bottle import template
-% multiple = (get("multiple", False) and 'multiple="true"') or ""
-<div class="pure-u-1">
-%  if defined("label"):
-%    include('label', id=id, label=label)
-%  end
-
-<select id="{{id}}" name="{{name}}" size="10" {{!multiple}} style="width: calc(100% - 1em); margin: 0 1em 1em">
-%  if defined("hint"):
-%    include('option', value=hint, disabled=True)
-%  end
-
-%  for child in children:
-%    if "optgroup" in child and child["optgroup"] is None:
-%      del child["optgroup"]
-%    end
-{{!template('options', **child)}}
-%  end
-</select>
-</div>

app/rest/templates/trend.tpl

@@ -1,77 +0,0 @@
-% setdefault("start", False)
-% setdefault("end", False)
-% setdefault("error", '')
-% if start:
-<html>
-  <head>
-    <title>Trend</title>
-    <meta name="viewport" content="width=device-width, initial-scale=1"/>
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/purecss@2.1.0/build/pure-min.css" integrity="sha384-yHIFVG6ClnONEA5yB5DJXfW2/KC173DIQrYoZMEtBvGzmf0PKiGyNEqe9N6BNDBH" crossorigin="anonymous"/>
-    <link rel="stylesheet" href="https://shandan.one/css/grids-responsive-min.css"/>
-    <link rel="stylesheet" href="https://shandan.one/css/responsive-visibility-collapse.css"/>
-    <link rel="stylesheet" href="/grocery/static/cloud-gears.css"/>
-    <link rel="manifest" href="/grocery/static/manifest.json"/>
-    <link rel="icon" type="image/png" href="/grocery/static/favicon.png"/>
-    <style>
-html {
-  --scrollbarBG: #333333;
-  --thumbBG: #080808;
-}
-svg {
-  max-height: min(100vh, calc(100vw * 9 / 16));
-  max-width: calc(100vw - 2em);
-}
-body {
-  background-color: #080808;
-  color: #cccccc;
-  text-align: center;
-}
-div.loader-container {
-  position: absolute;
-  left: 50vw;
-  top: 50vh;
-  margin-top: -5.5em;
-  margin-left: -87.5px;
-  padding-bottom: 2em;
-  height: 9em;
-  width: 175px;
-}
-div.loader-container:not(:has(+ .done)) {
-  display: block;
-}
-.loader-container:not(:last-child) {
-  display: none;
-}
-div.progress {
-  margin: 1em 0 1em;
-}
-div.progress:not(:has(+ .done)) {
-  display: block;
-}
-.progress label {
-  text-align:left;
-}
-.progress label:after {
-  content: "...";
-}
-.progress:not(:last-child) {
-  display: none;
-}
-    </style>
-  </head>
-  <body>
-    <div class="loader-container">
-    <span class="loader"></span>
-% end
-% if end:
-    </div>
-    <div class="done"></div>
-{{!form}}
-    % if error:
-    % include('error-500', error=error)
-    % else:
-{{!svg}}
-    % end
-    </body>
-</html>
-% end

app/rest/templates/volume.tpl

@@ -1,79 +0,0 @@
-% setdefault("start", False)
-% setdefault("end", False)
-% setdefault("error", '')
-% if start:
-<html>
-  <head>
-    <title>Trend</title>
-    <meta name="viewport" content="width=device-width, initial-scale=1"/>
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/purecss@2.1.0/build/pure-min.css" integrity="sha384-yHIFVG6ClnONEA5yB5DJXfW2/KC173DIQrYoZMEtBvGzmf0PKiGyNEqe9N6BNDBH" crossorigin="anonymous"/>
-    <link rel="stylesheet" href="https://shandan.one/css/grids-responsive-min.css"/>
-    <link rel="stylesheet" href="https://shandan.one/css/responsive-visibility-collapse.css"/>
-    <link rel="stylesheet" href="/grocery/static/cloud-gears.css"/>
-    <link rel="manifest" href="/grocery/static/manifest.json"/>
-    <link rel="icon" type="image/png" href="/grocery/static/favicon.png"/>
-    <style>
-html {
-  --scrollbarBG: #333333;
-  --thumbBG: #080808;
-}
-svg {
-  max-height: min(100vh, calc(100vw * 9 / 16));
-  max-width: calc(100vw - 2em);
-}
-body {
-  background-color: #080808;
-  color: #cccccc;
-  text-align: center;
-}
-div.loader-container {
-  position: absolute;
-  left: 50vw;
-  top: 50vh;
-  margin-top: -5.5em;
-  margin-left: -87.5px;
-  padding-bottom: 2em;
-  height: 9em;
-  width: 175px;
-}
-div.loader-container:not(:has(+ .done)) {
-  display: block;
-}
-.loader-container:not(:last-child) {
-  display: none;
-}
-div.progress {
-  margin: 1em 0 1em;
-}
-div.progress:not(:has(+ .done)) {
-  display: block;
-}
-.progress label {
-  text-align:left;
-}
-.progress label:after {
-  content: "...";
-}
-.progress:not(:last-child) {
-  display: none;
-}
-    </style>
-  </head>
-  <body>
-    <div class="loader-container">
-    <span class="loader"></span>
-% end
-% if end:
-    </div>
-    <div class="done"></div>
-{{!form}}
-    % if error:
-    % include('error-500', error=error)
-    % else:
-    %   for plt in svg:
-{{!plt}}
-    %    end
-    % end
-    </body>
-</html>
-% end

app/rest/trend.py

@@ -1,273 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-#
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from io import StringIO
-from datetime import date, datetime
-from queue import Queue
-from bottle import (
-    FormsDict,
-    HTTPError,
-    template,
-)
-import matplotlib.pyplot as plt
-import matplotlib
-import numpy as np
-from pandas import DataFrame
-import seaborn as sns
-from psycopg import Connection
-from psycopg.connection import TupleRow
-
-from . import ALL_UNITS, BOOLEAN, PARAMS
-from ..data.QueryManager import (
-    display_mapper,
-    QueryManager,
-)
-from ..data.filter import (
-    get_filter,
-    get_query_param,
-)
-from .form import(
-    get_form,
-)
-
-matplotlib.use('agg')
-
-plot_style = {
-    "lines.color": "#ffffff",
-    "patch.edgecolor": "#ffffff",
-    "text.color": "#ffffff",
-    "axes.facecolor": "#7f7f7f",
-    "axes.edgecolor": "#ffffff",
-    "axes.labelcolor": "#ffffff",
-    "xtick.color": "#ffffff",
-    "ytick.color": "#ffffff",
-    "grid.color": "#ffffff",
-    "figure.facecolor": "#7f7f7f",
-    "figure.edgecolor": "#7f7f7f",
-    "savefig.facecolor": "#7f7f7f",
-    "savefig.edgecolor": "#7f7f7f",
-}
-
-def get_data(query_manager: QueryManager, unit=None, **kwargs) -> DataFrame:
-    d = DataFrame(query_manager.get_historic_prices_data(unit, **kwargs))
-    if d.empty:
-        return d
-    d['ts_month'] = d['ts_raw'].apply(lambda x: date(x.date().year, x.date().month,1))
-    d[['price','quantity']] = d[['price','quantity']].apply(
-        lambda y: y.apply(lambda x: x and float(x)),
-    )
-    return d
-
-def abort(code, text):
-    raise HTTPError(code, text)
-
-def trend(queue: Queue, conn: Connection[TupleRow], path: str, query: FormsDict):
-    for item in trend_internal(conn, path, query):
-        queue.put(item, block=True)
-    queue.put(None)
-
-def trend_internal(conn: Connection[TupleRow], path: str, query: FormsDict):
-    progress = {
-        'stage': None,
-        'percent': None,
-    }
-    action = path.split('/')[-1]
-    organic = BOOLEAN.get(query.organic, None)
-    _filter = get_filter(query, allow=PARAMS)
-    yield template("trend", start=True)
-    try:
-        with conn.cursor() as cur:
-            query_manager = QueryManager(cur, display_mapper)
-            fields = {
-                k: get_query_param(*_filter[k])
-                for k in sorted(_filter) if k not in ('organic', 'unit') and _filter[k]
-            }
-            unit = fields['unit'] = query.unit or 'kg'
-            fields['organic'] = BOOLEAN.get(query.organic, None)
-            if unit and unit not in ALL_UNITS:
-                abort(400, f"Unsupported unit: {unit}")
-
-            progress.update({ "stage": "Querying database", "percent": "10"})
-            yield template("done") + template("progress", **progress)
-            data = get_data(query_manager, **fields)
-            
-            if data.empty:
-                abort(404, f"No data.")
-            
-            progress.update({ "stage": "Preparing data", "percent": "30"})
-            yield template("done") + template("progress", **progress)
-            
-            in_chart = data['$/unit'].apply(lambda x: (x or False) and True)
-            data = data[in_chart]
-            
-            pivot = data.pivot_table(index=['ts_raw',], columns=['product',], values=['$/unit'], aggfunc='mean')
-            pivot.columns = pivot.columns.droplevel()
-            sns.set_theme(style='darkgrid', palette='pastel', context="talk")
-            plt.style.use("dark_background")
-            plt.rcParams.update(plot_style)
-            plt.rcParams.update({"grid.linewidth":0.2, "grid.alpha":0.5})
-            plt.figure(figsize=[16, 9], layout="tight")
-            xlabel='Time'
-            ylabel=f'$ / {unit}'
-            if pivot.columns.size > 50:
-                ax = sns.scatterplot(data=pivot, markers=True)
-            else:
-                ax = sns.lineplot(data=pivot, markers=True)
-                legend = plt.figlegend(
-                    loc='upper center', ncol=6,
-                    title_fontsize="14", fontsize="12", labelcolor='#ffffff',
-                    framealpha=0.5
-                )
-                legend.set_title(title="Products")
-            ax.legend().set_visible(False)
-
-            ax.set_xlabel(xlabel, fontsize="14")
-            ax.set_ylabel(ylabel, fontsize="14")
-            ax.axes.tick_params(labelsize="12", which='both')
-            for _, spine in ax.spines.items():
-                spine.set_color('#ffffff')
-            
-            progress.update({ "stage": "Rendering chart", "percent": "50"})
-            yield template("done") + template("progress", **progress)
-            
-            f = StringIO()
-            plt.savefig(f, format='svg')
-            progress.update({ "stage": "Done", "percent": "100" })
-            yield template("done") + template("progress", **progress)
-            
-            form = get_form(action, 'post', _filter, organic, data)
-            
-            yield template("trend", end=True, form=form, svg=f.getvalue())
-
-    except HTTPError as e:
-        if 'data' not in locals():
-            data = DataFrame()
-        if 'form' not in locals():
-            form = get_form(action, 'post', _filter, organic, data)
-        yield template("done") + template("trend", end=True, form=form, error=e.body)
-
-    finally:
-        conn.commit()
-
-def volume(queue: Queue, conn: Connection[TupleRow], path: str, query: FormsDict):
-    for item in volume_internal(conn, path, query):
-        queue.put(item, block=True)
-    queue.put(None)
-
-def volume_internal(conn: Connection[TupleRow], path: str, query: FormsDict):
-    progress = {
-        'stage': None,
-        'percent': None,
-    }
-    action = path.split('/')[-1]
-    organic = BOOLEAN.get(query.organic, None)
-    _filter = get_filter(query, allow=PARAMS)
-    yield template("trend", start=True)
-    try:
-        with conn.cursor() as cur:
-            query_manager = QueryManager(cur, display_mapper)
-            fields = {
-                k: get_query_param(*_filter[k])
-                for k in sorted(_filter) if k not in ('organic', 'unit') and _filter[k]
-            }
-            unit = fields['unit'] = query.unit or 'kg'
-            fields['organic'] = BOOLEAN.get(query.organic, None)
-            if unit and unit not in ALL_UNITS:
-                abort(400, f"Unsupported unit: {unit}")
-
-            progress.update({ "stage": "Querying database", "percent": "10"})
-            yield template("done") + template("progress", **progress)
-            data = get_data(query_manager, **fields)
-
-            if data.empty:
-                abort(404, f"No data.")
-            
-            progress.update({ "stage": "Preparing data", "percent": "30"})
-            yield template("done") + template("progress", **progress)
-            
-            now = datetime.now().date()
-            prev_month = date(now.year,now.month-1,1) if now.month > 1 else date(now.year-1, 12, 1)
-            data = data[data['ts_month'] == prev_month ]
-            group = 'group'
-            for g, _g in zip(
-                ('category', 'group'),
-                ('product', 'category')
-            ):
-                if g and len(data[g].unique()) != 1:
-                    continue
-                group = _g
-                break
-
-            pivot = data[~(data['quantity'].isnull())].groupby([group,])[['price', 'quantity']].sum()
-            
-            if pivot.empty:
-                abort(404, f"No data.")
-
-            sns.set_theme(style='darkgrid', palette='pastel', context="talk")
-            plt.style.use("dark_background")
-            plt.rcParams.update(plot_style)
-            plt.rcParams.update({"grid.linewidth":0.2, "grid.alpha":0.5})
-            svg = []
-            for title, col, (pre, fmt, suf) in zip((
-                f"Expenditure ${pivot['price'].sum():0.2f}",
-                f"Quantity {pivot['quantity'].sum():0.1f} {unit}",
-            ), (
-                'price',
-                'quantity'
-            ), [
-                ('$', "{0:0.2f}", ''),
-                ('', "{0:0.1f}", f' {unit}')
-            ]):
-                plt.figure(figsize=[16, 9], layout="tight")
-                ax = plt.axes()
-                wedges, *_ = ax.pie(pivot[col].values, startangle=90)
-                bbox_props = dict(boxstyle="square,pad=0.3", lw=0.72, fc='#5f5f5f', ec=plot_style["axes.edgecolor"])
-                kw = dict(arrowprops=dict(arrowstyle="-"),
-                    bbox=bbox_props, zorder=0, va="center")
-                for i, p in enumerate(wedges):
-                    ang = (p.theta2 - p.theta1)/2. + p.theta1
-                    y = np.sin(np.deg2rad(ang))
-                    x = np.cos(np.deg2rad(ang))
-                    horizontalalignment = {-1: "right", 1: "left"}[int(np.sign(x))]
-                    connectionstyle = f"angle,angleA=0,angleB={ang}"
-                    kw["arrowprops"].update({"connectionstyle": connectionstyle})
-                    label = pivot.index[i]
-                    val = pivot.loc[label, col]
-                    label = f"{label} {pre}{fmt.format(val)}{suf} ({val*100/pivot[col].sum():0.1f}%)"
-                    ax.annotate(label, xy=(x, y), xytext=(1.35*np.sign(x), 1.4*y),
-                                horizontalalignment=horizontalalignment, **kw)
-                ax.set_title(title)
-                xlabel=''
-                ylabel=''
-                ax.legend().set_visible(False)
-
-                ax.set_xlabel(xlabel, fontsize="14")
-                ax.set_ylabel(ylabel, fontsize="14")
-                ax.axes.tick_params(labelsize="12", which='both')
-            
-                progress.update({ "stage": "Rendering chart", "percent": "50"})
-                yield template("done") + template("progress", **progress)
-                
-                f = StringIO()
-                plt.savefig(f, format='svg')
-                svg.append(f.getvalue())
-
-            progress.update({ "stage": "Done", "percent": "100" })
-            yield template("done") + template("progress", **progress)
-            
-            form = get_form(action, 'post', _filter, organic, data)
-            
-            yield template("volume", end=True, form=form, svg=svg)
-
-    except HTTPError as e:
-        if 'data' not in locals():
-            data = DataFrame()
-        if 'form' not in locals():
-            form = get_form(action, 'post', _filter, organic, data)
-        yield template("done") + template("trend", end=True, form=form, error=e.body)
-
-    finally:
-        conn.commit()

app/widgets.py

@@ -136,7 +136,7 @@ class AutoCompleteFloatEdit(FloatEdit):
         return super().keypress(size, key)
 
 class NoTabCheckBox(CheckBox):
-    
+
     def keypress(self, size, key):
         if not isinstance(key, tuple) and key == 'tab':
             return None
@@ -358,7 +358,7 @@ class FlowBarGraphWithVScale(Columns):
         labels = vscale or []
         caption = caption or u''
         top = self.height
-        scale = _max*(top+1)/top
+        scale = max(_max*(top+1)/top, 0.0001)
         self.graph.set_data(data, scale)
         self.graph_vscale._invalidate()
         self.graph_vscale.set_scale([

db/LICENSE

@@ -0,0 +1 @@
+This grocery database by Daniel Sheffield is marked with CC0 1.0 

example.sql

@@ -0,0 +1,6 @@
+-- Insert a tag if it got missed
+--INSERT INTO tags_map (transaction_id, tag_id) VALUES(2247, 11);
+SELECT * FROM transactions t
+	JOIN tags_map ON (transaction_id = t.id)
+	JOIN tags tg ON (tag_id = tg.id)
+	WHERE transaction_id = 2247;

grocery_transactions.py

@@ -1,12 +1,12 @@
 # Grocery Manager
 This a database driven app - meaning it's all about the data and nothing but the data.
 
-The database is assumed to be postgresql, but any modern database should work with some tweaks.
+The database is assumed to be postgresql, but any modern relational database should work with some tweaks.
 
 The following apps make the data useful:
 * ``grocery_transactions.py`` - facilitate data entry with a TUI
 * ``price_check.py`` - show price history graph per product (TUI)
-* ``rest`` - basic website to view price trends
+* ``SQLPage`` - basic website to view price trends
 
 
 ## Workflow
@@ -40,13 +40,15 @@ This is another ``python`` + ``urwid`` TUI to look up products to see their pric
 
 It's designed to be usable on mobile (I use termux and have a shortcut to ssh into a jumphost)
 
-### rest
+### SQLPage
 
-A small ``python`` + ``bottle`` website for things that are better viewed on the web:
+[SQLPage](https://sql.ophir.dev/) is a lightweight webserver written in Rust and generates web pages from SQL.
 
-* trending price data over all products (can be filtered) rendered with ``seaborn``
-* product listings (direct from database via xml + xsl)
-* tag listings and usage
+Features the SQLPage grocery website provides:
+
+* trending price data over all products (can be filtered) rendered with [appexcharts.js](https://apexcharts.com/)
+* product listings
+* tag listings and usage counts
 
 # Technology stack
 ## postgresql
@@ -70,22 +72,18 @@ Some window functions are used it the TUI apps but only because it was convienie
 
 Some client side data transforms are done with ``pandas``.
 
-## HTML + XSLT
-``postgreql``'s ``query_to_xml_and_xmlschema`` feature is used with an XSLT style sheet to generate the HTML pages.
-
-### bottle
-Thes web inerface uses ``bottle`` as a web server.
-
-``psycopg`` is used to get the XML data for the web page from the database.
+## SQLPage
 
-An HTML form is built client side and added to the page using ``bottle``'s templating feature.
+SQLPage provides web components out-of-the-box to generate beautiful web pages.
 
-The form is submitted using URL params so filters can be written by hand and bookmarked.
+I make heavy use of the table, form and chart components.
 
-Only CSS + HTML is used. No javascript.
+The filter options are internally stored as JSON format and stored in a cookie to preserve the filter on page changes.
 
-This is for portability reasons.
+* I took this approach because it was not very elegant to use buttons that post forms for navigation
+* The cookie only stores the user selections, so while there can be a lot of filter options (300+ products), only the selected options are stored in the cookie.
 
+I'll revisit this later with aim to avoid using cookies entirely.
 
 # Get Started
 ## Install Requirements
@@ -115,4 +113,4 @@ Check prices (TUI)
 python3 price_check.py
 ```
 
-Check the web inerface: https://shandan.one/grocery/trend?category=&group=Fish%2C+Meat%2C+Eggs&product=&tag=&unit=kg
+Check the web inerface: https://shandan.one/grocery/internal/apply.sql?title=Trend&apply=Apply&groups[]=Fish%2C+Meat%2C+Eggs

reconcile.py

@@ -1,23 +1,29 @@
 #!/usr/bin/python3
 #
-# Copyright (c) Daniel Sheffield 2021 - 2023
+# Copyright (c) Daniel Sheffield 2021 - 2024
+# Copyright (c) Shannon Sheffield 2021 - 2024
 #
 # All rights reserved
 #
 # THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
 #
-# usage:  ./reconcile.py ~/gnucash/merged.gnucash 2021-11-01 2021-12-10 1
-#
+import sys
+if sys.argv[1] == '-h':
+    print("""
+  usage:  ./reconcile.py ~/gnucash/merged.gnucash 2021-11-01 2021-12-10 1 Alcohol Sundries CatFood Cleaning
+""")
+    sys.exit(0)
+
 from datetime import datetime, timedelta
 from dateutil.parser import parse as parse_time
 import itertools
 import gnucash
-import sys
 import os
 import psycopg
 from app.data.QueryManager import cursor_as_dict
 from app.data.TransactionView import get_session_transactions_statement as get_statement
 
+
 try:
     from db_credentials import HOST, PASSWORD
     host = f'host={HOST}'
@@ -27,12 +33,14 @@ except:
     password = ''
 
 STORE_CODES = {
+    'WOOLWORTHS' : 'CD',
     'countdown': 'CD',
     'pak n save': 'PnS',
     'SEAFOOD BAZAAR': 'SB',
     'GORDONTON FARM SHOP': 'GFS',
     'THE FARM SHOP': 'GFS',
     'THE ORGANIC FOOD SHOP': 'TOFS',
+    'ORGANICFOOD': 'TOFS',
     'TOFS' : 'TOFS',
     'WHATAWHATA BERRY FARM': 'Farm',
     'TAUPIRI DAIRY TAUPIRI': 'TD',
@@ -49,6 +57,8 @@ STORE_CODES = {
     'Health Post' :  'HP',
     'Scotsburn' : 'SF',
     'Fruit King' : 'FK',
+    'Fresh Choice': 'FC',
+    'FC12-3189-0013895-00': 'DV',
 }
 
 user = os.getenv('USER')

sqlpage/categories.sql

@@ -0,0 +1,14 @@
+SET title = 'Categories';
+SET filter_config = '[
+  {"name": "products[]", "width": 4},
+  {"name": "categories[]", "width": 4},
+  {"name": "groups[]", "width": 4},
+  {"name": "tags[]", "type": "hidden"},
+  {"name": "log_scale", "type": "hidden"},
+  {"name": "start", "type": "hidden"},
+  {"name": "end", "type": "hidden"},
+  {"name": "unit_mass", "type": "hidden"},
+  {"name": "unit_volume", "type": "hidden"},
+  {"name": "unit_count[]", "type": "hidden"}
+]';
+SELECT 'dynamic' AS component, sqlpage.run_sql('sqlpage/internal/entry.sql') AS properties;

sqlpage/groups.sql

@@ -0,0 +1,14 @@
+SET title = 'Groups';
+SET filter_config = '[
+  {"name": "products[]", "width": 4},
+  {"name": "categories[]", "width": 4},
+  {"name": "groups[]", "width": 4},
+  {"name": "tags[]", "type": "hidden"},
+  {"name": "log_scale", "type": "hidden"},
+  {"name": "start", "type": "hidden"},
+  {"name": "end", "type": "hidden"},
+  {"name": "unit_mass", "type": "hidden"},
+  {"name": "unit_volume", "type": "hidden"},
+  {"name": "unit_count[]", "type": "hidden"}
+]';
+SELECT 'dynamic' AS component, sqlpage.run_sql('sqlpage/internal/entry.sql') AS properties;

sqlpage/internal/404.sql

@@ -0,0 +1,7 @@
+-- TODO set response code 404
+SET title = '404 '||COALESCE(''''||$title||'''','')||' not found';
+SELECT 'dynamic' AS component, sqlpage.run_sql('sqlpage/internal/theme.sql') AS properties;
+SELECT 'alert' AS component
+, '404 - Not found' AS title
+, 'Check the URL is correct.' AS description
+;

sqlpage/internal/apply.sql

@@ -0,0 +1,11 @@
+SET apply = TRUE;
+SELECT 'dynamic' AS component, sqlpage.run_sql('sqlpage/internal/cookie.sql') AS properties;
+SELECT 'redirect' AS component
+, (
+  SELECT CASE
+    WHEN $title IN (SELECT name FROM sqlpage_pages)
+    THEN '/grocery/'||lower($title)||'.sql'
+    ELSE '/grocery/internal/404.sql'
+  END
+) AS link
+;

sqlpage/internal/clear.sql

@@ -0,0 +1,11 @@
+SET clear = TRUE
+SELECT 'dynamic' AS component, sqlpage.run_sql('sqlpage/internal/cookie.sql') AS properties;
+SELECT 'redirect' AS component
+, (
+  SELECT CASE
+    WHEN $title IN (SELECT name FROM sqlpage_pages)
+    THEN '/grocery/internal/apply.sql?title='||$title
+    ELSE '/grocery/internal/404.sql'
+  END
+) AS link
+;

app/rest/static/manifest.json → sqlpage/manifest.json

@@ -3,18 +3,18 @@
   "name": "Grocery Manager",
   "short_name": "Grocery",
   "description": "View trending price data and tracked product info",
-  "start_url": "/grocery/trend",
+  "start_url": "/grocery/volume.sql",
   "theme_color": "firebrick",
   "background_color": "black",
   "display": "standalone",
   "icons": [
     {
-      "src": "/grocery/static/favicon.svg",
+      "src": "/grocery/favicon.svg",
       "sizes": "any",
       "purpose": "any"
     },
     {
-      "src": "/grocery/static/favicon_square.svg",
+      "src": "/grocery/favicon_square.svg",
       "sizes": "any",
       "purpose": "any"
     }

sqlpage/products.sql

@@ -0,0 +1,14 @@
+SET title = 'Products';
+SET filter_config = '[
+  {"name": "products[]", "width": 4},
+  {"name": "categories[]", "width": 4},
+  {"name": "groups[]", "width": 4},
+  {"name": "tags[]", "type": "hidden"},
+  {"name": "log_scale", "type": "hidden"},
+  {"name": "start", "type": "hidden"},
+  {"name": "end", "type": "hidden"},
+  {"name": "unit_mass", "type": "hidden"},
+  {"name": "unit_volume", "type": "hidden"},
+  {"name": "unit_count[]", "type": "hidden"}
+]';
+SELECT 'dynamic' AS component, sqlpage.run_sql('sqlpage/internal/entry.sql') AS properties;

sqlpage/sqlpage/data/products.sql

@@ -0,0 +1,21 @@
+DROP TABLE IF EXISTS sqlpage_products;
+CREATE TABLE IF NOT EXISTS sqlpage_products AS
+SELECT
+  count(DISTINCT p.id) AS "Products",
+  count(DISTINCT c.id) AS "Categories",
+  count(DISTINCT g.id) AS "Groups",
+  p.name AS "product",
+  c.name AS "category",
+  g.name AS "group"
+FROM products p
+JOIN categories c ON p.category_id = c.id
+JOIN groups g ON c.group_id = g.id
+WHERE
+  (p.name IN (SELECT v#>>'{}' FROM json_array_elements($products::json) j(v)) OR $products IS NULL)
+AND
+  (c.name IN (SELECT v#>>'{}' FROM json_array_elements($categories::json) j(v)) OR $categories IS NULL)
+AND
+  (g.name IN (SELECT v#>>'{}' FROM json_array_elements($groups::json) j(v)) OR $groups IS NULL)
+GROUP BY ROLLUP ("group", category, product)
+;
+GRANT SELECT ON sqlpage_products TO PUBLIC;

sqlpage/sqlpage/data/tags.sql

@@ -0,0 +1,24 @@
+DROP TABLE IF EXISTS sqlpage_tags;
+CREATE TABLE IF NOT EXISTS sqlpage_tags AS
+SELECT count(DISTINCT txn.id) AS "Uses", tg.name AS "Name"
+FROM tags tg
+JOIN tags_map tm ON tg.id = tm.tag_id
+JOIN transactions txn ON txn.id = tm.transaction_id
+JOIN products p ON p.id = txn.product_id
+JOIN categories c ON p.category_id = c.id
+JOIN groups g ON c.group_id = g.id
+WHERE
+  (p.name IN (SELECT v#>>'{}' FROM json_array_elements($products::json) j(v)) OR $products IS NULL)
+AND
+  (c.name IN (SELECT v#>>'{}' FROM json_array_elements($categories::json) j(v)) OR $categories IS NULL)
+AND
+  (g.name IN (SELECT v#>>'{}' FROM json_array_elements($groups::json) j(v)) OR $groups IS NULL)
+AND
+  (tg.name IN (SELECT v#>>'{}' FROM json_array_elements($tags::json) j(v)) OR $tags IS NULL)
+AND
+  ((ts AT TIME ZONE 'UTC') AT TIME ZONE 'Pacific/Auckland')::date
+  BETWEEN $start::date AND $end::date
+GROUP BY tg.name
+ORDER BY 1 DESC, 2
+;
+GRANT SELECT ON sqlpage_tags TO PUBLIC;

sqlpage/sqlpage/data/transactions.sql

@@ -0,0 +1,56 @@
+DROP TABLE IF EXISTS sqlpage_txn;
+CREATE TABLE IF NOT EXISTS sqlpage_txn AS (
+SELECT
+  t.ts AT TIME ZONE 'UTC' AS ts,
+  t.description,
+  s.code,
+  p.name AS product,
+  c.name AS category,
+  g.name AS "group",
+  t.organic,
+  t.price,
+  CASE
+    WHEN ut.name = 'Count' THEN t.quantity
+    ELSE t.quantity * convert_unit(u.name, CASE
+      WHEN ut.name = 'Volume' THEN $unit_volume
+      WHEN ut.name = 'Mass' THEN $unit_mass
+    END, p.name)
+  END AS quantity,
+  CASE
+    WHEN ut.name = 'Count' THEN u.name
+    WHEN ut.name = 'Volume' THEN $unit_volume
+    WHEN ut.name = 'Mass' THEN $unit_mass
+  END AS unit
+FROM transactions t
+JOIN products p ON (product_id = p.id)
+JOIN categories c ON (category_id = c.id)
+JOIN groups g ON (group_id = g.id)
+JOIN stores s ON (store_id = s.id)
+JOIN units u ON (t.unit_id = u.id)
+JOIN unit_types ut ON (u.unit_type_id = ut.id)
+WHERE
+  (p.name IN (SELECT v#>>'{}' FROM json_array_elements($products::json) j(v)) OR $products IS NULL)
+AND
+  (c.name IN (SELECT v#>>'{}' FROM json_array_elements($categories::json) j(v)) OR $categories IS NULL)
+AND
+  (g.name IN (SELECT v#>>'{}' FROM json_array_elements($groups::json) j(v)) OR $groups IS NULL)
+AND
+  CASE
+    WHEN ut.name = 'Count'
+    THEN (u.name IN (SELECT v#>>'{}' FROM json_array_elements($unit_count::json) j(v)) OR $unit_count IS NULL)
+    ELSE TRUE
+  END
+AND
+  ((ts AT TIME ZONE 'UTC') AT TIME ZONE 'Pacific/Auckland')::date
+  BETWEEN $start::date AND $end::date
+AND
+  (t.id IN (
+    SELECT transaction_id FROM tags_map
+    WHERE tag_id IN (
+      SELECT id FROM tags tg
+      WHERE tg.name IN (SELECT v#>>'{}' FROM json_array_elements($tags::json) j(v))
+    )
+  ) OR $tags IS NULL)
+ORDER BY ts DESC
+LIMIT 1000);
+GRANT SELECT ON sqlpage_txn TO PUBLIC;

sqlpage/sqlpage/data/units.sql

@@ -0,0 +1,6 @@
+DROP TABLE IF EXISTS sqlpage_units;
+CREATE TABLE IF NOT EXISTS sqlpage_units AS
+SELECT u.name, ut.name AS type FROM units u
+JOIN unit_types ut ON ut.id = u.unit_type_id
+;
+GRANT SELECT ON sqlpage_units TO PUBLIC;

sqlpage/sqlpage/internal/cookie.sql

@@ -0,0 +1,40 @@
+SELECT 'cookie' AS component
+, 'session' AS name
+, sqlpage.random_string(32) AS value
+WHERE sqlpage.cookie('session') IS NULL
+;
+
+SELECT 'cookie' AS component
+, name AS name
+, CASE
+    WHEN $apply::bool
+    THEN COALESCE(value, 'null')
+    WHEN $clear::bool
+    THEN 'null'
+  END::text AS value
+FROM (VALUES
+  ('products', $products),
+  ('categories', $categories),
+  ('groups', $groups),
+  ('tags', $tags),
+  ('unit_count', $unit_count)
+) AS vars(name, value)
+WHERE $apply::bool OR $clear::bool
+;
+SELECT 'cookie' AS component
+, name AS name
+, CASE
+    WHEN $apply::bool
+    THEN COALESCE(value, '')
+    WHEN $clear::bool
+    THEN ''
+  END AS value
+FROM (VALUES
+  ('start', $start),
+  ('end', $end),
+  ('unit_volume', $unit_volume),
+  ('unit_mass', $unit_mass),
+  ('log_scale', $log_scale)
+) AS vars(name, value)
+WHERE $apply::bool OR $clear::bool
+;

sqlpage/sqlpage/internal/entry.sql

@@ -0,0 +1,130 @@
+SET apply = CASE COALESCE($apply, '') WHEN 'Apply' THEN TRUE ELSE FALSE END;
+SET clear = CASE COALESCE($clear, '') WHEN 'Clear' THEN TRUE ELSE FALSE END;
+SELECT 'dynamic' AS component, sqlpage.run_sql('sqlpage/internal/cookie.sql') AS properties;
+SET products = sqlpage.cookie('products')::json;
+SET categories = sqlpage.cookie('categories')::json;
+SET groups = sqlpage.cookie('groups')::json;
+SET tags = sqlpage.cookie('tags')::json;
+SET start = sqlpage.cookie('start');
+SET end = sqlpage.cookie('end');
+SET unit_volume = sqlpage.cookie('unit_volume');
+SET unit_mass = sqlpage.cookie('unit_mass');
+SET unit_count = sqlpage.cookie('unit_count')::json;
+SET log_scale = sqlpage.cookie('log_scale');
+
+SET unit_volume = (CASE
+  WHEN $unit_volume = '' OR ($unit_volume IS NULL OR $unit_volume = '')
+  THEN 'L'
+  ELSE $unit_volume
+END);
+SET unit_mass = (CASE
+  WHEN $unit_mass = '' OR ($unit_mass IS NULL OR $unit_mass = '')
+  THEN 'kg'
+  ELSE $unit_mass
+END);
+SET start = (CASE
+  WHEN $start = '' OR $start IS NULL
+  THEN to_char(now()-'30 days'::interval, 'YYYY-MM-DD')
+  ELSE $start
+END);
+SET end = CASE
+  WHEN $end = '' OR $end IS NULL
+  THEN to_char(now(), 'YYYY-MM-DD')
+  ELSE $end
+END;
+SET log_scale = CASE
+  WHEN lower($log_scale) = 'true'
+  THEN 'true'
+  ELSE 'false'
+END;
+
+SET page = CASE
+  WHEN $title IN (
+    'Products', 'Categories', 'Groups', 'Tags', 'Transactions',
+    'Trend', 'Volume'
+  )
+  THEN $title||'.sql'
+  ELSE NULL
+END;
+
+SELECT 'redirect' AS component
+, '/grocery/internal/404.sql?title='||COALESCE($title, '') AS link
+WHERE ($page IS NULL OR $page = '')
+;
+
+SELECT 'dynamic' AS component, sqlpage.run_sql('sqlpage/data/tags.sql') AS properties
+WHERE $apply::bool OR $clear::bool;
+SELECT 'dynamic' AS component, sqlpage.run_sql('sqlpage/data/transactions.sql') AS properties
+WHERE $apply::bool OR $clear::bool;
+SELECT 'dynamic' AS component, sqlpage.run_sql('sqlpage/data/products.sql') AS properties
+WHERE $apply::bool OR $clear::bool;
+SELECT 'dynamic' AS component, sqlpage.run_sql('sqlpage/data/units.sql') AS properties
+WHERE $apply::bool OR $clear::bool;
+
+SET filter_options = (
+  SELECT json_agg(json_build_object('name', k, 'options', o))
+  FROM (
+    SELECT o.k||'[]', json_agg(
+      json_build_object('label', v, 'value', v, 'selected', s)
+      ORDER BY v)
+    FROM (
+      SELECT DISTINCT o.k, CASE o.k
+          WHEN 'products' THEN product
+          WHEN 'categories' THEN category
+          WHEN 'groups' THEN "group"
+        END, v.k IS NOT NULL --, v.v
+      FROM sqlpage_products
+      CROSS JOIN UNNEST(ARRAY['products', 'categories', 'groups']) o(k)
+      LEFT JOIN json_each_text(sqlpage.variables()::json) v(k,v)
+      ON v.k = o.k
+      AND (CASE o.k
+        WHEN 'products' THEN product
+        WHEN 'categories' THEN category
+        WHEN 'groups' THEN "group"
+      END) IN (
+        SELECT j.v#>>'{}' FROM json_array_elements(v.v::json) j(v)
+      )
+    ) AS o(k, v, s)
+    WHERE v IS NOT NULL
+    GROUP BY o.k
+    UNION ALL
+    SELECT o.k||'[]', json_agg(
+      json_build_object('label', v, 'value', v, 'selected', s)
+      ORDER BY v)
+    FROM (
+      SELECT DISTINCT 'tags', "Name", v.k IS NOT NULL
+      FROM sqlpage_tags
+      LEFT JOIN json_each_text(sqlpage.variables()::json) v(k,v)
+      ON v.k = 'tags' AND "Name" IN (
+        SELECT j.v#>>'{}' FROM json_array_elements(v.v::json) j(v)
+      )
+    ) AS o(k, v, s)
+    WHERE v IS NOT NULL
+    GROUP BY o.k
+    UNION ALL
+    SELECT CASE o.k
+      WHEN 'count' THEN 'unit_'||o.k||'[]'
+      ELSE 'unit_'||o.k END, json_agg(
+        json_build_object('label', v, 'value', v, 'selected', s)
+        ORDER BY v)
+    FROM (
+      SELECT DISTINCT o.k, name, v.k IS NOT NULL
+      FROM sqlpage_units
+      CROSS JOIN UNNEST(ARRAY['volume', 'mass', 'count']) o(k)
+      LEFT JOIN json_each_text(sqlpage.variables()::json) v(k,v)
+      ON v.k = 'unit_'||o.k AND (CASE o.k
+        WHEN 'count' THEN name IN (
+          SELECT j.v#>>'{}' FROM json_array_elements(v.v::json) j(v)
+        )
+        ELSE name = v.v END)
+      WHERE lower(type) = o.k
+    ) AS o(k, v, s)
+    WHERE v IS NOT NULL
+    GROUP BY o.k
+  ) q(k, o)
+);
+
+SELECT 'dynamic' AS component, sqlpage.run_sql('sqlpage/internal/theme.sql') AS properties;
+SELECT 'dynamic' AS component, sqlpage.run_sql('sqlpage/internal/nav.sql') AS properties;
+
+SELECT 'dynamic' AS component, sqlpage.run_sql('sqlpage/pages/'||$page) AS properties;

sqlpage/sqlpage/internal/nav.sql

@@ -0,0 +1,44 @@
+SELECT 'tab' AS component, TRUE AS center;
+
+SELECT title
+, lower(title) AS id
+, $title = title AS active
+, lower(title)||'.sql' AS link
+FROM (
+  SELECT name FROM sqlpage_pages
+) AS tabs(title)
+
+SELECT 'form' AS component
+, 'filter' AS id
+, 'get' AS method
+, '' AS title
+, '' AS validate
+;
+
+SELECT 'hidden' AS type
+, 'title' AS name
+, $title AS value
+;
+
+SELECT j.name
+--, fo.j#>>'{options}' AS label
+, j.label
+, COALESCE(c.type, j.type) AS type
+, j.dropdown AS dropdown
+, j.multiple
+, COALESCE(c.width, j.width) AS width
+, o.j#>'{options}' AS options
+, COALESCE(j.value, v.value) AS value
+, CASE j.type
+    WHEN 'checkbox' THEN v.value::bool
+    ELSE NULL
+  END AS checked
+, j.formaction
+FROM json_populate_recordset(null::sqlpage_filter_type, sqlpage.read_file_as_text('sqlpage/json/filters.json')::json) j
+LEFT JOIN json_populate_recordset(null::sqlpage_filter_type, $filter_config::json) c
+USING (name)
+LEFT JOIN json_array_elements($filter_options::json) o(j)
+ON (o.j#>>'{name}') = j.name
+LEFT JOIN json_each_text(sqlpage.variables()::json) v(key, value)
+ON v.key = j.name
+;

sqlpage/sqlpage/internal/theme.sql

@@ -0,0 +1,9 @@
+SELECT 'shell' AS component
+, 'dark' AS theme
+, 'fluid' AS layout
+, $title AS title
+, lower($title)||'.sql' AS link
+, 'favicon.svg' AS image
+, 'favicon.png' AS favicon
+, 'manifest.json' AS manifest
+;

sqlpage/sqlpage/json/filters.json

@@ -0,0 +1,50 @@
+[
+  { "name": "products[]", "label": "Products",
+    "type": "select", "dropdown": true, "multiple": true,
+    "width": 3
+  },
+  { "name": "categories[]", "label": "Categories",
+    "type": "select", "dropdown": true, "multiple": true,
+    "width": 3
+  },
+  { "name": "groups[]", "label": "Groups",
+    "type": "select", "dropdown": true, "multiple": true,
+    "width": 3
+  },
+  { "name": "tags[]", "label": "Tags",
+    "type": "select", "dropdown": true, "multiple": true,
+    "width": 3
+  },
+  { "name": "start", "label": "From",
+    "type": "date",
+    "width": 2
+  },
+  { "name": "end", "label": "To",
+    "type": "date",
+    "width": 2
+  },
+  { "name": "unit_volume", "label": "Unit (vol.)",
+    "type": "select", "multiple": false, "dropdown": true,
+    "width": 2
+  },
+  { "name": "unit_mass", "label": "Unit (mass)",
+    "type": "select", "multiple": false, "dropdown": true,
+    "width": 2
+  },
+  { "name": "unit_count[]", "label": "Unit (count)",
+    "type": "select", "multiple": true, "dropdown": true,
+    "width": 2
+  },
+  { "name": "log_scale", "label": "Log Scale", "value": "true",
+    "type": "checkbox",
+    "width": 2
+  },
+  { "name": "apply", "label": "", "value": "Apply",
+    "type": "submit",  "formaction": "/grocery/internal/apply.sql",
+    "width": 2
+  },
+  { "name": "clear", "label": "", "value": "Clear",
+    "type": "submit", "formaction": "/grocery/internal/clear.sql",
+    "width": 2
+  }
+]

sqlpage/sqlpage/migrations/000_schema.sql

@@ -0,0 +1,24 @@
+DROP TABLE IF EXISTS sqlpage_pages;
+CREATE TABLE sqlpage_pages
+(
+    name text PRIMARY KEY
+);
+INSERT INTO sqlpage_pages
+SELECT UNNEST(ARRAY[
+  'Products', 'Categories', 'Groups',
+  'Tags',
+  'Transactions', 'Trend', 'Volume'
+]);
+
+DROP TYPE IF EXISTS sqlpage_filter_type;
+CREATE TYPE sqlpage_filter_type AS (
+  name text,
+  label text,
+  value text,
+  "default" text,
+  type text,
+  dropdown bool,
+  multiple bool,
+  formaction text,
+  width int
+);

sqlpage/sqlpage/migrations/README.md

@@ -0,0 +1,39 @@
+# SQLPage migrations
+
+SQLPage migrations are SQL scripts that you can use to create or update the database schema.
+They are entirely optional: you can use SQLPage without them, and manage the database schema yourself with other tools.
+
+## Creating a migration
+
+To create a migration, create a file in the `sqlpage/migrations` directory with the following name:
+
+```
+<version>_<name>.sql
+```
+
+Where `<version>` is a number that represents the version of the migration, and `<name>` is a name for the migration.
+For example, `001_initial.sql` or `002_add_users.sql`.
+
+When you need to update the database schema, always create a **new** migration file with a new version number
+that is greater than the previous one.
+Use commands like `ALTER TABLE` to update the schema declaratively instead of modifying the existing `CREATE TABLE`
+statements.
+
+If you try to edit an existing migration, SQLPage will not run it again, will detect
+
+## Running migrations
+
+Migrations that need to be applied are run automatically when SQLPage starts.
+You need to restart SQLPage each time you create a new migration.
+
+## How does it work?
+
+SQLPage keeps track of the migrations that have been applied in a table called `_sqlx_migrations`.
+This table is created automatically when SQLPage starts for the first time, if you create migration files.
+If you don't create any migration files, SQLPage will never touch the database schema on its own.
+
+When SQLPage starts, it checks the `_sqlx_migrations` table to see which migrations have been applied.
+It checks the `sqlpage/migrations` directory to see which migrations are available.
+If the checksum of a migration file is different from the checksum of the migration that has been applied,
+SQLPage will return an error and refuse to start.
+If you end up in this situation, you can remove the `_sqlx_migrations` table: all your old migrations will be reapplied, and SQLPage will start again.

sqlpage/sqlpage/pages/Categories.sql

@@ -0,0 +1,19 @@
+SELECT
+  'table' AS component,
+  TRUE AS search,
+  TRUE AS striped_rows,
+  TRUE AS small,
+  'Notice' AS markdown
+;
+
+SELECT
+  "Products",
+  COALESCE("category", "Categories"||'') "Category",
+  COALESCE("group", "Groups"||'') "Group"
+FROM sqlpage_products q
+WHERE q.product IS NULL AND (q.category IS NOT NULL OR q.group IS NULL)
+ORDER BY product, category, "group"
+;
+
+SELECT '# No Data' AS "Notice"
+WHERE NOT EXISTS(SELECT * FROM sqlpage_products LIMIT 1);

sqlpage/sqlpage/pages/Groups.sql

@@ -0,0 +1,19 @@
+SELECT
+  'table' AS component,
+  TRUE AS search,
+  TRUE AS striped_rows,
+  TRUE AS small,
+  'Notice' AS markdown
+;
+
+SELECT
+  "Products",
+  "Categories",
+  COALESCE("group", "Groups"||'') "Group"
+FROM sqlpage_products q
+WHERE q.category IS NULL
+ORDER BY product, category, "group"
+;
+
+SELECT '# No Data' AS "Notice"
+WHERE NOT EXISTS(SELECT * FROM sqlpage_products LIMIT 1);

sqlpage/sqlpage/pages/Products.sql

@@ -0,0 +1,19 @@
+SELECT
+  'table' AS component,
+  TRUE AS search,
+  TRUE AS striped_rows,
+  TRUE AS small,
+  'Notice' AS markdown
+;
+
+SELECT
+  COALESCE("product", "Products"||'') "Product",
+  COALESCE("category", "Categories"||'') "Category",
+  COALESCE("group", "Groups"||'') "Group"
+FROM sqlpage_products q
+WHERE q.product IS NOT NULL OR q.group IS NULL
+ORDER BY product, category, "group"
+;
+
+SELECT '# No Data' AS "Notice"
+WHERE NOT EXISTS(SELECT * FROM sqlpage_products LIMIT 1);

sqlpage/sqlpage/pages/Tags.sql

@@ -0,0 +1,14 @@
+SELECT
+  'table' AS component,
+  TRUE AS search,
+  TRUE AS striped_rows,
+  TRUE AS small,
+  'Notice' AS markdown
+;
+SELECT * FROM sqlpage_tags;
+
+SELECT count("Uses") AS "Uses", count("Name")::text AS "Name"
+FROM sqlpage_tags;
+
+SELECT '# No Data' AS "Notice"
+WHERE NOT EXISTS(SELECT * FROM sqlpage_tags LIMIT 1);

sqlpage/sqlpage/pages/Transactions.sql

@@ -0,0 +1,29 @@
+SELECT
+  'table' AS component,
+  TRUE AS sort,
+  TRUE AS search,
+  TRUE AS striped_rows,
+  TRUE AS small,
+  'Price' AS align_right,
+  'Quantity' AS align_right,
+  'Unit' AS align_right,
+  'Notice' AS markdown
+;
+SELECT
+  to_char(ts
+     AT TIME ZONE 'Pacific/Auckland', --COALESCE('$ParameterTZ', 'Pacific/Auckland'),
+    'Mon DD HH24:MI:SS'
+  ) AS "Time",
+  description AS "Description",
+  code AS "Store",
+  product AS "Product",
+  category AS "Category",
+  "group" AS "Group",
+  CASE WHEN organic THEN 'yes' ELSE 'no' END AS "Organic",
+  to_char(price, 'FM999999999.00') AS "Price",
+  to_char(quantity, 'FM999999999.000') AS "Quantity",
+  unit AS "Unit"
+FROM sqlpage_txn;
+
+SELECT '# No Data' AS "Notice"
+WHERE NOT EXISTS(SELECT * FROM sqlpage_txn LIMIT 1);

sqlpage/sqlpage/pages/Trend.sql

@@ -0,0 +1,21 @@
+SELECT 'card' AS component
+, 'Cost by Quantity' AS title
+, 2 AS columns
+;
+
+SELECT 'trend/Quantity.sql?unit='||$unit_mass||'&log_scale='||$log_scale||'&_sqlpage_embed' AS embed
+, '' AS footer
+WHERE EXISTS (SELECT 1 FROM sqlpage_txn WHERE unit = $unit_mass LIMIT 1)
+;
+
+SELECT 'trend/Quantity.sql?unit='||$unit_volume||'&log_scale='||$log_scale||'&_sqlpage_embed' AS embed
+, '' AS footer
+WHERE EXISTS (SELECT 1 FROM sqlpage_txn WHERE unit = $unit_volume LIMIT 1)
+;
+
+SET unit_count_chart = (SELECT v#>>'{}' FROM json_array_elements($unit_count::json) j(v) LIMIT 1);
+SELECT 'trend/Quantity.sql?unit='||$unit_count_chart||'&log_scale='||$log_scale||'&_sqlpage_embed' AS embed
+, '' AS footer
+WHERE EXISTS (SELECT 1 FROM sqlpage_txn WHERE unit = $unit_count_chart LIMIT 1)
+AND (SELECT count(DISTINCT v#>>'{}') FROM json_array_elements($unit_count::json) j(v)) == 1
+;

sqlpage/sqlpage/pages/Volume.sql

@@ -0,0 +1,24 @@
+SELECT 'card' AS component
+, 'Volume by Expendature and Quantity' AS title
+, 2 AS columns
+;
+SELECT 'volume/Expense.sql?_sqlpage_embed' AS embed
+, '' AS footer
+WHERE EXISTS (SELECT 1 FROM sqlpage_txn LIMIT 1);
+
+SELECT 'volume/Quantity.sql?unit='||$unit_mass||'&_sqlpage_embed' AS embed
+, '' AS footer
+WHERE EXISTS (SELECT 1 FROM sqlpage_txn WHERE unit = $unit_mass LIMIT 1)
+;
+
+SELECT 'volume/Quantity.sql?unit='||$unit_volume||'&_sqlpage_embed' AS embed
+, '' AS footer
+WHERE EXISTS (SELECT 1 FROM sqlpage_txn WHERE unit = $unit_volume LIMIT 1)
+;
+
+SET unit_count_chart = (SELECT v#>>'{}' FROM json_array_elements($unit_count::json) j(v) LIMIT 1);
+SELECT 'volume/Quantity.sql?unit='||$unit_count_chart||'&_sqlpage_embed' AS embed
+, '' AS footer
+WHERE EXISTS (SELECT 1 FROM sqlpage_txn WHERE unit = $unit_count_chart LIMIT 1)
+AND (SELECT count(DISTINCT v#>>'{}') FROM json_array_elements($unit_count::json) j(v)) == 1
+;

sqlpage/sqlpage/sqlpage.json

@@ -0,0 +1,5 @@
+{
+  "database_url": "postgresql://grocery_ro@192.168.0.20/grocery",
+  "site_prefix": "/grocery",
+  "port": 6772
+}

sqlpage/sqlpage/templates/README.md

@@ -0,0 +1,20 @@
+# SQLPage component templates
+
+SQLPage templates are handlebars[^1] files that are used to render the results of SQL queries.
+
+[^1]: https://handlebarsjs.com/
+
+## Default components
+
+SQLPage comes with a set of default[^2] components that you can use without having to write any code.
+These are documented on https://sql.ophir.dev/components.sql
+
+## Custom components
+
+You can [write your own component templates](https://sql.ophir.dev/custom_components.sql)
+and place them in the `sqlpage/templates` directory.
+To override a default component, create a file with the same name as the default component.
+If you want to start from an existing component, you can copy it from the `sqlpage/templates` directory
+in the SQLPage source code[^2].
+
+[^2]: A simple component to start from: https://github.com/lovasoa/SQLpage/blob/main/sqlpage/templates/code.handlebars

sqlpage/tags.sql

@@ -0,0 +1,14 @@
+SET title = 'Tags';
+SET filter_config = '[
+  {"name": "products[]", "width": 4},
+  {"name": "categories[]", "width": 4},
+  {"name": "groups[]", "width": 4},
+  {"name": "tags[]", "width": 4},
+  {"name": "start", "width": 4},
+  {"name": "end", "width": 4},
+  {"name": "log_scale", "type": "hidden"},
+  {"name": "unit_mass", "type": "hidden"},
+  {"name": "unit_volume", "type": "hidden"},
+  {"name": "unit_count[]", "type": "hidden"}
+]';
+SELECT 'dynamic' AS component, sqlpage.run_sql('sqlpage/internal/entry.sql') AS properties;

sqlpage/transactions.sql

@@ -0,0 +1,6 @@
+SET title = 'Transactions';
+SET filter_config = '[
+  {"name": "unit_count[]", "width": 4},
+  {"name": "log_scale", "type": "hidden"}
+]';
+SELECT 'dynamic' AS component, sqlpage.run_sql('sqlpage/internal/entry.sql') AS properties;

sqlpage/trend.sql

@@ -0,0 +1,2 @@
+SET title = 'Trend';
+SELECT 'dynamic' AS component, sqlpage.run_sql('sqlpage/internal/entry.sql') AS properties;

sqlpage/trend/Quantity.sql

@@ -0,0 +1,38 @@
+SET height = '500';
+SET type = 'line';
+SET marker = 4;
+SET max_line_series = 50;
+
+
+SELECT 'chart' AS component
+, 'Cost per '||$unit AS title
+, $height AS height
+, '$/'||$unit AS ytitle
+, CASE
+    WHEN (
+      SELECT count(DISTINCT product)
+      FROM sqlpage_txn
+      WHERE unit = $unit
+    ) > $max_line_series::numeric
+    THEN 'scatter' ELSE $type
+  END AS type
+, $log_scale = 'true' AS logarithmic
+, TRUE AS time
+, CASE
+    WHEN $log_scale = 'true'
+    THEN 1 ELSE 0
+  END AS ymin
+, ((
+    SELECT max(price/quantity)
+    FROM sqlpage_txn
+    WHERE unit = $unit
+  )/10 + 1)::int AS ystep
+, $marker AS marker
+;
+SELECT product AS series
+, ts AS x
+, to_char(price/quantity, 'FM999999999.000') AS y
+FROM sqlpage_txn
+WHERE unit = $unit
+ORDER BY ts
+;

sqlpage/volume.sql

@@ -0,0 +1,6 @@
+SET title = 'Volume';
+SET filter_config = '[
+  {"name": "unit_count[]", "width": 4},
+  {"name": "log_scale", "type": "hidden"}
+]';
+SELECT 'dynamic' AS component, sqlpage.run_sql('sqlpage/internal/entry.sql') AS properties;

sqlpage/volume/Expense.sql

@@ -0,0 +1,35 @@
+SET height = '500';
+SET type = 'pie';
+
+SET group_by = (CASE
+WHEN (SELECT count(DISTINCT "group") FROM sqlpage_txn) > 2
+THEN 'Group'
+WHEN (SELECT count(DISTINCT category) FROM sqlpage_txn) > 2
+THEN 'Category'
+ELSE 'Product'
+END);
+SET total = (SELECT sum(price) FROM sqlpage_txn);
+SELECT 'chart' AS component
+, 'Share of $'||to_char($total::numeric, 'FM999999999.00') AS title
+, $height AS height
+, $type AS type
+, TRUE AS labels
+;
+SELECT "group" AS series
+, sum(price) AS value
+FROM sqlpage_txn
+WHERE $group_by = 'Group'
+GROUP BY "group"
+;
+SELECT category AS series
+, sum(price) AS value
+FROM sqlpage_txn
+WHERE $group_by = 'Category'
+GROUP BY category
+;
+SELECT product AS series
+, sum(price) AS value
+FROM sqlpage_txn
+WHERE $group_by = 'Product'
+GROUP BY product
+;

sqlpage/volume/Quantity.sql

@@ -0,0 +1,35 @@
+SET height = '500';
+SET type = 'pie';
+
+SET group_by = (CASE
+WHEN (SELECT count(DISTINCT "group") FROM sqlpage_txn WHERE unit = $unit) > 2
+THEN 'Group'
+WHEN (SELECT count(DISTINCT category) FROM sqlpage_txn WHERE unit = $unit) > 2
+THEN 'Category'
+ELSE 'Product'
+END);
+SET total = (SELECT sum(quantity) FROM sqlpage_txn WHERE unit = $unit);
+SELECT 'chart' AS component
+, 'Share of '||to_char($total::numeric, 'FM999999999.00')||' '||$unit AS title
+, $height AS height
+, $type AS type
+, TRUE AS labels
+;
+SELECT "group" AS series
+, sum(quantity) AS value
+FROM sqlpage_txn
+WHERE $group_by = 'Group' AND unit = $unit
+GROUP BY "group"
+;
+SELECT category AS series
+, sum(quantity) AS value
+FROM sqlpage_txn
+WHERE $group_by = 'Category' AND unit = $unit
+GROUP BY category
+;
+SELECT product AS series
+, sum(quantity) AS value
+FROM sqlpage_txn
+WHERE $group_by = 'Product' AND unit = $unit
+GROUP BY product
+;

test/__init__.py

@@ -0,0 +1 @@
+ALL_UNITS = { 'g', 'kg', 'mL', 'L', 'Pieces', 'Bunches', 'Bags' }

test/rest/templates/__init__.py

@@ -1,2 +0,0 @@
-from bottle import TEMPLATE_PATH
-TEMPLATE_PATH.append("app/rest/templates")

test/rest/templates/test_buttongroup-nav.py

@@ -1,42 +0,0 @@
-from bottle import template
-from pytest import mark
-
-@mark.parametrize('expected', [
-    """<div class="vertical-button-group">
-  <div class="pure-button-group vertical-button-group" role="nav" style="padding: 1em 0.25em 0;">
-    <button
-      class="pure-button "
-      type="submit"
-      formaction="trend"
-      style=""
-    > Trend </button>
-    <button
-      class="pure-button pure-button-active"
-      type="submit"
-      formaction="products"
-      style=""
-    > Products </button>
-    <button
-      class="pure-button "
-      type="submit"
-      formaction="categories"
-      style=""
-    > Categories </button>
-    <button
-      class="pure-button "
-      type="submit"
-      formaction="groups"
-      style=""
-    > Groups </button>
-    <button
-      class="pure-button "
-      type="submit"
-      formaction="tags"
-      style=""
-    > Tags </button>
-  </div>
-</div>"""])
-def test_form_nav_render_exact(expected):
-    assert template('buttongroup-nav', role='nav', action='products', targets=[
-        "trend", "products", "categories", "groups", "tags"
-    ]) == expected

test/rest/templates/test_include-exclude.py

@@ -1,56 +0,0 @@
-import pytest
-from pytest import mark, raises
-from bottle import template
-
-@mark.parametrize('expected, params', [
-    ("""<div class="pure-u-1-3 pure-u-lg-1-6">
-  <div class="pure-g">
-    <div class="pure-u-1">
-      <div class="l-box">
-        <h3>Item</h3>
-      </div>
-    </div>
-<div class="pure-u-1">
-
-<select id="item-include" name="item" size="10" multiple="true" style="width: calc(100% - 1em); margin: 0 1em 1em">
-  <option value="Include" disabled="true" >Include</option>
-
-  <option value="val1-to-backend"  >val1</option>
-  <option value="val2" disabled="true" >val2</option>
-</select>
-</div>
-<div class="pure-u-1">
-
-<select id="item-exclude" name="item" size="10" multiple="true" style="width: calc(100% - 1em); margin: 0 1em 1em">
-  <option value="Exclude" disabled="true" >Exclude</option>
-
-  <option value="val1-to-backend"  >val1</option>
-  <option value="val2" disabled="true" >val2</option>
-</select>
-</div>
-  </div>
-</div>""",{
-    "name": "item",
-    "_include": {
-        "options": [{
-            "value": "val1-to-backend",
-            "display": "val1",
-        },
-        {
-            "value": "val2",
-            "disabled": True,
-        },]
-    },
-    "_exclude": {
-        "options": [{
-            "value": "val1-to-backend",
-            "display": "val1",
-        },
-        {
-            "value": "val2",
-            "disabled": True,
-        },]
-    } }),
-])
-def test_include_exclude_render_exact(expected, params):
-    assert template('include-exclude', **params) == expected

test/rest/templates/test_options.py

@@ -1,108 +0,0 @@
-from pytest import mark, raises
-from bottle import template
-
-@mark.parametrize('expected, params', [
-    ("""<optgroup label="Group">
-  <option value="val-to-backend"  selected="true">val-displayed</option>
-</optgroup>""", {
-    "optgroup": "Group", "options": [
-    {
-        "value": "val-to-backend",
-        "selected": True,
-        "display": "val-displayed",
-    }, ] }),
-    ("""<optgroup label="Group">
-  <option value="val-to-backend" disabled="true" >val-to-backend</option>
-</optgroup>""", { "optgroup": "Group", "options": [
-    {
-        "value": "val-to-backend",
-        "disabled": True,
-    }, ] }),
-    ("""<optgroup label="Group">
-  <option value="val1-to-backend"  >val1</option>
-  <option value="val2-to-backend" disabled="true" >val2-to-backend</option>
-</optgroup>""", { "optgroup": "Group", "options": [
-    {
-        "value": "val1-to-backend",
-        "display": "val1",
-    },
-    {
-        "value": "val2-to-backend",
-        "disabled": True,
-    }, ] }),
-])
-def test_optgroup_render_exact(expected, params):
-    assert template('options', **params) == expected
-
-
-@mark.parametrize('name', [
-    '',
-    None,
-    'Group',
-])
-@mark.parametrize('value', [
-    *[ { "value": v } for v in ("", "val-to-backend", "!val-to-backend", None)],
-    ({}, NameError, "name 'value' is not defined"),
-])
-@mark.parametrize('display', [
-    *[ { "display": v } for v in ("", "val-displayed", None)],
-    {},
-])
-@mark.parametrize('disabled', [
-    *[ { "disabled": v } for v in (True, False, None)],
-    {},
-])
-@mark.parametrize('selected', [
-    *[ { "selected": v } for v in (True, False, None)],
-    {},
-])
-def test_optgroup_render_includes(name, value, selected, disabled, display):
-    options = dict()
-    exp_exceptions = []
-    for part, ex, msg in map(
-        lambda x: x if isinstance(x, tuple) else (x, None, None),
-        (value, selected, disabled, display)):
-        options.update(part)
-        if ex is not None:
-            exp_exceptions.append(ex)
-
-    if not exp_exceptions:
-        assert template('options', optgroup=name, options=[ options, ]) is not None
-        assert template('options', optgroup=name, options=[ options, options ]) is not None
-        return
-
-    with raises(tuple(exp_exceptions)):
-        template('options', optgroup=name, options=[ options, ]) is not None
-    
-    with raises(tuple(exp_exceptions)):
-        template('options', optgroup=name, options=[ options, options ]) is not None
-    
-    return
-
-@mark.parametrize('name', [
-    {},
-    *[ { "optgroup": v } for v in ("", "group", None)],
-])
-@mark.parametrize('options', [
-    ({}, NameError, "name 'options' is not defined"),
-    { "options": [{ "value": "anything" },] },
-    { "options": [{ "value": "opt1" }, { "value": "opt2" }] },
-])
-def test_optgroup_render(name, options):
-    params = dict()
-    exp_exceptions = []
-    for part, ex, msg in map(
-        lambda x: x if isinstance(x, tuple) else (x, None, None),
-        (name, options)):
-        params.update(part)
-        if ex is not None:
-            exp_exceptions.append(ex)
-
-    if not exp_exceptions:
-        assert template('options', **params) is not None
-        return
-
-    with raises(tuple(exp_exceptions)):
-        assert template('options', **params) is not None
-
-    return

test/rest/templates/test_select-one.py

@@ -1,43 +0,0 @@
-import pytest
-from pytest import mark, raises
-from bottle import template
-
-@mark.parametrize('expected, params', [
-    ("""<div class="pure-u-1">
-  <div class="l-box">
-    <h3>Unit</h3>
-  </div>
-</div>
-<div class="pure-u-1">
-
-<select id="unit-select-one" name="unit" size="10"  style="width: calc(100% - 1em); margin: 0 1em 1em">
-
-
-  <option value="Bags"  >Bags</option>
-  <option value="Bunches"  >Bunches</option>
-  <option value="L"  >L</option>
-  <option value="Pieces"  >Pieces</option>
-  <option value="g"  >g</option>
-  <option value="kg"  selected="true">kg</option>
-  <option value="mL"  >mL</option>
-</select>
-</div>""", {
-    "name": "unit", "options": [{
-        "value": "Bags",
-    }, {
-        "value": "Bunches",
-    }, {
-        "value": "L",
-    }, {
-        "value": "Pieces",
-    }, {
-        "value": "g",
-    }, {
-        "value": "kg",
-        "selected": True,
-    }, {
-        "value": "mL",
-    }, ]}, ),
-])
-def test_select_one_render_exact(expected, params):
-    assert template('select-one', **params) == expected

test/rest/templates/test_select.py

@@ -1,80 +0,0 @@
-import pytest
-from pytest import mark, raises
-from bottle import template
-
-@mark.parametrize('expected, params', [
-    ("""<div class="pure-u-1">
-<label for="select-id">Choose: </label>
-<select id="select-id" name="select-name" size="10" multiple="true" style="width: calc(100% - 1em); margin: 0 1em 1em">
-  <option value="hint" disabled="true" >hint</option>
-<optgroup label="Group">
-  <option value="val1-to-backend"  >val1</option>
-  <option value="val2" disabled="true" >val2</option>
-</optgroup>
-</select>
-</div>""", {
-    "id": "select-id", "label": "Choose: ", "name": "select-name",
-    "hint": "hint", "multiple": True, "children": [{"optgroup": "Group", "options": [
-    {
-        "value": "val1-to-backend",
-        "display": "val1",
-    },
-    {
-        "value": "val2",
-        "disabled": True,
-    }, ]}, ]}),
-    ("""<div class="pure-u-1">
-
-<select id="select-unit-id" name="unit" size="10"  style="width: calc(100% - 1em); margin: 0 1em 1em">
-
-
-  <option value="val1"  >val1</option>
-  <option value="val2"  selected="true">val2</option>
-</select>
-</div>""",{
-    "id": "select-unit-id", "name": "unit", "children": [{ "options": [
-    {
-        "value": "val1",
-    },
-    {
-        "value": "val2",
-        "selected": True,
-    }, ]}, ]}),
-    ("""<div class="pure-u-1">
-<label for="select-id">Choose: </label>
-<select id="select-id" name="select-name" size="10" multiple="true" style="width: calc(100% - 1em); margin: 0 1em 1em">
-  <option value="hint" disabled="true" >hint</option>
-<optgroup label="Group">
-  <option value="val1-to-backend"  >val1</option>
-  <option value="val2" disabled="true" >val2</option>
-</optgroup>
-
-  <option value="val1"  >val1</option>
-  <option value="val2"  selected="true">val2</option>
-</select>
-</div>""",{
-    "id": "select-id", "label": "Choose: ", "name": "select-name",
-    "hint": "hint", "multiple": True, "children": [
-        {"optgroup": "Group", "options": [
-            {
-                "value": "val1-to-backend",
-                "display": "val1",
-            },
-            {
-                "value": "val2",
-                "disabled": True,
-            }
-        ]},
-        { "options": [
-            {
-                "value": "val1",
-            },
-            {
-                "value": "val2",
-                "selected": True,
-            },
-        ]},
-    ]})
-])
-def test_select_render_exact(expected, params):
-    assert template('select', **params) == expected

test/rest/test_Cache.py

@@ -1,73 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-#
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from pytest import fixture
-from time import time
-from app.rest.PageCache import PageCache
-from app.rest.CachedLoadingPage import (
-    CachedLoadingPage,
-    STALE,
-)
-
-@fixture
-def cache():
-    return PageCache(0)
-
-def test_add(cache: PageCache):
-    val = 'test-cached-value'
-    key = 'test-key'
-    assert cache.add(key, CachedLoadingPage(val, lambda _: None, incremental=False)).value == val
-
-def test_get(cache: PageCache):
-    val = 'test-cached-value'
-    key = 'test-key'
-
-    assert cache.get(key) is None
-
-    assert cache.add(key, CachedLoadingPage(val, lambda q: q.put('next-val'), incremental=False)).value == val
-    assert cache.get(key).value == 'next-val'
-
-def test_remove(cache: PageCache):
-    val = 'test-cached-value'
-    key = 'test-key'
-
-    assert cache.get(key) is None
-    assert cache.add(key, CachedLoadingPage(val, lambda q: q.put('next-val'), incremental=False)).value == val
-    cache.remove(key)
-    assert cache.get(key) is None
-
-def test_enforce_limit(cache: PageCache):
-    val = 'test-cached-value'
-    key = 'test-key'
-    assert cache.add(key, CachedLoadingPage(val, lambda _: None, incremental=False)).value == val
-    # adding more exceeds limit
-    assert cache.add('other', CachedLoadingPage(val, lambda _: None, incremental=False)).value == val
-    assert cache.get(key) is None
-    assert cache.get('other').value == val
-
-def test_clean_stale(cache: PageCache):
-    val = 'test-cached-value'
-    key = 'test-key'
-    page = CachedLoadingPage(val, lambda _: None, incremental=False)
-    page._created = time() - STALE
-    # add stale page
-    assert cache.add(key, page).value == val
-    assert cache.get(key) is None
-
-    page = CachedLoadingPage(val, lambda _: None, incremental=False)
-    assert cache.add(key, page).value == val
-    # make page stale
-    page._created = time() - STALE
-    assert cache.get(key) is None
-
-    page = CachedLoadingPage(val, lambda _: None, incremental=False)
-    assert cache.add(key, page).value == val
-    # make page stale
-    page._created = time() - STALE
-    # stale page is rotated out on addition
-    assert cache.add('other', CachedLoadingPage(val, lambda _: None, incremental=False)).value == val
-    assert cache.get(key) is None
-    assert cache.get('other').value == val

test/rest/test_CachedLoadingPage.py

@@ -1,66 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-#
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from queue import Queue
-from typing import Any, Generator
-from pytest import mark, fixture
-from time import sleep, time
-from app.rest.CachedLoadingPage import (
-    CachedLoadingPage,
-    STALE,
-)
-
-
-@fixture
-def cache():
-    return CachedLoadingPage("start", lambda _: None, incremental=False)
-
-
-def test_get_age(cache: CachedLoadingPage):
-    sleep(0.1)
-    assert cache.age > 0.1
-
-def test_get_queue(cache: CachedLoadingPage):
-    assert type(cache.queue) == Queue
-    cache.queue.put("next", timeout=0.1)
-    assert cache.update() == "next"
-
-def test_get_loaded(cache: CachedLoadingPage):
-    assert cache.loaded is False
-    cache.queue.put("not None", timeout=0.1)
-    assert cache.update() == "not None"
-    assert cache.loaded is False
-    cache.queue.put(None, timeout=0.1)
-    assert cache.update() == "not None"
-    assert cache.loaded is True
-
-def test_update(cache: CachedLoadingPage):
-    cache.queue.put("not None", timeout=0.1)
-    assert cache.update() == "not None"
-    cache.queue.put("next value", timeout=0.1)
-    assert cache.update() == "next value"
-    assert cache.update() == "next value"
-    cache.queue.put("another value", timeout=0.1)
-    cache.queue.put("final value", timeout=0.1)
-    assert cache.update() == "another value"
-    assert cache.update() == "final value"
-    cache.queue.put(None, timeout=0.1)
-    assert cache.update() == "final value"
-    assert cache.loaded is True
-    assert cache.update() == "final value"
-
-def test_stale(cache: CachedLoadingPage):
-    cache._created = time() - STALE
-    assert cache.stale
-
-def test_lock(cache: CachedLoadingPage):
-    cache.queue.put("not None", timeout=0.1)
-    assert cache.update() == "not None"
-    assert cache._lock.acquire()
-    cache.queue.put("next value", timeout=0.1)
-    assert cache.update() == "not None"
-    cache._lock.release()
-    assert cache.update() == "next value"

test/rest/test_form.py

@@ -1,126 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-#
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from typing import Dict, Iterable, Tuple
-from pandas import DataFrame
-from pytest import fixture, mark
-from app.rest.form import get_option_groups
-
-@fixture
-def data():
-    return DataFrame({
-        'type': ['dog', 'cat', 'horse', 'sheep', 'cow'],
-        'category': ['pet', 'pet', 'utility', 'farm', 'farm'],
-        'group': ['small', 'small', 'large', 'medium', 'large',],
-        'tags': [['a'], ['a','b'], [], [], []],
-        '$/unit': ['1', '2', '3', '4', '5'],
-    })
-
-@mark.parametrize('filter_data, _type, key, parent_grouping, expected', [
-    ({
-        'type': [{'dog'}, {'cat'}]
-    }, 'include', 'type', 'category', [
-        {
-            "optgroup": 'pet', "options": [{
-                "selected": True,
-                "value": "dog",
-                "display": "dog",
-            }]
-        }, {
-            "optgroup": None, "options":[{
-                "selected": False,
-                "value": "cat",
-                "display": "cat",
-            }]
-        }
-    ]), ({
-        'type': [{'dog'}, {'cat'}]
-    }, 'exclude', 'type', 'category', [
-        {
-            "optgroup": 'pet', "options": [{
-                "selected": False,
-                "value": "!dog",
-                "display": "dog",
-            }],
-        }, {
-            "optgroup": None, "options":[{
-                "selected": True,
-                "value": "!cat",
-                "display": "cat",
-            }],
-        }
-    ]), ({
-        'group': [{'small'}, {'large'}]
-    }, 'include', 'group', None, [
-        {
-            "optgroup": None, "options": [ {
-                "selected": False,
-                "value": "large",
-                "display": "large",
-            },{
-                "selected": True,
-                "value": "small",
-                "display": "small",
-            }],
-        }
-    ]), ({
-        'group': [{'small'}, {'large'}]
-    }, 'exclude', 'group', None, [
-        {
-            "optgroup": None, "options": [{
-                "selected": True,
-                "value": "!large",
-                "display": "large",
-            },{
-                "selected": False,
-                "value": "!small",
-                "display": "small",
-            }],
-        }
-    ]),  ({
-        'tag': [{'a'}, {'b'}]
-    }, 'include', 'tag', None, [
-        {
-            "optgroup": None, "options": [ {
-                "selected": True,
-                "value": "a",
-                "display": "a",
-            },{
-                "selected": False,
-                "value": "b",
-                "display": "b",
-            }],
-        }
-    ]), ({
-        'tag': [{'a'}, {'b'}]
-    }, 'exclude', 'tag', None, [
-        {
-            "optgroup": None, "options": [{
-                "selected": False,
-                "value": "!a",
-                "display": "a",
-            },{
-                "selected": True,
-                "value": "!b",
-                "display": "b",
-            }],
-        }
-    ]),
-])
-def test_get_option_groups(
-    data: DataFrame,  _type: str,
-    filter_data: Dict[str, Tuple[set, set]],
-    key: str, parent_grouping: str,
-    expected: Iterable
-):
-
-    _filter = data['type'].apply(lambda _: False)
-    for k, (inc, exc) in filter_data.items():
-        for i in inc:
-            _filter |= (data['tags'].apply(lambda x: i in x) if k == 'tag' else data[k] == i)
-        for i in exc:
-            _filter &= ~(data['tags'].apply(lambda x: i in x) if k == 'tag' else data[k] == i)
-    assert list(get_option_groups(data[_filter], filter_data, key, parent_grouping, _type)) == expected

test/rest/test_route_decorators.py

@@ -1,76 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-#
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from bottle import FormsDict
-from pytest import fixture, mark, raises
-from app.rest import PARAMS
-from app.rest.route_decorators import (
-    cache,
-    cursor,
-    normalize_query,
-)
-
-def incr(_dict, key):
-    _dict[key] = _dict[key] + 1
-
-@fixture
-def counter():
-    return dict()
-
-@fixture
-def tracker():
-    return dict()
-
-@fixture
-def method(counter, tracker):
-    counter['method'] = 0
-    tracker['method'] = []
-    return lambda *args, **kwargs: incr(counter, 'method') or \
-        tracker['method'].push({'args': args, 'kwargs': kwargs})
-
-@mark.parametrize('form_data, expected', [
-    ([
-        ('product', 'kiwi'),
-        ('product', 'apple'),
-        ('organic', ''),
-    ], ('category=&group=&organic=0.5&product=apple%7Ckiwi&tag=&unit=', None)),
-    ([
-        ('product', 'kiwi'),
-        ('category', '!baking'),
-        ('organic', '1'),
-    ], ('category=%21baking&group=&organic=1&product=kiwi&tag=&unit=', None)),
-])
-def test_normalise_query(form_data, expected):
-    form = FormsDict()
-    for k,v in form_data:
-        form.append(k, v)
-    
-    assert normalize_query(form, allow=PARAMS) == expected
-
-
-# need to mock bottle request.params
-def test_cache(counter, tracker, method):
-    decorated = cache(query_cache=None, page_cache=None)
-    assert callable(decorated)
-    #decorated()
-    #assert counter['method'] == 1
-    #assert tracker['method'][0]['args'] == []
-    #assert tracker['method'][0]['kwargs'] == {'allow': None}
-    with raises(Exception) as ex:
-        cursor(method)
-    assert ex.value.args[0] == "decorator argument required"
-
-
-def test_cursor(counter, tracker, method):
-    decorated = cursor(cache=None)
-    assert callable(decorated)
-    #decorated()
-    #assert counter['method'] == 1
-    #assert tracker['method'][0]['args'] == []
-    #assert tracker['method'][0]['kwargs'] == {'allow': None}
-    with raises(Exception) as ex:
-        cursor(method)
-    assert ex.value.args[0] == "decorator argument required"

test/test_parse_recipe.py

@@ -7,7 +7,7 @@
 from pytest import fixture
 from app.parse_recipe import parse_recipe
 from app.data.QueryManager import QueryManager
-from app.rest import ALL_UNITS
+from . import ALL_UNITS
 
 @fixture
 def blank_recipe():

test/test_widgets.py

@@ -8,7 +8,7 @@ from typing import Dict, List, Tuple
 from pytest import fixture, mark
 from app.parse_recipe import parse_recipe
 from app.data.QueryManager import QueryManager
-from app.rest import ALL_UNITS
+from . import ALL_UNITS
 
 from app.widgets import (
     SuggestionPopup,
@@ -46,7 +46,7 @@ def test_suggestion_popup_choice(suggestion_popup: SuggestionPopup, tracker: Dic
     assert len(tracker.keys()) == 0
     suggestion_popup.keypress((0,0), 'enter')
     assert tracker['identifier'] == options[pos]
-    
+
 def test_suggestion_popup_banish(suggestion_popup: SuggestionPopup, tracker: Dict[str, str], options: List[str], signals):
     suggestion_popup.keypress((0,0), 'esc')
     assert len(tracker.keys()) == 0
@@ -82,4 +82,4 @@ def test_no_tab_checkbox(notabcheckbox: NoTabCheckBox):
     notabcheckbox.keypress((0,0), 'tab')
     assert notabcheckbox.get_state() == state
     notabcheckbox.keypress((0,0), 'enter')
-    assert notabcheckbox.get_state() != state
+    assert notabcheckbox.get_state() != state