remove old UI

app/rest/CachedLoadingPage.py

@@ -1,72 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from queue import Queue, Empty
-from time import time
-from threading import Lock
-from typing import Callable, Union
-
-STALE = 7*24*60*60
-
-class CachedLoadingPage():
-    
-    value: str
-
-    def __init__(self, initial_value: Union[str, list], provider: Callable[[Queue], None], incremental: bool = True):
-        self._created = time()
-        self._queue = Queue()
-        self._loaded = False
-        self.value = initial_value
-        self._lock = Lock()
-        self.provider = provider
-        self.incremental = incremental
-
-    @property
-    def age(self) -> float:
-        return time() - self._created
-
-    @property
-    def queue(self) -> Queue:
-        return self._queue
-
-    @property
-    def loaded(self) -> bool:
-        return self._loaded
-
-    def _set_loaded(self, value: bool) -> bool:
-        self._loaded = value
-        return self._loaded
-    
-    @property
-    def stale(self) -> bool:
-        return self.age > STALE
-    
-    def _start(self) -> None:
-        if not self.provider:
-            return
-
-        self.provider(self.queue)
-        self.provider = None
-
-    def update(self) -> Union[str, list]:
-        if not self._lock.acquire(blocking=True, timeout=0.5):
-            return self.value
-        try:
-            self._start()
-            item = self._queue.get(block=True, timeout=0.5)
-            if item is None:
-                self._queue.task_done()
-                self._set_loaded(True)
-            else:
-                if self.incremental:
-                    self.value.append(item)
-                else:
-                    self.value = item
-                self.queue.task_done()
-        except Empty:
-            pass
-        finally:
-            self._lock.release()
-        return self.value

app/rest/Dockerfile

@@ -1,7 +0,0 @@
-FROM python:3-slim
-WORKDIR /usr/src/app
-COPY app/rest/requirements.txt ./
-RUN python3 -m pip install --upgrade pip && \
-    python3 -m pip install --no-cache-dir -r requirements.txt
-COPY app ./app
-CMD [ "python", "-m", "app.rest.cherrypy" ]

app/rest/PageCache.py

@@ -1,151 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-
-import os
-from time import time
-from typing import Dict
-import cherrypy
-
-from .hash_util import blake, bytes_to_hash, hash_to_base32
-from .CachedLoadingPage import STALE, CachedLoadingPage
-
-def delete_page(name: str, root: str = 'app/rest/static/files'):
-    directory = f'{root}/{name}'
-    try:
-        os.remove(f'{directory}/{name}.file')
-    except FileNotFoundError:
-        pass
-
-
-def save_page(name: str, content: bytes, tool: str, root='app/rest/static/files') -> str:
-    directory = f'{root}/{name}'
-    try:
-        os.mkdir(directory, mode=0o700, dir_fd=None)
-    except FileExistsError:
-        pass
-
-    fd = os.open(f'{directory}/{name}.file', os.O_WRONLY | os.O_TRUNC | os.O_CREAT, 0o600)
-    with open(fd, "wb") as f:
-        f.write(content)
-
-
-def get_page(name: str, root: str = 'app/rest/static/files') -> str:
-    directory = f'{root}/{name}'
-
-    try:
-        mtime = os.stat(f'{directory}/{name}.file').st_mtime
-    except:
-        mtime = None
-    
-    if mtime is None:
-        return None
-    
-    if mtime and time() - mtime > STALE:
-        delete_page(name)
-        return None
-
-    # todo: store file hash and validate it
-    fd = os.open(f'{directory}/{name}.file', os.O_RDONLY, 0o600)
-    with open(fd, "rb") as f:
-        f.seek(0)
-        page = f.read()
-    return page.decode('utf-8')
-
-
-def key_to_hash(key):
-
-    if isinstance(key, tuple):
-        orig, _hash = key
-    else:
-        if isinstance(key, int):
-            orig, _hash = None, key
-        else:
-            orig, _hash = key, None
-
-    if None not in (orig, _hash):
-        if get_hash(orig) != _hash:
-            raise KeyError(f"Invalid key: {key}")
-
-        return _hash
-
-    if (_hash, orig) is (None, None):
-        raise KeyError(f"Invalid key: {key}")
-    
-    return get_hash(orig) if _hash is None else _hash
-
-
-def get_hash(key):
-    _bytes = blake(key.encode('utf-8'), person='grocery'.encode('utf-8'))
-    return bytes_to_hash(_bytes)
-
-
-class PageCache:
-    def __init__(self, limit) -> None:
-        self._cache: Dict[str, CachedLoadingPage] = dict()
-        self._limit = limit
-
-    def __delitem__(self, key):
-        key = key_to_hash(key)
-        self._cache.pop(key, None)
-        delete_page(hash_to_base32(key))
-
-    def __getitem__(self, key):
-        return self.get(key)
-
-    def __setitem__(self, key, value):
-        self._cache[key_to_hash(key)] = value
-
-    def get(self, key: str) -> CachedLoadingPage:
-        key = key_to_hash(key)
-
-        if key not in self._cache:
-            try:
-                existing = get_page(hash_to_base32(key))
-            except:
-                existing = None
-            if existing is None:
-                return None
-            
-            return self.add(key, CachedLoadingPage(existing, lambda q: q.put(None), incremental=True))
-        
-        page = self._cache[key]
-        if page.stale:
-            del self._cache[key]
-            delete_page(hash_to_base32(key))
-            return None
-
-        if not page.loaded:
-            page.update()
-
-        if page.loaded:
-            try:
-                existing = get_page(hash_to_base32(key))
-            except:
-                existing = None
-            if existing is None:
-                content = ''.join(page.value) if isinstance(page.value, list) else page.value
-                save_page(hash_to_base32(key), content.encode('utf-8'), tool='grocery')
-
-        return page
-
-    def _enforce_limit(self, limit):
-        for idx, (_, k) in enumerate(sorted([
-                (v.age, k) for k, v in self._cache.items()
-            ])):
-            if idx >= limit: del self[k]
-
-    def _clear_stale(self):
-        for k in [k for k, v in self._cache.items() if v.stale]:
-            del self[k]
-    
-    def add(self, key: str, page: CachedLoadingPage) -> CachedLoadingPage:
-        self._clear_stale()
-        self._enforce_limit(self._limit)
-        self._cache[key_to_hash(key)] = page
-        return page
-    
-    def remove(self, key: str):
-        del self[key]

app/rest/QueryCache.py

@@ -1,162 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-import os
-from typing import Dict, Iterable, Tuple
-from urllib.parse import urlencode
-
-from bottle import FormsDict
-from ..data.filter import get_filter, get_query_param
-
-from . import BOOLEAN, PARAMS
-from .hash_util import base32_to_hash, blake, bytes_to_hash, hash_to_base32, normalize_base32
-
-def delete_query(name: str, root: str = 'app/rest/static/files'):
-    directory = f'{root}/{name}'
-    try:
-        os.remove(f'{directory}/{name}.query')
-    except FileNotFoundError:
-        pass
-
-def save_query(name: str, content: bytes, tool: str, root='app/rest/static/files') -> str:
-    directory = f'{root}/{name}'
-    try:
-        os.mkdir(directory, mode=0o700, dir_fd=None)
-    except FileExistsError:
-        pass
-
-    fd = os.open(f'{directory}/{name}.query', os.O_WRONLY | os.O_TRUNC | os.O_CREAT, 0o600)
-    with open(fd, "wb") as f:
-        f.write(content)
-
-def get_query(name: str, root: str = 'app/rest/static/files') -> str:
-    directory = f'{root}/{name}'
-
-    try:
-        mtime = os.stat(f'{directory}/{name}.query').st_mtime
-    except:
-        mtime = None
-    
-    # if mtime and time() - mtime > STALE:
-    #     delete_query(name)
-    #     return None
-
-    # todo: store query hash and validate it
-    fd = os.open(f'{directory}/{name}.query', os.O_RDONLY, 0o600)
-    with open(fd, "rb") as f:
-        f.seek(0)
-        page = f.read()
-    return page.decode('utf-8')
-
-
-def norm(key):
-
-    if isinstance(key, tuple):
-        query, _hash = key
-    else:
-        if isinstance(key, int):
-            query, _hash = None, key
-        else:
-            query, _hash = key, None
-
-    if _hash and not isinstance(_hash, int):
-        _hash = base32_to_hash(normalize_base32(query.hash))
-        # TODO: normalize should be implicit
-        #_hash = base32_to_hash(query.hash)
-
-    if None not in (query, _hash):
-        if get_hash(query) != _hash:
-            raise KeyError(f"Invalid key: {key}")
-
-        return query, _hash
-
-    if (_hash, query) is (None, None):
-        raise KeyError(f"Invalid key: {key}")
-    
-    return query, _hash if _hash else get_hash(query)
-
-def get_hash(key):
-    _bytes = blake(key.encode('utf-8'), person='grocery'.encode('utf-8'))
-    return bytes_to_hash(_bytes)
-
-
-def normalize_query(query: FormsDict, allow: Iterable[str] = None) -> Tuple[str, str]:
-    _hash = query.hash
-    allow = allow or PARAMS
-    param = get_filter(query, allow=allow)
-    norm = urlencode(map(
-        lambda k: (
-            k, get_query_param(*param[k]) if k != 'organic' else BOOLEAN[
-                BOOLEAN.get(query.organic, None)
-            ]
-        ),
-        sorted(filter(bool, param))
-    ))
-    return norm, _hash
-
-
-class QueryCache:
-    def __init__(self, limit) -> None:
-        self._cache: Dict[int, str] = dict()
-        self._limit = limit
-
-    def __delitem__(self, key):
-        return self.remove(key)
-
-    def __getitem__(self, key):
-        return self.get(key)
-
-    def __setitem__(self, key, value):
-        return self.add(key, value)
-
-    def get(self, key: str) -> str:
-        query, _hash = norm(key)
-        if _hash not in self._cache:
-            if query:
-                return self.add(_hash, query)
-            
-            try:
-                existing = get_query(hash_to_base32(_hash))
-            except:
-                existing = None
-            
-            if existing:
-                return self.add(_hash, existing)
-            
-            return self.add(_hash, query)
-        
-        value = self._cache[_hash]
-        # if value.stale:
-        #     del self._cache[key]
-        #     delete_query(hash_to_base32(key))
-        #     return None
-        return value
-
-    # def _enforce_limit(self, limit):
-    #     for idx, (_, k) in enumerate(sorted([
-    #             (v.age, k) for k, v in self._cache.items()
-    #         ])):
-    #         if idx >= limit: del self[k]
-
-    # def _clear_stale(self):
-    #     for k in [k for k, v in self._cache.items() if v.stale]:
-    #         del self[k]
-    
-    def add(self, key: str, value: str) -> str:
-        #self._clear_stale()
-        #self._enforce_limit(self._limit)
-        query, _hash = norm(key)
-        value = value or query
-        
-        if not value:
-            raise ValueError("Invalid query string: {value}")
-        self._cache[_hash] = value
-        save_query(hash_to_base32(_hash), value.encode("utf-8"), 'query')
-        return value
-    
-    def remove(self, key: str):
-        key = norm(key)
-        self._cache.pop(key, None)
-        delete_query(hash_to_base32(key))

app/rest/__init__.py

@@ -1,18 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-#
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from bottle import TEMPLATE_PATH
-
-TEMPLATE_PATH.append("app/rest/templates")
-ALL_UNITS = { 'g', 'kg', 'mL', 'L', 'Pieces', 'Bunches', 'Bags' }
-PARAMS = { 'group', 'category', 'product', 'unit', 'tag', 'organic' }
-BOOLEAN = {
-    "1": True,
-    True: "1",
-    "0": False,
-    False: "0",
-    None: "0.5",
-}

app/rest/cherrypy.py

@@ -1,19 +0,0 @@
-import cherrypy
-import bottle
-from .pyapi import *
-
-application = bottle.default_app()
-
-cherrypy.config.update({'environment' : 'staging'})
-cherrypy.config.update({
-    'server.socket_host': "0.0.0.0",
-    'server.socket_port': 6772,
-    'engine.autoreload.on': True,
-    'request.show_tracebacks': True,
-    'request.show_mismatched_params': True,
-    'log.screen': True,
-})
-
-cherrypy.tree.graft(application, "/")
-cherrypy.engine.start()
-cherrypy.engine.block()

app/rest/dev-requirements.txt

@@ -1 +0,0 @@
-pytest

app/rest/form.py

@@ -1,109 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-#
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from typing import Dict, Tuple
-from bottle import template
-from pandas import DataFrame
-from itertools import chain
-from . import ALL_UNITS, BOOLEAN
-
-def get_option_groups(
-    data: DataFrame, filter_data: Dict[str, Tuple[set, set]],
-    k: str, g: str, _type: str
-):
-    if k in data or (k == 'tag' and 'tags' in data):
-        k_data_in_chart = chain(*data["tags"]) if k == "tag" else data[k]
-    else:
-        k_data_in_chart = []
-    
-    groups = sorted(set(data[g] if g is not None and g in data else []))
-    groups.append(None)
-    if _type == "exclude":
-        prefix = "!"
-    else:
-        prefix = ""
-    
-    for group in groups:
-        selected = []
-        unselected = []
-        if group is None:
-            if _type == "include":
-                selected.extend(filter_data[k][0] - (
-                    set(k_data_in_chart if g is not None else set())
-                ))
-                unselected.extend(((
-                    set(k_data_in_chart if g is None else set())
-                ) | filter_data[k][1]) - filter_data[k][0])
-            else:
-                selected.extend(filter_data[k][1])
-                unselected.extend((
-                    set(k_data_in_chart if g is None else set())
-                ) | (
-                    filter_data[k][0] - set(k_data_in_chart) - filter_data[k][1]
-                ))
-        else:
-            k_grouped_data_in_chart = set(data[data[g].apply(lambda x,axis=None: x == group)][k]) if k in data else set()
-            if _type == "include":
-                selected.extend(filter_data[k][0] & set(k_grouped_data_in_chart))
-                unselected.extend(k_grouped_data_in_chart - filter_data[k][0])
-            else:
-                unselected.extend(set(k_grouped_data_in_chart))
-        assert set(selected) - set(unselected) == set(selected), f"{set(selected)} {set(unselected)}"
-        options = sorted(map(lambda x: {
-            "selected": x[0],
-            "value": f"{prefix}{x[1]}",
-            "display": x[1]  
-        }, chain(
-            map(lambda x: (True, x), set(selected)),
-            map(lambda x: (False, x), set(unselected)),
-        )), key=lambda x: x["display"] if "display" in x else x["value"])
-        if len(options) > 0:
-            yield {
-                "optgroup": group,
-                "options": options,
-            }
-
-
-def get_form(action: str, method: str, filter_data: Dict[str, Tuple[set, set]], organic: bool, data: DataFrame):
-    keys = sorted(filter(lambda x: x not in ('unit', 'tag', 'organic'), filter_data), key=lambda x: {
-        'product': 0,
-        'category': 1,
-        'group': 2,
-    }[x])
-    return template('form-filter', action=action, method=method,
-        params=filter_data,
-        **{
-            k: {
-                "name": k,
-                "_include": {
-                    "option_groups": get_option_groups(data, filter_data, k, g, "include"),
-                },
-                "_exclude": {
-                    "option_groups": get_option_groups(data, filter_data, k, g, "exclude"),
-                }
-            } for k, g in zip(keys, [*keys[1:], None])
-        },
-        tags={
-            "name": "tag",
-            "_include": {
-                "option_groups": get_option_groups(data, filter_data, "tag", None, "include")
-            },
-            "_exclude": {
-                "option_groups": get_option_groups(data, filter_data, "tag", None, "exclude")
-            },
-        },
-        units={
-            "name": "unit",
-            "options": sorted(map(lambda x: {
-                "selected": x[0],
-                "value": x[1],
-            },chain(
-                map(lambda x: (True, x), filter_data['unit'][0]),
-                map(lambda x: (False, x), ALL_UNITS - filter_data['unit'][0])
-            )), key=lambda x: x["display"] if "display" in x else x["value"])
-        },
-        organic=BOOLEAN[organic],
-    )

app/rest/hash_util.py

@@ -1,180 +0,0 @@
-from hashlib import blake2b, shake_128, md5, sha256, sha1
-import os
-from base64 import b64encode, b64decode, b85encode, b85decode
-import base32_lib as b32
-
-DIGEST_SIZE_BYTES = 3
-DIGEST_SIZE_BITMASK = 0xffffff
-DIGEST_SIZE_SIGNED_TO_UNSIGNED_BITMASK = 0x1ffffff
-DIGEST_SIZE_SIGNED_TO_UNSIGNED_BIT = 0x1000000
-DIGEST_SIZE_NIBBLES = DIGEST_SIZE_BYTES * 2
-B64ALTCHARS = b'.-'
-
-def sha1hash(data: str):
-    return sha1(data.encode("utf-8"), usedforsecurity=False).hexdigest()[:DIGEST_SIZE_BYTES]
-
-def sha256hash(data: str):
-    return sha256(data.encode("utf-8"), usedforsecurity=False).hexdigest()[:DIGEST_SIZE_BYTES]
-
-def md5hash(data: str):
-    return md5(data.encode("utf-8"), usedforsecurity=False).hexdigest()[:DIGEST_SIZE_BYTES]
-
-def shake(data: str):
-    return shake_128(data.encode("utf-8"), usedforsecurity=False).hexdigest(DIGEST_SIZE_BYTES)
-
-def blake(data: bytes, person: bytes = None) -> bytes:
-    return blake2b(
-        data,
-        usedforsecurity=False,
-        digest_size=DIGEST_SIZE_BYTES,
-        person=person
-    ).digest()
-
-def blake_file(path: str, person: bytes = None, root: str ='rest/static') -> bytes:
-    fd = os.open(f'{root}/{path}', os.O_RDONLY, 0o600)
-    with open(fd, "rb") as f:
-        f.seek(0)
-        _blake = blake2b(usedforsecurity=False, digest_size=DIGEST_SIZE_BYTES, person=person)
-        while f.peek(1):
-            _blake.update(f.read(1024))
-    return _blake.digest()
-        
-
-def python(data: str):
-    return hash(data)
-
-def normalize_hash(_hash: int) -> int:
-    #hex = hash_to_hex(_hash)
-    #return int(hex, 16)
-    #_bytes = _hash.to_bytes(8, byteorder='big', signed=True)
-    #return bytes_to_hash(_bytes)
-    return _hash & DIGEST_SIZE_BITMASK
-
-def normalize_bytes(_bytes: bytes) -> bytes:
-    return (b'\x00' * DIGEST_SIZE_BYTES + _bytes)[-DIGEST_SIZE_BYTES:]
-
-def normalize_hex(_hex: str) -> str:
-    #_bytes = hex_to_bytes(hex)
-    #return _bytes.hex()
-    return _hex.zfill(DIGEST_SIZE_NIBBLES)[-DIGEST_SIZE_NIBBLES:]
-
-def hex_to_bytes(_hex: str) -> bytes:
-    _bytes = bytes.fromhex(_hex.zfill(DIGEST_SIZE_NIBBLES))
-    return normalize_bytes(_bytes)
-
-def bytes_to_hex(_bytes: bytes) -> str:
-    return normalize_bytes(_bytes).hex()
-
-def hash_to_bytes(_hash: int) -> bytes:
-    _bytes = _hash.to_bytes(8, byteorder='big', signed=True)
-    return normalize_bytes(_bytes)
-
-def bytes_to_hash(_bytes: bytes) -> int:
-    norm = normalize_bytes(_bytes)
-    return int.from_bytes(norm, byteorder='big', signed=False)
-
-def hash_to_hex(_hash: int) -> str:
-    #return hash_to_bytes(_hash).hex()
-    #return normalize_hex(
-    #return f"{_hash + (1 << 64):x}"[-4:]
-    #return hex(_hash + (1<<64))[2:][-4:]
-    #return f"{_hash & 0xffff:04x}"
-    return hex((_hash|DIGEST_SIZE_SIGNED_TO_UNSIGNED_BIT) & DIGEST_SIZE_SIGNED_TO_UNSIGNED_BITMASK)[3:]
-
-def hex_to_hash(_hex: str) -> int:
-    #_bytes = bytes.fromhex(hex.zfill(4))
-    #return bytes_to_hash(_bytes)
-    #return int(normalize_hex(hex), 16)
-    return int(_hex, 16) & DIGEST_SIZE_BITMASK
-
-def remove_padding(f):
-    def wrap(*args, **kwargs):
-        return f(*args, **kwargs).split('=')[0]
-    return wrap
-
-def fix_padding(f):
-    def wrap(_b64, *args, **kwargs):
-        pad = (4 - len(_b64)) % 4
-        fixed = _b64 + '='*pad
-        return f(fixed, *args, **kwargs)
-    return wrap
-
-def normalize_base32(_b32: str):
-    return b32.encode(b32.decode(_b32)).upper().zfill(DIGEST_SIZE_BYTES*8//5+1)
-
-def add_padding_base32(f):
-    def wrap(*args, **kwargs):
-        return normalize_base32(f(*args, **kwargs))
-    return wrap
-
-@remove_padding
-def hash_to_base64(_hash: int) -> str:
-    return b64encode(hash_to_bytes(_hash), altchars=B64ALTCHARS).decode("utf-8")
-
-@remove_padding
-def hex_to_base64(_hex: str) -> str:
-    return b64encode(hex_to_bytes(_hex), altchars=B64ALTCHARS).decode("utf-8")
-
-@remove_padding
-def bytes_to_base64(_bytes: str) -> str:
-    return b64encode(normalize_bytes(_bytes), altchars=B64ALTCHARS).decode("utf-8")
-
-@fix_padding
-def base64_to_hash(_b64: str) -> str:
-    return bytes_to_hash(b64decode(_b64, altchars=B64ALTCHARS))
-
-@fix_padding
-def base64_to_hex(_b64: str) -> str:
-    return bytes_to_hex(b64decode(_b64, altchars=B64ALTCHARS))
-
-@fix_padding
-def base64_to_bytes(_b64: str) -> str:
-    return normalize_bytes(b64decode(_b64, altchars=B64ALTCHARS))
-
-#@remove_padding
-def hash_to_base85(_hash: int) -> str:
-    return b85encode(hash_to_bytes(_hash)).decode("utf-8")
-
-#@remove_padding
-def hex_to_base85(_hex: str) -> str:
-    return b85encode(hex_to_bytes(_hex)).decode("utf-8")
-
-#@remove_padding
-def bytes_to_base85(_bytes: str) -> str:
-    return b85encode(normalize_bytes(_bytes)).decode("utf-8")
-
-#@fix_padding
-def base85_to_hash(_b64: str) -> str:
-    return bytes_to_hash(b85decode(_b64))
-
-#@fix_padding
-def base85_to_hex(_b64: str) -> str:
-    return bytes_to_hex(b85decode(_b64))
-
-#@fix_padding
-def base85_to_bytes(_b64: str) -> str:
-    return normalize_bytes(b85decode(_b64))
-
-@add_padding_base32
-def hash_to_base32(_hash: int) -> str:
-    return b32.encode(_hash & DIGEST_SIZE_BITMASK)
-
-@add_padding_base32
-def hex_to_base32(_hex: str) -> str:
-    return b32.encode(hex_to_hash(_hex))
-
-@add_padding_base32
-def bytes_to_base32(_bytes: bytes) -> str:
-    return b32.encode(bytes_to_hash(_bytes))
-
-#@fix_padding
-def base32_to_hash(_b64: str) -> str:
-    return b32.decode(_b64)
-
-#@fix_padding
-def base32_to_hex(_b64: str) -> str:
-    return hash_to_hex(base32_to_hash(_b64))
-
-#@fix_padding
-def base32_to_bytes(_b64: str) -> str:
-    return hash_to_bytes(base32_to_hash(_b64))

app/rest/pyapi.py

@@ -1,139 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-import os
-from threading import Thread
-from typing import Tuple
-from bottle import (
-    route, request, response,
-    static_file,
-    FormsDict,
-)
-from psycopg import Cursor, connect
-from psycopg.rows import TupleRow
-from urllib.parse import parse_qs
-
-from .QueryCache import QueryCache
-from .route_decorators import cache, cursor
-from .query_to_xml import get_categories, get_groups, get_products, get_tags
-from .CachedLoadingPage import CachedLoadingPage
-from .PageCache import PageCache
-from . import trend as worker
-
-host = f"host={os.getenv('HOST')}"
-db = f"dbname={os.getenv('DB', 'grocery')}"
-user = f"user={os.getenv('USER', 'das')}"
-password = f"password={os.getenv('PASSWORD','')}"
-if not password.split('=',1)[1]:
-    password = ''
-conn = connect(f"{host} {db} {user} {password}")
-
-@route('/grocery/static/<filename:path>')
-def send_static(filename):
-    return static_file(filename, root='app/rest/static')
-
-def new_thread(target, conn, path, forms):
-    def cb(queue):
-        return Thread(target=target, args=(
-            queue, conn, path, forms
-        )).start()
-    return cb
-
-PAGE_CACHE = PageCache(100)
-QUERY_CACHE = QueryCache(None)
-
-@route('/grocery/volume', method=['GET', 'POST'])
-@cache(query_cache=QUERY_CACHE, page_cache=PAGE_CACHE)
-def volume(key: Tuple[str, int], cache: PageCache):
-    _, _, path, *_ = request.urlparts
-
-    page = cache[key]
-    if page is None:
-        form = key_to_form(key)
-        page = cache.add(key, CachedLoadingPage([], new_thread(worker.volume, conn, path, form)))
-    
-    for i in iter_page(page):
-        yield i
-
-
-@route('/grocery/trend', method=['GET', 'POST'])
-@cache(query_cache=QUERY_CACHE, page_cache=PAGE_CACHE)
-def trend(key: Tuple[str, int], cache: PageCache):
-    _, _, path, *_ = request.urlparts
-
-    page = cache[key]
-    if page is None:
-        form = key_to_form(key)
-        page = cache.add(key, CachedLoadingPage([], new_thread(worker.trend, conn, path, form)))
-    
-    for i in iter_page(page):
-        yield i
-
-
-def query_to_form(query):
-    form = FormsDict()
-    for k, v in parse_qs(query).items():
-        for item in v:
-            form.append(k, item)
-    return form
-
-
-def key_to_form(key):
-    query, _ = key
-    return query_to_form(query.split('?', 1)[-1])
-
-
-def iter_page(page):
-    # copy first to avoid races
-    resp = list(page.value)
-    pos = len(resp)
-    yield ''.join(resp)
-    
-    while not page.loaded:
-        page.update()
-        # all changes since last yield
-        resp = list(page.value[pos:])
-        pos = pos + len(resp)
-        yield ''.join(resp)
-    
-    # possibly have not yielded the entire page
-    if pos < len(page.value):
-        yield ''.join(page.value[pos:])
-
-
-@route('/grocery/groups', method=['GET', 'POST'])
-@cache(query_cache=QUERY_CACHE, page_cache=PAGE_CACHE)
-@cursor(connection=conn)
-def groups(cur: Cursor[TupleRow], key: Tuple[str, int], cache: PageCache):
-    form = key_to_form(key)
-    response.content_type = 'application/xhtml+xml; charset=utf-8'
-    return get_groups(cur, form)
-
-
-@route('/grocery/categories', method=['GET', 'POST'])
-@cache(query_cache=QUERY_CACHE, page_cache=PAGE_CACHE)
-@cursor(connection=conn)
-def categories(cur: Cursor[TupleRow], key: Tuple[str, int], cache: PageCache):
-    form = key_to_form(key)
-    response.content_type = 'application/xhtml+xml; charset=utf-8'
-    return get_categories(cur, form)
-
-
-@route('/grocery/products', method=['GET', 'POST'])
-@cache(query_cache=QUERY_CACHE, page_cache=PAGE_CACHE)
-@cursor(connection=conn)
-def products(cur: Cursor[TupleRow], key: Tuple[str, int], cache: PageCache):
-    form = key_to_form(key)
-    response.content_type = 'application/xhtml+xml; charset=utf-8'
-    return get_products(cur, form)
-
-
-@route('/grocery/tags', method=['GET', 'POST'])
-@cache(query_cache=QUERY_CACHE, page_cache=PAGE_CACHE)
-@cursor(connection=conn)
-def tags(cur: Cursor[TupleRow], key: Tuple[str, int], cache: PageCache):
-    form = key_to_form(key)
-    response.content_type = 'application/xhtml+xml; charset=utf-8'
-    return get_tags(cur, form)

app/rest/query_to_xml.py

@@ -1,143 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from bottle import request, template, FormsDict
-from pandas import DataFrame
-from psycopg import Cursor
-from psycopg.sql import SQL, Literal, Identifier
-from typing import List
-
-from ..data.filter import get_filter
-from ..data.util import get_where_include_exclude
-from ..data.QueryManager import get_data
-from .form import get_form
-from . import BOOLEAN, PARAMS
-
-
-def get_product_rollup_statement(filters, orderby: List[str]) -> SQL:
-    _map = { k: k[0] for k in ('product', 'category', 'group') }
-    where = [ get_where_include_exclude(
-        _map[k], "name", list(include), list(exclude)
-    ) for k, (include, exclude) in filters.items() ]
-    return SQL("""
-SELECT
-  count(DISTINCT p.id) AS "Products",
-  count(DISTINCT c.id) AS "Categories",
-  count(DISTINCT g.id) AS "Groups",
-  p.name AS "product",
-  c.name AS "category",
-  g.name AS "group"
-FROM products p
-JOIN categories c ON p.category_id = c.id
-JOIN groups g ON c.group_id = g.id
-WHERE {where}
-GROUP BY ROLLUP (g.name, c.name, p.name)
-ORDER BY {orderby}
-""").format(
-        where=SQL("\nAND").join(where),
-        orderby=SQL(", ").join(map(Identifier,orderby)),
-    )
-
-
-def get_inner_query(query: FormsDict, orderby: List[str]) -> SQL:
-    filters = get_filter(query, allow=('group', 'category', 'product'))
-    inner = get_product_rollup_statement(filters, orderby)
-    return inner
-
-
-def render_form(cur: Cursor, inner: str, query: FormsDict):
-    _filter = get_filter(query, allow=PARAMS)
-    data = DataFrame(get_data(cur, inner)).dropna()
-    action = request.path.split('/')[-1]
-    organic = BOOLEAN.get(query.organic, None)
-    return get_form(action, 'post', _filter, organic, data)
-
-
-def get_xml(cur: Cursor, sql: str):
-    return cur.execute(SQL(
-        "SELECT query_to_xml_and_xmlschema({q}, false, false, ''::text)"
-    ).format(q=Literal(sql))).fetchone()[0]
-
-
-def get_products(cur: Cursor, query: FormsDict):
-    inner = get_inner_query(query, ['product', 'category', 'group'])
-    form = render_form(cur, inner, query)
-    sql = SQL("""
-SELECT
-    --"Transactions",
-    COALESCE("product", "Products"||'') "Product",
-    COALESCE("category", "Categories"||'') "Category",
-    COALESCE("group", "Groups"||'') "Group"
-FROM ({inner}) q
-WHERE q.product IS NOT NULL OR q.group IS NULL
-""").format(inner=inner).as_string(cur)
-    xml = get_xml(cur, sql)
-    return template("query-to-xml", title="Products", xml=xml, form=form)
-
-
-def get_categories(cur: Cursor, query: FormsDict):
-    inner = get_inner_query(query, ['category', 'group'])
-    form = render_form(cur, inner, query)
-    sql = SQL("""
-SELECT
-    "Products",
-    COALESCE("category", "Categories"||'') "Category",
-    COALESCE("group", "Groups"||'') "Group"
-FROM ({inner}) q
-WHERE q.product IS NULL AND (q.category IS NOT NULL OR q.group IS NULL)
-""").format(inner=inner).as_string(cur)
-    xml = get_xml(cur, sql)
-    return template("query-to-xml", title="Categories", xml=xml, form=form)
-
-
-def get_groups(cur: Cursor, query: FormsDict):
-    inner = get_inner_query(query, ['group',])
-    form = render_form(cur, inner, query)
-    sql = SQL("""
-SELECT
-    "Products",
-    "Categories",
-    COALESCE("group", "Groups"||'') "Group"
-FROM ({inner}) q
-WHERE q.category IS NULL
-""").format(inner=inner).as_string(cur)
-    xml = get_xml(cur, sql)
-    return template("query-to-xml", title="Groups", xml=xml, form=form)
-
-def get_tags_statement(filters) -> SQL:
-    _map = {
-        k: k[0] for k in ('product', 'category', 'group')
-    }
-    _map.update({ 'tag': 'tg' })
-    where = [ get_where_include_exclude(
-        _map[k], "name", list(include), list(exclude)
-    ) for k, (include, exclude) in filters.items() ]
-    return SQL("""
-SELECT * FROM (SELECT count(DISTINCT txn.id) AS "Uses", tg.name AS "Name"
-FROM tags tg
-JOIN tags_map tm ON tg.id = tm.tag_id
-JOIN transactions txn ON txn.id = tm.transaction_id
-WHERE {where}
-GROUP BY tg.name
-ORDER BY 1 DESC, 2) q
-UNION ALL
-SELECT count(DISTINCT txn.id) AS "Uses", count(DISTINCT tg.name)||'' AS "Name"
-FROM tags tg
-JOIN tags_map tm ON tg.id = tm.tag_id
-JOIN transactions txn ON txn.id = tm.transaction_id
-WHERE {where}
-""").format(where=SQL("\nAND").join(where))
-
-def get_inner_tags_query(query: FormsDict) -> SQL:
-    filters = get_filter(query, allow=('tag',))
-    inner = get_tags_statement(filters)
-    return inner
-
-def get_tags(cur: Cursor, query: FormsDict):
-    inner = get_inner_tags_query(query)
-    form = render_form(cur, inner, query)
-    sql = inner.as_string(cur)
-    xml = get_xml(cur, sql)
-    return template("query-to-xml", title="Tags", xml=xml, form=form)

app/rest/requirements.txt

@@ -1,5 +0,0 @@
-seaborn
-psycopg[binary]
-bottle
-cherrypy
-base32-lib

app/rest/route_decorators.py

@@ -1,93 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from typing import Callable, Iterable, Tuple
-from urllib.parse import urlencode
-from bottle import request, FormsDict, redirect, HTTPError
-from psycopg import Connection
-from psycopg.connection import TupleRow
-
-from .QueryCache import QueryCache, get_hash
-from ..data.filter import get_filter, get_query_param
-from . import BOOLEAN, PARAMS
-from .PageCache import PageCache
-from .hash_util import base32_to_hash, hash_to_base32, normalize_base32
-
-def normalize_query(query: FormsDict, allow: Iterable[str] = None) -> Tuple[str, str]:
-    if query.hash:
-        _hash = normalize_base32(query.hash)
-    else:
-        _hash = None
-
-    allow = allow or PARAMS
-    param = get_filter(query, allow=allow)
-    norm = urlencode([
-        (
-            k, get_query_param(*param[k])
-        ) if k != 'organic' else (
-            "organic", BOOLEAN[BOOLEAN.get(query.organic, None)]
-        ) for k in sorted(param) if param[k]
-    ])
-    return norm if not _hash or len(query.keys()) > 1 else None, _hash
-
-
-def _cache_decorator(func: Callable, query_cache: QueryCache = None, page_cache: PageCache = None):
-    def wrap(*args, **kwargs):
-        _, _, path, *_ = request.urlparts
-        
-        endpoint = path.split('/', 2)[-1]
-
-        query, _hash = normalize_query(request.params)
-        query = f"{endpoint}?{query}"
-        if not _hash:
-            _hashInt = get_hash(query)
-            _hash = hash_to_base32(_hashInt)
-            key = (query, _hashInt)
-        else:
-            key = (query, base32_to_hash(_hash))
-        
-        if request.params.reload == "true":
-            page_cache.remove(key)
-
-        # key with tuple to avoid ambiguity
-        cached = query_cache.get(key)
-        if not cached:
-            if query:
-                cached = query_cache.add(key, None)
-            else:
-                return HTTPError(404, f"No query found for hash: {_hash}")
-
-        if not request.params.hash:
-            if cached and len(cached) > 2000:
-                return redirect(f"{path}?hash={_hash}")
-
-            if cached and f"{endpoint}?{request.query_string}" != cached:
-                return redirect(cached)
-
-        return func((cached, key[1]), page_cache, *args, **kwargs)
-    return wrap
-
-
-def cache(*args, **kwargs):
-    if not len(args):
-        return lambda f: _cache_decorator(f, **kwargs)
-    
-    raise Exception("decorator argument required")
-
-
-def _cursor_decorator(func: Callable, connection: Connection[TupleRow] = None):
-    def wrap(*args, **kwargs):
-        try:
-            with connection.cursor() as cur:
-                return func(cur, *args, **kwargs)
-        finally:
-            connection.commit()
-    return wrap
-
-
-def cursor(*args, **kwargs):
-    if not len(args):
-        return lambda f: _cursor_decorator(f, **kwargs)
-    raise Exception("decorator argument required")

app/rest/static/cloud-gears.css

@@ -1,49 +0,0 @@
-.loader {
-  width: 175px;
-  height: 80px;
-  display: block;
-  margin:auto;
-  background-image: radial-gradient(circle 25px at 25px 25px, #FFF 100%, transparent 0), radial-gradient(circle 50px at 50px 50px, #FFF 100%, transparent 0), radial-gradient(circle 25px at 25px 25px, #FFF 100%, transparent 0), linear-gradient(#FFF 50px, transparent 0);
-  background-size: 50px 50px, 100px 76px, 50px 50px, 120px 40px;
-  background-position: 0px 30px, 37px 0px, 122px 30px, 25px 40px;
-  background-repeat: no-repeat;
-  position: relative;
-  box-sizing: border-box;
-}
-.loader::before {
-  content: '';  
-  left: 60px;
-  bottom: 18px;
-  position: absolute;
-  width: 36px;
-  height: 36px;
-  border-radius: 50%;
-  background-color: #555555;
-  background-image: radial-gradient(circle 8px at 18px 18px, #FFF 100%, transparent 0), radial-gradient(circle 4px at 18px 0px, #FFF 100%, transparent 0), radial-gradient(circle 4px at 0px 18px, #FFF 100%, transparent 0), radial-gradient(circle 4px at 36px 18px, #FFF 100%, transparent 0), radial-gradient(circle 4px at 18px 36px, #FFF 100%, transparent 0), radial-gradient(circle 4px at 30px 5px, #FFF 100%, transparent 0), radial-gradient(circle 4px at 30px 5px, #FFF 100%, transparent 0), radial-gradient(circle 4px at 30px 30px, #FFF 100%, transparent 0), radial-gradient(circle 4px at 5px 30px, #FFF 100%, transparent 0), radial-gradient(circle 4px at 5px 5px, #FFF 100%, transparent 0);
-  background-repeat: no-repeat;
-  box-sizing: border-box;
-  animation: rotationBack 3s linear infinite;
-}
-.loader::after {
-  content: '';  
-  left: 94px;
-  bottom: 15px;
-  position: absolute;
-  width: 24px;
-  height: 24px;
-  border-radius: 50%;
-  background-color: #555555;
-  background-image: radial-gradient(circle 5px at 12px 12px, #FFF 100%, transparent 0), radial-gradient(circle 2.5px at 12px 0px, #FFF 100%, transparent 0), radial-gradient(circle 2.5px at 0px 12px, #FFF 100%, transparent 0), radial-gradient(circle 2.5px at 24px 12px, #FFF 100%, transparent 0), radial-gradient(circle 2.5px at 12px 24px, #FFF 100%, transparent 0), radial-gradient(circle 2.5px at 20px 3px, #FFF 100%, transparent 0), radial-gradient(circle 2.5px at 20px 3px, #FFF 100%, transparent 0), radial-gradient(circle 2.5px at 20px 20px, #FFF 100%, transparent 0), radial-gradient(circle 2.5px at 3px 20px, #FFF 100%, transparent 0), radial-gradient(circle 2.5px at 3px 3px, #FFF 100%, transparent 0);
-  background-repeat: no-repeat;
-  box-sizing: border-box;
-  animation: rotationBack 4s linear infinite reverse;
-}
-
-@keyframes rotationBack {
-  0% {
-    transform: rotate(0deg);
-  }
-  100% {
-    transform: rotate(-360deg);
-  }
-}

app/rest/static/manifest.json

@@ -1,22 +0,0 @@
-{
-  "id": "/grocery",
-  "name": "Grocery Manager",
-  "short_name": "Grocery",
-  "description": "View trending price data and tracked product info",
-  "start_url": "/grocery/trend",
-  "theme_color": "firebrick",
-  "background_color": "black",
-  "display": "standalone",
-  "icons": [
-    {
-      "src": "/grocery/static/favicon.svg",
-      "sizes": "any",
-      "purpose": "any"
-    },
-    {
-      "src": "/grocery/static/favicon_square.svg",
-      "sizes": "any",
-      "purpose": "any"
-    }
-  ]
-}

app/rest/static/query-to-xml-xslt.xml

@@ -1,91 +0,0 @@
-<xsl:stylesheet version="1.1"
-  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-  xmlns:xsd="http://www.w3.org/2001/XMLSchema"
-  xmlns:xhtml="http://www.w3.org/1999/xhtml"
->
-
-  <xsl:output method="xml"
-    doctype-system="http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"
-    doctype-public="-//W3C/DTD XHTML 1.0 Strict//EN"
-    indent="yes"/>
-  
-  <!-- This is an identity template - it copies everything
-        that doesn't match another template -->
-  <xsl:template match="@* | node()">
-    <xsl:copy>
-      <xsl:apply-templates select="@* | node()"/>
-    </xsl:copy>
-  </xsl:template>
-  <xsl:template match="processing-instruction()">
-  </xsl:template>
-
-  <xsl:template match="//xhtml:div[@id='xmldata']/*">
-    <xsl:variable name="schema" select="//xsd:schema"/>
-    <xsl:variable name="tabletypename"
-                  select="$schema/xsd:element[@name=name(current())]/@type"/>
-    <xsl:variable name="rowtypename"
-                  select="$schema/xsd:complexType[@name=$tabletypename]/xsd:sequence/xsd:element[@name='row']/@type"/>
-    <xsl:variable name="fieldname"
-                  select="$schema/xsd:complexType[@name=$rowtypename]/xsd:sequence/xsd:element[@name='Product' or @name='Category' or @name='Group' or @name='Name'][1]/@name"/>
-    
-    <table xmlns="http://www.w3.org/1999/xhtml"
-      class="pure-table pure-table-bordered pure-table-striped" style="text-align: left; width: 100%;">
-    
-      <thead style="text-align: center">
-      <tr>
-        <th></th>
-        <xsl:for-each select="$schema/xsd:complexType[@name=$rowtypename]/xsd:sequence/xsd:element/@name">
-          <th><xsl:value-of select="."/></th>
-        </xsl:for-each>
-      </tr>
-      </thead>
-
-      <tbody>
-      <xsl:for-each select="xhtml:row">
-        <xsl:choose>
-          <xsl:when test="position() != last()">
-            <tr>
-              <td>
-                <input type="checkbox">
-                  <xsl:attribute name="name">
-                    <xsl:choose>
-                      <xsl:when test="$fieldname = 'Name'">tag</xsl:when>
-                      <xsl:otherwise>
-                        <xsl:value-of select="translate($fieldname,'PCG', 'pcg')"/>
-                      </xsl:otherwise>
-                    </xsl:choose>
-                  </xsl:attribute>
-                  <xsl:attribute name="value">
-                    <xsl:value-of select="*[name() = $fieldname][1]"/>
-                  </xsl:attribute>
-                  <xsl:attribute name="form">filter</xsl:attribute>
-                </input>
-              </td>
-              <xsl:for-each select="*">
-                <td><xsl:value-of select="."/></td>
-              </xsl:for-each>
-            </tr>
-          </xsl:when>
-        </xsl:choose>
-      </xsl:for-each>
-      </tbody>
-      <tfoot>
-      <xsl:for-each select="xhtml:row">
-        <xsl:choose>
-          <xsl:when test="position() = last()">
-              <tr>
-                <th></th>
-                <xsl:for-each select="*">
-                  <th style="text-align: center"><xsl:value-of select="."/></th>
-                </xsl:for-each>
-              </tr>
-          </xsl:when>
-        </xsl:choose>
-      </xsl:for-each>
-      </tfoot>
-
-    </table>
-
-  </xsl:template>
-
-</xsl:stylesheet>

app/rest/templates/button-action.tpl

@@ -1,5 +0,0 @@
-<div class="pure-button-group vertical-button-group" role="action" style="padding: 1em 0.25em 0;">
-  <button class="pure-button" type="submit"> Apply </button>
-  <button form="clear" class="pure-button" type="submit"> Clear </button>
-  <button form="reload" class="pure-button" type="submit"> Reload </button>
-</div>

app/rest/templates/button-style.tpl

@@ -1,16 +0,0 @@
-<style>
-.vertical-button-group .pure-button:first-child {
-    border-top-right-radius: 2px;
-    border-bottom-left-radius: 0px;
-}
-.vertical-button-group .pure-button:last-child {
-    border-top-right-radius: 0px;
-    border-bottom-left-radius: 2px;
-}
-.vertical-button-group .pure-button {
-    width: 100%;
-}
-.vertical-button-group .pure-button-hover, .pure-button:focus, .pure-button:hover {
-    background-image: linear-gradient(.25turn, rgba(0, 0, 0, .1), rgba(0, 0, 0, .05) 20%, rgba(0, 0, 0, .05) 80%, rgba(0, 0, 0, 0.1));
-}
-</style>

app/rest/templates/buttongroup-nav.tpl

@@ -1,13 +0,0 @@
-% setdefault('style', '')
-<div class="pure-button-group vertical-button-group" role="{{role}}" style="padding: 1em 0.25em 0;">
-% for target in targets:
-%   active = 'pure-button-active' if target == action else ''
-%   label = target.title()
-  <button
-    class="pure-button {{active}}"
-    type="submit"
-    formaction="{{target}}"
-    style="{{style}}"
-  > {{label}} </button>
-% end
-</div>

app/rest/templates/done.tpl

@@ -1 +0,0 @@
-<div class="done"></div>

app/rest/templates/error-500.tpl

@@ -1 +0,0 @@
-<span style="font-size: 3em">{{ error }}</span>

app/rest/templates/filter-set.tpl

@@ -1,17 +0,0 @@
-<div class="pure-u-lg-1-8"></div>
-<%
-  for filter in (product, category, group):
-    include('include-exclude', **filter)
-  end
-%>
-<%
-  include('include-exclude', **tags)
-
-%>
-  <div class="pure-u-1-3 pure-u-lg-1-12">
-    <div class="pure-g">
-    <% include('select-one', **units) %>
-    <% include('range-organic') %>
-    </div>
-  </div>
-<div class="pure-u-lg-1-8"></div>

app/rest/templates/form-clear.tpl

@@ -1,8 +0,0 @@
-<form id="clear" method="{{ method }}" action="{{ action }}">
-  <div style="width: 0; height: 0">
-  % for param in params:
-  %   include('hidden-input', name=param['name'])
-
-  % end
-  </div>
-</form>

app/rest/templates/form-filter.tpl

@@ -1,47 +0,0 @@
-% from app.data.filter import get_query_param
-<form id="filter" method="{{ method }}" action="{{ action }}">
-  <style>
-  select::-webkit-scrollbar {
-  width: 11px;
-}
-select {
-  color: #cccccc;
-  background-color: #080808;
-  scrollbar-width: thin;
-  scrollbar-color: var(--thumbBG) var(--scrollbarBG);
-}
-select::-webkit-scrollbar-track {
-  background: var(--scrollbarBG);
-}
-select::-webkit-scrollbar-thumb {
-  background-color: var(--thumbBG) ;
-  border-radius: 6px;
-  border: 3px solid var(--scrollbarBG);
-}
-  </style>
-  <div class="pure-g">
-    <div class="pure-u-1-24 pure-u-lg-1-5"></div>  
-    <div class="pure-u-11-12 pure-u-lg-3-5">
-    % include('button-style')
-    % include('menu', action=action)
-
-    </div>
-    <div class="pure-u-1-24 pure-u-lg-1-5"></div>    
-  </div>
-  <details style="padding: 1em 0">
-    <summary>Click to expand filter...</summary>
-    <div class="pure-g">
-      <%
-      include('filter-set',
-        product=product, category=category, group=group,
-        tags=tags, units=units)
-      %>
-    </div>
-  </details>
-</form>
-% params = [{
-%   'name': k,
-%   'value': get_query_param(inc, ex),
-% } for k, (inc, ex) in params.items()]
-% include('form-clear', params=params)
-% include('form-reload', params=params)

app/rest/templates/form-reload.tpl

@@ -1,10 +0,0 @@
-<form id="reload" method="{{ method }}" action="{{ action }}">
-  <div style="width: 0; height: 0">
-  % for param in params:
-  %   include('hidden-input', **param)
-
-  % end
-  % include('hidden-input', name='reload', value='true')
-
-  </div>
-</form>

app/rest/templates/hidden-input.tpl

@@ -1,2 +0,0 @@
-% setdefault("value", "")
-<input type="text" name="{{name}}" value="{{value}}" hidden="true"/>

app/rest/templates/include-exclude.tpl

@@ -1,26 +0,0 @@
-<div class="pure-u-1-3 pure-u-lg-{{"1-6" if name != "tag" else "1-8"}}">
-  <div class="pure-g">
-    <div class="pure-u-1">
-      <div class="l-box">
-        <h3>{{name.title()}}</h3>
-      </div>
-    </div>
-    <%
-      include('select', id=f"{name}-include", name=name,
-        children=_include["option_groups"] if "option_groups" in _include else [{
-          "options": _include["options"]
-        }],
-        hint="Include", multiple=True)
-    %>
-
-    <%
-    if defined("_exclude"):
-      include('select', id=f"{name}-exclude", name=name,
-        children=_exclude["option_groups"] if "option_groups" in _exclude else [{
-          "options": _exclude["options"]
-        }], hint="Exclude", multiple=True)
-    end
-    %>
-
-  </div>
-</div>

app/rest/templates/label.tpl

@@ -1 +0,0 @@
-<label for="{{id}}">{{label}}</label>

app/rest/templates/menu.tpl

@@ -1,19 +0,0 @@
-<div class="pure-g">
-  <div class="pure-u-1-3">
-  % include('buttongroup-nav', role='chart', style='background-color: firebrick', targets=[
-  %   "trend",
-  %   "volume",
-  % ])
-
-  </div>
-  <div class="pure-u-1-3">
-  % include('buttongroup-nav', role='data', targets=[
-  %   "products", "categories", "groups", "tags"
-  % ])
-
-  </div>
-  <div class="pure-u-1-3">
-    % include('button-action')
-
-  </div>
-</div>

app/rest/templates/optgroup.tpl

@@ -1,2 +0,0 @@
-<optgroup label="{{name}}">{{!base}}
-</optgroup>

app/rest/templates/option.tpl

@@ -1,5 +0,0 @@
-% disabled = (get("disabled", False) and 'disabled="true"') or ""
-% selected = (get("selected", False) and 'selected="true"') or ""
-% display = get("display", value)
-% setdefault("indent", "  ")
-{{indent}}<option value="{{value}}" {{!disabled}} {{!selected}}>{{display}}</option>

app/rest/templates/options.tpl

@@ -1,5 +0,0 @@
-% defined("optgroup") and rebase('optgroup', name=optgroup)
-% for opt in options:
-
-%   include('option', **opt)
-% end

app/rest/templates/progress.tpl

@@ -1,5 +0,0 @@
-<div class="progress">
-  <progress id="loading-{{stage}}" value="{{percent}}" max="100"></progress>
-  <br/>
-  <label for="loading-{{stage}}">{{stage}}</label>
-</div>

app/rest/templates/query-to-xml.tpl

@@ -1,47 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<?xml-stylesheet type="text/xsl" href="/grocery/static/query-to-xml-xslt.xml"?>
-<html xmlns="http://www.w3.org/1999/xhtml">
-  <head>
-    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
-    <title>{{ title }}</title>
-    <meta name="viewport" content="width=device-width, initial-scale=1"/>
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/purecss@2.1.0/build/pure-min.css" integrity="sha384-yHIFVG6ClnONEA5yB5DJXfW2/KC173DIQrYoZMEtBvGzmf0PKiGyNEqe9N6BNDBH" crossorigin="anonymous"/>
-    <link rel="stylesheet" href="https://shandan.one/css/grids-responsive-min.css"/>
-    <link rel="stylesheet" href="https://shandan.one/css/responsive-visibility-collapse.css"/>
-    <link rel="manifest" href="/grocery/static/manifest.json"/>
-    <link rel="icon" type="image/png" href="/grocery/static/favicon.png"/>
-    <style>
-body {
-  background-color: #080808;
-  color: #cccccc;
-  text-align: center;
-}
-.pure-table thead{
-  background-color:#a0a0a0;
-  color:#000;
-  text-align:left;
-  vertical-align:bottom;
-}
-.pure-table td{
-  background-color:transparent;
-}
-.pure-table-odd td{
-  background-color:#181818;
-}
-.pure-table-striped tr:nth-child(2n-1) td{
-  background-color:#181818;
-}
-    </style>
-  </head>
-  <body align="center">
-{{!form}}
-    <div class="pure-g">
-      <div class="pure-u-lg-1-3"> </div>
-      <div id="xmldata" class="pure-u-1 pure-u-lg-1-3">
-{{!xml}}
-      </div>
-      <div class="pure-u-lg-1-3"> </div>
-    </div>
-  </body>
-</html>
-

app/rest/templates/range-organic.tpl

@@ -1,14 +0,0 @@
-<div class="pure-u-1">
-  <div class="l-box">
-    <h3>Organic</h3>
-  </div>
-  </div>
-  <label for="organic-state" hidden="true">Organic</label>
-  <div class="pure-g">
-  <div class="pure-u-1-3">No</div>
-  <div class="pure-u-1-3">Any</div>
-  <div class="pure-u-1-3">Yes</div>
-  <div class="pure-u-1">
-    <input type="range" id="organic-state" name="organic" min="0" max="1" step="0.5" value="{{organic}}" />
-  </div>
-</div>

app/rest/templates/select-one.tpl

@@ -1,10 +0,0 @@
-<div class="pure-u-1">
-  <div class="l-box">
-    <h3>{{name.title()}}</h3>
-  </div>
-</div>
-<% include('select', id=f"{name}-select-one", name=name,
-   children=[{
-      "options": options
-   }])
-%>

app/rest/templates/select.tpl

@@ -1,20 +0,0 @@
-% from bottle import template
-% multiple = (get("multiple", False) and 'multiple="true"') or ""
-<div class="pure-u-1">
-%  if defined("label"):
-%    include('label', id=id, label=label)
-%  end
-
-<select id="{{id}}" name="{{name}}" size="10" {{!multiple}} style="width: calc(100% - 1em); margin: 0 1em 1em">
-%  if defined("hint"):
-%    include('option', value=hint, disabled=True)
-%  end
-
-%  for child in children:
-%    if "optgroup" in child and child["optgroup"] is None:
-%      del child["optgroup"]
-%    end
-{{!template('options', **child)}}
-%  end
-</select>
-</div>

app/rest/templates/trend.tpl

@@ -1,77 +0,0 @@
-% setdefault("start", False)
-% setdefault("end", False)
-% setdefault("error", '')
-% if start:
-<html>
-  <head>
-    <title>Trend</title>
-    <meta name="viewport" content="width=device-width, initial-scale=1"/>
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/purecss@2.1.0/build/pure-min.css" integrity="sha384-yHIFVG6ClnONEA5yB5DJXfW2/KC173DIQrYoZMEtBvGzmf0PKiGyNEqe9N6BNDBH" crossorigin="anonymous"/>
-    <link rel="stylesheet" href="https://shandan.one/css/grids-responsive-min.css"/>
-    <link rel="stylesheet" href="https://shandan.one/css/responsive-visibility-collapse.css"/>
-    <link rel="stylesheet" href="/grocery/static/cloud-gears.css"/>
-    <link rel="manifest" href="/grocery/static/manifest.json"/>
-    <link rel="icon" type="image/png" href="/grocery/static/favicon.png"/>
-    <style>
-html {
-  --scrollbarBG: #333333;
-  --thumbBG: #080808;
-}
-svg {
-  max-height: min(100vh, calc(100vw * 9 / 16));
-  max-width: calc(100vw - 2em);
-}
-body {
-  background-color: #080808;
-  color: #cccccc;
-  text-align: center;
-}
-div.loader-container {
-  position: absolute;
-  left: 50vw;
-  top: 50vh;
-  margin-top: -5.5em;
-  margin-left: -87.5px;
-  padding-bottom: 2em;
-  height: 9em;
-  width: 175px;
-}
-div.loader-container:not(:has(+ .done)) {
-  display: block;
-}
-.loader-container:not(:last-child) {
-  display: none;
-}
-div.progress {
-  margin: 1em 0 1em;
-}
-div.progress:not(:has(+ .done)) {
-  display: block;
-}
-.progress label {
-  text-align:left;
-}
-.progress label:after {
-  content: "...";
-}
-.progress:not(:last-child) {
-  display: none;
-}
-    </style>
-  </head>
-  <body>
-    <div class="loader-container">
-    <span class="loader"></span>
-% end
-% if end:
-    </div>
-    <div class="done"></div>
-{{!form}}
-    % if error:
-    % include('error-500', error=error)
-    % else:
-{{!svg}}
-    % end
-    </body>
-</html>
-% end

app/rest/templates/volume.tpl

@@ -1,79 +0,0 @@
-% setdefault("start", False)
-% setdefault("end", False)
-% setdefault("error", '')
-% if start:
-<html>
-  <head>
-    <title>Trend</title>
-    <meta name="viewport" content="width=device-width, initial-scale=1"/>
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/purecss@2.1.0/build/pure-min.css" integrity="sha384-yHIFVG6ClnONEA5yB5DJXfW2/KC173DIQrYoZMEtBvGzmf0PKiGyNEqe9N6BNDBH" crossorigin="anonymous"/>
-    <link rel="stylesheet" href="https://shandan.one/css/grids-responsive-min.css"/>
-    <link rel="stylesheet" href="https://shandan.one/css/responsive-visibility-collapse.css"/>
-    <link rel="stylesheet" href="/grocery/static/cloud-gears.css"/>
-    <link rel="manifest" href="/grocery/static/manifest.json"/>
-    <link rel="icon" type="image/png" href="/grocery/static/favicon.png"/>
-    <style>
-html {
-  --scrollbarBG: #333333;
-  --thumbBG: #080808;
-}
-svg {
-  max-height: min(100vh, calc(100vw * 9 / 16));
-  max-width: calc(100vw - 2em);
-}
-body {
-  background-color: #080808;
-  color: #cccccc;
-  text-align: center;
-}
-div.loader-container {
-  position: absolute;
-  left: 50vw;
-  top: 50vh;
-  margin-top: -5.5em;
-  margin-left: -87.5px;
-  padding-bottom: 2em;
-  height: 9em;
-  width: 175px;
-}
-div.loader-container:not(:has(+ .done)) {
-  display: block;
-}
-.loader-container:not(:last-child) {
-  display: none;
-}
-div.progress {
-  margin: 1em 0 1em;
-}
-div.progress:not(:has(+ .done)) {
-  display: block;
-}
-.progress label {
-  text-align:left;
-}
-.progress label:after {
-  content: "...";
-}
-.progress:not(:last-child) {
-  display: none;
-}
-    </style>
-  </head>
-  <body>
-    <div class="loader-container">
-    <span class="loader"></span>
-% end
-% if end:
-    </div>
-    <div class="done"></div>
-{{!form}}
-    % if error:
-    % include('error-500', error=error)
-    % else:
-    %   for plt in svg:
-{{!plt}}
-    %    end
-    % end
-    </body>
-</html>
-% end

app/rest/trend.py

@@ -1,273 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-#
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from io import StringIO
-from datetime import date, datetime
-from queue import Queue
-from bottle import (
-    FormsDict,
-    HTTPError,
-    template,
-)
-import matplotlib.pyplot as plt
-import matplotlib
-import numpy as np
-from pandas import DataFrame
-import seaborn as sns
-from psycopg import Connection
-from psycopg.connection import TupleRow
-
-from . import ALL_UNITS, BOOLEAN, PARAMS
-from ..data.QueryManager import (
-    display_mapper,
-    QueryManager,
-)
-from ..data.filter import (
-    get_filter,
-    get_query_param,
-)
-from .form import(
-    get_form,
-)
-
-matplotlib.use('agg')
-
-plot_style = {
-    "lines.color": "#ffffff",
-    "patch.edgecolor": "#ffffff",
-    "text.color": "#ffffff",
-    "axes.facecolor": "#7f7f7f",
-    "axes.edgecolor": "#ffffff",
-    "axes.labelcolor": "#ffffff",
-    "xtick.color": "#ffffff",
-    "ytick.color": "#ffffff",
-    "grid.color": "#ffffff",
-    "figure.facecolor": "#7f7f7f",
-    "figure.edgecolor": "#7f7f7f",
-    "savefig.facecolor": "#7f7f7f",
-    "savefig.edgecolor": "#7f7f7f",
-}
-
-def get_data(query_manager: QueryManager, unit=None, **kwargs) -> DataFrame:
-    d = DataFrame(query_manager.get_historic_prices_data(unit, **kwargs))
-    if d.empty:
-        return d
-    d['ts_month'] = d['ts_raw'].apply(lambda x: date(x.date().year, x.date().month,1))
-    d[['price','quantity']] = d[['price','quantity']].apply(
-        lambda y: y.apply(lambda x: x and float(x)),
-    )
-    return d
-
-def abort(code, text):
-    raise HTTPError(code, text)
-
-def trend(queue: Queue, conn: Connection[TupleRow], path: str, query: FormsDict):
-    for item in trend_internal(conn, path, query):
-        queue.put(item, block=True)
-    queue.put(None)
-
-def trend_internal(conn: Connection[TupleRow], path: str, query: FormsDict):
-    progress = {
-        'stage': None,
-        'percent': None,
-    }
-    action = path.split('/')[-1]
-    organic = BOOLEAN.get(query.organic, None)
-    _filter = get_filter(query, allow=PARAMS)
-    yield template("trend", start=True)
-    try:
-        with conn.cursor() as cur:
-            query_manager = QueryManager(cur, display_mapper)
-            fields = {
-                k: get_query_param(*_filter[k])
-                for k in sorted(_filter) if k not in ('organic', 'unit') and _filter[k]
-            }
-            unit = fields['unit'] = query.unit or 'kg'
-            fields['organic'] = BOOLEAN.get(query.organic, None)
-            if unit and unit not in ALL_UNITS:
-                abort(400, f"Unsupported unit: {unit}")
-
-            progress.update({ "stage": "Querying database", "percent": "10"})
-            yield template("done") + template("progress", **progress)
-            data = get_data(query_manager, **fields)
-            
-            if data.empty:
-                abort(404, f"No data.")
-            
-            progress.update({ "stage": "Preparing data", "percent": "30"})
-            yield template("done") + template("progress", **progress)
-            
-            in_chart = data['$/unit'].apply(lambda x: (x or False) and True)
-            data = data[in_chart]
-            
-            pivot = data.pivot_table(index=['ts_raw',], columns=['product',], values=['$/unit'], aggfunc='mean')
-            pivot.columns = pivot.columns.droplevel()
-            sns.set_theme(style='darkgrid', palette='pastel', context="talk")
-            plt.style.use("dark_background")
-            plt.rcParams.update(plot_style)
-            plt.rcParams.update({"grid.linewidth":0.2, "grid.alpha":0.5})
-            plt.figure(figsize=[16, 9], layout="tight")
-            xlabel='Time'
-            ylabel=f'$ / {unit}'
-            if pivot.columns.size > 50:
-                ax = sns.scatterplot(data=pivot, markers=True)
-            else:
-                ax = sns.lineplot(data=pivot, markers=True)
-                legend = plt.figlegend(
-                    loc='upper center', ncol=6,
-                    title_fontsize="14", fontsize="12", labelcolor='#ffffff',
-                    framealpha=0.5
-                )
-                legend.set_title(title="Products")
-            ax.legend().set_visible(False)
-
-            ax.set_xlabel(xlabel, fontsize="14")
-            ax.set_ylabel(ylabel, fontsize="14")
-            ax.axes.tick_params(labelsize="12", which='both')
-            for _, spine in ax.spines.items():
-                spine.set_color('#ffffff')
-            
-            progress.update({ "stage": "Rendering chart", "percent": "50"})
-            yield template("done") + template("progress", **progress)
-            
-            f = StringIO()
-            plt.savefig(f, format='svg')
-            progress.update({ "stage": "Done", "percent": "100" })
-            yield template("done") + template("progress", **progress)
-            
-            form = get_form(action, 'post', _filter, organic, data)
-            
-            yield template("trend", end=True, form=form, svg=f.getvalue())
-
-    except HTTPError as e:
-        if 'data' not in locals():
-            data = DataFrame()
-        if 'form' not in locals():
-            form = get_form(action, 'post', _filter, organic, data)
-        yield template("done") + template("trend", end=True, form=form, error=e.body)
-
-    finally:
-        conn.commit()
-
-def volume(queue: Queue, conn: Connection[TupleRow], path: str, query: FormsDict):
-    for item in volume_internal(conn, path, query):
-        queue.put(item, block=True)
-    queue.put(None)
-
-def volume_internal(conn: Connection[TupleRow], path: str, query: FormsDict):
-    progress = {
-        'stage': None,
-        'percent': None,
-    }
-    action = path.split('/')[-1]
-    organic = BOOLEAN.get(query.organic, None)
-    _filter = get_filter(query, allow=PARAMS)
-    yield template("trend", start=True)
-    try:
-        with conn.cursor() as cur:
-            query_manager = QueryManager(cur, display_mapper)
-            fields = {
-                k: get_query_param(*_filter[k])
-                for k in sorted(_filter) if k not in ('organic', 'unit') and _filter[k]
-            }
-            unit = fields['unit'] = query.unit or 'kg'
-            fields['organic'] = BOOLEAN.get(query.organic, None)
-            if unit and unit not in ALL_UNITS:
-                abort(400, f"Unsupported unit: {unit}")
-
-            progress.update({ "stage": "Querying database", "percent": "10"})
-            yield template("done") + template("progress", **progress)
-            data = get_data(query_manager, **fields)
-
-            if data.empty:
-                abort(404, f"No data.")
-            
-            progress.update({ "stage": "Preparing data", "percent": "30"})
-            yield template("done") + template("progress", **progress)
-            
-            now = datetime.now().date()
-            prev_month = date(now.year,now.month-1,1) if now.month > 1 else date(now.year-1, 12, 1)
-            data = data[data['ts_month'] == prev_month ]
-            group = 'group'
-            for g, _g in zip(
-                ('category', 'group'),
-                ('product', 'category')
-            ):
-                if g and len(data[g].unique()) != 1:
-                    continue
-                group = _g
-                break
-
-            pivot = data[~(data['quantity'].isnull())].groupby([group,])[['price', 'quantity']].sum()
-            
-            if pivot.empty:
-                abort(404, f"No data.")
-
-            sns.set_theme(style='darkgrid', palette='pastel', context="talk")
-            plt.style.use("dark_background")
-            plt.rcParams.update(plot_style)
-            plt.rcParams.update({"grid.linewidth":0.2, "grid.alpha":0.5})
-            svg = []
-            for title, col, (pre, fmt, suf) in zip((
-                f"Expenditure ${pivot['price'].sum():0.2f}",
-                f"Quantity {pivot['quantity'].sum():0.1f} {unit}",
-            ), (
-                'price',
-                'quantity'
-            ), [
-                ('$', "{0:0.2f}", ''),
-                ('', "{0:0.1f}", f' {unit}')
-            ]):
-                plt.figure(figsize=[16, 9], layout="tight")
-                ax = plt.axes()
-                wedges, *_ = ax.pie(pivot[col].values, startangle=90)
-                bbox_props = dict(boxstyle="square,pad=0.3", lw=0.72, fc='#5f5f5f', ec=plot_style["axes.edgecolor"])
-                kw = dict(arrowprops=dict(arrowstyle="-"),
-                    bbox=bbox_props, zorder=0, va="center")
-                for i, p in enumerate(wedges):
-                    ang = (p.theta2 - p.theta1)/2. + p.theta1
-                    y = np.sin(np.deg2rad(ang))
-                    x = np.cos(np.deg2rad(ang))
-                    horizontalalignment = {-1: "right", 1: "left"}[int(np.sign(x))]
-                    connectionstyle = f"angle,angleA=0,angleB={ang}"
-                    kw["arrowprops"].update({"connectionstyle": connectionstyle})
-                    label = pivot.index[i]
-                    val = pivot.loc[label, col]
-                    label = f"{label} {pre}{fmt.format(val)}{suf} ({val*100/pivot[col].sum():0.1f}%)"
-                    ax.annotate(label, xy=(x, y), xytext=(1.35*np.sign(x), 1.4*y),
-                                horizontalalignment=horizontalalignment, **kw)
-                ax.set_title(title)
-                xlabel=''
-                ylabel=''
-                ax.legend().set_visible(False)
-
-                ax.set_xlabel(xlabel, fontsize="14")
-                ax.set_ylabel(ylabel, fontsize="14")
-                ax.axes.tick_params(labelsize="12", which='both')
-            
-                progress.update({ "stage": "Rendering chart", "percent": "50"})
-                yield template("done") + template("progress", **progress)
-                
-                f = StringIO()
-                plt.savefig(f, format='svg')
-                svg.append(f.getvalue())
-
-            progress.update({ "stage": "Done", "percent": "100" })
-            yield template("done") + template("progress", **progress)
-            
-            form = get_form(action, 'post', _filter, organic, data)
-            
-            yield template("volume", end=True, form=form, svg=svg)
-
-    except HTTPError as e:
-        if 'data' not in locals():
-            data = DataFrame()
-        if 'form' not in locals():
-            form = get_form(action, 'post', _filter, organic, data)
-        yield template("done") + template("trend", end=True, form=form, error=e.body)
-
-    finally:
-        conn.commit()