|
@@ -3,10 +3,7 @@
|
|
# All rights reserved
|
|
# All rights reserved
|
|
#
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
|
|
# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
|
|
-from itertools import chain
|
|
|
|
-from time import time
|
|
|
|
from typing import Iterable
|
|
from typing import Iterable
|
|
-from io import StringIO
|
|
|
|
import os
|
|
import os
|
|
from urllib.parse import urlencode
|
|
from urllib.parse import urlencode
|
|
from bottle import (
|
|
from bottle import (
|
|
@@ -14,35 +11,30 @@ from bottle import (
|
|
request,
|
|
request,
|
|
run,
|
|
run,
|
|
response,
|
|
response,
|
|
- abort,
|
|
|
|
FormsDict,
|
|
FormsDict,
|
|
redirect,
|
|
redirect,
|
|
template,
|
|
template,
|
|
- HTTPError,
|
|
|
|
static_file,
|
|
static_file,
|
|
TEMPLATE_PATH,
|
|
TEMPLATE_PATH,
|
|
)
|
|
)
|
|
from matplotlib.axes import Axes
|
|
from matplotlib.axes import Axes
|
|
-TEMPLATE_PATH.append("app/rest/templates")
|
|
|
|
from psycopg import connect
|
|
from psycopg import connect
|
|
from psycopg.sql import SQL, Literal
|
|
from psycopg.sql import SQL, Literal
|
|
-from pandas import DataFrame
|
|
|
|
-import matplotlib.pyplot as plt
|
|
|
|
import seaborn as sns
|
|
import seaborn as sns
|
|
from multiprocessing import Lock
|
|
from multiprocessing import Lock
|
|
-from ..activities.Plot import (
|
|
|
|
- get_data,
|
|
|
|
|
|
+
|
|
|
|
+from ..data.filter import(
|
|
|
|
+ get_filter,
|
|
)
|
|
)
|
|
-from ..data.QueryManager import QueryManager, display_mapper
|
|
|
|
|
|
+
|
|
from ..data.util import(
|
|
from ..data.util import(
|
|
- get_include_exclude,
|
|
|
|
get_where_include_exclude
|
|
get_where_include_exclude
|
|
)
|
|
)
|
|
|
|
+from .trend import trend_internal
|
|
|
|
+from . import PARAMS
|
|
import matplotlib
|
|
import matplotlib
|
|
matplotlib.use('agg')
|
|
matplotlib.use('agg')
|
|
|
|
|
|
-ALL_UNITS = {'g', 'kg', 'mL', 'L', 'Pieces', 'Bunches', 'Bags'}
|
|
|
|
-PARAMS = { 'group', 'category', 'product', 'unit', 'tag' }
|
|
|
|
CACHE = dict()
|
|
CACHE = dict()
|
|
|
|
|
|
host = f"host={os.getenv('HOST')}"
|
|
host = f"host={os.getenv('HOST')}"
|
|
@@ -83,12 +75,6 @@ ORDER BY "Product", "Category", "Group"
|
|
""").format(having=having) if having else SQL('')
|
|
""").format(having=having) if having else SQL('')
|
|
])
|
|
])
|
|
|
|
|
|
-def get_filter(query: FormsDict, allow: Iterable[str] = None):
|
|
|
|
- return {
|
|
|
|
- k: list(get_include_exclude(
|
|
|
|
- (query[k] or 'kg' if k == 'unit' else query.getall(k)) if k in query else None
|
|
|
|
- )) for k in sorted(set(query.keys()) | set(allow)) if allow is None or k in allow
|
|
|
|
- }
|
|
|
|
|
|
|
|
def get_query_param(include, exclude):
|
|
def get_query_param(include, exclude):
|
|
return '!'.join([
|
|
return '!'.join([
|
|
@@ -96,112 +82,19 @@ def get_query_param(include, exclude):
|
|
*([ '|'.join(sorted(exclude)) ] if exclude else [ ]),
|
|
*([ '|'.join(sorted(exclude)) ] if exclude else [ ]),
|
|
])
|
|
])
|
|
|
|
|
|
-def get_query(**param):
|
|
|
|
|
|
+def normalize_query(query: FormsDict, allow: Iterable[str] = None) -> str:
|
|
|
|
+ param = get_filter(query, allow=allow)
|
|
return urlencode([
|
|
return urlencode([
|
|
(k, get_query_param(*param[k])) for k in sorted(param) if param[k]
|
|
(k, get_query_param(*param[k])) for k in sorted(param) if param[k]
|
|
])
|
|
])
|
|
|
|
|
|
-def normalize_query(query: FormsDict, allow: Iterable[str] = None):
|
|
|
|
- return get_query(**get_filter(query, allow=allow))
|
|
|
|
-
|
|
|
|
-def get_option_groups(data, filter_data, k, g, _type):
|
|
|
|
- in_chart = data['$/unit'].apply(lambda x: (x or False) and True)
|
|
|
|
- groups = sorted(set(data[g] if g is not None else []))
|
|
|
|
- groups.append(None)
|
|
|
|
- if _type == "exclude":
|
|
|
|
- prefix = "!"
|
|
|
|
- else:
|
|
|
|
- prefix = ""
|
|
|
|
-
|
|
|
|
- for group in groups:
|
|
|
|
- selected = []
|
|
|
|
- unselected = []
|
|
|
|
- if group is None:
|
|
|
|
- if _type == "include":
|
|
|
|
- selected.extend(filter_data[k][0] - (
|
|
|
|
- set((chain(*data[in_chart]["tags"]) if k == "tag" else data[in_chart][k]) if g is not None else set())
|
|
|
|
- ))
|
|
|
|
- unselected.extend(((
|
|
|
|
- set((chain(*data[in_chart]["tags"]) if k == "tag" else data[in_chart][k]) if g is None else set())
|
|
|
|
- ) | filter_data[k][1]) - filter_data[k][0])
|
|
|
|
- else:
|
|
|
|
- selected.extend(filter_data[k][1])
|
|
|
|
- unselected.extend((
|
|
|
|
- set((chain(*data[in_chart]["tags"]) if k == "tag" else data[in_chart][k]) if g is None else set())
|
|
|
|
- ) | (
|
|
|
|
- filter_data[k][0] - set(chain(*data[in_chart]["tags"]) if k == "tag" else data[in_chart][k]) - filter_data[k][1]
|
|
|
|
- ))
|
|
|
|
- else:
|
|
|
|
- if _type == "include":
|
|
|
|
- selected.extend(filter_data[k][0] & set(
|
|
|
|
- data[in_chart & (data[g].apply(lambda x,axis=None: x == group))][k]
|
|
|
|
- ))
|
|
|
|
- unselected.extend(set(
|
|
|
|
- data[in_chart & (data[g].apply(lambda x,axis=None: x == group))][k]
|
|
|
|
- ) - filter_data[k][0])
|
|
|
|
- else:
|
|
|
|
- unselected.extend(set(
|
|
|
|
- data[in_chart & (data[g].apply(lambda x,axis=None: x == group))][k]
|
|
|
|
- ))
|
|
|
|
- assert set(selected) - set(unselected) == set(selected), f"{set(selected)} {set(unselected)}"
|
|
|
|
-
|
|
|
|
- yield {
|
|
|
|
- "optgroup": group,
|
|
|
|
- "options": sorted(map(lambda x: {
|
|
|
|
- "selected": x[0],
|
|
|
|
- "value": f"{prefix}{x[1]}",
|
|
|
|
- "display": x[1]
|
|
|
|
- }, chain(
|
|
|
|
- map(lambda x: (True, x), set(selected)),
|
|
|
|
- map(lambda x: (False, x), set(unselected)),
|
|
|
|
- )), key=lambda x: x["display"] if "display" in x else x["value"])
|
|
|
|
- }
|
|
|
|
|
|
|
|
-def get_form(action, method, filter_data, data):
|
|
|
|
- keys = sorted(filter(lambda x: x not in ('unit', 'tag'), filter_data), key=lambda x: {
|
|
|
|
- 'product': 0,
|
|
|
|
- 'category': 1,
|
|
|
|
- 'group': 2,
|
|
|
|
- }[x])
|
|
|
|
- return template('form', action=action, method=method,
|
|
|
|
- **{
|
|
|
|
- k: {
|
|
|
|
- "name": k,
|
|
|
|
- "_include": {
|
|
|
|
- "option_groups": get_option_groups(data, filter_data, k, g, "include"),
|
|
|
|
- },
|
|
|
|
- "_exclude": {
|
|
|
|
- "option_groups": get_option_groups(data, filter_data, k, g, "exclude"),
|
|
|
|
- }
|
|
|
|
- } for k, g in zip(keys, [*keys[1:], None])
|
|
|
|
- },
|
|
|
|
- tags={
|
|
|
|
- "name": "tag",
|
|
|
|
- "_include": {
|
|
|
|
- "option_groups": get_option_groups(data, filter_data, "tag", None, "include")
|
|
|
|
- },
|
|
|
|
- "_exclude": {
|
|
|
|
- "option_groups": get_option_groups(data, filter_data, "tag", None, "exclude")
|
|
|
|
- },
|
|
|
|
- },
|
|
|
|
- units={
|
|
|
|
- "name": "unit",
|
|
|
|
- "options": sorted(map(lambda x: {
|
|
|
|
- "selected": x[0],
|
|
|
|
- "value": x[1],
|
|
|
|
- },chain(
|
|
|
|
- map(lambda x: (True, x), filter_data['unit'][0]),
|
|
|
|
- map(lambda x: (False, x), ALL_UNITS - filter_data['unit'][0])
|
|
|
|
- )), key=lambda x: x["display"] if "display" in x else x["value"])
|
|
|
|
- }
|
|
|
|
- )
|
|
|
|
|
|
|
|
@route('/grocery/static/<filename:path>')
|
|
@route('/grocery/static/<filename:path>')
|
|
def send_static(filename):
|
|
def send_static(filename):
|
|
return static_file(filename, root='app/rest/static')
|
|
return static_file(filename, root='app/rest/static')
|
|
|
|
|
|
|
|
|
|
-
|
|
|
|
global LOCK
|
|
global LOCK
|
|
LOCK = Lock()
|
|
LOCK = Lock()
|
|
|
|
|
|
@@ -234,86 +127,13 @@ def trend():
|
|
return CACHE[request.query_string]["state"]
|
|
return CACHE[request.query_string]["state"]
|
|
try:
|
|
try:
|
|
CACHE[request.query_string] = {
|
|
CACHE[request.query_string] = {
|
|
- "iter": trend_internal(path, request.query),
|
|
|
|
|
|
+ "iter": trend_internal(conn, path, request.query),
|
|
"state": loading,
|
|
"state": loading,
|
|
}
|
|
}
|
|
finally:
|
|
finally:
|
|
LOCK.release()
|
|
LOCK.release()
|
|
return loading
|
|
return loading
|
|
|
|
|
|
-def trend_internal(path, query):
|
|
|
|
- progress = []
|
|
|
|
- try:
|
|
|
|
- with conn.cursor() as cur:
|
|
|
|
- query_manager = QueryManager(cur, display_mapper)
|
|
|
|
- fields = { k: query[k] or None for k in query.keys() if k in PARAMS }
|
|
|
|
- unit = fields['unit'] = fields['unit'] or 'kg' if 'unit' in fields else 'kg'
|
|
|
|
- if unit and unit not in ALL_UNITS:
|
|
|
|
- raise abort(400, f"Unsupported unit {unit}")
|
|
|
|
-
|
|
|
|
- progress.append({ "name": "Loading data", "status": ""})
|
|
|
|
- yield template("loading", progress=progress)
|
|
|
|
- data = get_data(query_manager, **fields)
|
|
|
|
-
|
|
|
|
- progress[-1]["status"] = "done"
|
|
|
|
- yield template("loading", progress=progress)
|
|
|
|
-
|
|
|
|
- if data.empty:
|
|
|
|
- raise abort(404, f"No data for {fields}")
|
|
|
|
-
|
|
|
|
- progress.append({ "name": "Loading chart", "status": ""})
|
|
|
|
- yield template("loading", progress=progress)
|
|
|
|
-
|
|
|
|
- pivot = data.pivot_table(index=['ts_raw',], columns=['product',], values=['$/unit'], aggfunc='mean')
|
|
|
|
- pivot.columns = pivot.columns.droplevel()
|
|
|
|
-
|
|
|
|
- sns.set(style="darkgrid", palette='pastel', context="talk")
|
|
|
|
- plt.style.use("dark_background")
|
|
|
|
- plt.rcParams.update({"grid.linewidth":0.1, "grid.alpha":0.5})
|
|
|
|
- plt.figure(figsize=[16, 9], layout="tight")
|
|
|
|
- xlabel='Time'
|
|
|
|
- ylabel=f'$ / {unit}'
|
|
|
|
- if pivot.columns.size > 50:
|
|
|
|
- ax = sns.scatterplot(data=pivot, markers=True)
|
|
|
|
- else:
|
|
|
|
- ax = sns.lineplot(data=pivot, markers=True)
|
|
|
|
- legend = plt.figlegend(
|
|
|
|
- loc='upper center', ncol=6,
|
|
|
|
- title_fontsize="14", fontsize="12", labelcolor='#a0a0a0',
|
|
|
|
- framealpha=0.5
|
|
|
|
- )
|
|
|
|
- legend.set_title(title="Products")
|
|
|
|
- ax.legend().set_visible(False)
|
|
|
|
-
|
|
|
|
- ax.set_xlabel(xlabel, color='#a0a0a0', fontsize="14")
|
|
|
|
- ax.set_ylabel(ylabel, color='#a0a0a0', fontsize="14")
|
|
|
|
- ax.axes.tick_params(colors='#a0a0a0', labelsize="12", which='both')
|
|
|
|
- for _, spine in ax.spines.items():
|
|
|
|
- spine.set_color('#a0a0a0')
|
|
|
|
-
|
|
|
|
- progress[-1]["status"] = "done"
|
|
|
|
- yield template("loading", progress=progress)
|
|
|
|
-
|
|
|
|
- progress.append({ "name": "Rendering chart", "status": ""})
|
|
|
|
- yield template("loading", progress=progress)
|
|
|
|
-
|
|
|
|
- f = StringIO()
|
|
|
|
- plt.savefig(f, format='svg')
|
|
|
|
- form = get_form(path.split('/')[-1], 'get', get_filter(request.query, allow=PARAMS), data)
|
|
|
|
-
|
|
|
|
- progress[-1]["status"] = "done"
|
|
|
|
- yield template("loading", progress=progress)
|
|
|
|
-
|
|
|
|
- resp = lambda: template("trend", form=form, svg=f.getvalue())
|
|
|
|
-
|
|
|
|
- except HTTPError as e:
|
|
|
|
- resp = lambda exception=e: exception
|
|
|
|
-
|
|
|
|
- finally:
|
|
|
|
- conn.commit()
|
|
|
|
-
|
|
|
|
- yield from iter(resp, lambda started=time(): time() - started > 600)
|
|
|
|
-
|
|
|
|
|
|
|
|
@route('/grocery/groups')
|
|
@route('/grocery/groups')
|
|
def groups():
|
|
def groups():
|