Home Explore Help
Sign In
gogsadmin
/
grocery-manager
1
0
Fork 0
Files Issues 1 Pull Requests 0 Wiki
Browse Source

try incremenally yielding the page

Daniel Sheffield 1 year ago
parent
da126b6105
commit
42031897d0
8 changed files with 67 additions and 52 deletions
Unified View Show Diff Stats
  1. 9 5
      app/rest/Cache.py
  2. 8 4
      app/rest/CachedLoadingPage.py
  3. 19 6
      app/rest/pyapi.py
  4. 9 4
      app/rest/templates/loading.tpl
  5. 1 15
      app/rest/templates/progress.tpl
  6. 5 2
      app/rest/trend.py
  7. 15 15
      test/rest/test_Cache.py
  8. 1 1
      test/rest/test_CachedLoadingPage.py

+ 9 - 5
app/rest/Cache.py
View File 

@@ -40,7 +40,7 @@ class Cache:
 
     def __setitem__(self, key, value):
 
     def __setitem__(self, key, value):
 
         self._cache[key_to_hash(key)] = value
 
         self._cache[key_to_hash(key)] = value
 
 
 
-    def get(self, key: str) -> str:
 
+    def get(self, key: str) -> CachedLoadingPage:
 
         key = key_to_hash(key)
 
         key = key_to_hash(key)
 
         if key not in self._cache:
 
         if key not in self._cache:
 
             return None
 
             return None
 
@@ -49,8 +49,12 @@ class Cache:
 
         if page.stale:
 
         if page.stale:
 
             del self._cache[key]
 
             del self._cache[key]
 
             return None
 
             return None
 
-        return page.value if page.loaded else page.update()
 
-    
+
 
+        if not page.loaded:
 
+            page.update()
 
+
 
+        return page
 
+
 
     def _enforce_limit(self, limit):
 
     def _enforce_limit(self, limit):
 
         for idx, (_, k) in enumerate(sorted([
 
         for idx, (_, k) in enumerate(sorted([
 
                 (v.age, k) for k, v in self._cache.items()
 
                 (v.age, k) for k, v in self._cache.items()
 
@@ -61,11 +65,11 @@ class Cache:
 
         for k in [k for k, v in self._cache.items() if v.stale]:
 
         for k in [k for k, v in self._cache.items() if v.stale]:
 
             del self._cache[k]
 
             del self._cache[k]
 
     
     
-    def add(self, key: str, page: CachedLoadingPage) -> str:
 
+    def add(self, key: str, page: CachedLoadingPage) -> CachedLoadingPage:
 
         self._clear_stale()
 
         self._clear_stale()
 
         self._enforce_limit(self._limit)
 
         self._enforce_limit(self._limit)
 
         self._cache[key_to_hash(key)] = page
 
         self._cache[key_to_hash(key)] = page
 
-        return page.value
 
+        return page
 
     
     
     def remove(self, key: str):
 
     def remove(self, key: str):
 
         key = key_to_hash(key)
 
         key = key_to_hash(key)

+ 8 - 4
app/rest/CachedLoadingPage.py
View File 

@@ -6,19 +6,20 @@
 
 from queue import Queue, Empty
 
 from queue import Queue, Empty
 
 from time import time
 
 from time import time
 
 from threading import Lock
 
 from threading import Lock
 
-from typing import Callable
 
+from typing import Callable, Union
 
 
 
 class CachedLoadingPage():
 
 class CachedLoadingPage():
 
     
     
     value: str
 
     value: str
 
 
 
-    def __init__(self, initial_value: str, provider: Callable[[Queue], None]):
 
+    def __init__(self, initial_value: Union[str, list], provider: Callable[[Queue], None], incremental: bool = True):
 
         self._created = time()
 
         self._created = time()
 
         self._queue = Queue()
 
         self._queue = Queue()
 
         self._loaded = False
 
         self._loaded = False
 
         self.value = initial_value
 
         self.value = initial_value
 
         self._lock = Lock()
 
         self._lock = Lock()
 
         self.provider = provider
 
         self.provider = provider
 
+        self.incremental = incremental
 
 
 
     @property
 
     @property
 
     def age(self) -> float:
 
     def age(self) -> float:
 
@@ -47,7 +48,7 @@ class CachedLoadingPage():
 
         self.provider(self.queue)
 
         self.provider(self.queue)
 
         self.provider = None
 
         self.provider = None
 
 
 
-    def update(self) -> str:
 
+    def update(self) -> Union[str, list]:
 
         if not self._lock.acquire(blocking=True, timeout=0.5):
 
         if not self._lock.acquire(blocking=True, timeout=0.5):
 
             return self.value
 
             return self.value
 
         try:
 
         try:
 
@@ -57,7 +58,10 @@ class CachedLoadingPage():
 
                 self._queue.task_done()
 
                 self._queue.task_done()
 
                 self._set_loaded(True)
 
                 self._set_loaded(True)
 
             else:
 
             else:
 
-                self.value = item
 
+                if self.incremental:
 
+                    self.value.append(item)
 
+                else:
 
+                    self.value = item
 
                 self.queue.task_done()
 
                 self.queue.task_done()
 
         except Empty:
 
         except Empty:
 
             pass
 
             pass

+ 19 - 6
app/rest/pyapi.py
View File 

@@ -36,19 +36,32 @@ def send_static(filename):
 
 @poison(cache=Cache(10))
 
 @poison(cache=Cache(10))
 
 @normalize
 
 @normalize
 
 def trend(key: str, forms: FormsDict, cache: Cache):
 
 def trend(key: str, forms: FormsDict, cache: Cache):
 
-    page = cache[key]
 
-    if page:
 
-        return page
 
-
 
     _, _, path, *_ = request.urlparts
 
     _, _, path, *_ = request.urlparts
 
 
 
-    return cache.add(key, CachedLoadingPage(
 
-        template("loading", progress=[]),
 
+    page = cache[key]
 
+    if page is None:
 
+        page = cache.add(key, CachedLoadingPage(
 
+        [],
 
         lambda queue: Thread(target=worker.trend, args=(
 
         lambda queue: Thread(target=worker.trend, args=(
 
             queue, conn, path, forms
 
             queue, conn, path, forms
 
         )).start()
 
         )).start()
 
     ))
 
     ))
 
 
 
+    # copy first to avoid races
 
+    resp = list(page.value)
 
+    pos = len(resp)
 
+    yield ''.join(resp)
 
+    
+    while not page.loaded:
 
+        # all changes since last yield
 
+        resp = list(page.value[pos:])
 
+        pos = pos + len(resp)
 
+        yield ''.join(resp)
 
+    
+    # possibly have not yielded the entire page
 
+    if pos < len(page.value):
 
+        yield ''.join(page.value[pos:])
 
+
 
 
 
 @route('/grocery/groups', method=['GET', 'POST'])
 
 @route('/grocery/groups', method=['GET', 'POST'])
 
 @poison(cache=Cache(10))
 
 @poison(cache=Cache(10))

+ 9 - 4
app/rest/templates/loading.tpl
View File 

@@ -1,3 +1,6 @@
 
+% setdefault("start", False)
 
+% setdefault("end", False)
 
+% if start:
 
 <html>
 
 <html>
 
   <head>
 
   <head>
 
     <meta name="viewport" content="width=device-width, initial-scale=1"/>
 
     <meta name="viewport" content="width=device-width, initial-scale=1"/>
 
@@ -19,8 +22,10 @@ body {
 
   </head>
 
   </head>
 
   <body>
 
   <body>
 
     <div class="loader-container">
 
     <div class="loader-container">
 
-    % include('progress', **progress)
 
-    </div>
 
-    <meta http-equiv="Refresh" content="0.2;" />
 
+% end
 
+% include('progress', **progress)
 
+% if end:
 
+    <meta http-equiv="Refresh" content="0;" />
 
   </body>
 
   </body>
 
-</html>
 
+</html>
 
+% end

+ 1 - 15
app/rest/templates/progress.tpl
View File 

@@ -1,15 +1 @@
 
-<style>
 
-div.progress {
 
-    display:grid;
 
-    grid-template-columns: max-content max-content;
 
-    grid-gap:5px;
 
-}
 
-div.progress label       { text-align:left; }
 
-div.progress label:after { content: {{"..." if not complete else ""}}; }
 
-</style>
 
-<p>
 
-  <div class="progress">
 
-    <label for="loading">{{stage}}</label>
 
-    <progress id="loading" value="{{percent}}" max="100"></progress>
 
-  </div>
 
-</p>
 
+<p>{{stage}} ... </p>

+ 5 - 2
app/rest/trend.py
View File 

@@ -64,7 +64,7 @@ def trend_internal(conn: Connection[TupleRow], path: str, query: FormsDict):
 
                 return
 
                 return
 
 
 
             progress.update({ "stage": "Querying database", "percent": "10"})
 
             progress.update({ "stage": "Querying database", "percent": "10"})
 
-            yield template("loading", progress=progress)
 
+            yield template("loading", start=True, progress=progress)
 
             data = get_data(query_manager, **fields)
 
             data = get_data(query_manager, **fields)
 
             
             
             if data.empty:
 
             if data.empty:
 
@@ -123,7 +123,7 @@ def trend_internal(conn: Connection[TupleRow], path: str, query: FormsDict):
 
             f = StringIO()
 
             f = StringIO()
 
             plt.savefig(f, format='svg')
 
             plt.savefig(f, format='svg')
 
             progress.update({ "stage": "Done", "percent": "100", "complete": True})
 
             progress.update({ "stage": "Done", "percent": "100", "complete": True})
 
-            yield template("loading", progress=progress)
 
+            yield template("loading", end=True, progress=progress)
 
             
             
             organic = BOOLEAN.get(query.organic, None)
 
             organic = BOOLEAN.get(query.organic, None)
 
             action = path.split('/')[-1]
 
             action = path.split('/')[-1]
 
@@ -131,5 +131,8 @@ def trend_internal(conn: Connection[TupleRow], path: str, query: FormsDict):
 
             
             
             yield template("trend", form=form, svg=f.getvalue())
 
             yield template("trend", form=form, svg=f.getvalue())
 
 
 
+    except:
 
+        yield abort(500, f"Failed to render page")
 
+
 
     finally:
 
     finally:
 
         conn.commit()
 
         conn.commit()

+ 15 - 15
test/rest/test_Cache.py
View File 

@@ -18,7 +18,7 @@ def cache():
 
 def test_add(cache: Cache):
 
 def test_add(cache: Cache):
 
     val = 'test-cached-value'
 
     val = 'test-cached-value'
 
     key = 'test-key'
 
     key = 'test-key'
 
-    assert cache.add(key, CachedLoadingPage(val, lambda _: None)) == val
 
+    assert cache.add(key, CachedLoadingPage(val, lambda _: None, incremental=False)).value == val
 
 
 
 def test_get(cache: Cache):
 
 def test_get(cache: Cache):
 
     val = 'test-cached-value'
 
     val = 'test-cached-value'
 
@@ -26,47 +26,47 @@ def test_get(cache: Cache):
 
 
 
     assert cache.get(key) is None
 
     assert cache.get(key) is None
 
 
 
-    assert cache.add(key, CachedLoadingPage(val, lambda q: q.put('next-val'))) == val
 
-    assert cache.get(key) == 'next-val'
 
+    assert cache.add(key, CachedLoadingPage(val, lambda q: q.put('next-val'), incremental=False)).value == val
 
+    assert cache.get(key).value == 'next-val'
 
 
 
 def test_remove(cache: Cache):
 
 def test_remove(cache: Cache):
 
     val = 'test-cached-value'
 
     val = 'test-cached-value'
 
     key = 'test-key'
 
     key = 'test-key'
 
 
 
     assert cache.get(key) is None
 
     assert cache.get(key) is None
 
-    assert cache.add(key, CachedLoadingPage(val, lambda q: q.put('next-val'))) == val
 
+    assert cache.add(key, CachedLoadingPage(val, lambda q: q.put('next-val'), incremental=False)).value == val
 
     cache.remove(key)
 
     cache.remove(key)
 
     assert cache.get(key) is None
 
     assert cache.get(key) is None
 
 
 
 def test_enforce_limit(cache: Cache):
 
 def test_enforce_limit(cache: Cache):
 
     val = 'test-cached-value'
 
     val = 'test-cached-value'
 
     key = 'test-key'
 
     key = 'test-key'
 
-    assert cache.add(key, CachedLoadingPage(val, lambda _: None)) == val
 
+    assert cache.add(key, CachedLoadingPage(val, lambda _: None, incremental=False)).value == val
 
     # adding more exceeds limit
 
     # adding more exceeds limit
 
-    assert cache.add('other', CachedLoadingPage(val, lambda _: None)) == val
 
+    assert cache.add('other', CachedLoadingPage(val, lambda _: None, incremental=False)).value == val
 
     assert cache.get(key) is None
 
     assert cache.get(key) is None
 
-    assert cache.get('other') == val
 
+    assert cache.get('other').value == val
 
 
 
 def test_clean_stale(cache: Cache):
 
 def test_clean_stale(cache: Cache):
 
     val = 'test-cached-value'
 
     val = 'test-cached-value'
 
     key = 'test-key'
 
     key = 'test-key'
 
-    page = CachedLoadingPage(val, lambda _: None)
 
+    page = CachedLoadingPage(val, lambda _: None, incremental=False)
 
     page._created = time() - 10*60
 
     page._created = time() - 10*60
 
     # add stale page
 
     # add stale page
 
-    assert cache.add(key, page) == val
 
+    assert cache.add(key, page).value == val
 
     assert cache.get(key) is None
 
     assert cache.get(key) is None
 
     
     
-    page = CachedLoadingPage(val, lambda _: None)
 
-    assert cache.add(key, page) == val
 
+    page = CachedLoadingPage(val, lambda _: None, incremental=False)
 
+    assert cache.add(key, page).value == val
 
     # make page stale
 
     # make page stale
 
     page._created = time() - 10*60
 
     page._created = time() - 10*60
 
     assert cache.get(key) is None
 
     assert cache.get(key) is None
 
 
 
-    page = CachedLoadingPage(val, lambda _: None)
 
-    assert cache.add(key, page) == val
 
+    page = CachedLoadingPage(val, lambda _: None, incremental=False)
 
+    assert cache.add(key, page).value == val
 
     # make page stale
 
     # make page stale
 
     page._created = time() - 10*60
 
     page._created = time() - 10*60
 
     # stale page is rotated out on addition
 
     # stale page is rotated out on addition
 
-    assert cache.add('other', CachedLoadingPage(val, lambda _: None)) == val
 
+    assert cache.add('other', CachedLoadingPage(val, lambda _: None, incremental=False)).value == val
 
     assert cache.get(key) is None
 
     assert cache.get(key) is None
 
-    assert cache.get('other') == val
 
+    assert cache.get('other').value == val

+ 1 - 1
test/rest/test_CachedLoadingPage.py
View File 

@@ -15,7 +15,7 @@ from app.rest.CachedLoadingPage import (
 
 
 
 @fixture
 
 @fixture
 
 def cache():
 
 def cache():
 
-    return CachedLoadingPage("start", lambda _: None)
 
+    return CachedLoadingPage("start", lambda _: None, incremental=False)
 
 
 
 
 
 def test_get_age(cache: CachedLoadingPage):
 
 def test_get_age(cache: CachedLoadingPage):