cache saves to file - WIP

app/rest/Cache.py

@@ -3,10 +3,44 @@
 # All rights reserved
 #
 # THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+import os
 from typing import Dict
+
+from .hash_util import blake, blake_file, bytes_to_base32, bytes_to_hash, hash_to_base32
 from .CachedLoadingPage import CachedLoadingPage
 
+def delete_page(name: str, root: str = 'app/rest/static/files'):
+    directory = f'{root}/{name}'
+    try:
+        os.remove(f'{directory}/{name}.file')
+    except FileNotFoundError:
+        pass
+
+def save_page(name: str, content: bytes, tool: str, root='app/rest/static/files') -> str:
+    directory = f'{root}/{name}'
+    try:
+        os.mkdir(directory, mode=0o700, dir_fd=None)
+    except FileExistsError:
+        pass
+
+    fd = os.open(f'{directory}/{name}.file', os.O_WRONLY | os.O_TRUNC | os.O_CREAT, 0o600)
+    with open(fd, "wb") as f:
+        f.write(content)
+
+def get_page(name: str, root: str = 'app/rest/static/files') -> str:
+    directory = f'{root}/{name}'
+
+    # todo: if page is stale, delete it
+
+    # todo: store file hash and validate it
+    fd = os.open(f'{directory}/{name}.file', os.O_RDONLY, 0o600)
+    with open(fd, "rb") as f:
+        f.seek(0)
+        page = f.read()
+    return page.decode('utf-8')
+
 def key_to_hash(key):
+
     if isinstance(key, tuple):
         orig, _hash = key
     else:
@@ -16,15 +50,20 @@ def key_to_hash(key):
             orig, _hash = key, None
 
     if None not in (orig, _hash):
-        if hash(orig) != _hash:
+        if get_hash(orig) != _hash:
             raise KeyError(f"Invalid key: {key}")
 
         return _hash
 
     if (_hash, orig) is (None, None):
         raise KeyError(f"Invalid key: {key}")
+    
+    return get_hash(orig) if _hash is None else _hash
+
+def get_hash(key):
+    _bytes = blake(key.encode('utf-8'), person='grocery'.encode('utf-8'))
+    return bytes_to_hash(_bytes)
 
-    return hash(orig) if _hash is None else _hash
 
 class Cache:
     def __init__(self, limit) -> None:
@@ -32,26 +71,39 @@ class Cache:
         self._limit = limit
 
     def __delitem__(self, key):
-        self._cache.pop(key_to_hash(key), None)
+        key = key_to_hash(key)
+        self._cache.pop(key, None)
+        delete_page(hash_to_base32(key))
 
     def __getitem__(self, key):
-        return self.get(key_to_hash(key))
+        return self.get(key)
 
     def __setitem__(self, key, value):
         self._cache[key_to_hash(key)] = value
 
     def get(self, key: str) -> CachedLoadingPage:
         key = key_to_hash(key)
+
         if key not in self._cache:
-            return None
+            try:
+                existing = get_page(hash_to_base32(key))
+            except:
+                return None
+            
+            return self.add(key, CachedLoadingPage(existing, lambda q: q.put(None), incremental=True))
         
         page = self._cache[key]
         if page.stale:
             del self._cache[key]
+            delete_page(key)
             return None
 
         if not page.loaded:
             page.update()
+        
+        if page.loaded:
+            content = ''.join(page.value) if isinstance(page.value, list) else page.value
+            save_page(hash_to_base32(key), content.encode('utf-8'), tool='grocery')
 
         return page
 
@@ -72,7 +124,4 @@ class Cache:
         return page
     
     def remove(self, key: str):
-        key = key_to_hash(key)
-        if key in self._cache:
-            del self._cache[key]
-
+        del self[key]

app/rest/hash_util.py

@@ -30,11 +30,11 @@ def blake(data: bytes, person: bytes = None) -> bytes:
         person=person
     ).digest()
 
-def blake_file(path: str, person: bytes = None, root: str ='app/rest/static') -> bytes:
+def blake_file(path: str, person: bytes = None, root: str ='rest/static') -> bytes:
     fd = os.open(f'{root}/{path}', os.O_RDONLY, 0o600)
     with open(fd, "rb") as f:
         f.seek(0)
-        _blake = blake2b(usedforsecurity=False, digest_size=DIGEST_SIZE_BYTES, person='upload'.encode('utf-8'))
+        _blake = blake2b(usedforsecurity=False, digest_size=DIGEST_SIZE_BYTES, person=person)
         while f.peek(1):
             _blake.update(f.read(1024))
     return _blake.digest()
@@ -100,7 +100,7 @@ def fix_padding(f):
     return wrap
 
 def normalize_base32(_b32: str):
-    return _b32.upper().zfill(DIGEST_SIZE_BYTES*8//5+1)
+    return b32.encode(b32.decode(_b32)).upper().zfill(DIGEST_SIZE_BYTES*8//5+1)
 
 def add_padding_base32(f):
     def wrap(*args, **kwargs):

app/rest/pyapi.py

@@ -51,7 +51,6 @@ def trend(key: str, forms: FormsDict, cache: Cache):
     
     for i in iter_page(page):
         yield i
-        sleep(0.5)
 
 
 def iter_page(page):