config/clip.json

@@ -2,5 +2,8 @@
   "site_prefix": "/clip",
   "max_database_pool_connections": 16,
   "database_url": "sqlite:///var/sqlpage/clip.db",
+  "sqlite_extensions": [
+	  "/usr/lib/sha1.so"
+  ],
   "compress_responses": false
 }

config/code.json

@@ -2,5 +2,8 @@
   "site_prefix": "/code",
   "max_database_pool_connections": 16,
   "database_url": "sqlite:///var/sqlpage/code.db",
+  "sqlite_extensions": [
+	  "/usr/lib/sha1.so"
+  ],
   "compress_responses": false
 }

config/goto.json

@@ -2,5 +2,8 @@
   "site_prefix": "/goto",
   "max_database_pool_connections": 16,
   "database_url": "sqlite:///var/sqlpage/goto.db",
+  "sqlite_extensions": [
+	  "/usr/lib/sha1.so"
+  ],
   "compress_responses": false
 }

docker-compose.yml

@@ -13,6 +13,7 @@ services:
       - ./config/templates:/etc/sqlpage/templates
       - ./config/clip.json:/etc/sqlpage/sqlpage.json
       - ./data/clip.db:/var/sqlpage/clip.db
+      - ./sha1.so:/usr/lib/sha1.so:ro
     expose:
       - 8080
     networks:
@@ -34,6 +35,7 @@ services:
       - ./config/templates:/etc/sqlpage/templates
       - ./config/goto.json:/etc/sqlpage/sqlpage.json
       - ./data/goto.db:/var/sqlpage/goto.db
+      - ./sha1.so:/usr/lib/sha1.so:ro
     expose:
       - 8080
     networks:
@@ -78,6 +80,7 @@ services:
       - ./config/templates:/etc/sqlpage/templates
       - ./config/code.json:/etc/sqlpage/sqlpage.json
       - ./data/code.db:/var/sqlpage/code.db
+      - ./sha1.so:/usr/lib/sha1.so:ro
     expose:
       - 8080
     networks:

rest/hash_util.py

@@ -1,181 +0,0 @@
-from hashlib import blake2b, shake_128, md5, sha256, sha1
-import os
-from base64 import b64encode, b64decode, b85encode, b85decode
-import base32_lib as b32
-
-DIGEST_SIZE_BYTES = 3
-DIGEST_SIZE_BITMASK = 0xffffff
-DIGEST_SIZE_SIGNED_TO_UNSIGNED_BITMASK = 0x1ffffff
-DIGEST_SIZE_SIGNED_TO_UNSIGNED_BIT = 0x1000000
-DIGEST_SIZE_NIBBLES = DIGEST_SIZE_BYTES * 2
-B64ALTCHARS = b'.-'
-B32REGEX = r'[0-9a-tv-zA-TV-Z]'
-
-def sha1hash(data: str):
-    return sha1(data.encode("utf-8"), usedforsecurity=False).hexdigest()[:DIGEST_SIZE_BYTES]
-
-def sha256hash(data: str):
-    return sha256(data.encode("utf-8"), usedforsecurity=False).hexdigest()[:DIGEST_SIZE_BYTES]
-
-def md5hash(data: str):
-    return md5(data.encode("utf-8"), usedforsecurity=False).hexdigest()[:DIGEST_SIZE_BYTES]
-
-def shake(data: str):
-    return shake_128(data.encode("utf-8"), usedforsecurity=False).hexdigest(DIGEST_SIZE_BYTES)
-
-def blake(data: bytes, person: bytes = None) -> bytes:
-    return blake2b(
-        data,
-        usedforsecurity=False,
-        digest_size=DIGEST_SIZE_BYTES,
-        person=person
-    ).digest()
-
-def blake_file(path: str, person: bytes = None, root: str ='rest/static') -> bytes:
-    fd = os.open(f'{root}/{path}', os.O_RDONLY, 0o600)
-    with open(fd, "rb") as f:
-        f.seek(0)
-        _blake = blake2b(usedforsecurity=False, digest_size=DIGEST_SIZE_BYTES, person=person)
-        while f.peek(1):
-            _blake.update(f.read(1024))
-    return _blake.digest()
-        
-
-def python(data: str):
-    return hash(data)
-
-def normalize_hash(_hash: int) -> int:
-    #hex = hash_to_hex(_hash)
-    #return int(hex, 16)
-    #_bytes = _hash.to_bytes(8, byteorder='big', signed=True)
-    #return bytes_to_hash(_bytes)
-    return _hash & DIGEST_SIZE_BITMASK
-
-def normalize_bytes(_bytes: bytes) -> bytes:
-    return (b'\x00' * DIGEST_SIZE_BYTES + _bytes)[-DIGEST_SIZE_BYTES:]
-
-def normalize_hex(_hex: str) -> str:
-    #_bytes = hex_to_bytes(hex)
-    #return _bytes.hex()
-    return _hex.zfill(DIGEST_SIZE_NIBBLES)[-DIGEST_SIZE_NIBBLES:]
-
-def hex_to_bytes(_hex: str) -> bytes:
-    _bytes = bytes.fromhex(_hex.zfill(DIGEST_SIZE_NIBBLES))
-    return normalize_bytes(_bytes)
-
-def bytes_to_hex(_bytes: bytes) -> str:
-    return normalize_bytes(_bytes).hex()
-
-def hash_to_bytes(_hash: int) -> bytes:
-    _bytes = _hash.to_bytes(8, byteorder='big', signed=True)
-    return normalize_bytes(_bytes)
-
-def bytes_to_hash(_bytes: bytes) -> int:
-    norm = normalize_bytes(_bytes)
-    return int.from_bytes(norm, byteorder='big', signed=False)
-
-def hash_to_hex(_hash: int) -> str:
-    #return hash_to_bytes(_hash).hex()
-    #return normalize_hex(
-    #return f"{_hash + (1 << 64):x}"[-4:]
-    #return hex(_hash + (1<<64))[2:][-4:]
-    #return f"{_hash & 0xffff:04x}"
-    return hex((_hash|DIGEST_SIZE_SIGNED_TO_UNSIGNED_BIT) & DIGEST_SIZE_SIGNED_TO_UNSIGNED_BITMASK)[3:]
-
-def hex_to_hash(_hex: str) -> int:
-    #_bytes = bytes.fromhex(hex.zfill(4))
-    #return bytes_to_hash(_bytes)
-    #return int(normalize_hex(hex), 16)
-    return int(_hex, 16) & DIGEST_SIZE_BITMASK
-
-def remove_padding(f):
-    def wrap(*args, **kwargs):
-        return f(*args, **kwargs).split('=')[0]
-    return wrap
-
-def fix_padding(f):
-    def wrap(_b64, *args, **kwargs):
-        pad = (4 - len(_b64)) % 4
-        fixed = _b64 + '='*pad
-        return f(fixed, *args, **kwargs)
-    return wrap
-
-def normalize_base32(_b32: str):
-    return b32.encode(b32.decode(_b32)).upper().zfill(DIGEST_SIZE_BYTES*8//5+1)
-
-def add_padding_base32(f):
-    def wrap(*args, **kwargs):
-        return normalize_base32(f(*args, **kwargs))
-    return wrap
-
-@remove_padding
-def hash_to_base64(_hash: int) -> str:
-    return b64encode(hash_to_bytes(_hash), altchars=B64ALTCHARS).decode("utf-8")
-
-@remove_padding
-def hex_to_base64(_hex: str) -> str:
-    return b64encode(hex_to_bytes(_hex), altchars=B64ALTCHARS).decode("utf-8")
-
-@remove_padding
-def bytes_to_base64(_bytes: str) -> str:
-    return b64encode(normalize_bytes(_bytes), altchars=B64ALTCHARS).decode("utf-8")
-
-@fix_padding
-def base64_to_hash(_b64: str) -> str:
-    return bytes_to_hash(b64decode(_b64, altchars=B64ALTCHARS))
-
-@fix_padding
-def base64_to_hex(_b64: str) -> str:
-    return bytes_to_hex(b64decode(_b64, altchars=B64ALTCHARS))
-
-@fix_padding
-def base64_to_bytes(_b64: str) -> str:
-    return normalize_bytes(b64decode(_b64, altchars=B64ALTCHARS))
-
-#@remove_padding
-def hash_to_base85(_hash: int) -> str:
-    return b85encode(hash_to_bytes(_hash)).decode("utf-8")
-
-#@remove_padding
-def hex_to_base85(_hex: str) -> str:
-    return b85encode(hex_to_bytes(_hex)).decode("utf-8")
-
-#@remove_padding
-def bytes_to_base85(_bytes: str) -> str:
-    return b85encode(normalize_bytes(_bytes)).decode("utf-8")
-
-#@fix_padding
-def base85_to_hash(_b64: str) -> str:
-    return bytes_to_hash(b85decode(_b64))
-
-#@fix_padding
-def base85_to_hex(_b64: str) -> str:
-    return bytes_to_hex(b85decode(_b64))
-
-#@fix_padding
-def base85_to_bytes(_b64: str) -> str:
-    return normalize_bytes(b85decode(_b64))
-
-@add_padding_base32
-def hash_to_base32(_hash: int) -> str:
-    return b32.encode(_hash & DIGEST_SIZE_BITMASK)
-
-@add_padding_base32
-def hex_to_base32(_hex: str) -> str:
-    return b32.encode(hex_to_hash(_hex))
-
-@add_padding_base32
-def bytes_to_base32(_bytes: bytes) -> str:
-    return b32.encode(bytes_to_hash(_bytes))
-
-#@fix_padding
-def base32_to_hash(_b64: str) -> str:
-    return b32.decode(_b64)
-
-#@fix_padding
-def base32_to_hex(_b64: str) -> str:
-    return hash_to_hex(base32_to_hash(_b64))
-
-#@fix_padding
-def base32_to_bytes(_b64: str) -> str:
-    return hash_to_bytes(base32_to_hash(_b64))

rest/pyapi.py

@@ -4,60 +4,85 @@
 #
 # THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
 
+import base32_lib as b32
 from bottle import (
     route, request, response,
     redirect, abort, static_file,
     HTTPResponse,
 )
 from linkpreview import link_preview
-
-from .hash_util import B32REGEX, normalize_base32, blake, bytes_to_base32
 from .qr import get_qr_code
 from .bar import get_bar_code
 from json import dumps, load
 
+DIGEST_SIZE_BYTES = 3
+DIGEST_SIZE_BITMASK = 0xffffff
+B32REGEX = r'[0-9a-tv-zA-TV-Z]'
+
 SCHEME = "https://"
 HOST = ""
 DOMAIN = "shandan.one"
 PORT = ""
 LOCATION = SCHEME + (f"{HOST}." if HOST else "") + DOMAIN + (f":{PORT}" if PORT else "")
 
+def normalize_base32(_b32: str):
+    return b32.encode(b32.decode(_b32)).upper().zfill(DIGEST_SIZE_BYTES*8//5+1)
+
+def add_padding_base32(f):
+    def wrap(*args, **kwargs):
+        return normalize_base32(f(*args, **kwargs))
+    return wrap
+
+@add_padding_base32
+def hex_to_base32(_hex: str) -> str:
+    return b32.encode(hex_to_hash(_hex))
+
+def hex_to_hash(_hex: str) -> int:
+    return int(_hex, 16) & DIGEST_SIZE_BITMASK
+
+@route('/<any>/', method='GET')
+def redirect_trailing_slash(any): return redirect(f'/{any}')
+
+@route('/static/<filename:path>')
+def send_static(filename): return static_file(filename, root='rest/static')
+
+@route('/<route:re:(clip|goto|upload)>/normalize', method=['GET'])
+def normalize(route):
+    response.content_type = 'application/json'
+    _hash = request.params.hash
+    normalized = _hash and normalize_base32(_hash)
+    return dumps({ 'i': _hash, 'o': normalized })
+
+@route(f'/<route:re:(clip|goto|upload|code)>/<hash:re:{B32REGEX}{{1,5}}>', method='GET')
+def get_data(route, hash):
+    hash = hash and normalize_base32(hash)
+    return redirect(f'/{route}/?hash={hash}&go=true')
+
 @route('/<route:re:(clip|goto|upload|code)>/meta', method=['POST'])
 def get_meta(route):
     response.content_type = 'application/json'
     body = load(request.body)
-    person = route
-    if route == 'upload':
-        data = None
-        _bytes = bytes.fromhex(body['data'])
-    elif route == 'code':
-        data = dumps(body['data'], sort_keys=True).encode('utf-8')
-        _bytes = blake(data, person = person and person.encode('utf-8'))
-    else:
-        data = body['data'].encode('utf-8')
-        _bytes = blake(data, person = person and person.encode('utf-8'))
-
-    hash = bytes_to_base32(_bytes)
+    data = body['data']
+    hash = hex_to_base32(body['sha1'])
     fallback = f'https://shandan.one/{route}/{hash}'
 
     preview = None
     if route == 'goto':
-        link = data.decode('utf-8')
         try:
-            page = link_preview(link, parser="lxml")
+            page = link_preview(data, parser="lxml")
             preview = {
                 'title': page.title,
                 'img': page.absolute_image,
                 'domain': page.site_name,
-                'link': link,
+                'link': data,
             }
         except:
             pass
     elif route == 'code':
-        if body['data']['format'] == 'QR_CODE':
-            preview = b64encode(get_qr_code(body['data']['content'], err_lvl=body['data']['errorCorrectionLevel']))
+        if data['format'] == 'QR_CODE':
+            preview = b64encode(get_qr_code(data['content'], err_lvl=data['errorCorrectionLevel']))
         else:
-            preview = b64encode(get_bar_code(body['data']))
+            preview = b64encode(get_bar_code(data))
         preview = preview.decode('utf-8')
 
     qr = None
@@ -69,24 +94,3 @@ def get_meta(route):
         'qr': qr,
         'preview': preview,
     })
-
-@route('/<route:re:(clip|goto|upload)>/normalize', method=['GET'])
-def normalize(route):
-    _hash = request.params.hash
-    response.content_type = 'application/json'
-    return dumps({
-        'i': _hash,
-        'o': normalize_base32(_hash) if _hash else None,
-    })
-
-@route('/static/<filename:path>')
-def send_static(filename):
-    return static_file(filename, root='rest/static')
-
-@route(f'/<route:re:(clip|goto|upload|code)>/<hash:re:{B32REGEX}{{1,5}}>', method='GET')
-def get_clip(route, hash):
-    hash = hash and normalize_base32(hash)
-    return redirect(f'/{route}/?hash={hash}&go=true')
-
-@route('/<any>/', method='GET')
-def redirect_trailing_slash(any): return redirect(f'/{any}')

site/clip/save.sql

@@ -5,4 +5,3 @@ UPDATE SET
   created = excluded.created,
   qr = excluded.qr
 WHERE excluded.created > clip.created;
-SELECT 'dynamic' AS component, sqlpage.run_sql('sqlpage/link.sql') AS properties;

site/goto/save.sql

@@ -6,4 +6,3 @@ UPDATE SET
   qr = excluded.qr,
   preview = excluded.preview
 WHERE excluded.created > goto.created;
-SELECT 'dynamic' AS component, sqlpage.run_sql('sqlpage/link.sql') AS properties;

site/shared/save.sql

@@ -4,8 +4,19 @@ SET ":request" = json_object(
     'headers', json_object(),
     'timeout_ms', 15000,
     'body', json_object(
-        'data', CASE :tool
+        'sha1', CASE :tool
+            WHEN 'code' THEN ''||sha1((SELECT
+                json_group_object(q.key, q.value) FROM (
+                    SELECT *
+                    FROM json_each(:content) AS j
+                    ORDER BY j.key
+                ) q))
+            WHEN 'upload' THEN :content
+            ELSE ''||sha1(:content)
+        END,
+	'data', CASE :tool
             WHEN 'code' THEN json(:content)
+            WHEN 'upload' THEN NULL
             ELSE :content
         END
     )
@@ -15,3 +26,5 @@ SET ":qr" = :meta->>'qr';
 SET ":hash" = :meta->>'hash';
 SET ":preview" = :meta->>'preview';
 SELECT 'dynamic' AS component, sqlpage.run_sql('save.sql') AS properties;
+SELECT 'dynamic' AS component, sqlpage.run_sql('sqlpage/link.sql') AS properties
+WHERE :tool <> 'code';

site/shared/validate.sql

@@ -10,7 +10,7 @@ SET ":content" = (
   SELECT content
   FROM upload
   WHERE hash = :hash AND :tool = 'upload'
-  AND substr(:mime_type, 0, instr(:mime_type, '/')) = 'image'
+  AND substr(mime, 0, instr(mime, '/')) = 'image'
 );
 
 SET ":qr" = (

site/upload/save.sql

@@ -12,5 +12,3 @@ WHERE excluded.created > upload.created
 AND NULL NOT IN (:file_name, :mime_type, :qr);
 
 DELETE FROM upload_temp WHERE rowid = :rowid;
-
-SELECT 'dynamic' AS component, sqlpage.run_sql('sqlpage/link.sql') AS properties;