move utils from grocery-manager repo into home-launcher repo

.gitignore

@@ -1,2 +1,3 @@
 media/
 .env
+__pycache__/

.gitmodules

@@ -6,4 +6,5 @@
 [submodule "grocery-transactions"]
 	path = grocery-transactions
 	url = /home/pi/git/grocery_transactions.git
-	branch = master
+	#branch = filter-tables
+	branch = limit-url-length

docker-compose.yml

@@ -60,6 +60,18 @@ services:
       - traefik
     restart: always
 
+  util-pyapi:
+    image: util-pyapi
+    hostname: util-pyapi
+    build:
+      context: .
+      dockerfile: rest/Dockerfile
+    expose:
+      - 6772
+    networks:
+      - traefik
+    restart: always
+
 networks:
   traefik:
     external: true

grocery-transactions

@@ -1 +1 @@
-Subproject commit 745ed5e02cfd74681b731baf149e50c15f4f37d9
+Subproject commit 470b53d1f1a44f60729d2b2140d7c70dbc5b0155

index.html

@@ -120,6 +120,54 @@
   </div>
 </div>
 <hr/>
+<div class="pure-g">
+  <div class="pure-u-1-2 pure-u-sm-1-2 pure-u-md-1-3 pure-u-lg-1-4 l-box">
+    <form action="/clip" id="clip" rel="external" method="get" target="_self">
+      <div class="card">
+        <button form="clip" style="color: darkgreen; border-color: darkgreen">
+          <div class="image">
+                  <a title="Nanodudek, CC BY-SA 3.0 &lt;https://creativecommons.org/licenses/by-sa/3.0&gt;, via Wikimedia Commons" href="https://commons.wikimedia.org/wiki/File:Clipboard_Symbol.svg"><img width="64" alt="Clipboard Symbol" src="https://upload.wikimedia.org/wikipedia/commons/thumb/9/9c/Clipboard_Symbol.svg/512px-Clipboard_Symbol.svg.png"></a>
+          </div>
+          <div class="text">
+            <h3>Clipboard</h3>
+            <p>Paste snippets to share via tiny URL</p>
+          </div>
+        </button>
+      </div>
+    </form>
+  </div>
+  <div class="pure-u-1-2 pure-u-sm-1-2 pure-u-md-1-3 pure-u-lg-1-4 l-box">
+    <form action="/goto" id="goto" rel="external" method="get" target="_self">
+      <div class="card">
+        <button form="goto" style="color: darkred; border-color: darkred">
+          <div class="image">
+            <a title="Mdowdell, CC BY-SA 3.0 &lt;https://creativecommons.org/licenses/by-sa/3.0&gt;, via Wikimedia Commons" href="https://commons.wikimedia.org/wiki/File:Chain_link_icon.png"><img width="64" alt="Chain link icon" src="https://upload.wikimedia.org/wikipedia/commons/thumb/e/ee/Chain_link_icon.png/512px-Chain_link_icon.png"></a>
+          </div>
+          <div class="text">
+            <h3>Short URL</h3>
+            <p>Make long URLs shorter and easy to type and pronounce</p>
+          </div>
+        </button>
+      </div>
+    </form>
+  </div>
+  <div class="pure-u-1-2 pure-u-sm-1-2 pure-u-md-1-3 pure-u-lg-1-4 l-box">
+    <form action="/upload" id="upload" rel="external" method="get" target="_self">
+      <div class="card">
+        <button form="upload" style="color: olive; border-color: olive">
+          <div class="image">
+                  <a title="Dave Gandy, Galaksidekiotostopcu, CC BY-SA 3.0 &lt;https://creativecommons.org/licenses/by-sa/3.0&gt;, via Wikimedia Commons" href="https://commons.wikimedia.org/wiki/File:Upload_icon_-_Font_Awesome_-_Red.svg"><img width="64" alt="Upload icon - Font Awesome - Red" src="https://upload.wikimedia.org/wikipedia/commons/thumb/1/17/Upload_icon_-_Font_Awesome_-_Red.svg/512px-Upload_icon_-_Font_Awesome_-_Red.svg.png"></a>
+          </div>
+          <div class="text">
+            <h3>Share a file</h3>
+            <p>Upload a file and generate a short URL to share easily</p>
+          </div>
+        </button>
+      </div>
+    </form>
+  </div>
+</div>
+<hr/>
 <div class="pure-g">
   <div class="pure-u-1-2 pure-u-sm-1-2 pure-u-md-1-3 pure-u-lg-1-4 l-box">
     <form action="https://wol.shandan.one"

rest/Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3-slim
+WORKDIR /usr/src/app
+COPY rest/requirements.txt ./
+RUN python3 -m pip install --upgrade pip && \
+    python3 -m pip install --no-cache-dir -r requirements.txt
+COPY rest rest
+CMD [ "python", "-m", "rest.cherrypy" ]

rest/__init__.py

@@ -0,0 +1,18 @@
+#
+# Copyright (c) Daniel Sheffield 2023
+#
+# All rights reserved
+#
+# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+from bottle import TEMPLATE_PATH
+
+TEMPLATE_PATH.append("rest/templates")
+ALL_UNITS = { 'g', 'kg', 'mL', 'L', 'Pieces', 'Bunches', 'Bags' }
+PARAMS = { 'group', 'category', 'product', 'unit', 'tag', 'organic' }
+BOOLEAN = {
+    "1": True,
+    True: "1",
+    "0": False,
+    False: "0",
+    None: "0.5",
+}

rest/cherrypy.py

@@ -0,0 +1,16 @@
+import cherrypy
+import wsgigzip
+import bottle
+from .pyapi import *
+
+application = wsgigzip.GzipMiddleware(bottle.default_app())
+
+cherrypy.config.update({
+    'server.socket_host': "0.0.0.0",
+    'server.socket_port': 6772,
+})
+
+cherrypy.tree.graft(application, "/")
+
+cherrypy.engine.start()
+cherrypy.engine.block()

rest/dev-requirements.txt

@@ -0,0 +1 @@
+pytest

rest/hash_util.py

@@ -0,0 +1,180 @@
+from hashlib import blake2b, shake_128, md5, sha256, sha1
+import os
+from base64 import b64encode, b64decode, b85encode, b85decode
+import base32_lib as b32
+
+DIGEST_SIZE_BYTES = 3
+DIGEST_SIZE_BITMASK = 0xffffff
+DIGEST_SIZE_SIGNED_TO_UNSIGNED_BITMASK = 0x1ffffff
+DIGEST_SIZE_SIGNED_TO_UNSIGNED_BIT = 0x1000000
+DIGEST_SIZE_NIBBLES = DIGEST_SIZE_BYTES * 2
+B64ALTCHARS = b'.-'
+
+def sha1hash(data: str):
+    return sha1(data.encode("utf-8"), usedforsecurity=False).hexdigest()[:DIGEST_SIZE_BYTES]
+
+def sha256hash(data: str):
+    return sha256(data.encode("utf-8"), usedforsecurity=False).hexdigest()[:DIGEST_SIZE_BYTES]
+
+def md5hash(data: str):
+    return md5(data.encode("utf-8"), usedforsecurity=False).hexdigest()[:DIGEST_SIZE_BYTES]
+
+def shake(data: str):
+    return shake_128(data.encode("utf-8"), usedforsecurity=False).hexdigest(DIGEST_SIZE_BYTES)
+
+def blake(data: bytes, person: bytes = None) -> bytes:
+    return blake2b(
+        data,
+        usedforsecurity=False,
+        digest_size=DIGEST_SIZE_BYTES,
+        person=person
+    ).digest()
+
+def blake_file(path: str, person: bytes = None, root: str ='rest/static') -> bytes:
+    fd = os.open(f'{root}/{path}', os.O_RDONLY, 0o600)
+    with open(fd, "rb") as f:
+        f.seek(0)
+        _blake = blake2b(usedforsecurity=False, digest_size=DIGEST_SIZE_BYTES, person='upload'.encode('utf-8'))
+        while f.peek(1):
+            _blake.update(f.read(1024))
+    return _blake.digest()
+        
+
+def python(data: str):
+    return hash(data)
+
+def normalize_hash(_hash: int) -> int:
+    #hex = hash_to_hex(_hash)
+    #return int(hex, 16)
+    #_bytes = _hash.to_bytes(8, byteorder='big', signed=True)
+    #return bytes_to_hash(_bytes)
+    return _hash & DIGEST_SIZE_BITMASK
+
+def normalize_bytes(_bytes: bytes) -> bytes:
+    return (b'\x00' * DIGEST_SIZE_BYTES + _bytes)[-DIGEST_SIZE_BYTES:]
+
+def normalize_hex(_hex: str) -> str:
+    #_bytes = hex_to_bytes(hex)
+    #return _bytes.hex()
+    return _hex.zfill(DIGEST_SIZE_NIBBLES)[-DIGEST_SIZE_NIBBLES:]
+
+def hex_to_bytes(_hex: str) -> bytes:
+    _bytes = bytes.fromhex(_hex.zfill(DIGEST_SIZE_NIBBLES))
+    return normalize_bytes(_bytes)
+
+def bytes_to_hex(_bytes: bytes) -> str:
+    return normalize_bytes(_bytes).hex()
+
+def hash_to_bytes(_hash: int) -> bytes:
+    _bytes = _hash.to_bytes(8, byteorder='big', signed=True)
+    return normalize_bytes(_bytes)
+
+def bytes_to_hash(_bytes: bytes) -> int:
+    norm = normalize_bytes(_bytes)
+    return int.from_bytes(norm, byteorder='big', signed=False)
+
+def hash_to_hex(_hash: int) -> str:
+    #return hash_to_bytes(_hash).hex()
+    #return normalize_hex(
+    #return f"{_hash + (1 << 64):x}"[-4:]
+    #return hex(_hash + (1<<64))[2:][-4:]
+    #return f"{_hash & 0xffff:04x}"
+    return hex((_hash|DIGEST_SIZE_SIGNED_TO_UNSIGNED_BIT) & DIGEST_SIZE_SIGNED_TO_UNSIGNED_BITMASK)[3:]
+
+def hex_to_hash(_hex: str) -> int:
+    #_bytes = bytes.fromhex(hex.zfill(4))
+    #return bytes_to_hash(_bytes)
+    #return int(normalize_hex(hex), 16)
+    return int(_hex, 16) & DIGEST_SIZE_BITMASK
+
+def remove_padding(f):
+    def wrap(*args, **kwargs):
+        return f(*args, **kwargs).split('=')[0]
+    return wrap
+
+def fix_padding(f):
+    def wrap(_b64, *args, **kwargs):
+        pad = (4 - len(_b64)) % 4
+        fixed = _b64 + '='*pad
+        return f(fixed, *args, **kwargs)
+    return wrap
+
+def normalize_base32(_b32: str):
+    return _b32.upper().zfill(DIGEST_SIZE_BYTES*8//5+1)
+
+def add_padding_base32(f):
+    def wrap(*args, **kwargs):
+        return normalize_base32(f(*args, **kwargs))
+    return wrap
+
+@remove_padding
+def hash_to_base64(_hash: int) -> str:
+    return b64encode(hash_to_bytes(_hash), altchars=B64ALTCHARS).decode("utf-8")
+
+@remove_padding
+def hex_to_base64(_hex: str) -> str:
+    return b64encode(hex_to_bytes(_hex), altchars=B64ALTCHARS).decode("utf-8")
+
+@remove_padding
+def bytes_to_base64(_bytes: str) -> str:
+    return b64encode(normalize_bytes(_bytes), altchars=B64ALTCHARS).decode("utf-8")
+
+@fix_padding
+def base64_to_hash(_b64: str) -> str:
+    return bytes_to_hash(b64decode(_b64, altchars=B64ALTCHARS))
+
+@fix_padding
+def base64_to_hex(_b64: str) -> str:
+    return bytes_to_hex(b64decode(_b64, altchars=B64ALTCHARS))
+
+@fix_padding
+def base64_to_bytes(_b64: str) -> str:
+    return normalize_bytes(b64decode(_b64, altchars=B64ALTCHARS))
+
+#@remove_padding
+def hash_to_base85(_hash: int) -> str:
+    return b85encode(hash_to_bytes(_hash)).decode("utf-8")
+
+#@remove_padding
+def hex_to_base85(_hex: str) -> str:
+    return b85encode(hex_to_bytes(_hex)).decode("utf-8")
+
+#@remove_padding
+def bytes_to_base85(_bytes: str) -> str:
+    return b85encode(normalize_bytes(_bytes)).decode("utf-8")
+
+#@fix_padding
+def base85_to_hash(_b64: str) -> str:
+    return bytes_to_hash(b85decode(_b64))
+
+#@fix_padding
+def base85_to_hex(_b64: str) -> str:
+    return bytes_to_hex(b85decode(_b64))
+
+#@fix_padding
+def base85_to_bytes(_b64: str) -> str:
+    return normalize_bytes(b85decode(_b64))
+
+@add_padding_base32
+def hash_to_base32(_hash: int) -> str:
+    return b32.encode(_hash & DIGEST_SIZE_BITMASK)
+
+@add_padding_base32
+def hex_to_base32(_hex: str) -> str:
+    return b32.encode(hex_to_hash(_hex))
+
+@add_padding_base32
+def bytes_to_base32(_bytes: bytes) -> str:
+    return b32.encode(bytes_to_hash(_bytes))
+
+#@fix_padding
+def base32_to_hash(_b64: str) -> str:
+    return b32.decode(_b64)
+
+#@fix_padding
+def base32_to_hex(_b64: str) -> str:
+    return hash_to_hex(base32_to_hash(_b64))
+
+#@fix_padding
+def base32_to_bytes(_b64: str) -> str:
+    return hash_to_bytes(base32_to_hash(_b64))

rest/pyapi.py

@@ -0,0 +1,237 @@
+#
+# Copyright (c) Daniel Sheffield 2023
+# All rights reserved
+#
+# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+from io import BytesIO
+from bottle import (
+    route, request, response,
+    redirect, abort,
+    template, static_file,
+    HTTPResponse, HTTPError,
+)
+from linkpreview import link_preview
+
+from .validate import CLIP_SIZE_LIMIT, get_file_mimetype, get_file_size, get_filename, validate, validate_file, validate_parameter, validate_url
+from .hash_util import normalize_base32
+from .save import save, save_upload
+
+
+SCHEME = "https://"
+HOST = ""
+DOMAIN = "shandan.one"
+PORT = ""
+LOCATION = SCHEME + (f"{HOST}." if HOST else "") + DOMAIN + (f":{PORT}" if PORT else "")
+
+
+@route('/clip', method=['GET', 'POST'])
+def clip():
+
+    if request.method == 'GET':
+        _hash = request.params.hash
+        if _hash:
+            _hash = normalize_base32(_hash)
+            content = validate(_hash).decode('utf-8')
+        else:
+            content = None
+        link = f'{LOCATION}/clip/{_hash}' if content else f'{LOCATION}/clip'
+
+        response.content_type = 'text/html; charset=utf-8'
+        form = template(
+            'clip-form',
+            action='/clip',
+            method='post',
+            content=content,
+            disabled=True if content else False
+        )
+        return template(
+            'paste',
+            form=form,
+            qr=f'{LOCATION}/clip/{_hash}.qr' if content else f'{LOCATION}/grocery/static/clip-qr.svg',
+            link=link,
+            disabled=True if content else False,
+            download=f'/clip/{_hash}' if content else None
+        )
+
+    if request.method == 'POST':
+        content = validate_parameter(request, 'paste')
+        if request.params.copy != 'true':
+            _b32 = save(content, LOCATION)
+            return redirect(f'/clip?hash={_b32}')
+
+        response.content_type = 'text/html; charset=utf-8'
+        form = template(
+            'clip-form',
+            action='/clip',
+            method='post',
+            content=content,
+            disabled=False
+        )
+        link = f'{LOCATION}/clip'
+        return template(
+            'paste',
+            form=form,
+            qr=f'{LOCATION}/grocery/static/clip-qr.svg',
+            link=link,
+            disabled=False,
+            download=None
+        )
+
+
+@route('/clip/<filename:path>', method='GET')
+def get_clip(filename):
+    ext = 'file'
+    if filename and filename.endswith('.qr'):
+        filename, ext = filename.split('.', 1)
+    
+    filename = filename and normalize_base32(filename)
+    path = f'{filename}/{filename}.{ext}'
+    
+    if ext == 'qr':
+        return static_file(path, root='rest/static', mimetype='image/svg+xml')
+    
+    filename = filename and normalize_base32(filename)
+    if not request.params.raw.lower() == 'true':
+        # TODO: return a form with timeout to a GET instead ?
+        return redirect(f'/clip?hash={filename}')
+
+    ret = validate(filename)
+    if isinstance(ret, HTTPError):
+        return ret
+
+    return static_file(path, root='rest/static')
+
+
+@route('/upload', method=['GET', 'POST'])
+def upload():
+
+    if request.method == 'GET':
+        _hash = request.params.hash
+        mimetype = None
+        if _hash:
+            _hash = normalize_base32(_hash)
+            size = get_file_size(_hash)
+            name = get_filename(_hash)
+            mimetype = get_file_mimetype(name)
+            if mimetype is not True and mimetype.startswith('text'):
+                mimetype = None if size > CLIP_SIZE_LIMIT else mimetype
+        
+        link = f'{LOCATION}/upload/{_hash}' if _hash else f'{LOCATION}/upload'
+        response.content_type = 'text/html; charset=utf-8'
+        disabled = True if _hash else False
+        form = template('file-form', action='/upload', method='post', disabled=disabled)
+        
+        return template(
+            'upload',
+            form=form,
+            qr=f'{LOCATION}/upload/{_hash}.qr' if _hash else f'{LOCATION}/grocery/static/upload-qr.svg',
+            link=link,
+            mimetype=mimetype,
+            disabled=disabled
+        )
+
+    if request.method == 'POST':
+        if 'paste' not in request.files:
+            return abort(400, "Parameter 'paste' must be specified")
+
+        upload = request.files['paste']
+        if isinstance(upload.file, BytesIO):
+            if len(upload.file.read()) == 0:
+                return abort(400, "File is empty")
+
+        _b32 = save_upload(upload.raw_filename, LOCATION, upload.file, root='rest/static')
+        return redirect(f'/upload?hash={_b32}')
+
+
+@route('/upload/<filename:path>', method='GET')
+def get_upload(filename):
+    ext = 'file'
+    if filename and filename.endswith('.qr'):
+        filename, ext = filename.split('.', 1)
+    
+    filename = filename and normalize_base32(filename)
+    path = f'{filename}/{filename}.{ext}'
+    
+    if ext == 'qr':
+        return static_file(path, root='rest/static', mimetype='image/svg+xml')
+    
+    download = True
+    mimetype = True
+    if request.params.download == "false":
+        download = False
+    mimetype = request.params.mimetype or None
+
+    return validate_file(filename, download=download, mimetype=mimetype)
+
+
+@route('/goto', method=['GET', 'POST'])
+def goto():
+
+    if request.method == 'GET':
+        _hash = request.params.hash
+        if _hash:
+            _hash = normalize_base32(_hash)
+            content = validate(_hash).decode('utf-8')
+        else:
+            content = None
+
+        if content and request.params.go == 'true':
+            target = validate_url(content)
+            return redirect(target)
+
+        link = f'{LOCATION}/goto/{_hash}' if content else f'{LOCATION}/goto'
+        disabled = True if content else False
+        response.content_type = 'text/html; charset=utf-8'
+        form = template(
+            'goto-form',
+            action='/goto',
+            method='post',
+            content=content,
+            disabled=disabled
+        )
+        preview = dict()
+        if content:
+            try:
+                page = link_preview(link, parser="lxml")
+                preview['title'] = page.title
+                preview['img'] = page.absolute_image
+                preview['domain'] = page.site_name
+                preview['link'] = content
+            except:
+                pass
+
+        return template(
+            'goto',
+            form=form,
+            qr=f'{LOCATION}/goto/{_hash}.qr' if content else f'{LOCATION}/grocery/static/goto-qr.svg',
+            link=link,
+            disabled=disabled,
+            preview=preview,
+        )
+
+    if request.method == 'POST':
+        content = validate_parameter(request, 'url')
+        _b32 = save(content, LOCATION)
+
+        # validate but save content unmodified
+        _ = validate_url(content.decode('utf-8'))
+        return redirect(f'/goto?hash={_b32}')
+
+
+@route('/goto/<filename:path>', method='GET')
+def redirect_goto(filename):
+    ext = 'file'
+    if filename and filename.endswith('.qr'):
+        filename, ext = filename.split('.', 1)
+    
+    filename = filename and normalize_base32(filename)
+    path = f'{filename}/{filename}.{ext}'
+    
+    if ext == 'qr':
+        return static_file(path, root='rest/static', mimetype='image/svg+xml')
+    
+    return redirect(f'/goto?hash={filename}&go=true')
+
+
+@route('/<any>/', method='GET')
+def redirect_trailing_slash(any): return redirect(f'/{any}')

rest/qr.py

@@ -0,0 +1,34 @@
+from io import BytesIO
+from typing import Union
+from qrcode import QRCode
+from qrcode.constants import ERROR_CORRECT_L, ERROR_CORRECT_M, ERROR_CORRECT_Q, ERROR_CORRECT_H
+from qrcode.image.styledpil import StyledPilImage
+from qrcode.image.svg import SvgPathImage
+from qrcode.image.styles.moduledrawers.svg import SvgCircleDrawer
+from qrcode.image.styles.colormasks import RadialGradiantColorMask
+
+QR_MAX_BYTES = {
+    ERROR_CORRECT_H: 1273,
+    ERROR_CORRECT_Q: 1663,
+    ERROR_CORRECT_M: 2331,
+    ERROR_CORRECT_L: 2953,
+}
+def get_qr_code(data: Union[bytes, str], fallback: Union[bytes, str] = None):
+    err_lvl = ERROR_CORRECT_H
+    qr = QRCode(error_correction=err_lvl)
+    if data is not None and isinstance(data, str):
+        data = data.encode('utf-8')
+    if fallback is not None and isinstance(fallback, str):
+        fallback = fallback.encode('utf-8')
+    if fallback is not None and data and len(data) > QR_MAX_BYTES[err_lvl]:
+        qr.add_data(fallback, optimize=0)
+    else:
+        qr.add_data(data or fallback, optimize=0)
+
+    img_1 = qr.make_image(image_factory=SvgPathImage)
+    with BytesIO() as f:
+        img_1.save(f)
+        f.flush()
+        ret = f.getvalue()
+    return ret
+    

rest/requirements.txt

@@ -0,0 +1,7 @@
+bottle
+wsgigzip
+cherrypy
+base32-lib
+lxml
+linkpreview
+qrcode

rest/save.py

@@ -0,0 +1,84 @@
+#
+# Copyright (c) Daniel Sheffield 2023
+# All rights reserved
+#
+# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+
+from hashlib import blake2b
+from io import BufferedRandom, BytesIO
+import os
+import shutil
+from uuid import uuid4
+
+from .hash_util import DIGEST_SIZE_BYTES, blake, bytes_to_base32
+from .qr import get_qr_code
+
+def save_qr(qr: bytes, _b32:str , directory: str) -> str:
+    fd = os.open(f'{directory}/{_b32}.qr', os.O_WRONLY | os.O_TRUNC | os.O_CREAT, 0o600)
+    with open(fd, "wb") as f:
+        f.write(qr)
+
+
+def save(content: bytes, location: str, root='rest/static') -> str:
+    _bytes = blake(content, person='clip'.encode('utf-8'))
+    _b32 = bytes_to_base32(_bytes)
+    directory = f'{root}/{_b32}'
+    try:
+        os.mkdir(directory, mode=0o700, dir_fd=None)
+    except FileExistsError:
+        pass
+    fd = os.open(f'{directory}/{_b32}.file', os.O_WRONLY | os.O_TRUNC | os.O_CREAT, 0o600)
+    with open(fd, "wb") as f:
+        f.write(content)
+    
+    link = f'{location}/clip/{_b32}'
+    svg = get_qr_code(content, fallback=link)
+    save_qr(svg, _b32, directory)
+    return _b32
+
+
+def save_filename(name: str, _b32: str, directory: str):
+    fd = os.open(f'{directory}/{_b32}.name', os.O_WRONLY | os.O_TRUNC | os.O_CREAT, 0o600)
+    with open(fd, "w") as f:
+        f.write(name)
+
+
+def save_upload(name: str, location: str, content: BufferedRandom, root='rest/static') -> str:
+    tmpdir = '/tmp/upload'
+    try:
+        os.mkdir(tmpdir, mode=0o700, dir_fd=None)
+    except FileExistsError:
+        pass
+    
+    unique = uuid4()
+    fd = os.open(f'{tmpdir}/{unique.hex}', os.O_WRONLY | os.O_TRUNC | os.O_CREAT, 0o600)
+    with open(fd, "wb") as f:
+        done = False
+        if isinstance(content, BytesIO):
+            f.write(content.getvalue())
+            done = True
+        while not done and content.peek(1):
+            seg = content.read(1024)
+            f.write(seg)
+    
+    with open(f'{tmpdir}/{unique.hex}', "rb") as f:
+        _blake = blake2b(usedforsecurity=False, digest_size=DIGEST_SIZE_BYTES, person='upload'.encode('utf-8'))
+        while f.peek(1):
+            _blake.update(f.read(1024))
+        
+        _bytes = _blake.digest()
+        _b32 = bytes_to_base32(_bytes)
+    
+    directory = f'{root}/{_b32}'
+    try:
+        os.mkdir(directory, mode=0o700, dir_fd=None)
+    except FileExistsError:
+        pass
+
+    shutil.move(f'{tmpdir}/{unique.hex}', f'{directory}/{_b32}.file')
+    save_filename(name, _b32, directory)
+    link = f'{location}/clip/{_b32}'
+    svg = get_qr_code(link, fallback=link)
+    save_qr(svg, _b32, directory)
+
+    return _b32

rest/templates/button-style.tpl

@@ -0,0 +1,15 @@
+<style>
+.button-resize { font-size: 80%; }
+@media screen and (min-width:35.5em){
+    .button-resize { font-size: 80%; }
+}
+@media screen and (min-width:40em){
+    .button-resize { font-size: 85%; }
+}
+@media screen and (min-width:64em){
+    .button-resize { font-size: 100%; }
+}
+@media screen and (min-width:80em){
+    .button-resize { font-size: 110%; }
+}
+</style>

rest/templates/card-goto.tpl

@@ -0,0 +1,22 @@
+<style>
+.card img {
+    color: floralwhite;
+    object-fit: cover;
+    width: 100%;
+    height: 100%;
+}
+</style>
+<a href="{{link}}">
+<article class="card">
+  <header>
+    <h2>{{title}}</h2>
+  </header>
+  <figure> 
+    <img src="{{img}}" alt="{{title}}">
+    <figcaption>{{domain}}</figcaption>
+  </figure>
+  <div class="content">
+    {{link}}
+  </div>       
+</article>
+</a>

rest/templates/clip-form.tpl

@@ -0,0 +1,38 @@
+% content = setdefault("content", "") or ""
+% disabled = (setdefault("disabled", False) and 'readonly="true"') or ""
+<form id="paste" method="{{ method }}" action="{{ action }}" enctype="multipart/form-data">
+  <style>
+textarea::-webkit-scrollbar {
+  width: 11px;
+}
+textarea {
+  color: #cccccc;
+  background-color: #080808;
+  scrollbar-width: thin;
+  scrollbar-color: var(--thumbBG) var(--scrollbarBG);
+}
+textarea::-webkit-scrollbar-track {
+  background: var(--scrollbarBG);
+}
+textarea::-webkit-scrollbar-thumb {
+  background-color: var(--thumbBG) ;
+  border-radius: 6px;
+  border: 3px solid var(--scrollbarBG);
+}
+  </style>
+  <textarea
+    style="width: 80%"
+    id="paste-text-area"
+    name="paste"
+    rows="30"
+    {{!disabled}}
+    required="true"
+    autofocus="true"
+    placeholder="Paste something here..."
+  >{{ content }}</textarea>
+</form>
+<form id="copy" method="post" action="{{ action }}">
+  <input id="copy-paste" name="paste" type="text" value="{{ content }}" hidden="true" />
+  <input id="copy" name="copy" type="text" value="true" hidden="true" />
+</form>
+<form id="new" method="get" action="{{ action }}"></form>

rest/templates/file-form.tpl

@@ -0,0 +1,6 @@
+% content = setdefault("content", "") or ""
+% disabled = setdefault("disabled", "") and 'hidden="true"'
+<form id="upload" method="{{ method }}" action="{{ action }}" enctype="multipart/form-data">
+  <input type="file" name="paste" required="true" {{disabled}} />
+</form>
+<form id="new" method="get" action="{{ action }}"></form>

rest/templates/goto-form.tpl

@@ -0,0 +1,34 @@
+% content = setdefault("content", "") or ""
+% disabled = (setdefault("disabled", False) and 'readonly="true"') or ""
+<form id="goto" method="{{ method }}" action="{{ action }}">
+  <style>
+input[type="url"]::-webkit-scrollbar {
+  width: 11px;
+}
+input[type="url"] {
+  color: #cccccc;
+  background-color: #080808;
+  scrollbar-width: thin;
+  scrollbar-color: var(--thumbBG) var(--scrollbarBG);
+}
+input[type="url"]::-webkit-scrollbar-track {
+  background: var(--scrollbarBG);
+}
+input[type="url"]::-webkit-scrollbar-thumb {
+  background-color: var(--thumbBG) ;
+  border-radius: 6px;
+  border: 3px solid var(--scrollbarBG);
+}
+  </style>
+  <input type="url"
+    style="width: 80%"
+    id="input-url"
+    name="url"
+    {{!disabled}}
+    required="true"
+    autofocus="true"
+    placeholder="Paste URL here..."
+    value="{{ content }}"
+  ></input>
+</form>
+<form id="new" method="get" action="{{ action }}"></form>

rest/templates/goto.tpl

@@ -0,0 +1,56 @@
+% link = setdefault("link", "") or ""
+% disabled = setdefault("disabled", "") and 'disabled="true"'
+% preview = setdefault("preview", None)
+<!DOCTYPE html>
+<html>
+  <head>
+    <style>
+html {
+  --scrollbarBG: #333333;
+  --thumbBG: #080808;
+}
+body {
+  background-color: #080808;
+  color: #cccccc;
+}
+img {
+  background-color: floralwhite;
+  color: black;
+  max-height: min(100vh, calc(100vw * 9 / 16));
+  max-width: calc(100vw - 2em);
+}
+    </style>
+    <title>Go to ...</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1"/>
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/purecss@2.1.0/build/pure-min.css" integrity="sha384-yHIFVG6ClnONEA5yB5DJXfW2/KC173DIQrYoZMEtBvGzmf0PKiGyNEqe9N6BNDBH" crossorigin="anonymous"/>
+    <link rel="stylesheet" href="https://shandan.one/css/grids-responsive-min.css"/>
+    <link rel="stylesheet" href="https://shandan.one/css/responsive-visibility-collapse.css"/>
+  </head>
+  <body align="center" style="text-align: center">
+    <div class="pure-g">
+      % include('button-style')
+      <div class="pure-u-1">
+        <div class="pure-button-group" role="action" style="padding: 1em 0 0;">
+          <button class="button-resize pure-button" type="submit" form="new"> New </button>
+          <button class="button-resize pure-button" type="submit" form="goto" {{!disabled}}> Shrtn It! </button>
+        </div>
+      </div>
+      <div class="pure-u-1">
+        <div class="pure-button" style="margin: 1em 0 0; background: #8f4f4f">
+          <a href="{{!link}}" style="color: floralwhite;">{{ link }}</a>
+        </div>
+      </div>
+      <div class="pure-u-1">
+        <p><details><summary> Show QR code ...</summary><img src="{{qr}}"/></details></p>
+{{!form}}
+      </div>
+      <div class="pure-u-1-3"></div>
+      <div class="pure-u-1-3">
+      % if preview:
+      %   include('card-goto', **preview)
+      % end
+      </div>
+      <div class="pure-u-1-3"></div>
+    </div>
+  </body>
+</html>

rest/templates/paste.tpl

@@ -0,0 +1,55 @@
+% link = setdefault("link", "") or ""
+% disabled = setdefault("disabled", "") and 'disabled="true"'
+% download = setdefault("download", "") or ""
+% download_disabled = "" if download else 'disabled="true"'
+<!DOCTYPE html>
+<html>
+  <head>
+    <style>
+html {
+  --scrollbarBG: #333333;
+  --thumbBG: #080808;
+}
+body {
+  background-color: #080808;
+  color: #cccccc;
+}
+img {
+  background-color: floralwhite;
+  color: black;
+  max-height: min(100vh, calc(100vw * 9 / 16));
+  max-width: calc(100vw - 2em);
+}
+    </style>
+    <title>Paste</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1"/>
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/purecss@2.1.0/build/pure-min.css" integrity="sha384-yHIFVG6ClnONEA5yB5DJXfW2/KC173DIQrYoZMEtBvGzmf0PKiGyNEqe9N6BNDBH" crossorigin="anonymous"/>
+    <link rel="stylesheet" href="https://shandan.one/css/grids-responsive-min.css"/>
+    <link rel="stylesheet" href="https://shandan.one/css/responsive-visibility-collapse.css"/>
+  </head>
+  <body align="center" style="text-align: center">
+    <div class="pure-g">
+      % include('button-style')
+      <div class="pure-u-1">
+        <div class="pure-button-group" role="action" style="padding: 1em 0 0;">
+          <button class="button-resize pure-button" type="submit" form="new"> New </button>
+          <button class="button-resize pure-button" type="submit" form="copy"> Edit as New </button>
+          <button class="button-resize pure-button" type="submit" form="paste" {{!disabled}}> Paste </button>
+          <button class="button-resize pure-button" type="submit" form="download" {{!download_disabled}}> Download </button>
+        </div>
+      </div>
+      <div class="pure-u-1">
+        <div class="pure-button" style="margin: 1em 0 0; background: #4f8f4f;">
+          <a href="{{!link}}" style="color: floralwhite;">{{ link }}</a>
+        </div>
+      </div>
+      <div class="pure-u-1">
+        <p><details><summary> Show QR code ...</summary><img src="{{qr}}"/></details></p>
+{{!form}}
+<form id="download" method="get" action="{{ download }}">
+<input name="raw" type="text" hidden="true" value="true" />
+</form>
+      </div>
+    </div>
+  </body>
+</html>

rest/templates/upload.tpl

@@ -0,0 +1,72 @@
+% link = setdefault("link", "") or ""
+% disabled = setdefault("disabled", "") and 'disabled="true"'
+% download_disabled = "" if disabled else 'disabled="true"'
+% mimetype = (setdefault("mimetype", None) is not True and mimetype) or None
+<!DOCTYPE html>
+<html>
+  <head>
+    <style>
+html {
+  --scrollbarBG: #333333;
+  --thumbBG: #080808;
+}
+body {
+  background-color: #080808;
+  color: #cccccc;
+}
+img {
+  background-color: floralwhite;
+  color: black;
+  max-height: min(100vh, calc(100vw * 9 / 16));
+  max-width: calc(100vw - 2em);
+}
+object {
+  width: 80%;
+  margin: 1em 0 0;
+}
+object.other {
+  height: 60vh;
+}
+object.text {
+  height: 60vh;
+  border: floralwhite;
+  border-style: solid;
+  border-width: thin;
+}
+    </style>
+    <title>Upload</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1"/>
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/purecss@2.1.0/build/pure-min.css" integrity="sha384-yHIFVG6ClnONEA5yB5DJXfW2/KC173DIQrYoZMEtBvGzmf0PKiGyNEqe9N6BNDBH" crossorigin="anonymous"/>
+    <link rel="stylesheet" href="https://shandan.one/css/grids-responsive-min.css"/>
+    <link rel="stylesheet" href="https://shandan.one/css/responsive-visibility-collapse.css"/>
+  </head>
+  <body align="center" style="text-align: center">
+    <div class="pure-g">
+      % include('button-style')
+      <div class="pure-u-1">
+        <div class="pure-button-group" role="action" style="padding: 1em 0 0;">
+          <button class="button-resize pure-button" type="submit" form="new"> New </button>
+          <button class="button-resize pure-button" type="submit" form="upload" {{!disabled}}> Upload </button>
+          <button class="button-resize pure-button" type="submit" form="download" {{!download_disabled}}> Download </button>
+        </div>
+      </div>
+      <div class="pure-u-1">
+        <div class="pure-button" style="margin: 1em 0 0; background: #afaf0f;">
+          <a href="{{link}}" style="color: floralwhite;">{{ link }}</a>
+        </div>
+      </div>
+      <div class="pure-u-1">
+        <p><details><summary> Show QR code ...</summary><img src="{{qr}}"/></details></p>
+{{!form}}
+<form id="download" method="get" action="{{link}}"></form>
+        % if mimetype and mimetype.startswith('text'):
+        <object class="text" data="{{link}}?download=false&mimetype=text/plain" type={{mimetype}}><p>Unable to display {{mimetype}}</p></object>
+        % elif mimetype and mimetype.startswith('image'):
+        <object data="{{link}}?download=false" type="{{mimetype}}"><p>Unable to display {{mimetype}}</p></object>
+        % elif mimetype:
+        <object class="other" data="{{link}}?download=false" type={{mimetype}}><p>Unable to display {{mimetype}}</p></object>
+        % end
+      </div>
+    </div>
+  </body>
+</html>

rest/validate.py

@@ -0,0 +1,153 @@
+#
+# Copyright (c) Daniel Sheffield 2023
+# All rights reserved
+#
+# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+
+# https://www.ietf.org/rfc/rfc3696.txt
+
+"""
+   Without quotes, local-parts may consist of any combination of
+   alphabetic characters, digits, or any of the special characters
+
+      ! # $ % & ' * + - / = ?  ^ _ ` . { | } ~
+
+   period (".") may also appear, but may not be used to start or end the
+   local part, nor may two or more consecutive periods appear.  Stated
+   differently, any ASCII graphic (printing) character other than the
+   at-sign ("@"), backslash, double quote, comma, or square brackets may
+   appear without quoting.  If any of that list of excluded characters
+   are to appear, they must be quoted.
+"""
+
+from io import BufferedReader
+import mimetypes
+from itertools import chain
+import os
+from bottle import static_file, response, HTTPError, abort, LocalRequest, HTTPResponse
+from urllib.parse import urlparse, quote, quote_plus
+from .hash_util import blake_file, bytes_to_base32, blake
+
+# according to rfc3696
+URL_MUST_ESCAPE = bytes([
+    x for x in chain(
+        # control characters
+        range(int('0x1F', 0)+1),
+        # 0x7F and non 7bit-ASCII
+        range(int('0x7F', 0,), int('0xFF', 0)+1),
+        # specifically excluded
+        b'@\\",[]'
+    )
+])
+# so give this list to urllib.parse.quote which follows rfc3986
+URL_SAFE = bytes(( i for i in range(int('0xff',0)+1) if i not in map(int, URL_MUST_ESCAPE) ))
+
+CLIP_SIZE_LIMIT = 65535
+def validate(filename: str) -> bytes:
+    ret = static_file('/'.join([filename,]*2) + '.file', root='rest/static')
+    if isinstance(ret, HTTPError):
+        return abort(404, f"No such paste: {filename}")
+
+    if ret.content_length > CLIP_SIZE_LIMIT:
+        return abort(418, f"Paste size exceeds {CLIP_SIZE_LIMIT}")
+
+    content: bytes = ret.body.read() if isinstance(ret.body, BufferedReader) else ret.body.encode('utf-8')
+
+    _bytes = blake(content, person='clip'.encode('utf-8'))
+    _b32 = bytes_to_base32(_bytes)
+    if _b32 != filename:
+        return abort(410, f"Paste content differs")
+    return content
+
+
+def get_filename(filename: str, root: str = 'rest/static'):
+    path = '/'.join([filename,]*2)
+    try:
+        with open(f'{root}/{path}.name', "r") as f:
+            name = f.read()
+        return name
+    except:
+        pass
+
+def get_file_size(filename: str, root: str = 'rest/static'):
+    path = '/'.join([filename,]*2)
+    return os.stat(f'{root}/{path}.file').st_size
+
+def get_file_mimetype(name):
+    mimetype = mimetypes.guess_type(name, strict=False)[0] if name else True
+    return mimetype
+
+def validate_file(filename: str, root: str = 'rest/static', download=True, mimetype=True) -> HTTPResponse:
+    path = '/'.join([filename,]*2)
+
+    name = get_filename(filename)
+    mimetype = mimetype if mimetype and mimetype is not True else get_file_mimetype(name)
+    
+    ret = static_file(f'{path}.file', root=root, download=name if name and download else download, mimetype=mimetype)
+    if isinstance(ret, HTTPError):
+        return abort(404, f"No such upload: {filename}")
+
+    _bytes = blake_file(f'{path}.file', person='upload'.encode('utf-8'), root=root)
+    _b32 = bytes_to_base32(_bytes)
+    if _b32 != filename:
+        return abort(410, f"Uploaded content differs")
+    return ret
+
+
+def validate_parameter(request: LocalRequest, name: str) -> bytes:
+    if name not in request.params:
+        return abort(400, f"Missing parameter: '{name}'")
+    
+    # TODO: what is correct overhead for form content?
+    OVERHEAD = 1024
+    content: bytes = request.query.get(name, None)
+    content_length = request.content_length
+    if content_length == -1:
+        return abort(418, f"Content-Length must be specified")
+    if content_length > CLIP_SIZE_LIMIT + OVERHEAD:
+        return abort(418, f"Content-Length can not exceed {CLIP_SIZE_LIMIT*3} bytes")
+
+    # TODO: add test for both query/form param
+    if 'multipart/form-data' in request.content_type:
+        # TODO: what about binary data ?
+        content: bytes = (content or request.params[name].encode('utf-8'))
+    else:
+        content: bytes = (content or request.params[name].encode('latin-1'))
+    
+    if len(content) > CLIP_SIZE_LIMIT:
+        return abort(418, f"Paste can not exceed {CLIP_SIZE_LIMIT} bytes")
+    return content
+
+def validate_url(url: str) -> str:
+    scheme, netloc, path, params, query, fragment = urlparse(url)
+
+    if not scheme: return abort(400, "URL has no scheme")
+
+    if scheme == 'file' and not path: return abort(400, "File URL has no path")
+
+    if scheme in ('http', 'https') and not netloc: return abort(400, "HTTP(S) URL has no netloc")
+
+    if netloc:
+        try:
+            user_info, loc = netloc.rsplit('@', 1)
+        except ValueError:
+            user_info = ''
+            loc = ''
+        if user_info:
+            user_info = quote(user_info, safe=URL_SAFE)
+            netloc = f"{user_info}@{''.join(loc)}"
+        else:
+            # TODO: do this properly, ie, valid dns-name/ip/port etc
+            netloc = quote(netloc, safe=URL_SAFE)
+    
+    path = quote(path, safe=URL_SAFE)
+    params = quote_plus(params, safe=URL_SAFE)
+    query = quote(query, safe=URL_SAFE)
+    fragment = quote(fragment, safe=URL_SAFE)
+    
+    url = f'{scheme}://{netloc}{path}{params}'
+    if query:
+        url = f'{url}?{query}'
+    if fragment:
+        url = f'{url}#{fragment}'
+    return url

test/rest/rfc2396.py

@@ -0,0 +1,112 @@
+# https://www.ietf.org/rfc/rfc2396.txt
+"""   
+The following definitions are common to many elements:
+
+      alpha    = lowalpha | upalpha
+
+      lowalpha = "a" | "b" | "c" | "d" | "e" | "f" | "g" | "h" | "i" |
+                 "j" | "k" | "l" | "m" | "n" | "o" | "p" | "q" | "r" |
+                 "s" | "t" | "u" | "v" | "w" | "x" | "y" | "z"
+
+      upalpha  = "A" | "B" | "C" | "D" | "E" | "F" | "G" | "H" | "I" |
+                 "J" | "K" | "L" | "M" | "N" | "O" | "P" | "Q" | "R" |
+                 "S" | "T" | "U" | "V" | "W" | "X" | "Y" | "Z"
+
+      digit    = "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" |
+                 "8" | "9"
+                 
+      alphanum = alpha | digit
+
+      uric          = reserved | unreserved | escaped
+
+      reserved    = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" |
+                    "$" | ","
+
+      unreserved  = alphanum | mark
+
+      mark        = "-" | "_" | "." | "!" | "~" | "*" | "'" | "(" | ")"
+
+      escaped     = "%" hex hex
+      hex         = digit | "A" | "B" | "C" | "D" | "E" | "F" |
+                            "a" | "b" | "c" | "d" | "e" | "f"
+
+      control     = <US-ASCII coded characters 00-1F and 7F hexadecimal>
+
+      space       = <US-ASCII coded character 20 hexadecimal>
+
+      delims      = "<" | ">" | "#" | "%" | <">
+
+      unwise      = "{" | "}" | "|" | "\" | "^" | "[" | "]" | "`"
+
+      <scheme>:<scheme-specific-part>
+
+      <scheme>://<authority><path>?<query>
+
+      absoluteURI   = scheme ":" ( hier_part | opaque_part )
+
+      hier_part     = ( net_path | abs_path ) [ "?" query ]
+
+      net_path      = "//" authority [ abs_path ]
+
+      abs_path      = "/"  path_segments
+
+      opaque_part   = uric_no_slash *uric
+
+      uric_no_slash = unreserved | escaped | ";" | "?" | ":" | "@" |
+                      "&" | "=" | "+" | "$" | ","
+    
+      scheme        = alpha *( alpha | digit | "+" | "-" | "." )
+
+      authority     = server | reg_name
+
+      Within the authority component, the characters ";", ":",
+   "@", "?", and "/" are reserved.
+      
+      reg_name      = 1*( unreserved | escaped | "$" | "," |
+                          ";" | ":" | "@" | "&" | "=" | "+" )
+    
+      <userinfo>@<host>:<port>
+
+      server        = [ [ userinfo "@" ] hostport ]
+
+      userinfo      = *( unreserved | escaped |
+                         ";" | ":" | "&" | "=" | "+" | "$" | "," )
+    
+      hostport      = host [ ":" port ]
+      host          = hostname | IPv4address
+      hostname      = *( domainlabel "." ) toplabel [ "." ]
+      domainlabel   = alphanum | alphanum *( alphanum | "-" ) alphanum
+      toplabel      = alpha | alpha *( alphanum | "-" ) alphanum
+
+      IPv4address   = 1*digit "." 1*digit "." 1*digit "." 1*digit
+      port          = *digit
+
+"""
+# uric := URI characters
+lowalpha = 'a-z'
+upalpha = 'A-Z'
+alpha = f'{lowalpha}{upalpha}'
+digit = '0-9'
+alphanum = f'{alpha}{digit}'
+reserved_no_slash = r';\?:@&=\+\$,'
+reserved = f'/{reserved_no_slash}'
+_mark = r"\-_\.!~*'()"
+hex = f'a-fA-F{digit}'
+escaped = f"%[{hex}][{hex}]"
+unreserved = f'{alphanum}{_mark}'
+control = b''
+#for i in chain(range(int('0x1F', 0)+1), [int('0x7F', 0)]):
+#    control += control + bytes(i)
+control = control.decode('ascii')
+space = ' '
+delims = r'\<\>#%"'
+unwise = r'{}|\\\^\[\]`'
+excluded = f'{control}{space}{delims}{unwise}'
+scheme = f'[{alpha}][{alphanum}+.-]*'
+authority_reserved = r';:@\?/'
+reg_name_aux = r'\$,;:@&=\+'
+reg_name = f'([{unreserved}]|{escaped}|{reg_name_aux})+'
+user_info_aux = r'\$,;:&=\+'
+user_info = f'([{unreserved}]|{escaped}|{user_info_aux})+'
+server = r'({user_info}@)?'
+# ... gave up

test/rest/test_url.py

@@ -0,0 +1,72 @@
+#
+# Copyright (c) Daniel Sheffield 2023
+#
+# All rights reserved
+#
+# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
+
+from pytest import mark, raises
+from bottle import HTTPError
+from rest.validate import validate_url
+
+@mark.parametrize('url, expected', [
+    ['file:///',]*2,
+    ['file:///a/b/c',]*2,
+    ['https://shandan.one',]*2,
+    ['https://www.shandan.one',]*2,
+    ['https://www.shandan.one/clip?id=123',]*2,
+    
+    # empty query
+    ['https://www.shandan.one?', 'https://www.shandan.one',],
+
+    # empty fragment
+    ['https://www.shandan.one/clip?id=123#', 'https://www.shandan.one/clip?id=123',],
+
+    # no double slash
+    #['file:/a/b/c', (HTTPError, ""),],
+    #['file:/abc', (HTTPError, ""),],
+
+    # no scheme
+    ['/a/b/c', (HTTPError, 400, "URL has no scheme"),],
+
+    # no file path scheme
+    ['file:', (HTTPError, 400, "File URL has no path"),],
+    
+    # no HTTPS domain
+    #['https://abc?id=1:', (HTTPError, 400, "HTTP(S) URL has no netloc"),],
+    
+    # conecutive dots
+    #['https://shandan.one/abc..id', 'https://shandan.one/abc..id',],
+
+    # unescaped char in reg_name
+    # TODO: should be invalid because netloc must be a domain name or ip ?
+    ['https://🌚.shandan.one', 'https://%F0%9F%8C%9A.shandan.one',],
+
+    # @ in user_info not allowed
+    # TODO: check this - final @ should not be encoded ?
+    ['https://user@mail@www.shandan.one','https://user%40mail@www.shandan.one'],
+
+    # delimiters
+    # TODO: should < be translated to %3C ?
+    ['https://www.shandan.one?a<b', 'https://www.shandan.one?a<b'],
+
+    # more delimiters
+    ['https://www.shandan.one/clip?proportion=69%', 'https://www.shandan.one/clip?proportion=69%'],
+
+    # fragment before end of reference URI
+    ['https://www.shandan.one/tiny#url?id=123', 'https://www.shandan.one/tiny#url?id=123'],
+])
+def test_validate_url_invalid(url: str, expected: str):
+    if isinstance(expected, tuple):
+        exp_exception, *ex_args = expected
+    else:
+        exp_exception = None
+    
+    if not exp_exception:
+        assert validate_url(url) == expected
+        return
+
+    with raises(exp_exception) as ex:
+        validate_url(url)
+    
+    assert list(ex.value.args) == ex_args