config/upload.json

@@ -3,5 +3,6 @@
   "max_uploaded_file_size": 5242880000,
   "max_database_pool_connections": 16,
   "database_url": "sqlite:///var/sqlpage/upload.db",
+  "sqlite_extensions": ["/var/www/fileio.so"],
   "compress_responses": false
 }

docker-compose.yml

@@ -16,7 +16,9 @@ services:
     expose:
       - 8080
     networks:
-      - priv
+      priv:
+        aliases:
+          - clip
     restart: unless-stopped
 
   goto:
@@ -35,7 +37,9 @@ services:
     expose:
       - 8080
     networks:
-      - priv
+      priv:
+        aliases:
+          - goto
     restart: unless-stopped
 
   upload:
@@ -50,11 +54,14 @@ services:
       - ./config/migrations:/etc/sqlpage/migrations
       - ./config/templates:/etc/sqlpage/templates
       - ./config/upload.json:/etc/sqlpage/sqlpage.json
-      - ./data/upload.db:/var/sqlpage/upload.db
+      - ./data/upload2.db:/var/sqlpage/upload.db
+      - ./fileio.so:/var/www/fileio.so:ro
     expose:
       - 8080
     networks:
-      - priv
+      priv:
+        aliases:
+          - upload
     restart: unless-stopped
 
   code:
@@ -73,7 +80,9 @@ services:
     expose:
       - 8080
     networks:
-      - priv
+      priv:
+        aliases:
+          - code
     restart: unless-stopped
 
 
@@ -84,11 +93,13 @@ services:
       context: .
       dockerfile: rest/Dockerfile
     volumes:
-      - ./data/upload.db:/usr/src/app/util.db:ro
+      - ./data/upload2.db:/usr/src/app/util.db:ro
     expose:
       - 6772
     networks:
-      - priv
+      priv:
+        aliases:
+          - util-proxy
     restart: unless-stopped
 
 networks:

rest/__init__.py

@@ -3,16 +3,4 @@
 #
 # All rights reserved
 #
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from bottle import TEMPLATE_PATH
-
-TEMPLATE_PATH.append("rest/templates")
-ALL_UNITS = { 'g', 'kg', 'mL', 'L', 'Pieces', 'Bunches', 'Bags' }
-PARAMS = { 'group', 'category', 'product', 'unit', 'tag', 'organic' }
-BOOLEAN = {
-    "1": True,
-    True: "1",
-    "0": False,
-    False: "0",
-    None: "0.5",
-}
+# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY

rest/pyapi.py

@@ -27,25 +27,6 @@ DOMAIN = "shandan.one"
 PORT = ""
 LOCATION = SCHEME + (f"{HOST}." if HOST else "") + DOMAIN + (f":{PORT}" if PORT else "")
 
-def parse_data_uri(content):
-    # extract bytes from data url
-    _, *media, data = chain.from_iterable(map(
-        lambda x: x.split(',', 1), content.split(':', 1)
-    ))
-    media = media and media[0]
-    mimetype, *params, encoding = media.split(';')
-    if '=' in encoding:
-        params.append(encoding)
-        encoding = None
-    
-    return {
-        'mimetype': mimetype,
-        'params': dict(map(lambda x: x.split('='), params)),
-        'encoding': encoding,
-        'data': data,
-    }
-
-
 def parse_upload_placeholder(rowid):
     rowid = int(rowid)
     con = connect('util.db')
@@ -56,11 +37,8 @@ SELECT content FROM upload_temp WHERE rowid = ? LIMIT 1;
 """, (rowid,)).fetchall()[0][0]
     finally:
         con.close()
-    
-    data = parse_data_uri(content)
-    assert data['encoding'] == 'base64', f"unsupported encoding: {data['encoding']}"
-    data = b64decode(data['data'] + '==')
-    return data
+
+    return content
 
 
 @route('/<route:re:(clip|goto|upload|code)>/meta', method=['POST'])
@@ -122,45 +100,10 @@ def normalize(route):
 def send_static(filename):
     return static_file(filename, root='rest/static')
 
-@route(f'/<route:re:(clip|goto|code)>/<hash:re:{B32REGEX}{{1,5}}>', method='GET')
+@route(f'/<route:re:(clip|goto|upload|code)>/<hash:re:{B32REGEX}{{1,5}}>', method='GET')
 def get_clip(route, hash):
     hash = hash and normalize_base32(hash)
     return redirect(f'/{route}/?hash={hash}&go=true')
 
-
-@route(f'/upload/<hash:re:{B32REGEX}{{1,5}}>', method='GET')
-def get_upload(hash):
-    hash = hash and normalize_base32(hash)
-    con = connect('util.db')
-    fname, mimetype, content = (None, None, None)
-    try:
-        fname, mimetype, content, created = con.cursor().execute("""
-SELECT name, mime, content, created FROM upload WHERE hash = ? LIMIT 1;
-""", (hash,)).fetchall()[0]
-    finally:
-        con.close()
-
-    created = datetime.strptime(created, '%Y-%m-%d %H:%M:%S').replace(tzinfo=timezone.utc).timestamp()
-    data = parse_data_uri(content)
-    assert data['mimetype'].split(';', 1)[0] == mimetype.split(';', 1)[0].split('+')[0], f"mimetype in db and data uri differ"
-    charset = data['params'].get('charset', None)
-    assert data['encoding'] == 'base64', f"unsupported encoding: {data['encoding']}"
-    content = b64decode(data['data'] + '==')
-
-    headers = {}
-    headers['Content-Length'] = len(content)
-
-    # TODO: create ext from mime type?
-    headers['Content-Disposition'] = 'attachment; filename="%s"' % (fname or hash)
-    headers['Content-Encoding'] = 'application/octet-stream'
-
-    if mimetype:
-        if mimetype[:5] == 'text/' and charset and 'charset' not in mimetype:
-            mimetype += '; charset=%s' % charset
-        headers['Content-Type'] = mimetype
-    lm = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(created))
-    headers['Last-Modified'] = lm
-    return HTTPResponse(content, **headers)
-
 @route('/<any>/', method='GET')
 def redirect_trailing_slash(any): return redirect(f'/{any}')

rest/tool_color.py

@@ -1,5 +0,0 @@
-color = {
-    'clip': '#4f8f4f',
-    'goto': '#8f4f4f',
-    'upload': '#afaf0f',
-}

rest/validate.py

@@ -1,146 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2024
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-
-# https://www.ietf.org/rfc/rfc3696.txt
-
-"""
-   Without quotes, local-parts may consist of any combination of
-   alphabetic characters, digits, or any of the special characters
-
-      ! # $ % & ' * + - / = ?  ^ _ ` . { | } ~
-
-   period (".") may also appear, but may not be used to start or end the
-   local part, nor may two or more consecutive periods appear.  Stated
-   differently, any ASCII graphic (printing) character other than the
-   at-sign ("@"), backslash, double quote, comma, or square brackets may
-   appear without quoting.  If any of that list of excluded characters
-   are to appear, they must be quoted.
-"""
-
-from io import BufferedReader
-import mimetypes
-from itertools import chain
-import os
-from bottle import static_file, HTTPError, abort, LocalRequest, HTTPResponse
-from urllib.parse import urlparse, quote, quote_plus
-from .hash_util import blake_file, bytes_to_base32, blake
-
-# according to rfc3696
-URL_MUST_ESCAPE = bytes([
-    x for x in chain(
-        # control characters
-        range(int('0x1F', 0)+1),
-        # 0x7F and non 7bit-ASCII
-        range(int('0x7F', 0,), int('0xFF', 0)+1),
-        # specifically excluded
-        b'@\\",[]'
-    )
-])
-# so give this list to urllib.parse.quote which follows rfc3986
-URL_SAFE = bytes(( i for i in range(int('0xff',0)+1) if i not in map(int, URL_MUST_ESCAPE) ))
-
-CLIP_SIZE_LIMIT = 65535
-
-def get_filename(filename: str, root: str = 'rest/static/files'):
-    path = '/'.join([filename,]*2)
-    try:
-        with open(f'{root}/{path}.name', "r") as f:
-            name = f.read()
-        return name
-    except:
-        pass
-
-def get_file_size(filename: str, root: str = 'rest/static/files'):
-    path = '/'.join([filename,]*2)
-    try:
-        return os.stat(f'{root}/{path}.file').st_sizea
-    except:
-        pass
-
-def get_file_mimetype(name):
-    mimetype = mimetypes.guess_type(name, strict=False)[0] if name else True
-    return mimetype
-
-def validate_file(filename: str, root: str = 'rest/static/files', download=True, mimetype=True) -> HTTPResponse:
-    path = '/'.join([filename,]*2)
-
-    name = get_filename(filename)
-    mimetype = mimetype if mimetype and mimetype is not True else get_file_mimetype(name)
-    
-    ret = static_file(
-        f'{path}.file',
-        root=root,
-        download=name if name and download else download,
-        mimetype='auto' if mimetype is True else mimetype
-    )
-    if isinstance(ret, HTTPError):
-        return abort(404, f"No such `Upload`: {filename}")
-
-    _bytes = blake_file(f'{path}.file', person='upload'.encode('utf-8'), root=root)
-    _b32 = bytes_to_base32(_bytes)
-    if _b32 != filename:
-        return abort(410, f"Uploaded content differs")
-    return ret
-
-
-def validate_parameter(request: LocalRequest, name: str) -> bytes:
-    if name not in request.params:
-        return abort(400, f"Missing parameter: '{name}'")
-
-    # TODO: what is correct overhead for form content?
-    OVERHEAD = 1024
-    content: bytes = request.query.get(name, None)
-    content_length = request.content_length
-    if content_length == -1:
-        return abort(418, f"Content-Length must be specified")
-    if content_length > CLIP_SIZE_LIMIT + OVERHEAD:
-        return abort(418, f"Content-Length can not exceed {CLIP_SIZE_LIMIT*3} bytes")
-
-    # TODO: add test for both query/form param
-    if 'multipart/form-data' in request.content_type:
-        # TODO: what about binary data ?
-        content: bytes = (content or request.params[name].encode('utf-8'))
-    else:
-        content: bytes = (content or request.params[name].encode('latin-1'))
-
-    if len(content) > CLIP_SIZE_LIMIT:
-        return abort(418, f"Paste can not exceed {CLIP_SIZE_LIMIT} bytes")
-    return content
-
-
-def validate_url(url: str) -> str:
-    scheme, netloc, path, params, query, fragment = urlparse(url)
-
-    if not scheme: return abort(400, "URL has no scheme")
-
-    if scheme == 'file' and not path: return abort(400, "File URL has no path")
-
-    if scheme in ('http', 'https') and not netloc: return abort(400, "HTTP(S) URL has no netloc")
-
-    if netloc:
-        try:
-            user_info, loc = netloc.rsplit('@', 1)
-        except ValueError:
-            user_info = ''
-            loc = ''
-        if user_info:
-            user_info = quote(user_info, safe=URL_SAFE)
-            netloc = f"{user_info}@{''.join(loc)}"
-        else:
-            # TODO: do this properly, ie, valid dns-name/ip/port etc
-            netloc = quote(netloc, safe=URL_SAFE)
-    
-    path = quote(path, safe=URL_SAFE)
-    params = quote_plus(params, safe=URL_SAFE)
-    query = quote(query, safe=URL_SAFE)
-    fragment = quote(fragment, safe=URL_SAFE)
-    
-    url = f'{scheme}://{netloc}{path}{params}'
-    if query:
-        url = f'{url}?{query}'
-    if fragment:
-        url = f'{url}#{fragment}'
-    return url

site/clip/form.sql

@@ -2,7 +2,7 @@ SET ":view" = COALESCE(:content, '') <> '' AND COALESCE(:action, '') <> 'Edit as
 SELECT 'button' AS component;
 SELECT 'Open' AS title
 , 1 AS width
-, '/clip?action=open' AS link
+, '/clip/?action=open' AS link
 ;
 SELECT 'New' AS title
 , 1 AS width

site/code/recent.sql

@@ -29,7 +29,7 @@ SELECT 'dynamic' AS component, sqlpage.run_sql('form.sql') AS properties;
 SELECT 'list' AS component;
 SELECT COALESCE(type||' ','') || COALESCE(store||' ', '') || COALESCE(expiry, created) AS title
 , COALESCE(content->>'content'||' ', '') || COALESCE(content->>'type', '') AS description
-, '/code?hash='||c.hash AS link
+, '/code/?hash='||c.hash AS link
 FROM code c
 LEFT JOIN code_detail cd
 ON c.hash = cd.hash

site/goto/form.sql

@@ -2,7 +2,7 @@ SET ":view" = COALESCE(:content, '') <> '';
 SELECT 'button' AS component;
 SELECT 'Open' AS title
 , 1 AS width
-, '/goto?action=open' AS link
+, '/goto/?action=open' AS link
 ;
 SELECT 'New' AS title
 , 1 AS width

site/upload/download.sql

@@ -0,0 +1,3 @@
+SELECT 'download' AS component, content AS data_url
+FROM upload WHERE hash = :hash
+;

site/upload/form.sql

@@ -3,7 +3,7 @@ SET ":view" = COALESCE(:content, '') <> '';
 SELECT 'button' AS component;
 SELECT 'Open' AS title
 , 1 AS width
-, '/upload?action=open' AS link
+, '/upload/?action=open' AS link
 ;
 SELECT 'New' AS title
 , 1 AS width
@@ -48,7 +48,7 @@ WHERE :view
 ;
 SELECT 'Preview' as title
 , :tabler_color AS color
-, CASE WHEN substr(:mime_type, 0, instr(:mime_type, '/')) = 'image' THEN :content ELSE NULL END AS top_image
+, :content AS top_image
 , '
 Uploaded file type: ' || COALESCE(:mime_type, 'null') ||'
 

site/upload/index.sql

@@ -14,12 +14,13 @@ SET ":inner" = (CASE :action
 END);
 SET ":file_name" = sqlpage.uploaded_file_name('content');
 SET ":mime_type" = sqlpage.uploaded_file_mime_type('content');
--- although using a variable works, docs say to pass the function as first argument
--- https://sql.ophir.dev/functions.sql?function=read_file_as_data_url#function
-SET ":content" = sqlpage.read_file_as_data_url(sqlpage.uploaded_file_path('content'));
+SET ":file_path" = sqlpage.uploaded_file_path('content');
+
+SELECT 'dynamic' AS component, sqlpage.run_sql('download.sql') AS properties
+WHERE :hash IS NOT NULL AND $go = 'true';
 
 SELECT 'dynamic' AS component, sqlpage.run_sql('temp.sql') AS properties
-WHERE :content IS NOT NULL AND $rowid IS NULL;
+WHERE :file_path IS NOT NULL AND $rowid IS NULL;
 
 SET ":file_name" = (SELECT name FROM upload_temp WHERE rowid = $rowid);
 SET ":mime_type" = (SELECT mime FROM upload_temp WHERE rowid = $rowid);

site/upload/save.sql

@@ -1,11 +1,11 @@
 SET ":rowid" = :content;
-SET ":content" = (SELECT content FROM upload_temp WHERE rowid = :rowid);
+SET ":content" = (SELECT CASE WHEN substr(:mime_type, 0, instr(:mime_type, '/')) = 'image' THEN content ELSE NULL END FROM upload_temp WHERE rowid = :rowid);
 
 INSERT INTO upload (hash, content, name, mime, qr, created)
 VALUES (:hash, :content, :file_name, :mime_type, :qr, CURRENT_TIMESTAMP)
 ON CONFLICT DO
 UPDATE SET
-  content = excluded.content,
+  content = (SELECT content FROM upload_temp WHERE rowid = :rowid),
   name = excluded.name,
   mime = excluded.mime,
   created = excluded.created,

site/upload/temp.sql

@@ -1,3 +1,9 @@
 DELETE FROM upload_temp WHERE name IS NULL OR mime IS NULL;
-INSERT INTO upload_temp(name, mime, content) VALUES (:file_name, :mime_type, :content)
-RETURNING 'redirect' AS component, '/upload?rowid='||rowid AS link;
+INSERT INTO upload_temp(name, mime, content)
+VALUES (
+    :file_name,
+    :mime_type,
+    -- readfile() is provided by fileio sqlite extension
+    readfile(:file_path)
+)
+RETURNING 'redirect' AS component, '/upload/?rowid='||rowid AS link;

test/rest/test_url.py

@@ -1,72 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-#
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-
-from pytest import mark, raises
-from bottle import HTTPError
-from rest.validate import validate_url
-
-@mark.parametrize('url, expected', [
-    ['file:///',]*2,
-    ['file:///a/b/c',]*2,
-    ['https://shandan.one',]*2,
-    ['https://www.shandan.one',]*2,
-    ['https://www.shandan.one/clip?id=123',]*2,
-    
-    # empty query
-    ['https://www.shandan.one?', 'https://www.shandan.one',],
-
-    # empty fragment
-    ['https://www.shandan.one/clip?id=123#', 'https://www.shandan.one/clip?id=123',],
-
-    # no double slash
-    #['file:/a/b/c', (HTTPError, ""),],
-    #['file:/abc', (HTTPError, ""),],
-
-    # no scheme
-    ['/a/b/c', (HTTPError, 400, "URL has no scheme"),],
-
-    # no file path scheme
-    ['file:', (HTTPError, 400, "File URL has no path"),],
-    
-    # no HTTPS domain
-    #['https://abc?id=1:', (HTTPError, 400, "HTTP(S) URL has no netloc"),],
-    
-    # conecutive dots
-    #['https://shandan.one/abc..id', 'https://shandan.one/abc..id',],
-
-    # unescaped char in reg_name
-    # TODO: should be invalid because netloc must be a domain name or ip ?
-    ['https://🌚.shandan.one', 'https://%F0%9F%8C%9A.shandan.one',],
-
-    # @ in user_info not allowed
-    # TODO: check this - final @ should not be encoded ?
-    ['https://user@mail@www.shandan.one','https://user%40mail@www.shandan.one'],
-
-    # delimiters
-    # TODO: should < be translated to %3C ?
-    ['https://www.shandan.one?a<b', 'https://www.shandan.one?a<b'],
-
-    # more delimiters
-    ['https://www.shandan.one/clip?proportion=69%', 'https://www.shandan.one/clip?proportion=69%'],
-
-    # fragment before end of reference URI
-    ['https://www.shandan.one/tiny#url?id=123', 'https://www.shandan.one/tiny#url?id=123'],
-])
-def test_validate_url_invalid(url: str, expected: str):
-    if isinstance(expected, tuple):
-        exp_exception, *ex_args = expected
-    else:
-        exp_exception = None
-    
-    if not exp_exception:
-        assert validate_url(url) == expected
-        return
-
-    with raises(exp_exception) as ex:
-        validate_url(url)
-    
-    assert list(ex.value.args) == ex_args