move utilities into home-launcher repo

app/rest/pyapi.py

@@ -4,27 +4,20 @@
 #
 # THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
 import os
-from io import BytesIO
 from threading import Thread
 from typing import Union
 from bottle import (
     route, request, response,
-    redirect, abort,
     template, static_file,
-    FormsDict, HTTPError,
+    FormsDict,
 )
 from psycopg import Cursor, connect
 from psycopg.rows import TupleRow
-from linkpreview import link_preview
-
-from .validate import get_file_mimetype, get_file_size, get_filename, validate, validate_file, validate_parameter, validate_url
-from .hash_util import normalize_base32
 from .route_decorators import normalize, poison, cursor
 from .query_to_xml import get_categories, get_groups, get_products, get_tags
 from .CachedLoadingPage import CachedLoadingPage
 from .Cache import Cache
 from . import trend as worker
-from .save import save, save_upload
 
 host = f"host={os.getenv('HOST')}"
 db = f"dbname={os.getenv('DB', 'grocery')}"
@@ -91,222 +84,3 @@ def products(cur: Cursor[TupleRow], key: Union[int, str], forms: FormsDict, cach
 def tags(cur: Cursor[TupleRow], key: Union[int, str], forms: FormsDict, cache: Cache):
     response.content_type = 'application/xhtml+xml; charset=utf-8'
     return get_tags(cur, forms)
-
-
-SCHEME = "https://"
-HOST = ""
-DOMAIN = "shandan.one"
-PORT = ""
-LOCATION = SCHEME + (f"{HOST}." if HOST else "") + DOMAIN + (f":{PORT}" if PORT else "")
-
-
-@route('/clip', method=['GET', 'POST'])
-def clip():
-
-    if request.method == 'GET':
-        _hash = request.params.hash
-        if _hash:
-            _hash = normalize_base32(_hash)
-            content = validate(_hash).decode('utf-8')
-        else:
-            content = None
-        link = f'{LOCATION}/clip/{_hash}' if content else f'{LOCATION}/clip'
-
-        response.content_type = 'text/html; charset=utf-8'
-        form = template(
-            'clip-form',
-            action='/clip',
-            method='post',
-            content=content,
-            disabled=True if content else False
-        )
-        return template(
-            'paste',
-            form=form,
-            qr=f'{LOCATION}/clip/{_hash}.qr' if content else f'{LOCATION}/grocery/static/clip-qr.svg',
-            link=link,
-            disabled=True if content else False,
-            download=f'/clip/{_hash}' if content else None
-        )
-
-    if request.method == 'POST':
-        content = validate_parameter(request, 'paste')
-        if request.params.copy != 'true':
-            _b32 = save(content, LOCATION)
-            return redirect(f'/clip?hash={_b32}')
-
-        response.content_type = 'text/html; charset=utf-8'
-        form = template(
-            'clip-form',
-            action='/clip',
-            method='post',
-            content=content,
-            disabled=False
-        )
-        link = f'{LOCATION}/clip'
-        return template(
-            'paste',
-            form=form,
-            qr=f'{LOCATION}/grocery/static/clip-qr.svg',
-            link=link,
-            disabled=False,
-            download=None
-        )
-
-
-@route('/clip/<filename:path>', method='GET')
-def get_clip(filename):
-    ext = 'file'
-    if filename and filename.endswith('.qr'):
-        filename, ext = filename.split('.', 1)
-    
-    filename = filename and normalize_base32(filename)
-    path = f'{filename}/{filename}.{ext}'
-    
-    if ext == 'qr':
-        return static_file(path, root='app/rest/static', mimetype='image/svg+xml')
-    
-    filename = filename and normalize_base32(filename)
-    if not request.params.raw.lower() == 'true':
-        # TODO: return a form with timeout to a GET instead ?
-        return redirect(f'/clip?hash={filename}')
-
-    ret = validate(filename)
-    if isinstance(ret, HTTPError):
-        return ret
-
-    return static_file(path, root='app/rest/static')
-
-
-@route('/upload', method=['GET', 'POST'])
-def upload():
-
-    if request.method == 'GET':
-        _hash = request.params.hash
-        mimetype = None
-        if _hash:
-            _hash = normalize_base32(_hash)
-            size = get_file_size(_hash)
-            name = get_filename(_hash)
-            mimetype = get_file_mimetype(name)
-            if mimetype is not True and mimetype.startswith('text'):
-                mimetype = None if size > CLIP_SIZE_LIMIT else mimetype
-        
-        link = f'{LOCATION}/upload/{_hash}' if _hash else f'{LOCATION}/upload'
-        response.content_type = 'text/html; charset=utf-8'
-        disabled = True if _hash else False
-        form = template('file-form', action='/upload', method='post', disabled=disabled)
-        
-        return template(
-            'upload',
-            form=form,
-            qr=f'{LOCATION}/upload/{_hash}.qr' if _hash else f'{LOCATION}/grocery/static/upload-qr.svg',
-            link=link,
-            mimetype=mimetype,
-            disabled=disabled
-        )
-
-    if request.method == 'POST':
-        if 'paste' not in request.files:
-            return abort(400, "Parameter 'paste' must be specified")
-
-        upload = request.files['paste']
-        if isinstance(upload.file, BytesIO):
-            if len(upload.file.read()) == 0:
-                return abort(400, "File is empty")
-
-        _b32 = save_upload(upload.raw_filename, LOCATION, upload.file, root='app/rest/static')
-        return redirect(f'/upload?hash={_b32}')
-
-
-@route('/upload/<filename:path>', method='GET')
-def get_upload(filename):
-    ext = 'file'
-    if filename and filename.endswith('.qr'):
-        filename, ext = filename.split('.', 1)
-    
-    filename = filename and normalize_base32(filename)
-    path = f'{filename}/{filename}.{ext}'
-    
-    if ext == 'qr':
-        return static_file(path, root='app/rest/static', mimetype='image/svg+xml')
-    
-    download = True
-    mimetype = True
-    if request.params.download == "false":
-        download = False
-    mimetype = request.params.mimetype or None
-
-    return validate_file(filename, download=download, mimetype=mimetype)
-
-
-@route('/goto', method=['GET', 'POST'])
-def goto():
-
-    if request.method == 'GET':
-        _hash = request.params.hash
-        if _hash:
-            _hash = normalize_base32(_hash)
-            content = validate(_hash).decode('utf-8')
-        else:
-            content = None
-
-        if content and request.params.go == 'true':
-            target = validate_url(content)
-            return redirect(target)
-
-        link = f'{LOCATION}/goto/{_hash}' if content else f'{LOCATION}/goto'
-        disabled = True if content else False
-        response.content_type = 'text/html; charset=utf-8'
-        form = template(
-            'goto-form',
-            action='/goto',
-            method='post',
-            content=content,
-            disabled=disabled
-        )
-        preview = dict()
-        if content:
-            try:
-                page = link_preview(link, parser="lxml")
-                preview['title'] = page.title
-                preview['img'] = page.absolute_image
-                preview['domain'] = page.site_name
-                preview['link'] = content
-            except:
-                pass
-
-        return template(
-            'goto',
-            form=form,
-            qr=f'{LOCATION}/goto/{_hash}.qr' if content else f'{LOCATION}/grocery/static/goto-qr.svg',
-            link=link,
-            disabled=disabled,
-            preview=preview,
-        )
-
-    if request.method == 'POST':
-        content = validate_parameter(request, 'url')
-        _b32 = save(content, LOCATION)
-
-        # validate but save content unmodified
-        _ = validate_url(content.decode('utf-8'))
-        return redirect(f'/goto?hash={_b32}')
-
-
-@route('/goto/<filename:path>', method='GET')
-def redirect_goto(filename):
-    ext = 'file'
-    if filename and filename.endswith('.qr'):
-        filename, ext = filename.split('.', 1)
-    
-    filename = filename and normalize_base32(filename)
-    path = f'{filename}/{filename}.{ext}'
-    
-    if ext == 'qr':
-        return static_file(path, root='app/rest/static', mimetype='image/svg+xml')
-    
-    return redirect(f'/goto?hash={filename}&go=true')
-
-@route('/<any>/', method='GET')
-def redirect_trailing_slash(any): return redirect(f'/{any}')

app/rest/qr.py

@@ -1,34 +0,0 @@
-from io import BytesIO
-from typing import Union
-from qrcode import QRCode
-from qrcode.constants import ERROR_CORRECT_L, ERROR_CORRECT_M, ERROR_CORRECT_Q, ERROR_CORRECT_H
-from qrcode.image.styledpil import StyledPilImage
-from qrcode.image.svg import SvgPathImage
-from qrcode.image.styles.moduledrawers.svg import SvgCircleDrawer
-from qrcode.image.styles.colormasks import RadialGradiantColorMask
-
-QR_MAX_BYTES = {
-    ERROR_CORRECT_H: 1273,
-    ERROR_CORRECT_Q: 1663,
-    ERROR_CORRECT_M: 2331,
-    ERROR_CORRECT_L: 2953,
-}
-def get_qr_code(data: Union[bytes, str], fallback: Union[bytes, str] = None):
-    err_lvl = ERROR_CORRECT_H
-    qr = QRCode(error_correction=err_lvl)
-    if data is not None and isinstance(data, str):
-        data = data.encode('utf-8')
-    if fallback is not None and isinstance(fallback, str):
-        fallback = fallback.encode('utf-8')
-    if fallback is not None and data and len(data) > QR_MAX_BYTES[err_lvl]:
-        qr.add_data(fallback, optimize=0)
-    else:
-        qr.add_data(data or fallback, optimize=0)
-
-    img_1 = qr.make_image(image_factory=SvgPathImage)
-    with BytesIO() as f:
-        img_1.save(f)
-        f.flush()
-        ret = f.getvalue()
-    return ret
-    

app/rest/requirements.txt

@@ -4,6 +4,3 @@ bottle
 wsgigzip
 cherrypy
 base32-lib
-lxml
-linkpreview
-qrcode

app/rest/save.py

@@ -1,83 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-
-from hashlib import blake2b
-from io import BufferedRandom, BytesIO
-import os
-from uuid import uuid4
-
-from .hash_util import DIGEST_SIZE_BYTES, blake, bytes_to_base32
-from .qr import get_qr_code
-
-def save_qr(qr: bytes, _b32:str , directory: str) -> str:
-    fd = os.open(f'{directory}/{_b32}.qr', os.O_WRONLY | os.O_TRUNC | os.O_CREAT, 0o600)
-    with open(fd, "wb") as f:
-        f.write(qr)
-
-
-def save(content: bytes, location: str, root='app/rest/static') -> str:
-    _bytes = blake(content, person='clip'.encode('utf-8'))
-    _b32 = bytes_to_base32(_bytes)
-    directory = f'{root}/{_b32}'
-    try:
-        os.mkdir(directory, mode=0o700, dir_fd=None)
-    except FileExistsError:
-        pass
-    fd = os.open(f'{directory}/{_b32}.file', os.O_WRONLY | os.O_TRUNC | os.O_CREAT, 0o600)
-    with open(fd, "wb") as f:
-        f.write(content)
-    
-    link = f'{location}/clip/{_b32}'
-    svg = get_qr_code(content, fallback=link)
-    save_qr(svg, _b32, directory)
-    return _b32
-
-
-def save_filename(name: str, _b32: str, directory: str):
-    fd = os.open(f'{directory}/{_b32}.name', os.O_WRONLY | os.O_TRUNC | os.O_CREAT, 0o600)
-    with open(fd, "w") as f:
-        f.write(name)
-
-
-def save_upload(name: str, location: str, content: BufferedRandom, root='app/rest/static') -> str:
-    tmpdir = '/tmp/upload'
-    try:
-        os.mkdir(tmpdir, mode=0o700, dir_fd=None)
-    except FileExistsError:
-        pass
-    
-    unique = uuid4()
-    fd = os.open(f'{tmpdir}/{unique.hex}', os.O_WRONLY | os.O_TRUNC | os.O_CREAT, 0o600)
-    with open(fd, "wb") as f:
-        done = False
-        if isinstance(content, BytesIO):
-            f.write(content.getvalue())
-            done = True
-        while not done and content.peek(1):
-            seg = content.read(1024)
-            f.write(seg)
-    
-    with open(f'{tmpdir}/{unique.hex}', "rb") as f:
-        _blake = blake2b(usedforsecurity=False, digest_size=DIGEST_SIZE_BYTES, person='upload'.encode('utf-8'))
-        while f.peek(1):
-            _blake.update(f.read(1024))
-        
-        _bytes = _blake.digest()
-        _b32 = bytes_to_base32(_bytes)
-    
-    directory = f'{root}/{_b32}'
-    try:
-        os.mkdir(directory, mode=0o700, dir_fd=None)
-    except FileExistsError:
-        pass
-
-    os.replace(f'{tmpdir}/{unique.hex}', f'{directory}/{_b32}.file')
-    save_filename(name, _b32, directory)
-    link = f'{location}/clip/{_b32}'
-    svg = get_qr_code(link, fallback=link)
-    save_qr(svg, _b32, directory)
-
-    return _b32

app/rest/templates/card-goto.tpl

@@ -1,22 +0,0 @@
-<style>
-.card img {
-    color: floralwhite;
-    object-fit: cover;
-    width: 100%;
-    height: 100%;
-}
-</style>
-<a href="{{link}}">
-<article class="card">
-  <header>
-    <h2>{{title}}</h2>
-  </header>
-  <figure> 
-    <img src="{{img}}" alt="{{title}}">
-    <figcaption>{{domain}}</figcaption>
-  </figure>
-  <div class="content">
-    {{link}}
-  </div>       
-</article>
-</a>

app/rest/templates/clip-form.tpl

@@ -1,39 +0,0 @@
-% from app.data.filter import get_query_param
-% content = setdefault("content", "") or ""
-% disabled = (setdefault("disabled", False) and 'readonly="true"') or ""
-<form id="paste" method="{{ method }}" action="{{ action }}" enctype="multipart/form-data">
-  <style>
-textarea::-webkit-scrollbar {
-  width: 11px;
-}
-textarea {
-  color: #cccccc;
-  background-color: #080808;
-  scrollbar-width: thin;
-  scrollbar-color: var(--thumbBG) var(--scrollbarBG);
-}
-textarea::-webkit-scrollbar-track {
-  background: var(--scrollbarBG);
-}
-textarea::-webkit-scrollbar-thumb {
-  background-color: var(--thumbBG) ;
-  border-radius: 6px;
-  border: 3px solid var(--scrollbarBG);
-}
-  </style>
-  <textarea
-    style="width: 80%"
-    id="paste-text-area"
-    name="paste"
-    rows="30"
-    {{!disabled}}
-    required="true"
-    autofocus="true"
-    placeholder="Paste something here..."
-  >{{ content }}</textarea>
-</form>
-<form id="copy" method="post" action="{{ action }}">
-  <input id="copy-paste" name="paste" type="text" value="{{ content }}" hidden="true" />
-  <input id="copy" name="copy" type="text" value="true" hidden="true" />
-</form>
-<form id="new" method="get" action="{{ action }}"></form>

app/rest/templates/file-form.tpl

@@ -1,7 +0,0 @@
-% from app.data.filter import get_query_param
-% content = setdefault("content", "") or ""
-% disabled = setdefault("disabled", "") and 'hidden="true"'
-<form id="upload" method="{{ method }}" action="{{ action }}" enctype="multipart/form-data">
-  <input type="file" name="paste" required="true" {{disabled}} />
-</form>
-<form id="new" method="get" action="{{ action }}"></form>

app/rest/templates/goto-form.tpl

@@ -1,35 +0,0 @@
-% from app.data.filter import get_query_param
-% content = setdefault("content", "") or ""
-% disabled = (setdefault("disabled", False) and 'readonly="true"') or ""
-<form id="goto" method="{{ method }}" action="{{ action }}">
-  <style>
-input[type="url"]::-webkit-scrollbar {
-  width: 11px;
-}
-input[type="url"] {
-  color: #cccccc;
-  background-color: #080808;
-  scrollbar-width: thin;
-  scrollbar-color: var(--thumbBG) var(--scrollbarBG);
-}
-input[type="url"]::-webkit-scrollbar-track {
-  background: var(--scrollbarBG);
-}
-input[type="url"]::-webkit-scrollbar-thumb {
-  background-color: var(--thumbBG) ;
-  border-radius: 6px;
-  border: 3px solid var(--scrollbarBG);
-}
-  </style>
-  <input type="url"
-    style="width: 80%"
-    id="input-url"
-    name="url"
-    {{!disabled}}
-    required="true"
-    autofocus="true"
-    placeholder="Paste URL here..."
-    value="{{ content }}"
-  ></input>
-</form>
-<form id="new" method="get" action="{{ action }}"></form>

app/rest/templates/goto.tpl

@@ -1,56 +0,0 @@
-% link = setdefault("link", "") or ""
-% disabled = setdefault("disabled", "") and 'disabled="true"'
-% preview = setdefault("preview", None)
-<!DOCTYPE html>
-<html>
-  <head>
-    <style>
-html {
-  --scrollbarBG: #333333;
-  --thumbBG: #080808;
-}
-body {
-  background-color: #080808;
-  color: #cccccc;
-}
-img {
-  background-color: floralwhite;
-  color: black;
-  max-height: min(100vh, calc(100vw * 9 / 16));
-  max-width: calc(100vw - 2em);
-}
-    </style>
-    <title>Go to ...</title>
-    <meta name="viewport" content="width=device-width, initial-scale=1"/>
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/purecss@2.1.0/build/pure-min.css" integrity="sha384-yHIFVG6ClnONEA5yB5DJXfW2/KC173DIQrYoZMEtBvGzmf0PKiGyNEqe9N6BNDBH" crossorigin="anonymous"/>
-    <link rel="stylesheet" href="https://shandan.one/css/grids-responsive-min.css"/>
-    <link rel="stylesheet" href="https://shandan.one/css/responsive-visibility-collapse.css"/>
-  </head>
-  <body align="center" style="text-align: center">
-    <div class="pure-g">
-      % include('button-style')
-      <div class="pure-u-1">
-        <div class="pure-button-group" role="action" style="padding: 1em 0 0;">
-          <button class="button-resize pure-button" type="submit" form="new"> New </button>
-          <button class="button-resize pure-button" type="submit" form="goto" {{!disabled}}> Shrtn It! </button>
-        </div>
-      </div>
-      <div class="pure-u-1">
-        <div class="pure-button" style="margin: 1em 0 0; background: #8f4f4f">
-          <a href="{{!link}}" style="color: floralwhite;">{{ link }}</a>
-        </div>
-      </div>
-      <div class="pure-u-1">
-        <p><details><summary> Show QR code ...</summary><img src="{{qr}}"/></details></p>
-{{!form}}
-      </div>
-      <div class="pure-u-1-3"></div>
-      <div class="pure-u-1-3">
-      % if preview:
-      %   include('card-goto', **preview)
-      % end
-      </div>
-      <div class="pure-u-1-3"></div>
-    </div>
-  </body>
-</html>

app/rest/templates/paste.tpl

@@ -1,55 +0,0 @@
-% link = setdefault("link", "") or ""
-% disabled = setdefault("disabled", "") and 'disabled="true"'
-% download = setdefault("download", "") or ""
-% download_disabled = "" if download else 'disabled="true"'
-<!DOCTYPE html>
-<html>
-  <head>
-    <style>
-html {
-  --scrollbarBG: #333333;
-  --thumbBG: #080808;
-}
-body {
-  background-color: #080808;
-  color: #cccccc;
-}
-img {
-  background-color: floralwhite;
-  color: black;
-  max-height: min(100vh, calc(100vw * 9 / 16));
-  max-width: calc(100vw - 2em);
-}
-    </style>
-    <title>Paste</title>
-    <meta name="viewport" content="width=device-width, initial-scale=1"/>
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/purecss@2.1.0/build/pure-min.css" integrity="sha384-yHIFVG6ClnONEA5yB5DJXfW2/KC173DIQrYoZMEtBvGzmf0PKiGyNEqe9N6BNDBH" crossorigin="anonymous"/>
-    <link rel="stylesheet" href="https://shandan.one/css/grids-responsive-min.css"/>
-    <link rel="stylesheet" href="https://shandan.one/css/responsive-visibility-collapse.css"/>
-  </head>
-  <body align="center" style="text-align: center">
-    <div class="pure-g">
-      % include('button-style')
-      <div class="pure-u-1">
-        <div class="pure-button-group" role="action" style="padding: 1em 0 0;">
-          <button class="button-resize pure-button" type="submit" form="new"> New </button>
-          <button class="button-resize pure-button" type="submit" form="copy"> Edit as New </button>
-          <button class="button-resize pure-button" type="submit" form="paste" {{!disabled}}> Paste </button>
-          <button class="button-resize pure-button" type="submit" form="download" {{!download_disabled}}> Download </button>
-        </div>
-      </div>
-      <div class="pure-u-1">
-        <div class="pure-button" style="margin: 1em 0 0; background: #4f8f4f;">
-          <a href="{{!link}}" style="color: floralwhite;">{{ link }}</a>
-        </div>
-      </div>
-      <div class="pure-u-1">
-        <p><details><summary> Show QR code ...</summary><img src="{{qr}}"/></details></p>
-{{!form}}
-<form id="download" method="get" action="{{ download }}">
-<input name="raw" type="text" hidden="true" value="true" />
-</form>
-      </div>
-    </div>
-  </body>
-</html>

app/rest/templates/upload.tpl

@@ -1,72 +0,0 @@
-% link = setdefault("link", "") or ""
-% disabled = setdefault("disabled", "") and 'disabled="true"'
-% download_disabled = "" if disabled else 'disabled="true"'
-% mimetype = (setdefault("mimetype", None) is not True and mimetype) or None
-<!DOCTYPE html>
-<html>
-  <head>
-    <style>
-html {
-  --scrollbarBG: #333333;
-  --thumbBG: #080808;
-}
-body {
-  background-color: #080808;
-  color: #cccccc;
-}
-img {
-  background-color: floralwhite;
-  color: black;
-  max-height: min(100vh, calc(100vw * 9 / 16));
-  max-width: calc(100vw - 2em);
-}
-object {
-  width: 80%;
-  margin: 1em 0 0;
-}
-object.other {
-  height: 60vh;
-}
-object.text {
-  height: 60vh;
-  border: floralwhite;
-  border-style: solid;
-  border-width: thin;
-}
-    </style>
-    <title>Upload</title>
-    <meta name="viewport" content="width=device-width, initial-scale=1"/>
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/purecss@2.1.0/build/pure-min.css" integrity="sha384-yHIFVG6ClnONEA5yB5DJXfW2/KC173DIQrYoZMEtBvGzmf0PKiGyNEqe9N6BNDBH" crossorigin="anonymous"/>
-    <link rel="stylesheet" href="https://shandan.one/css/grids-responsive-min.css"/>
-    <link rel="stylesheet" href="https://shandan.one/css/responsive-visibility-collapse.css"/>
-  </head>
-  <body align="center" style="text-align: center">
-    <div class="pure-g">
-      % include('button-style')
-      <div class="pure-u-1">
-        <div class="pure-button-group" role="action" style="padding: 1em 0 0;">
-          <button class="button-resize pure-button" type="submit" form="new"> New </button>
-          <button class="button-resize pure-button" type="submit" form="upload" {{!disabled}}> Upload </button>
-          <button class="button-resize pure-button" type="submit" form="download" {{!download_disabled}}> Download </button>
-        </div>
-      </div>
-      <div class="pure-u-1">
-        <div class="pure-button" style="margin: 1em 0 0; background: #afaf0f;">
-          <a href="{{link}}" style="color: floralwhite;">{{ link }}</a>
-        </div>
-      </div>
-      <div class="pure-u-1">
-        <p><details><summary> Show QR code ...</summary><img src="{{qr}}"/></details></p>
-{{!form}}
-<form id="download" method="get" action="{{link}}"></form>
-        % if mimetype and mimetype.startswith('text'):
-        <object class="text" data="{{link}}?download=false&mimetype=text/plain" type={{mimetype}}><p>Unable to display {{mimetype}}</p></object>
-        % elif mimetype and mimetype.startswith('image'):
-        <object data="{{link}}?download=false" type="{{mimetype}}"><p>Unable to display {{mimetype}}</p></object>
-        % elif mimetype:
-        <object class="other" data="{{link}}?download=false" type={{mimetype}}><p>Unable to display {{mimetype}}</p></object>
-        % end
-      </div>
-    </div>
-  </body>
-</html>

app/rest/validate.py

@@ -1,153 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-
-# https://www.ietf.org/rfc/rfc3696.txt
-
-"""
-   Without quotes, local-parts may consist of any combination of
-   alphabetic characters, digits, or any of the special characters
-
-      ! # $ % & ' * + - / = ?  ^ _ ` . { | } ~
-
-   period (".") may also appear, but may not be used to start or end the
-   local part, nor may two or more consecutive periods appear.  Stated
-   differently, any ASCII graphic (printing) character other than the
-   at-sign ("@"), backslash, double quote, comma, or square brackets may
-   appear without quoting.  If any of that list of excluded characters
-   are to appear, they must be quoted.
-"""
-
-from io import BufferedReader
-import mimetypes
-from itertools import chain
-import os
-from bottle import static_file, response, HTTPError, abort, LocalRequest, HTTPResponse
-from urllib.parse import urlparse, quote, quote_plus
-from .hash_util import blake_file, bytes_to_base32, blake
-
-# according to rfc3696
-URL_MUST_ESCAPE = bytes([
-    x for x in chain(
-        # control characters
-        range(int('0x1F', 0)+1),
-        # 0x7F and non 7bit-ASCII
-        range(int('0x7F', 0,), int('0xFF', 0)+1),
-        # specifically excluded
-        b'@\\",[]'
-    )
-])
-# so give this list to urllib.parse.quote which follows rfc3986
-URL_SAFE = bytes(( i for i in range(int('0xff',0)+1) if i not in map(int, URL_MUST_ESCAPE) ))
-
-CLIP_SIZE_LIMIT = 65535
-def validate(filename: str) -> bytes:
-    ret = static_file('/'.join([filename,]*2) + '.file', root='app/rest/static')
-    if isinstance(ret, HTTPError):
-        return abort(404, f"No such paste: {filename}")
-
-    if ret.content_length > CLIP_SIZE_LIMIT:
-        return abort(418, f"Paste size exceeds {CLIP_SIZE_LIMIT}")
-
-    content: bytes = ret.body.read() if isinstance(ret.body, BufferedReader) else ret.body.encode('utf-8')
-
-    _bytes = blake(content, person='clip'.encode('utf-8'))
-    _b32 = bytes_to_base32(_bytes)
-    if _b32 != filename:
-        return abort(410, f"Paste content differs")
-    return content
-
-
-def get_filename(filename: str, root: str = 'app/rest/static'):
-    path = '/'.join([filename,]*2)
-    try:
-        with open(f'{root}/{path}.name', "r") as f:
-            name = f.read()
-        return name
-    except:
-        pass
-
-def get_file_size(filename: str, root: str = 'app/rest/static'):
-    path = '/'.join([filename,]*2)
-    return os.stat(f'{root}/{path}.file').st_size
-
-def get_file_mimetype(name):
-    mimetype = mimetypes.guess_type(name, strict=False)[0] if name else True
-    return mimetype
-
-def validate_file(filename: str, root: str = 'app/rest/static', download=True, mimetype=True) -> HTTPResponse:
-    path = '/'.join([filename,]*2)
-
-    name = get_filename(filename)
-    mimetype = mimetype if mimetype and mimetype is not True else get_file_mimetype(name)
-    
-    ret = static_file(f'{path}.file', root=root, download=name if name and download else download, mimetype=mimetype)
-    if isinstance(ret, HTTPError):
-        return abort(404, f"No such upload: {filename}")
-
-    _bytes = blake_file(f'{path}.file', person='upload'.encode('utf-8'), root=root)
-    _b32 = bytes_to_base32(_bytes)
-    if _b32 != filename:
-        return abort(410, f"Uploaded content differs")
-    return ret
-
-
-def validate_parameter(request: LocalRequest, name: str) -> bytes:
-    if name not in request.params:
-        return abort(400, f"Missing parameter: '{name}'")
-    
-    # TODO: what is correct overhead for form content?
-    OVERHEAD = 1024
-    content: bytes = request.query.get(name, None)
-    content_length = request.content_length
-    if content_length == -1:
-        return abort(418, f"Content-Length must be specified")
-    if content_length > CLIP_SIZE_LIMIT + OVERHEAD:
-        return abort(418, f"Content-Length can not exceed {CLIP_SIZE_LIMIT*3} bytes")
-
-    # TODO: add test for both query/form param
-    if 'multipart/form-data' in request.content_type:
-        # TODO: what about binary data ?
-        content: bytes = (content or request.params[name].encode('utf-8'))
-    else:
-        content: bytes = (content or request.params[name].encode('latin-1'))
-    
-    if len(content) > CLIP_SIZE_LIMIT:
-        return abort(418, f"Paste can not exceed {CLIP_SIZE_LIMIT} bytes")
-    return content
-
-def validate_url(url: str) -> str:
-    scheme, netloc, path, params, query, fragment = urlparse(url)
-
-    if not scheme: return abort(400, "URL has no scheme")
-
-    if scheme == 'file' and not path: return abort(400, "File URL has no path")
-
-    if scheme in ('http', 'https') and not netloc: return abort(400, "HTTP(S) URL has no netloc")
-
-    if netloc:
-        try:
-            user_info, loc = netloc.rsplit('@', 1)
-        except ValueError:
-            user_info = ''
-            loc = ''
-        if user_info:
-            user_info = quote(user_info, safe=URL_SAFE)
-            netloc = f"{user_info}@{''.join(loc)}"
-        else:
-            # TODO: do this properly, ie, valid dns-name/ip/port etc
-            netloc = quote(netloc, safe=URL_SAFE)
-    
-    path = quote(path, safe=URL_SAFE)
-    params = quote_plus(params, safe=URL_SAFE)
-    query = quote(query, safe=URL_SAFE)
-    fragment = quote(fragment, safe=URL_SAFE)
-    
-    url = f'{scheme}://{netloc}{path}{params}'
-    if query:
-        url = f'{url}?{query}'
-    if fragment:
-        url = f'{url}#{fragment}'
-    return url

test/rest/test_hash_util.py

@@ -329,34 +329,34 @@ def find_collision(f=python, norm=normalize_hash, how_many=None, **kwargs):
             seen[_hash] = url
     return None, None, seen
 
-trials = 5 #20
-for name, args, kwargs in [
-    #("Python", [], {'norm': normalize_hash}),
-    #("Shake", [shake], {'norm': hex_to_hash}),
-    ("Blake", [blake], {'norm': hex_to_hash, 'person': 'trend'.encode("utf-8")}),
-    #("Python", [], {'norm': hash_to_base32}),
-    #("Shake", [shake], {'norm': hex_to_base32}),
-    #("Blake", [blake], {'norm': hex_to_base32, 'person': 'trend'.encode("utf-8")}),
-]:
-    start = time.time()
-    random.seed(0)
-    score = 0
-    _min = 0
-    print(name)
-    for i in range(trials):
-        url, _hash, seen = find_collision(*args, **kwargs, how_many=list(range(len(products))))
-        n = len(seen)
-        print(n)
-        if n < 30000:
-            print(url)
-            print(seen[_hash])
-        score += n
-        _min = _min if _min < n and _min else n
-    end = time.time()
-    print(f"Score: {_min} / {score/trials}")
-    print(f"Time: {end - start}")
-
-exit(0)
+# trials = 5 #20
+# for name, args, kwargs in [
+#     #("Python", [], {'norm': normalize_hash}),
+#     #("Shake", [shake], {'norm': hex_to_hash}),
+#     ("Blake", [blake], {'norm': hex_to_hash, 'person': 'trend'.encode("utf-8")}),
+#     #("Python", [], {'norm': hash_to_base32}),
+#     #("Shake", [shake], {'norm': hex_to_base32}),
+#     #("Blake", [blake], {'norm': hex_to_base32, 'person': 'trend'.encode("utf-8")}),
+# ]:
+#     start = time.time()
+#     random.seed(0)
+#     score = 0
+#     _min = 0
+#     print(name)
+#     for i in range(trials):
+#         url, _hash, seen = find_collision(*args, **kwargs, how_many=list(range(len(products))))
+#         n = len(seen)
+#         print(n)
+#         if n < 30000:
+#             print(url)
+#             print(seen[_hash])
+#         score += n
+#         _min = _min if _min < n and _min else n
+#     end = time.time()
+#     print(f"Score: {_min} / {score/trials}")
+#     print(f"Time: {end - start}")
+
+# exit(0)
 
 
 if __name__ == '__main__':

test/rest/test_url.py

@@ -1,72 +0,0 @@
-#
-# Copyright (c) Daniel Sheffield 2023
-#
-# All rights reserved
-#
-# THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-
-from pytest import mark, raises
-from bottle import HTTPError
-from app.rest.validate import validate_url
-
-@mark.parametrize('url, expected', [
-    ['file:///',]*2,
-    ['file:///a/b/c',]*2,
-    ['https://shandan.one',]*2,
-    ['https://www.shandan.one',]*2,
-    ['https://www.shandan.one/clip?id=123',]*2,
-    
-    # empty query
-    ['https://www.shandan.one?', 'https://www.shandan.one',],
-
-    # empty fragment
-    ['https://www.shandan.one/clip?id=123#', 'https://www.shandan.one/clip?id=123',],
-
-    # no double slash
-    #['file:/a/b/c', (HTTPError, ""),],
-    #['file:/abc', (HTTPError, ""),],
-
-    # no scheme
-    ['/a/b/c', (HTTPError, 400, "URL has no scheme"),],
-
-    # no file path scheme
-    ['file:', (HTTPError, 400, "File URL has no path"),],
-    
-    # no HTTPS domain
-    #['https://abc?id=1:', (HTTPError, 400, "HTTP(S) URL has no netloc"),],
-    
-    # conecutive dots
-    #['https://shandan.one/abc..id', 'https://shandan.one/abc..id',],
-
-    # unescaped char in reg_name
-    # TODO: should be invalid because netloc must be a domain name or ip ?
-    ['https://🌚.shandan.one', 'https://%F0%9F%8C%9A.shandan.one',],
-
-    # @ in user_info not allowed
-    # TODO: check this - final @ should not be encoded ?
-    ['https://user@mail@www.shandan.one','https://user%40mail@www.shandan.one'],
-
-    # delimiters
-    # TODO: should < be translated to %3C ?
-    ['https://www.shandan.one?a<b', 'https://www.shandan.one?a<b'],
-
-    # more delimiters
-    ['https://www.shandan.one/clip?proportion=69%', 'https://www.shandan.one/clip?proportion=69%'],
-
-    # fragment before end of reference URI
-    ['https://www.shandan.one/tiny#url?id=123', 'https://www.shandan.one/tiny#url?id=123'],
-])
-def test_validate_url_invalid(url: str, expected: str):
-    if isinstance(expected, tuple):
-        exp_exception, *ex_args = expected
-    else:
-        exp_exception = None
-    
-    if not exp_exception:
-        assert validate_url(url) == expected
-        return
-
-    with raises(exp_exception) as ex:
-        validate_url(url)
-    
-    assert list(ex.value.args) == ex_args