From be796b9b5bf38e9c0292e544534d35059b392c3e Mon Sep 17 00:00:00 2001 From: Ember Hearth Date: Sat, 26 Nov 2022 22:56:38 -0500 Subject: [PATCH 01/23] Add example configuration file See #73. --- README.rst | 12 +-- instance/config.example.py | 163 +++++++++++++++++++++++++++++++++++++ 2 files changed, 169 insertions(+), 6 deletions(-) create mode 100644 instance/config.example.py diff --git a/README.rst b/README.rst index 0e1c1cc..f13167c 100644 --- a/README.rst +++ b/README.rst @@ -7,13 +7,13 @@ This is a no-bullshit file hosting and URL shortening service that also runs Configuration ------------- -To configure 0x0, create ``instance/config.py``. -The defaults are at the start of ``fhost.py``. To change them, -add them to ``instance/config.py``— for example:: +To configure 0x0, copy ``instance/config.example.py`` to ``instance/config.py``, then edit +it. Resonable defaults are set, but there's a couple options you'll need to change +before running 0x0 for the first time. - SQLALCHEMY_DATABASE_URI = "sqlite:///some/path/db.sqlite" - -For more information on instance configuration, see +By default, the configuration is stored in the Flask instance directory. +Normally, this is in `./instance`, but it might be different for your system. +For details, see `the Flask documentation `_. To customize the home and error pages, simply create a ``templates`` directory diff --git a/instance/config.example.py b/instance/config.example.py new file mode 100644 index 0000000..cd78dde --- /dev/null +++ b/instance/config.example.py @@ -0,0 +1,163 @@ + + + ################################################################################ + # This is a configuration file for 0x0 / The Null Pointer # + # # + # The default values here are set to generally reasonable defaults, but a # + # couple of things need your attention. Specifically, make sure you set # + # SQLALCHEMY_DATABASE_URI. You'll also probably want to configure # + # FHOST_USE_X_SENDFILE and FHOST_USE_X_ACCEL_REDIRECT to match your webserver. # + # # + # Need help, or find anything confusing? Try opening up an issue! # + # https://git.0x0.st/mia/0x0/issues/new # + ################################################################################ + + + +# The database URL for the database 0x0 should use +# +# See https://docs.sqlalchemy.org/en/20/core/engines.html#backend-specific-urls +# for help configuring these for your database. +# +# For small and medium servers, it's plenty sufficient to just use an sqlite +# database. In this case, the database URI you want to use is just +# +# sqlite:/// + /path/to/your/database.db +# +# Until https://git.0x0.st/mia/0x0/issues/70 is resolved, it's recommended that +# any sqlite databases use an absolute path, as relative paths aren't consistently +# resolved. +SQLALCHEMY_DATABASE_URI = 'sqlite:///' + '/path/to/database.sqlite' + + +# The maximum allowable upload size, in bytes +# +# Keep in mind that this affects the expiration of files as well! The closer a +# file is to the max content length, the less time it will last before being +# deleted. +MAX_CONTENT_LENGTH = 256 * 1024 * 1024 # Default: 256MiB + + +# The maximum length of URLs we'll shorten, in characters +# +# If a user tries to submit a URL longer than this, we'll reject their request +# with a 414 REQUEST URI TOO LONG. +MAX_URL_LENGTH = 4096 + + +# Use the X-SENDFILE header to speed up serving files w/ compatible webservers +# +# Some webservers can be configured use the X-Sendfile header to handle sending +# large files on behalf of the application. If your server is setup to do +# this, set this variable to True +USE_X_SENDFILE = False + + +# Use X-Accel-Redirect to speed up serving files w/ compatible webservers +# +# Other webservers, like nginx and Caddy, use the X-Accel-Redirect header to +# accomplish a very similar thing to X-Sendfile (above). If your webserver is +# configured to do this, set this variable to True +# +# Note: It's recommended that you use either X-Sendfile or X-Accel-Redirect +# when you deploy in production. +FHOST_USE_X_ACCEL_REDIRECT = True # expect nginx by default + + +# The directory that 0x0 should store uploaded files in +# +# Whenever a file is uploaded to 0x0, we store it here! Relative paths are +# resolved relative to the working directory that 0x0 is being run from. +FHOST_STORAGE_PATH = "up" + + +# The maximum acceptable user-specified file extension +# +# When a user uploads a file, in most cases, we keep the file extension they +# provide. But! If the specified file extension is longer than +# FHOST_MAX_EXT_LENGTH, we truncate it. So if a user tries to upload the file +# "myfile.withareallongext", but FHOST_MAX_EXT_LENGTH is set to 9, then the +# extension that we keep is ".withareal" +FHOST_MAX_EXT_LENGTH = 9 + + +# A list of filetypes to use when the uploader doesn't specify one +# +# When a user uploads a file with no file extension, we try to find an extension that +# works for that file. This configuration option is the first thing that we check. If +# the type of a file without an extension is in this dict, then it'll be used as the file +# extension for that file. +# +# For example, if the user uploads "myfile" with no extension, and the file is a jpeg +# image, the file will get a URL like "eAa.jpg" +# +# For a list of MIME types you can use in this list, check +# https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types +FHOST_EXT_OVERRIDE = { + "audio/flac" : ".flac", + "image/gif" : ".gif", + "image/jpeg" : ".jpg", + "image/png" : ".png", + "image/svg+xml" : ".svg", + "video/webm" : ".webm", + "video/x-matroska" : ".mkv", + "application/octet-stream" : ".bin", + "text/plain" : ".log", + "text/plain" : ".txt", + "text/x-diff" : ".diff", +} + + +# Control which files aren't allowed to be uploaded +# +# Certain kinds of files are never accepted. If the file claims to be one of +# these types of files, or if we look at the contents of the file and it looks +# like one of these filetypes, then we reject the file outright with a 415 +# UNSUPPORTED MEDIA EXCEPTION +FHOST_MIME_BLACKLIST = [ + "application/x-dosexec", + "application/java-archive", + "application/java-vm" +] + + +# A list of IP addresses which are blacklisted from uploading files +# +# Can be set to the path of a file with an IP address on each line. The file +# can also include comment lines using a pound sign (#). Paths are resolved +# relative to the instance/ directory. +# +# If this is set to None, then no IP blacklist will be consulted. +FHOST_UPLOAD_BLACKLIST = None + + +# Enables support for detecting NSFW images +# +# Consult README.md for additional dependencies before setting to True +NSFW_DETECT = False + + +# The cutoff for when an image is considered NFSW +# +# When the NSFW detection algorithm generates an output higher than this +# number, an image is considered to be NSFW. NSFW images aren't declined, but +# are marked as NSFW. +# +# If NSFW_DETECT is set to False, then this has no effect. +NSFW_THRESHOLD = 0.608 + + +# A list of all characters which can appear in a URL +# +# If this list is too short, then URLs can very quickly become long. +# Generally, the default value for this should work for basically all usecases. +URL_ALPHABET = "DEQhd2uFteibPwq0SWBInTpA_jcZL5GKz3YCR14Ulk87Jors9vNHgfaOmMXy6Vx-" + + + ################################################################################# + # CONGRATULATIONS! You made it all the way through! # + # If you want to go even further to customize your instance, try checking out # + # the templates in the templates/ directory to customize your landing page, 404 # + # page, and other error pages. # + ################################################################################# + From 00dba0e189125706528068459ec35a1895f20736 Mon Sep 17 00:00:00 2001 From: Mia Herkt Date: Mon, 28 Nov 2022 22:25:52 +0100 Subject: [PATCH 02/23] config.example.py: Clarify MIME ext mapping --- instance/config.example.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/instance/config.example.py b/instance/config.example.py index cd78dde..64c977c 100644 --- a/instance/config.example.py +++ b/instance/config.example.py @@ -86,7 +86,8 @@ FHOST_MAX_EXT_LENGTH = 9 # When a user uploads a file with no file extension, we try to find an extension that # works for that file. This configuration option is the first thing that we check. If # the type of a file without an extension is in this dict, then it'll be used as the file -# extension for that file. +# extension for that file. Otherwise, we try to pick something sensible from libmagic's +# database. # # For example, if the user uploads "myfile" with no extension, and the file is a jpeg # image, the file will get a URL like "eAa.jpg" From af4b3b06c0d75a01c1fa59f1f6a4bae12e69daa5 Mon Sep 17 00:00:00 2001 From: Emi Simpson Date: Tue, 22 Nov 2022 16:15:50 -0500 Subject: [PATCH 03/23] Add support for expiring files SUPPLEMENTALLY: - Add an `expiration` field to the `file` table of the database - Produce a migration for the above change - Overhaul the cleanup script, and integrate into fhost.py (now run using FLASK_APP=fhost flask prune) - Replace the old cleanup script with a deprecation notice - Add information about how to expire files to the index - Update the README with information about the new script Squashed commits: Add a note explaining that expired files aren't immediately removed Show correct times on the index page graph Improve the migration script, removing the need for --legacy Use automap in place of an explicit file map in migration Remove vestigial `touch()` Don't crash when upgrading a fresh database Remove vestigial warning about legacy files More efficiently filter to unexpired files when migrating https://git.0x0.st/mia/0x0/pulls/72#issuecomment-224 Coalesce updates to the database during migration https://git.0x0.st/mia/0x0/pulls/72#issuecomment-226 Remove vestigial database model https://git.0x0.st/mia/0x0/pulls/72#issuecomment-261 prune: Stream expired files from the database (as opposed to collecting them all first) config.example.py: Add min & max expiration + description --- README.rst | 4 +- cleanup.py | 48 +------- fhost.py | 165 +++++++++++++++++++++++++-- instance/config.example.py | 13 +++ migrations/versions/939a08e1d6e5_.py | 81 +++++++++++++ templates/index.html | 15 ++- 6 files changed, 269 insertions(+), 57 deletions(-) create mode 100644 migrations/versions/939a08e1d6e5_.py diff --git a/README.rst b/README.rst index f13167c..af9074d 100644 --- a/README.rst +++ b/README.rst @@ -35,8 +35,8 @@ downsides, one of them being that range requests will not work. This is a problem for example when streaming media files: It won’t be possible to seek, and some ISOBMFF (MP4) files will not play at all. -To make files expire, simply create a cronjob that runs ``cleanup.py`` every -now and then. +To make files expire, simply create a cronjob that runs ``FLASK_APP=fhost +flask prune`` every now and then. Before running the service for the first time, run ``FLASK_APP=fhost flask db upgrade``. diff --git a/cleanup.py b/cleanup.py index 0f9a5ce..14fbc61 100755 --- a/cleanup.py +++ b/cleanup.py @@ -1,44 +1,8 @@ #!/usr/bin/env python3 -""" - Copyright © 2020 Mia Herkt - Licensed under the EUPL, Version 1.2 or - as soon as approved - by the European Commission - subsequent versions of the EUPL - (the "License"); - You may not use this work except in compliance with the License. - You may obtain a copy of the license at: - - https://joinup.ec.europa.eu/software/page/eupl - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - either express or implied. - See the License for the specific language governing permissions - and limitations under the License. -""" - -import os -import sys -import time -import datetime -from fhost import app - -os.chdir(os.path.dirname(sys.argv[0])) -os.chdir(app.config["FHOST_STORAGE_PATH"]) - -files = [f for f in os.listdir(".")] - -maxs = app.config["MAX_CONTENT_LENGTH"] -mind = 30 -maxd = 365 - -for f in files: - stat = os.stat(f) - systime = time.time() - age = datetime.timedelta(seconds=(systime - stat.st_mtime)).days - - maxage = mind + (-maxd + mind) * (stat.st_size / maxs - 1) ** 3 - - if age >= maxage: - os.remove(f) +print("This script has been replaced!!") +print("Instead, please run") +print("") +print(" $ FLASK_APP=fhost flask prune") +print("") +exit(1); diff --git a/fhost.py b/fhost.py index 9c2b94b..bda680f 100755 --- a/fhost.py +++ b/fhost.py @@ -22,12 +22,17 @@ from flask import Flask, abort, make_response, redirect, request, send_from_directory, url_for, Response, render_template from flask_sqlalchemy import SQLAlchemy from flask_migrate import Migrate +from sqlalchemy import and_ from jinja2.exceptions import * from jinja2 import ChoiceLoader, FileSystemLoader from hashlib import sha256 from magic import Magic from mimetypes import guess_extension +import click +import os import sys +import time +import typing import requests from validators import url as url_valid from pathlib import Path @@ -121,12 +126,14 @@ class File(db.Model): addr = db.Column(db.UnicodeText) removed = db.Column(db.Boolean, default=False) nsfw_score = db.Column(db.Float) + expiration = db.Column(db.BigInteger) - def __init__(self, sha256, ext, mime, addr): + def __init__(self, sha256, ext, mime, addr, expiration): self.sha256 = sha256 self.ext = ext self.mime = mime self.addr = addr + self.expiration = expiration def getname(self): return u"{0}{1}".format(su.enbase(self.id), self.ext) @@ -139,7 +146,16 @@ class File(db.Model): else: return url_for("get", path=n, _external=True) + "\n" - def store(file_, addr): + """ + requested_expiration can be: + - None, to use the longest allowed file lifespan + - a duration (in hours) that the file should live for + - a timestamp in epoch millis that the file should expire at + + Any value greater that the longest allowed file lifespan will be rounded down to that + value. + """ + def store(file_, requested_expiration: typing.Optional[int], addr): data = file_.read() digest = sha256(data).hexdigest() @@ -175,15 +191,51 @@ class File(db.Model): return ext[:app.config["FHOST_MAX_EXT_LENGTH"]] or ".bin" - f = File.query.filter_by(sha256=digest).first() + # Returns the epoch millisecond that this file should expire + # + # Uses the expiration time provided by the user (requested_expiration) + # upper-bounded by an algorithm that computes the size based on the size of the + # file. + # + # That is, all files are assigned a computed expiration, which can voluntarily + # shortened by the user either by providing a timestamp in epoch millis or a + # duration in hours. + def get_expiration() -> int: + current_epoch_millis = time.time() * 1000; + # Maximum lifetime of the file in milliseconds + this_files_max_lifespan = get_max_lifespan(len(data)); + + # The latest allowed expiration date for this file, in epoch millis + this_files_max_expiration = this_files_max_lifespan + 1000 * time.time(); + + if requested_expiration is None: + return this_files_max_expiration + elif requested_expiration < 1650460320000: + # Treat the requested expiration time as a duration in hours + requested_expiration_ms = requested_expiration * 60 * 60 * 1000 + return min(this_files_max_expiration, current_epoch_millis + requested_expiration_ms) + else: + # Treat the requested expiration time as a timestamp in epoch millis + return min(this_files_max_expiration, requested_expiration); + + f = File.query.filter_by(sha256=digest).first() if f: + # If the file already exists if f.removed: + # The file was removed by moderation, so don't accept it back abort(451) + if f.expiration is None: + # The file has expired, so give it a new expiration date + f.expiration = get_expiration() + else: + # The file already exists, update the expiration if needed + f.expiration = max(f.expiration, get_expiration()) else: mime = get_mime() ext = get_ext(mime) - f = File(digest, ext, mime, addr) + expiration = get_expiration() + f = File(digest, ext, mime, addr, expiration) f.addr = addr @@ -194,8 +246,6 @@ class File(db.Model): if not p.is_file(): with open(p, "wb") as of: of.write(data) - else: - p.touch() if not f.nsfw_score and app.config["NSFW_DETECT"]: f.nsfw_score = nsfw.detect(p) @@ -260,11 +310,20 @@ def in_upload_bl(addr): return False -def store_file(f, addr): +""" +requested_expiration can be: + - None, to use the longest allowed file lifespan + - a duration (in hours) that the file should live for + - a timestamp in epoch millis that the file should expire at + +Any value greater that the longest allowed file lifespan will be rounded down to that +value. +""" +def store_file(f, requested_expiration: typing.Optional[int], addr): if in_upload_bl(addr): return "Your host is blocked from uploading files.\n", 451 - sf = File.store(f, addr) + sf = File.store(f, requested_expiration, addr) return sf.geturl() @@ -289,7 +348,7 @@ def store_url(url, addr): f = urlfile(read=r.raw.read, content_type=r.headers["content-type"], filename="") - return store_file(f, addr) + return store_file(f, None, addr) else: abort(413) else: @@ -336,7 +395,23 @@ def fhost(): sf = None if "file" in request.files: - return store_file(request.files["file"], request.remote_addr) + try: + # Store the file with the requested expiration date + return store_file( + request.files["file"], + int(request.form["expires"]), + request.remote_addr + ) + except ValueError: + # The requested expiration date wasn't properly formed + abort(400) + except KeyError: + # No expiration date was requested, store with the max lifespan + return store_file( + request.files["file"], + None, + request.remote_addr + ) elif "url" in request.form: return store_url(request.form["url"], request.remote_addr) elif "shorten" in request.form: @@ -364,3 +439,73 @@ def ehandler(e): return render_template(f"{e.code}.html", id=id), e.code except TemplateNotFound: return "Segmentation fault\n", e.code + +@app.cli.command("prune") +def prune(): + """ + Clean up expired files + + Deletes any files from the filesystem which have hit their expiration time. This + doesn't remove them from the database, only from the filesystem. It's recommended + that server owners run this command regularly, or set it up on a timer. + """ + current_time = time.time() * 1000; + + # The path to where uploaded files are stored + storage = Path(app.config["FHOST_STORAGE_PATH"]) + + # A list of all files who've passed their expiration times + expired_files = File.query\ + .where( + and_( + File.expiration.is_not(None), + File.expiration < current_time + ) + ) + + files_removed = 0; + + # For every expired file... + for file in expired_files: + # Log the file we're about to remove + file_name = file.getname() + file_hash = file.sha256 + file_path = storage / file_hash + print(f"Removing expired file {file_name} [{file_hash}]") + + # Remove it from the file system + try: + os.remove(file_path) + files_removed += 1; + except FileNotFoundError: + pass # If the file was already gone, we're good + except OSError as e: + print(e) + print( + "\n------------------------------------" + "Encountered an error while trying to remove file {file_path}. Double" + "check to make sure the server is configured correctly, permissions are" + "okay, and everything is ship shape, then try again.") + return; + + # Finally, mark that the file was removed + file.expiration = None; + db.session.commit() + + print(f"\nDone! {files_removed} file(s) removed") + +""" For a file of a given size, determine the largest allowed lifespan of that file + +Based on the current app's configuration: Specifically, the MAX_CONTENT_LENGTH, as well +as FHOST_{MIN,MAX}_EXPIRATION. + +This lifespan may be shortened by a user's request, but no files should be allowed to +expire at a point after this number. + +Value returned is a duration in milliseconds. +""" +def get_max_lifespan(filesize: int) -> int: + min_exp = app.config.get("FHOST_MIN_EXPIRATION", 30 * 24 * 60 * 60 * 1000) + max_exp = app.config.get("FHOST_MAX_EXPIRATION", 365 * 24 * 60 * 60 * 1000) + max_size = app.config.get("MAX_CONTENT_LENGTH", 256 * 1024 * 1024) + return min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3) diff --git a/instance/config.example.py b/instance/config.example.py index 64c977c..019ec11 100644 --- a/instance/config.example.py +++ b/instance/config.example.py @@ -45,6 +45,19 @@ MAX_CONTENT_LENGTH = 256 * 1024 * 1024 # Default: 256MiB MAX_URL_LENGTH = 4096 +# The minimum and maximum amount of time we'll retain a file for +# +# Small files (nearing zero bytes) are stored for the longest possible expiration date, +# while larger files (nearing MAX_CONTENT_LENGTH bytes) are stored for the shortest amount +# of time. Values between these two extremes are interpolated with an exponential curve, +# like the one shown on the index page. +# +# All times are in milliseconds. If you want all files to be stored for the same amount +# of time, set these to the same value. +FHOST_MIN_EXPIRATION = 30 * 24 * 60 * 60 * 1000 +FHOST_MAX_EXPIRATION = 365 * 24 * 60 * 60 * 1000 + + # Use the X-SENDFILE header to speed up serving files w/ compatible webservers # # Some webservers can be configured use the X-Sendfile header to handle sending diff --git a/migrations/versions/939a08e1d6e5_.py b/migrations/versions/939a08e1d6e5_.py new file mode 100644 index 0000000..f86dcb3 --- /dev/null +++ b/migrations/versions/939a08e1d6e5_.py @@ -0,0 +1,81 @@ +"""add file expirations + +Revision ID: 939a08e1d6e5 +Revises: 7e246705da6a +Create Date: 2022-11-22 12:16:32.517184 + +""" + +# revision identifiers, used by Alembic. +revision = '939a08e1d6e5' +down_revision = '7e246705da6a' + +from alembic import op +from flask import current_app +from flask_sqlalchemy import SQLAlchemy +from pathlib import Path +import sqlalchemy as sa +from sqlalchemy.ext.automap import automap_base +from sqlalchemy.orm import Session + +import os +import time + +""" For a file of a given size, determine the largest allowed lifespan of that file + +Based on the current app's configuration: Specifically, the MAX_CONTENT_LENGTH, as well +as FHOST_{MIN,MAX}_EXPIRATION. + +This lifespan may be shortened by a user's request, but no files should be allowed to +expire at a point after this number. + +Value returned is a duration in milliseconds. +""" +def get_max_lifespan(filesize: int) -> int: + min_exp = current_app.config.get("FHOST_MIN_EXPIRATION", 30 * 24 * 60 * 60 * 1000) + max_exp = current_app.config.get("FHOST_MAX_EXPIRATION", 365 * 24 * 60 * 60 * 1000) + max_size = current_app.config.get("MAX_CONTENT_LENGTH", 256 * 1024 * 1024) + return min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3) + +Base = automap_base() + +def upgrade(): + op.add_column('file', sa.Column('expiration', sa.BigInteger())) + + bind = op.get_bind() + Base.prepare(autoload_with=bind) + File = Base.classes.file + session = Session(bind=bind) + + storage = Path(current_app.config["FHOST_STORAGE_PATH"]) + current_time = time.time() * 1000; + + # List of file hashes which have not expired yet + # This could get really big for some servers + try: + unexpired_files = os.listdir(storage) + except FileNotFoundError: + return # There are no currently unexpired files + + # Calculate an expiration date for all existing files + files = session.scalars( + sa.select(File) + .where( + sa.not_(File.removed), + File.sha256.in_(unexpired_files) + ) + ) + updates = [] # We coalesce updates to the database here + for file in files: + file_path = storage / file.sha256 + stat = os.stat(file_path) + max_age = get_max_lifespan(stat.st_size) # How long the file is allowed to live, in ms + file_birth = stat.st_mtime * 1000 # When the file was created, in ms + updates.append({'id': file.id, 'expiration': int(file_birth + max_age)}) + + # Apply coalesced updates + session.bulk_update_mappings(File, updates) + session.commit() + +def downgrade(): + op.drop_column('file', 'expiration') diff --git a/templates/index.html b/templates/index.html index cef9de2..6f84f59 100644 --- a/templates/index.html +++ b/templates/index.html @@ -11,6 +11,15 @@ Or you can shorten URLs: File URLs are valid for at least 30 days and up to a year (see below). Shortened URLs do not expire. + +Files can be set to expire sooner by adding an "expires" parameter (in hours) + curl -F'file=@yourfile.png' -F'expires=24' {{ fhost_url }} +OR by setting "expires" to a timestamp in epoch milliseconds + curl -F'file=@yourfile.png' -F'expires=1681996320000' {{ fhost_url }} + +Expired files won't be removed immediately, but will be removed as part of +the next purge. + {% set max_size = config["MAX_CONTENT_LENGTH"]|filesizeformat(True) %} Maximum file size: {{ max_size }} Not allowed: {{ config["FHOST_MIME_BLACKLIST"]|join(", ") }} @@ -22,7 +31,7 @@ FILE RETENTION PERIOD retention = min_age + (-max_age + min_age) * pow((file_size / max_size - 1), 3) days - 365 | \\ + {{'{: 6}'.format(config.get("FHOST_MAX_EXPIRATION", 31536000000)//86400000)}} | \\ | \\ | \\ | \\ @@ -30,7 +39,7 @@ retention = min_age + (-max_age + min_age) * pow((file_size / max_size - 1), 3) | \\ | .. | \\ - 197.5 | ----------..------------------------------------------- + {{'{: 6.1f}'.format((config.get("FHOST_MIN_EXPIRATION", 2592000000)/2 + config.get("FHOST_MAX_EXPIRATION", 31536000000)/2)/86400000)}} | ----------..------------------------------------------- | .. | \\ | .. @@ -39,7 +48,7 @@ retention = min_age + (-max_age + min_age) * pow((file_size / max_size - 1), 3) | ... | .... | ...... - 30 | .................... + {{'{: 6}'.format(config.get("FHOST_MIN_EXPIRATION", 2592000000)//86400000)}} | .................... 0{{ ((config["MAX_CONTENT_LENGTH"]/2)|filesizeformat(True)).split(" ")[0].rjust(27) }}{{ max_size.split(" ")[0].rjust(27) }} {{ max_size.split(" ")[1].rjust(54) }} From f25619b7e3f7f6618b5e3fecc57cb08fdb67c2a9 Mon Sep 17 00:00:00 2001 From: Mia Herkt Date: Tue, 29 Nov 2022 13:31:35 +0100 Subject: [PATCH 04/23] nsfw_detect: Tolerate score computation failure --- nsfw_detect.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nsfw_detect.py b/nsfw_detect.py index 6bd9219..225d8a7 100755 --- a/nsfw_detect.py +++ b/nsfw_detect.py @@ -77,11 +77,11 @@ class NSFWDetector: "-cpng", "-i", fpath ], stdout=PIPE, stderr=DEVNULL, check=True) image_data = ff.stdout + + scores = self._compute(image_data) except: return -1.0 - scores = self._compute(image_data) - return scores[1] From db9a20c94d395b273caf24dfcc6a48ab3d020768 Mon Sep 17 00:00:00 2001 From: Mia Herkt Date: Tue, 29 Nov 2022 17:23:30 +0100 Subject: [PATCH 05/23] Add example systemd unit files for prune job --- 0x0-prune.service | 22 ++++++++++++++++++++++ 0x0-prune.timer | 9 +++++++++ README.rst | 10 ++++++++-- 3 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 0x0-prune.service create mode 100644 0x0-prune.timer diff --git a/0x0-prune.service b/0x0-prune.service new file mode 100644 index 0000000..b28fb2d --- /dev/null +++ b/0x0-prune.service @@ -0,0 +1,22 @@ +[Unit] +Description=Prune 0x0 files +After=remote-fs.target + +[Service] +Type=oneshot +User=nullptr +WorkingDirectory=/path/to/0x0 +BindPaths=/path/to/0x0 + +Environment=FLASK_APP=fhost +ExecStart=/usr/bin/flask prune +ProtectProc=noaccess +ProtectSystem=strict +ProtectHome=tmpfs +PrivateTmp=true +PrivateUsers=true +ProtectKernelLogs=true +LockPersonality=true + +[Install] +WantedBy=multi-user.target diff --git a/0x0-prune.timer b/0x0-prune.timer new file mode 100644 index 0000000..df6a594 --- /dev/null +++ b/0x0-prune.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Prune 0x0 files + +[Timer] +OnCalendar=hourly +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/README.rst b/README.rst index af9074d..916a6f3 100644 --- a/README.rst +++ b/README.rst @@ -35,8 +35,14 @@ downsides, one of them being that range requests will not work. This is a problem for example when streaming media files: It won’t be possible to seek, and some ISOBMFF (MP4) files will not play at all. -To make files expire, simply create a cronjob that runs ``FLASK_APP=fhost -flask prune`` every now and then. +To make files expire, simply run ``FLASK_APP=fhost flask prune`` every +now and then. You can use the provided systemd unit files for this:: + + 0x0-prune.service + 0x0-prune.timer + +Make sure to edit them to match your system configuration. In particular, +set the user and paths in ``0x0-prune.service``. Before running the service for the first time, run ``FLASK_APP=fhost flask db upgrade``. From aa443178e1fdd30504850fb1ebde138491cd3a48 Mon Sep 17 00:00:00 2001 From: Mia Herkt Date: Tue, 29 Nov 2022 17:23:56 +0100 Subject: [PATCH 06/23] README: Also run db upgrade after git pull! --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 916a6f3..e84f2b1 100644 --- a/README.rst +++ b/README.rst @@ -44,7 +44,8 @@ now and then. You can use the provided systemd unit files for this:: Make sure to edit them to match your system configuration. In particular, set the user and paths in ``0x0-prune.service``. -Before running the service for the first time, run ``FLASK_APP=fhost flask db upgrade``. +Before running the service for the first time and every time you update it +from this git repository, run ``FLASK_APP=fhost flask db upgrade``. NSFW Detection From 14cfe3da58a8194eb850a28e4170e684d1444831 Mon Sep 17 00:00:00 2001 From: Mia Herkt Date: Tue, 29 Nov 2022 21:42:46 +0100 Subject: [PATCH 07/23] nsfw_detect: Use pathlib, fix deprecation warning Also fix glog suppression --- nsfw_detect.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/nsfw_detect.py b/nsfw_detect.py index 225d8a7..eddf9cb 100755 --- a/nsfw_detect.py +++ b/nsfw_detect.py @@ -23,20 +23,19 @@ import os import sys from io import BytesIO from subprocess import run, PIPE, DEVNULL - -import caffe +from pathlib import Path os.environ["GLOG_minloglevel"] = "2" # seriously :| - +import caffe class NSFWDetector: def __init__(self): - - npath = os.path.join(os.path.dirname(__file__), "nsfw_model") + npath = Path(__file__).parent / "nsfw_model" self.nsfw_net = caffe.Net( - os.path.join(npath, "deploy.prototxt"), - os.path.join(npath, "resnet_50_1by2_nsfw.caffemodel"), - caffe.TEST) + str(npath / "deploy.prototxt"), + caffe.TEST, + weights = str(npath / "resnet_50_1by2_nsfw.caffemodel") + ) self.caffe_transformer = caffe.io.Transformer({ 'data': self.nsfw_net.blobs['data'].data.shape }) From eb0b1d2f6992740e3b09bf383d8c0fa442faca55 Mon Sep 17 00:00:00 2001 From: Mia Herkt Date: Tue, 29 Nov 2022 21:46:33 +0100 Subject: [PATCH 08/23] nsfw_detect: Use PyAV instead of ffmpegthumbnailer --- README.rst | 2 +- nsfw_detect.py | 27 +++++++++++++++++++-------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index e84f2b1..5b34b50 100644 --- a/README.rst +++ b/README.rst @@ -56,7 +56,7 @@ neural network model. This works for images and video files and requires the following: * Caffe Python module (built for Python 3) -* ``ffmpegthumbnailer`` executable in ``$PATH`` +* `PyAV `_ Network Security Considerations diff --git a/nsfw_detect.py b/nsfw_detect.py index eddf9cb..032f7e4 100755 --- a/nsfw_detect.py +++ b/nsfw_detect.py @@ -22,11 +22,12 @@ import numpy as np import os import sys from io import BytesIO -from subprocess import run, PIPE, DEVNULL from pathlib import Path os.environ["GLOG_minloglevel"] = "2" # seriously :| import caffe +import av +av.logging.set_level(av.logging.PANIC) class NSFWDetector: def __init__(self): @@ -49,7 +50,7 @@ class NSFWDetector: self.caffe_transformer.set_channel_swap('data', (2, 1, 0)) def _compute(self, img): - image = caffe.io.load_image(BytesIO(img)) + image = caffe.io.load_image(img) H, W, _ = image.shape _, _, h, w = self.nsfw_net.blobs["data"].data.shape @@ -71,13 +72,23 @@ class NSFWDetector: def detect(self, fpath): try: - ff = run([ - "ffmpegthumbnailer", "-m", "-o-", "-s256", "-t50%", "-a", - "-cpng", "-i", fpath - ], stdout=PIPE, stderr=DEVNULL, check=True) - image_data = ff.stdout + with av.open(fpath) as container: + try: container.seek(int(container.duration / 2)) + except: container.seek(0) - scores = self._compute(image_data) + frame = next(container.decode(video=0)) + + if frame.width >= frame.height: + w = 256 + h = int(frame.height * (256 / frame.width)) + else: + w = int(frame.width * (256 / frame.height)) + h = 256 + frame = frame.reformat(width=w, height=h, format="rgb24") + img = BytesIO() + frame.to_image().save(img, format="ppm") + + scores = self._compute(img) except: return -1.0 From a182b6199b7f55eeb573dccd6ce7926c40043680 Mon Sep 17 00:00:00 2001 From: Mia Herkt Date: Wed, 30 Nov 2022 01:42:49 +0100 Subject: [PATCH 09/23] Allow management operations like deleting files This introduces the X-Token header field in the response of newly uploaded files as a simple way for users to manage their own files. It does not need to be particularly secure. --- fhost.py | 60 +++++++++++++++++++++++----- migrations/versions/0659d7b9eea8_.py | 26 ++++++++++++ templates/401.html | 2 + templates/index.html | 5 +++ 4 files changed, 84 insertions(+), 9 deletions(-) create mode 100644 migrations/versions/0659d7b9eea8_.py create mode 100644 templates/401.html diff --git a/fhost.py b/fhost.py index bda680f..c771d3e 100755 --- a/fhost.py +++ b/fhost.py @@ -34,6 +34,7 @@ import sys import time import typing import requests +import secrets from validators import url as url_valid from pathlib import Path @@ -127,13 +128,15 @@ class File(db.Model): removed = db.Column(db.Boolean, default=False) nsfw_score = db.Column(db.Float) expiration = db.Column(db.BigInteger) + mgmt_token = db.Column(db.String) - def __init__(self, sha256, ext, mime, addr, expiration): + def __init__(self, sha256, ext, mime, addr, expiration, mgmt_token): self.sha256 = sha256 self.ext = ext self.mime = mime self.addr = addr self.expiration = expiration + self.mgmt_token = mgmt_token def getname(self): return u"{0}{1}".format(su.enbase(self.id), self.ext) @@ -146,6 +149,15 @@ class File(db.Model): else: return url_for("get", path=n, _external=True) + "\n" + def getpath(self) -> Path: + return Path(app.config["FHOST_STORAGE_PATH"]) / self.sha256 + + def delete(self, permanent=False): + self.expiration = None + self.mgmt_token = None + self.removed = permanent + self.getpath().unlink(missing_ok=True) + """ requested_expiration can be: - None, to use the longest allowed file lifespan @@ -218,6 +230,7 @@ class File(db.Model): else: # Treat the requested expiration time as a timestamp in epoch millis return min(this_files_max_expiration, requested_expiration); + isnew = True f = File.query.filter_by(sha256=digest).first() if f: @@ -228,14 +241,19 @@ class File(db.Model): if f.expiration is None: # The file has expired, so give it a new expiration date f.expiration = get_expiration() + + # Also generate a new management token + f.mgmt_token = secrets.token_urlsafe() else: # The file already exists, update the expiration if needed f.expiration = max(f.expiration, get_expiration()) + isnew = False else: mime = get_mime() ext = get_ext(mime) expiration = get_expiration() - f = File(digest, ext, mime, addr, expiration) + mgmt_token = secrets.token_urlsafe() + f = File(digest, ext, mime, addr, expiration, mgmt_token) f.addr = addr @@ -252,8 +270,7 @@ class File(db.Model): db.session.add(f) db.session.commit() - return f - + return f, isnew class UrlEncoder(object): @@ -323,9 +340,14 @@ def store_file(f, requested_expiration: typing.Optional[int], addr): if in_upload_bl(addr): return "Your host is blocked from uploading files.\n", 451 - sf = File.store(f, requested_expiration, addr) + sf, isnew = File.store(f, requested_expiration, addr) - return sf.geturl() + response = make_response(sf.geturl()) + + if isnew: + response.headers["X-Token"] = sf.mgmt_token + + return response def store_url(url, addr): if is_fhost_url(url): @@ -354,7 +376,20 @@ def store_url(url, addr): else: abort(411) -@app.route("/") +def manage_file(f): + try: + assert(request.form["token"] == f.mgmt_token) + except: + abort(401) + + if "delete" in request.form: + f.delete() + db.session.commit() + return "" + + abort(400) + +@app.route("/", methods=["GET", "POST"]) def get(path): path = Path(path.split("/", 1)[0]) sufs = "".join(path.suffixes[-2:]) @@ -368,11 +403,14 @@ def get(path): if f.removed: abort(451) - fpath = Path(app.config["FHOST_STORAGE_PATH"]) / f.sha256 + fpath = f.getpath() if not fpath.is_file(): abort(404) + if request.method == "POST": + return manage_file(f) + if app.config["FHOST_USE_X_ACCEL_REDIRECT"]: response = make_response() response.headers["Content-Type"] = f.mime @@ -382,6 +420,9 @@ def get(path): else: return send_from_directory(app.config["FHOST_STORAGE_PATH"], f.sha256, mimetype = f.mime) else: + if request.method == "POST": + abort(405) + u = URL.query.get(id) if u: @@ -428,6 +469,7 @@ Disallow: / """ @app.errorhandler(400) +@app.errorhandler(401) @app.errorhandler(404) @app.errorhandler(411) @app.errorhandler(413) @@ -436,7 +478,7 @@ Disallow: / @app.errorhandler(451) def ehandler(e): try: - return render_template(f"{e.code}.html", id=id), e.code + return render_template(f"{e.code}.html", id=id, request=request), e.code except TemplateNotFound: return "Segmentation fault\n", e.code diff --git a/migrations/versions/0659d7b9eea8_.py b/migrations/versions/0659d7b9eea8_.py new file mode 100644 index 0000000..2ef2151 --- /dev/null +++ b/migrations/versions/0659d7b9eea8_.py @@ -0,0 +1,26 @@ +"""add file management token + +Revision ID: 0659d7b9eea8 +Revises: 939a08e1d6e5 +Create Date: 2022-11-30 01:06:53.362973 + +""" + +# revision identifiers, used by Alembic. +revision = '0659d7b9eea8' +down_revision = '939a08e1d6e5' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('file', sa.Column('mgmt_token', sa.String(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('file', 'mgmt_token') + # ### end Alembic commands ### diff --git a/templates/401.html b/templates/401.html new file mode 100644 index 0000000..672c7e4 --- /dev/null +++ b/templates/401.html @@ -0,0 +1,2 @@ +rm: cannot remove '{{ request.path.split("/")[1] }}': Permission denied + diff --git a/templates/index.html b/templates/index.html index 6f84f59..f315daf 100644 --- a/templates/index.html +++ b/templates/index.html @@ -20,6 +20,11 @@ OR by setting "expires" to a timestamp in epoch milliseconds Expired files won't be removed immediately, but will be removed as part of the next purge. +Whenever a file that does not already exist or has expired is uploaded, +the HTTP response header includes an X-Token field. You can use this +to delete the file immediately: + curl -Ftoken=token_here -Fdelete= {{ fhost_url }}/abc.txt + {% set max_size = config["MAX_CONTENT_LENGTH"]|filesizeformat(True) %} Maximum file size: {{ max_size }} Not allowed: {{ config["FHOST_MIME_BLACKLIST"]|join(", ") }} From afe2329bf54c6c3e4783899ad4e953d2ba8f2a43 Mon Sep 17 00:00:00 2001 From: Mia Herkt Date: Wed, 30 Nov 2022 01:46:48 +0100 Subject: [PATCH 10/23] templates/index: Remove unnecessary escaping --- templates/index.html | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/templates/index.html b/templates/index.html index f315daf..9d5e398 100644 --- a/templates/index.html +++ b/templates/index.html @@ -36,17 +36,17 @@ FILE RETENTION PERIOD retention = min_age + (-max_age + min_age) * pow((file_size / max_size - 1), 3) days - {{'{: 6}'.format(config.get("FHOST_MAX_EXPIRATION", 31536000000)//86400000)}} | \\ - | \\ - | \\ - | \\ - | \\ - | \\ + {{'{: 6}'.format(config.get("FHOST_MAX_EXPIRATION", 31536000000)//86400000)}} | \ + | \ + | \ + | \ + | \ + | \ | .. - | \\ + | \ {{'{: 6.1f}'.format((config.get("FHOST_MIN_EXPIRATION", 2592000000)/2 + config.get("FHOST_MAX_EXPIRATION", 31536000000)/2)/86400000)}} | ----------..------------------------------------------- | .. - | \\ + | \ | .. | ... | .. From e1685342581604b83b4b1b6ab9b10bcde5ef9d01 Mon Sep 17 00:00:00 2001 From: Mia Herkt Date: Wed, 30 Nov 2022 02:16:19 +0100 Subject: [PATCH 11/23] Allow changing expiration date --- fhost.py | 71 +++++++++++++++++++++++++------------------- templates/index.html | 10 +++++-- 2 files changed, 48 insertions(+), 33 deletions(-) diff --git a/fhost.py b/fhost.py index c771d3e..2e2f8af 100755 --- a/fhost.py +++ b/fhost.py @@ -158,6 +158,34 @@ class File(db.Model): self.removed = permanent self.getpath().unlink(missing_ok=True) + # Returns the epoch millisecond that a file should expire + # + # Uses the expiration time provided by the user (requested_expiration) + # upper-bounded by an algorithm that computes the size based on the size of the + # file. + # + # That is, all files are assigned a computed expiration, which can voluntarily + # shortened by the user either by providing a timestamp in epoch millis or a + # duration in hours. + def get_expiration(requested_expiration, size) -> int: + current_epoch_millis = time.time() * 1000; + + # Maximum lifetime of the file in milliseconds + this_files_max_lifespan = get_max_lifespan(size); + + # The latest allowed expiration date for this file, in epoch millis + this_files_max_expiration = this_files_max_lifespan + 1000 * time.time(); + + if requested_expiration is None: + return this_files_max_expiration + elif requested_expiration < 1650460320000: + # Treat the requested expiration time as a duration in hours + requested_expiration_ms = requested_expiration * 60 * 60 * 1000 + return min(this_files_max_expiration, current_epoch_millis + requested_expiration_ms) + else: + # Treat the requested expiration time as a timestamp in epoch millis + return min(this_files_max_expiration, requested_expiration) + """ requested_expiration can be: - None, to use the longest allowed file lifespan @@ -203,33 +231,7 @@ class File(db.Model): return ext[:app.config["FHOST_MAX_EXT_LENGTH"]] or ".bin" - # Returns the epoch millisecond that this file should expire - # - # Uses the expiration time provided by the user (requested_expiration) - # upper-bounded by an algorithm that computes the size based on the size of the - # file. - # - # That is, all files are assigned a computed expiration, which can voluntarily - # shortened by the user either by providing a timestamp in epoch millis or a - # duration in hours. - def get_expiration() -> int: - current_epoch_millis = time.time() * 1000; - - # Maximum lifetime of the file in milliseconds - this_files_max_lifespan = get_max_lifespan(len(data)); - - # The latest allowed expiration date for this file, in epoch millis - this_files_max_expiration = this_files_max_lifespan + 1000 * time.time(); - - if requested_expiration is None: - return this_files_max_expiration - elif requested_expiration < 1650460320000: - # Treat the requested expiration time as a duration in hours - requested_expiration_ms = requested_expiration * 60 * 60 * 1000 - return min(this_files_max_expiration, current_epoch_millis + requested_expiration_ms) - else: - # Treat the requested expiration time as a timestamp in epoch millis - return min(this_files_max_expiration, requested_expiration); + expiration = File.get_expiration(requested_expiration, len(data)) isnew = True f = File.query.filter_by(sha256=digest).first() @@ -240,18 +242,17 @@ class File(db.Model): abort(451) if f.expiration is None: # The file has expired, so give it a new expiration date - f.expiration = get_expiration() + f.expiration = expiration # Also generate a new management token f.mgmt_token = secrets.token_urlsafe() else: # The file already exists, update the expiration if needed - f.expiration = max(f.expiration, get_expiration()) + f.expiration = max(f.expiration, expiration) isnew = False else: mime = get_mime() ext = get_ext(mime) - expiration = get_expiration() mgmt_token = secrets.token_urlsafe() f = File(digest, ext, mime, addr, expiration, mgmt_token) @@ -386,6 +387,16 @@ def manage_file(f): f.delete() db.session.commit() return "" + if "expires" in request.form: + try: + requested_expiration = int(request.form["expires"]) + except ValueError: + abort(400) + + fsize = f.getpath().stat().st_size + f.expiration = File.get_expiration(requested_expiration, fsize) + db.session.commit() + return "", 202 abort(400) diff --git a/templates/index.html b/templates/index.html index 9d5e398..d98482c 100644 --- a/templates/index.html +++ b/templates/index.html @@ -13,17 +13,21 @@ File URLs are valid for at least 30 days and up to a year (see below). Shortened URLs do not expire. Files can be set to expire sooner by adding an "expires" parameter (in hours) - curl -F'file=@yourfile.png' -F'expires=24' {{ fhost_url }} + curl -F'file=@yourfile.png' -Fexpires=24 {{ fhost_url }} OR by setting "expires" to a timestamp in epoch milliseconds - curl -F'file=@yourfile.png' -F'expires=1681996320000' {{ fhost_url }} + curl -F'file=@yourfile.png' -Fexpires=1681996320000 {{ fhost_url }} Expired files won't be removed immediately, but will be removed as part of the next purge. Whenever a file that does not already exist or has expired is uploaded, the HTTP response header includes an X-Token field. You can use this -to delete the file immediately: +to perform management operations on the file. + +To delete the file immediately: curl -Ftoken=token_here -Fdelete= {{ fhost_url }}/abc.txt +To change the expiration date (see above): + curl -Ftoken=token_here -Fexpires=3 {{ fhost_url }}/abc.txt {% set max_size = config["MAX_CONTENT_LENGTH"]|filesizeformat(True) %} Maximum file size: {{ max_size }} From 9214bb483289cbcba8268654129e08d1369c4b24 Mon Sep 17 00:00:00 2001 From: Mia Herkt Date: Wed, 30 Nov 2022 02:28:19 +0100 Subject: [PATCH 12/23] Add X-Expires to file response headers Tells clients when files will expire, in milliseconds since Unix epoch. Closes #50. --- fhost.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fhost.py b/fhost.py index 2e2f8af..026e3bd 100755 --- a/fhost.py +++ b/fhost.py @@ -344,6 +344,7 @@ def store_file(f, requested_expiration: typing.Optional[int], addr): sf, isnew = File.store(f, requested_expiration, addr) response = make_response(sf.geturl()) + response.headers["X-Expires"] = sf.expiration if isnew: response.headers["X-Token"] = sf.mgmt_token @@ -427,9 +428,11 @@ def get(path): response.headers["Content-Type"] = f.mime response.headers["Content-Length"] = fpath.stat().st_size response.headers["X-Accel-Redirect"] = "/" + str(fpath) - return response else: - return send_from_directory(app.config["FHOST_STORAGE_PATH"], f.sha256, mimetype = f.mime) + response = send_from_directory(app.config["FHOST_STORAGE_PATH"], f.sha256, mimetype = f.mime) + + response.headers["X-Expires"] = f.expiration + return response else: if request.method == "POST": abort(405) From 7661216bc00e072aec72a0047c867fbf90dbf9bf Mon Sep 17 00:00:00 2001 From: Mia Herkt Date: Thu, 1 Dec 2022 01:19:05 +0100 Subject: [PATCH 13/23] Fix handling double file name extensions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Long names would get truncated at the end, causing problems including unresolvable file URLs. Example with default settings: .package.lst → .package. Fixes #61 --- fhost.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fhost.py b/fhost.py index 026e3bd..70e1109 100755 --- a/fhost.py +++ b/fhost.py @@ -218,6 +218,8 @@ class File(db.Model): def get_ext(mime): ext = "".join(Path(file_.filename).suffixes[-2:]) + if len(ext) > app.config["FHOST_MAX_EXT_LENGTH"]: + ext = Path(file_.filename).suffixes[-1] gmime = mime.split(";")[0] guess = guess_extension(gmime) From ed84d3752c56f06e73876af0ab162999562a9d61 Mon Sep 17 00:00:00 2001 From: Mia Herkt Date: Thu, 1 Dec 2022 01:26:32 +0100 Subject: [PATCH 14/23] Fix 500 on invalid paths --- fhost.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fhost.py b/fhost.py index 70e1109..8770010 100755 --- a/fhost.py +++ b/fhost.py @@ -408,6 +408,10 @@ def get(path): path = Path(path.split("/", 1)[0]) sufs = "".join(path.suffixes[-2:]) name = path.name[:-len(sufs) or None] + + if "." in name: + abort(404) + id = su.debase(name) if sufs: From 0b80a62f80ebe6b5ad6e9db99036020b8d7d0cea Mon Sep 17 00:00:00 2001 From: Mia Herkt Date: Thu, 1 Dec 2022 02:49:28 +0100 Subject: [PATCH 15/23] =?UTF-8?q?Add=20support=20for=20=E2=80=9Csecret?= =?UTF-8?q?=E2=80=9D=20file=20URLs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #47 --- fhost.py | 48 ++++++++++++++++++++-------- instance/config.example.py | 7 ++++ migrations/versions/e2e816056589_.py | 26 +++++++++++++++ templates/index.html | 3 ++ 4 files changed, 70 insertions(+), 14 deletions(-) create mode 100644 migrations/versions/e2e816056589_.py diff --git a/fhost.py b/fhost.py index 8770010..6f11db7 100755 --- a/fhost.py +++ b/fhost.py @@ -48,6 +48,7 @@ app.config.update( FHOST_USE_X_ACCEL_REDIRECT = True, # expect nginx by default FHOST_STORAGE_PATH = "up", FHOST_MAX_EXT_LENGTH = 9, + FHOST_SECRET_BYTES = 16, FHOST_EXT_OVERRIDE = { "audio/flac" : ".flac", "image/gif" : ".gif", @@ -129,6 +130,7 @@ class File(db.Model): nsfw_score = db.Column(db.Float) expiration = db.Column(db.BigInteger) mgmt_token = db.Column(db.String) + secret = db.Column(db.String) def __init__(self, sha256, ext, mime, addr, expiration, mgmt_token): self.sha256 = sha256 @@ -145,9 +147,9 @@ class File(db.Model): n = self.getname() if self.nsfw_score and self.nsfw_score > app.config["NSFW_THRESHOLD"]: - return url_for("get", path=n, _external=True, _anchor="nsfw") + "\n" + return url_for("get", path=n, secret=self.secret, _external=True, _anchor="nsfw") + "\n" else: - return url_for("get", path=n, _external=True) + "\n" + return url_for("get", path=n, secret=self.secret, _external=True) + "\n" def getpath(self) -> Path: return Path(app.config["FHOST_STORAGE_PATH"]) / self.sha256 @@ -195,7 +197,7 @@ class File(db.Model): Any value greater that the longest allowed file lifespan will be rounded down to that value. """ - def store(file_, requested_expiration: typing.Optional[int], addr): + def store(file_, requested_expiration: typing.Optional[int], addr, secret: bool): data = file_.read() digest = sha256(data).hexdigest() @@ -260,6 +262,11 @@ class File(db.Model): f.addr = addr + if isnew: + f.secret = None + if secret: + f.secret = secrets.token_urlsafe(app.config["FHOST_SECRET_BYTES"]) + storage = Path(app.config["FHOST_STORAGE_PATH"]) storage.mkdir(parents=True, exist_ok=True) p = storage / digest @@ -339,11 +346,11 @@ requested_expiration can be: Any value greater that the longest allowed file lifespan will be rounded down to that value. """ -def store_file(f, requested_expiration: typing.Optional[int], addr): +def store_file(f, requested_expiration: typing.Optional[int], addr, secret: bool): if in_upload_bl(addr): return "Your host is blocked from uploading files.\n", 451 - sf, isnew = File.store(f, requested_expiration, addr) + sf, isnew = File.store(f, requested_expiration, addr, secret) response = make_response(sf.geturl()) response.headers["X-Expires"] = sf.expiration @@ -353,7 +360,7 @@ def store_file(f, requested_expiration: typing.Optional[int], addr): return response -def store_url(url, addr): +def store_url(url, addr, secret: bool): if is_fhost_url(url): abort(400) @@ -374,7 +381,7 @@ def store_url(url, addr): f = urlfile(read=r.raw.read, content_type=r.headers["content-type"], filename="") - return store_file(f, None, addr) + return store_file(f, None, addr, secret) else: abort(413) else: @@ -404,10 +411,11 @@ def manage_file(f): abort(400) @app.route("/", methods=["GET", "POST"]) -def get(path): - path = Path(path.split("/", 1)[0]) - sufs = "".join(path.suffixes[-2:]) - name = path.name[:-len(sufs) or None] +@app.route("/s//", methods=["GET", "POST"]) +def get(path, secret=None): + p = Path(path.split("/", 1)[0]) + sufs = "".join(p.suffixes[-2:]) + name = p.name[:-len(sufs) or None] if "." in name: abort(404) @@ -416,6 +424,8 @@ def get(path): if sufs: f = File.query.get(id) + if f.secret != secret: + abort(404) if f and f.ext == sufs: if f.removed: @@ -443,6 +453,9 @@ def get(path): if request.method == "POST": abort(405) + if "/" in path: + abort(404) + u = URL.query.get(id) if u: @@ -454,6 +467,7 @@ def get(path): def fhost(): if request.method == "POST": sf = None + secret = "secret" in request.form if "file" in request.files: try: @@ -461,7 +475,8 @@ def fhost(): return store_file( request.files["file"], int(request.form["expires"]), - request.remote_addr + request.remote_addr, + secret ) except ValueError: # The requested expiration date wasn't properly formed @@ -471,10 +486,15 @@ def fhost(): return store_file( request.files["file"], None, - request.remote_addr + request.remote_addr, + secret ) elif "url" in request.form: - return store_url(request.form["url"], request.remote_addr) + return store_url( + request.form["url"], + request.remote_addr, + secret + ) elif "shorten" in request.form: return shorten(request.form["shorten"]) diff --git a/instance/config.example.py b/instance/config.example.py index 019ec11..2315b75 100644 --- a/instance/config.example.py +++ b/instance/config.example.py @@ -94,6 +94,13 @@ FHOST_STORAGE_PATH = "up" FHOST_MAX_EXT_LENGTH = 9 +# The number of bytes used for "secret" URLs +# +# When a user uploads a file with the "secret" option, 0x0 generates a string +# from this many bytes of random data. It is base64-encoded, so on average +# each byte results in approximately 1.3 characters. +FHOST_SECRET_BYTES = 16 + # A list of filetypes to use when the uploader doesn't specify one # # When a user uploads a file with no file extension, we try to find an extension that diff --git a/migrations/versions/e2e816056589_.py b/migrations/versions/e2e816056589_.py new file mode 100644 index 0000000..7c31ba9 --- /dev/null +++ b/migrations/versions/e2e816056589_.py @@ -0,0 +1,26 @@ +"""add URL secret + +Revision ID: e2e816056589 +Revises: 0659d7b9eea8 +Create Date: 2022-12-01 02:16:15.976864 + +""" + +# revision identifiers, used by Alembic. +revision = 'e2e816056589' +down_revision = '0659d7b9eea8' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('file', sa.Column('secret', sa.String(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('file', 'secret') + # ### end Alembic commands ### diff --git a/templates/index.html b/templates/index.html index d98482c..f36fb54 100644 --- a/templates/index.html +++ b/templates/index.html @@ -6,6 +6,9 @@ HTTP POST files here: curl -F'file=@yourfile.png' {{ fhost_url }} You can also POST remote URLs: curl -F'url=http://example.com/image.jpg' {{ fhost_url }} +If you don't want the resulting URL to be easy to guess: + curl -F'file=@yourfile.png' -Fsecret= {{ fhost_url }} + curl -F'url=http://example.com/image.jpg' -Fsecret= {{ fhost_url }} Or you can shorten URLs: curl -F'shorten=http://example.com/some/long/url' {{ fhost_url }} From da30c8f8ff0e2f23a974d95923273a25bc3a2a23 Mon Sep 17 00:00:00 2001 From: Mia Herkt Date: Thu, 1 Dec 2022 03:28:25 +0100 Subject: [PATCH 16/23] index.html: Document appending file names --- templates/index.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/templates/index.html b/templates/index.html index f36fb54..a2add39 100644 --- a/templates/index.html +++ b/templates/index.html @@ -12,6 +12,9 @@ If you don't want the resulting URL to be easy to guess: Or you can shorten URLs: curl -F'shorten=http://example.com/some/long/url' {{ fhost_url }} +It is possible to append your own file name to the URL: + {{ fhost_url }}/aaa.jpg/image.jpeg + File URLs are valid for at least 30 days and up to a year (see below). Shortened URLs do not expire. From a904922cbd6f6478d5e7fd773adc368b7652adaa Mon Sep 17 00:00:00 2001 From: Mia Herkt Date: Mon, 12 Dec 2022 07:25:30 +0100 Subject: [PATCH 17/23] Add support for ClamAV --- 0x0-vscan.service | 22 ++++++++++ 0x0-vscan.timer | 9 ++++ README.rst | 16 +++++++ fhost.py | 64 +++++++++++++++++++++++++++- instance/config.example.py | 31 ++++++++++++++ migrations/versions/5cee97aab219_.py | 26 +++++++++++ 6 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 0x0-vscan.service create mode 100644 0x0-vscan.timer create mode 100644 migrations/versions/5cee97aab219_.py diff --git a/0x0-vscan.service b/0x0-vscan.service new file mode 100644 index 0000000..6a48b1c --- /dev/null +++ b/0x0-vscan.service @@ -0,0 +1,22 @@ +[Unit] +Description=Scan 0x0 files with ClamAV +After=remote-fs.target clamd.service + +[Service] +Type=oneshot +User=nullptr +WorkingDirectory=/path/to/0x0 +BindPaths=/path/to/0x0 + +Environment=FLASK_APP=fhost +ExecStart=/usr/bin/flask vscan +ProtectProc=noaccess +ProtectSystem=strict +ProtectHome=tmpfs +PrivateTmp=true +PrivateUsers=true +ProtectKernelLogs=true +LockPersonality=true + +[Install] +WantedBy=multi-user.target diff --git a/0x0-vscan.timer b/0x0-vscan.timer new file mode 100644 index 0000000..d2c6486 --- /dev/null +++ b/0x0-vscan.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Scan 0x0 files with ClamAV + +[Timer] +OnCalendar=hourly +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/README.rst b/README.rst index 5b34b50..fd7067d 100644 --- a/README.rst +++ b/README.rst @@ -59,6 +59,22 @@ the following: * `PyAV `_ +Virus Scanning +-------------- + +0x0 can scan its files with ClamAV’s daemon. As this can take a long time +for larger files, this does not happen immediately but instead every time +you run the ``vscan`` command. It is recommended to configure a systemd +timer or cronjob to do this periodically. Examples are included:: + + 0x0-vscan.service + 0x0-vscan.timer + +Remember to adjust your size limits in clamd.conf! + +This feature requires the `clamd module `_. + + Network Security Considerations ------------------------------- diff --git a/fhost.py b/fhost.py index 6f11db7..75702a1 100755 --- a/fhost.py +++ b/fhost.py @@ -22,7 +22,7 @@ from flask import Flask, abort, make_response, redirect, request, send_from_directory, url_for, Response, render_template from flask_sqlalchemy import SQLAlchemy from flask_migrate import Migrate -from sqlalchemy import and_ +from sqlalchemy import and_, or_ from jinja2.exceptions import * from jinja2 import ChoiceLoader, FileSystemLoader from hashlib import sha256 @@ -32,6 +32,7 @@ import click import os import sys import time +import datetime import typing import requests import secrets @@ -70,6 +71,13 @@ app.config.update( FHOST_UPLOAD_BLACKLIST = None, NSFW_DETECT = False, NSFW_THRESHOLD = 0.608, + VSCAN_SOCKET = None, + VSCAN_QUARANTINE_PATH = "quarantine", + VSCAN_IGNORE = [ + "Eicar-Test-Signature", + "PUA.Win.Packer.XmMusicFile", + ], + VSCAN_INTERVAL = datetime.timedelta(days=7), URL_ALPHABET = "DEQhd2uFteibPwq0SWBInTpA_jcZL5GKz3YCR14Ulk87Jors9vNHgfaOmMXy6Vx-", ) @@ -131,6 +139,7 @@ class File(db.Model): expiration = db.Column(db.BigInteger) mgmt_token = db.Column(db.String) secret = db.Column(db.String) + last_vscan = db.Column(db.DateTime) def __init__(self, sha256, ext, mime, addr, expiration, mgmt_token): self.sha256 = sha256 @@ -591,3 +600,56 @@ def get_max_lifespan(filesize: int) -> int: max_exp = app.config.get("FHOST_MAX_EXPIRATION", 365 * 24 * 60 * 60 * 1000) max_size = app.config.get("MAX_CONTENT_LENGTH", 256 * 1024 * 1024) return min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3) + +def do_vscan(f): + if f["path"].is_file(): + with open(f["path"], "rb") as scanf: + try: + f["result"] = list(app.config["VSCAN_SOCKET"].instream(scanf).values())[0] + except: + f["result"] = ("SCAN FAILED", None) + else: + f["result"] = ("FILE NOT FOUND", None) + + return f + +@app.cli.command("vscan") +def vscan(): + if not app.config["VSCAN_SOCKET"]: + print("""Error: Virus scanning enabled but no connection method specified. +Please set VSCAN_SOCKET.""") + sys.exit(1) + + qp = Path(app.config["VSCAN_QUARANTINE_PATH"]) + qp.mkdir(parents=True, exist_ok=True) + + from multiprocessing import Pool + with Pool() as p: + if isinstance(app.config["VSCAN_INTERVAL"], datetime.timedelta): + scandate = datetime.datetime.now() - app.config["VSCAN_INTERVAL"] + res = File.query.filter(or_(File.last_vscan < scandate, + File.last_vscan == None), + File.removed == False) + else: + res = File.query.filter(File.last_vscan == None, File.removed == False) + + work = [{"path" : f.getpath(), "name" : f.getname(), "id" : f.id} for f in res] + + results = [] + for i, r in enumerate(p.imap_unordered(do_vscan, work)): + if r["result"][0] != "OK": + print(f"{r['name']}: {r['result'][0]} {r['result'][1] or ''}") + + found = False + if r["result"][0] == "FOUND": + if not r["result"][1] in app.config["VSCAN_IGNORE"]: + r["path"].rename(qp / r["name"]) + found = True + + results.append({ + "id" : r["id"], + "last_vscan" : None if r["result"][0] == "SCAN FAILED" else datetime.datetime.now(), + "removed" : found}) + + db.session.bulk_update_mappings(File, results) + db.session.commit() diff --git a/instance/config.example.py b/instance/config.example.py index 2315b75..5674eea 100644 --- a/instance/config.example.py +++ b/instance/config.example.py @@ -168,6 +168,37 @@ NSFW_DETECT = False NSFW_THRESHOLD = 0.608 +# If you want to scan files for viruses using ClamAV, specify the socket used +# for connections here. You will need the clamd module. +# Since this can take a very long time on larger files, it is not done +# immediately but every time you run the vscan command. It is recommended to +# configure a systemd timer or cronjob to do this periodically. +# Remember to adjust your size limits in clamd.conf! +# +# Example: +# from clamd import ClamdUnixSocket +# VSCAN_SOCKET = ClamdUnixSocket("/run/clamav/clamd-socket") + +# This is the directory that files flagged as malicious are moved to. +# Relative paths are resolved relative to the working directory +# of the 0x0 process. +VSCAN_QUARANTINE_PATH = "quarantine" + +# Since updated virus definitions might catch some files that were previously +# reported as clean, you may want to rescan old files periodically. +# Set this to a datetime.timedelta to specify the frequency, or None to +# disable rescanning. +from datetime import timedelta +VSCAN_INTERVAL = timedelta(days=7) + +# Some files flagged by ClamAV are usually not malicious, especially if the +# DetectPUA option is enabled in clamd.conf. This is a list of signatures +# that will be ignored. +VSCAN_IGNORE = [ + "Eicar-Test-Signature", + "PUA.Win.Packer.XmMusicFile", +] + # A list of all characters which can appear in a URL # # If this list is too short, then URLs can very quickly become long. diff --git a/migrations/versions/5cee97aab219_.py b/migrations/versions/5cee97aab219_.py new file mode 100644 index 0000000..6c1a16b --- /dev/null +++ b/migrations/versions/5cee97aab219_.py @@ -0,0 +1,26 @@ +"""add date of last virus scan + +Revision ID: 5cee97aab219 +Revises: e2e816056589 +Create Date: 2022-12-10 16:39:56.388259 + +""" + +# revision identifiers, used by Alembic. +revision = '5cee97aab219' +down_revision = 'e2e816056589' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('file', sa.Column('last_vscan', sa.DateTime(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('file', 'last_vscan') + # ### end Alembic commands ### From b1ed63c4019228cf94deb7b524d586955ec44bb8 Mon Sep 17 00:00:00 2001 From: Mia Herkt Date: Mon, 12 Dec 2022 07:40:38 +0100 Subject: [PATCH 18/23] README: Add note about StreamMaxLength in clamd.conf --- README.rst | 3 ++- instance/config.example.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index fd7067d..8c512de 100644 --- a/README.rst +++ b/README.rst @@ -70,7 +70,8 @@ timer or cronjob to do this periodically. Examples are included:: 0x0-vscan.service 0x0-vscan.timer -Remember to adjust your size limits in clamd.conf! +Remember to adjust your size limits in clamd.conf, including +``StreamMaxLength``! This feature requires the `clamd module `_. diff --git a/instance/config.example.py b/instance/config.example.py index 5674eea..825afcb 100644 --- a/instance/config.example.py +++ b/instance/config.example.py @@ -173,7 +173,7 @@ NSFW_THRESHOLD = 0.608 # Since this can take a very long time on larger files, it is not done # immediately but every time you run the vscan command. It is recommended to # configure a systemd timer or cronjob to do this periodically. -# Remember to adjust your size limits in clamd.conf! +# Remember to adjust your size limits in clamd.conf, including StreamMaxLength! # # Example: # from clamd import ClamdUnixSocket From 6055a509485598afa6aea9cbf841424d48f5e761 Mon Sep 17 00:00:00 2001 From: Mia Herkt Date: Tue, 13 Dec 2022 21:51:39 +0100 Subject: [PATCH 19/23] File: Add is_nsfw property --- fhost.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fhost.py b/fhost.py index 75702a1..641729d 100755 --- a/fhost.py +++ b/fhost.py @@ -149,13 +149,17 @@ class File(db.Model): self.expiration = expiration self.mgmt_token = mgmt_token + @property + def is_nsfw(self) -> bool: + return self.nsfw_score and self.nsfw_score > app.config["NSFW_THRESHOLD"] + def getname(self): return u"{0}{1}".format(su.enbase(self.id), self.ext) def geturl(self): n = self.getname() - if self.nsfw_score and self.nsfw_score > app.config["NSFW_THRESHOLD"]: + if self.is_nsfw: return url_for("get", path=n, secret=self.secret, _external=True, _anchor="nsfw") + "\n" else: return url_for("get", path=n, secret=self.secret, _external=True) + "\n" From aaf0e4492adb89394ee2acd88bda12413107c5be Mon Sep 17 00:00:00 2001 From: Mia Herkt Date: Tue, 13 Dec 2022 23:02:41 +0100 Subject: [PATCH 20/23] Record file sizes in db Moderation interface is going to use this. --- fhost.py | 8 ++-- .../30bfe33aa328_add_file_size_field.py | 46 +++++++++++++++++++ 2 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 migrations/versions/30bfe33aa328_add_file_size_field.py diff --git a/fhost.py b/fhost.py index 641729d..eb30a9d 100755 --- a/fhost.py +++ b/fhost.py @@ -140,6 +140,7 @@ class File(db.Model): mgmt_token = db.Column(db.String) secret = db.Column(db.String) last_vscan = db.Column(db.DateTime) + size = db.Column(db.BigInteger) def __init__(self, sha256, ext, mime, addr, expiration, mgmt_token): self.sha256 = sha256 @@ -288,6 +289,8 @@ class File(db.Model): with open(p, "wb") as of: of.write(data) + f.size = len(data) + if not f.nsfw_score and app.config["NSFW_DETECT"]: f.nsfw_score = nsfw.detect(p) @@ -416,8 +419,7 @@ def manage_file(f): except ValueError: abort(400) - fsize = f.getpath().stat().st_size - f.expiration = File.get_expiration(requested_expiration, fsize) + f.expiration = File.get_expiration(requested_expiration, f.size) db.session.commit() return "", 202 @@ -455,7 +457,7 @@ def get(path, secret=None): if app.config["FHOST_USE_X_ACCEL_REDIRECT"]: response = make_response() response.headers["Content-Type"] = f.mime - response.headers["Content-Length"] = fpath.stat().st_size + response.headers["Content-Length"] = f.size response.headers["X-Accel-Redirect"] = "/" + str(fpath) else: response = send_from_directory(app.config["FHOST_STORAGE_PATH"], f.sha256, mimetype = f.mime) diff --git a/migrations/versions/30bfe33aa328_add_file_size_field.py b/migrations/versions/30bfe33aa328_add_file_size_field.py new file mode 100644 index 0000000..e6ac279 --- /dev/null +++ b/migrations/versions/30bfe33aa328_add_file_size_field.py @@ -0,0 +1,46 @@ +"""add file size field + +Revision ID: 30bfe33aa328 +Revises: 5cee97aab219 +Create Date: 2022-12-13 22:32:12.242394 + +""" + +# revision identifiers, used by Alembic. +revision = '30bfe33aa328' +down_revision = '5cee97aab219' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.ext.automap import automap_base +from sqlalchemy.orm import Session +from flask import current_app +from pathlib import Path + +Base = automap_base() + +def upgrade(): + op.add_column('file', sa.Column('size', sa.BigInteger(), nullable=True)) + bind = op.get_bind() + Base.prepare(autoload_with=bind) + File = Base.classes.file + session = Session(bind=bind) + + storage = Path(current_app.config["FHOST_STORAGE_PATH"]) + + updates = [] + files = session.scalars(sa.select(File).where(sa.not_(File.removed))) + for f in files: + p = storage / f.sha256 + if p.is_file(): + updates.append({ + "id" : f.id, + "size" : p.stat().st_size + }) + + session.bulk_update_mappings(File, updates) + session.commit() + + +def downgrade(): + op.drop_column('file', 'size') From d5763a985491547a715a006ec3ccdc371c81623d Mon Sep 17 00:00:00 2001 From: Mia Herkt Date: Tue, 13 Dec 2022 23:17:56 +0100 Subject: [PATCH 21/23] File: Fix 404 case with secret URLs --- fhost.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fhost.py b/fhost.py index eb30a9d..faf6d03 100755 --- a/fhost.py +++ b/fhost.py @@ -439,10 +439,11 @@ def get(path, secret=None): if sufs: f = File.query.get(id) - if f.secret != secret: - abort(404) if f and f.ext == sufs: + if f.secret != secret: + abort(404) + if f.removed: abort(451) From 77801efd216e0725e913942d2014f4d5e6600bd5 Mon Sep 17 00:00:00 2001 From: Mia Herkt Date: Tue, 13 Dec 2022 23:18:40 +0100 Subject: [PATCH 22/23] Fix URL test issue --- tests/test_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_client.py b/tests/test_client.py index 40041ce..0b29e00 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -56,8 +56,6 @@ def test_client(client): ]), (302, [ "E", - "E/test", - "E/test.bin", ]), (404, [ "test.bin", @@ -67,6 +65,8 @@ def test_client(client): "test/test", "test.bin/test.py", "E.bin", + "E/test", + "E/test.bin", ]), (451, [ "Q.truncate", From 57c4b6853f1b33ab55a630c5b0810ba667ba017a Mon Sep 17 00:00:00 2001 From: Mia Herkt Date: Tue, 13 Dec 2022 23:41:12 +0100 Subject: [PATCH 23/23] Prevent unreasonably long MIME types --- fhost.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fhost.py b/fhost.py index faf6d03..c1fd7cc 100755 --- a/fhost.py +++ b/fhost.py @@ -227,6 +227,9 @@ class File(db.Model): if mime in app.config["FHOST_MIME_BLACKLIST"] or guess in app.config["FHOST_MIME_BLACKLIST"]: abort(415) + if len(mime) > 128: + abort(400) + if mime.startswith("text/") and not "charset" in mime: mime += "; charset=utf-8"