Merge branch 'master' into boop

This commit is contained in:
Siina Mashek 2022-12-29 02:04:27 +02:00
commit 8b4258c032
13 changed files with 719 additions and 10 deletions

View file

@ -48,6 +48,50 @@ Before running the service for the first time and every time you update it
from this git repository, run ``FLASK_APP=fhost flask db upgrade``.
Moderation UI
-------------
.. image:: modui.webp
:height: 300
0x0 features a TUI program for file moderation. With it, you can view a list
of uploaded files, as well as extended information on them. It allows you to
take actions like removing files temporarily or permanently, as well as
blocking IP addresses and associated files.
If a sufficiently recent version of python-mpv with libmpv is present and
your terminal supports it, you also get graphical file previews, including
video playback. Upstream mpv currently supports sixels and the
`kitty graphics protocol <https://sw.kovidgoyal.net/kitty/graphics-protocol/>`_.
For this to work, set the ``MOD_PREVIEW_PROTO`` option in ``instance/config.py``.
Requirements:
* `Textual <https://textual.textualize.io/>`_
Optional:
* `python-mpv <https://github.com/jaseg/python-mpv>`_
(graphical previews)
* `PyAV <https://github.com/PyAV-Org/PyAV>`_
(information on multimedia files)
* `PyMuPDF <https://github.com/pymupdf/PyMuPDF>`_
(previews and file information for PDF, XPS, EPUB, MOBI and FB2)
* `libarchive-c <https://github.com/Changaco/python-libarchive-c>`_
(archive content listing)
.. note::
`Mosh <https://mosh.org/>`_ currently does not support sixels or kitty graphics.
.. hint::
You may need to set the ``COLORTERM`` environment variable to
``truecolor``.
.. tip::
Using compression with SSH (``-C`` option) can significantly
reduce the bandwidth requirements for graphics.
NSFW Detection
--------------

View file

@ -296,7 +296,7 @@ class File(db.Model):
f.size = len(data)
if not f.nsfw_score and app.config["NSFW_DETECT"]:
f.nsfw_score = nsfw.detect(p)
f.nsfw_score = nsfw.detect(str(p))
db.session.add(f)
db.session.commit()

View file

@ -58,6 +58,17 @@ FHOST_MIN_EXPIRATION = 30 * 24 * 60 * 60 * 1000
FHOST_MAX_EXPIRATION = 365 * 24 * 60 * 60 * 1000
# This should be detected automatically when running behind a reverse proxy, but needs
# to be set for URL resolution to work in e.g. the moderation UI.
# SERVER_NAME = "example.com"
# Specifies which graphics protocol to use for the media previews in the moderation UI.
# Requires pympv with libmpv >= 0.36.0 and terminal support.
# Available choices are "sixel" and "kitty".
# MOD_PREVIEW_PROTO = "sixel"
# Use the X-SENDFILE header to speed up serving files w/ compatible webservers
#
# Some webservers can be configured use the X-Sendfile header to handle sending

View file

@ -58,14 +58,19 @@ def upgrade():
return # There are no currently unexpired files
# Calculate an expiration date for all existing files
files = session.scalars(
q = session.scalars(
sa.select(File)
.where(
sa.not_(File.removed),
File.sha256.in_(unexpired_files)
sa.not_(File.removed)
)
)
updates = [] # We coalesce updates to the database here
# SQLite has a hard limit on the number of variables so we
# need to do this the slow way
files = [f for f in q if f.sha256 in unexpired_files]
for file in files:
file_path = storage / file.sha256
stat = os.stat(file_path)

56
mod.css Normal file
View file

@ -0,0 +1,56 @@
#ftable {
width: 1fr;
}
#infopane {
width: 50%;
outline-top: hkey $primary;
background: $panel;
}
#finfo {
background: $boost;
height: 12;
width: 1fr;
box-sizing: content-box;
}
#mpv {
display: none;
height: 20%;
width: 1fr;
content-align: center middle;
}
#ftextlog {
height: 1fr;
width: 1fr;
}
#filter_container {
height: auto;
display: none;
}
#filter_label {
content-align: right middle;
height: 1fr;
width: 20%;
margin: 0 1 0 2;
}
#filter_input {
width: 1fr;
}
Notification {
dock: bottom;
layer: notification;
width: auto;
margin: 2 4;
padding: 1 2;
background: $background;
color: $text;
height: auto;
}

279
mod.py Executable file
View file

@ -0,0 +1,279 @@
#!/usr/bin/env python3
from itertools import zip_longest
from sys import stdout
import time
from textual.app import App, ComposeResult
from textual.widgets import DataTable, Header, Footer, TextLog, Static, Input
from textual.containers import Horizontal, Vertical
from textual.screen import Screen
from textual import log
from rich.text import Text
from jinja2.filters import do_filesizeformat
from fhost import db, File, su, app as fhost_app, in_upload_bl
from modui import *
fhost_app.app_context().push()
class NullptrMod(Screen):
BINDINGS = [
("q", "quit_app", "Quit"),
("f1", "filter(1, 'Lookup name:')", "Lookup name"),
("f2", "filter(2, 'Filter IP address:')", "Filter IP"),
("f3", "filter(3, 'Filter MIME Type:')", "Filter MIME"),
("f4", "filter(4, 'Filter extension:')", "Filter Ext."),
("f5", "refresh", "Refresh"),
("f6", "filter_clear", "Clear filter"),
("r", "remove_file(False)", "Remove file"),
("ctrl+r", "remove_file(True)", "Ban file"),
("p", "ban_ip(False)", "Ban IP"),
("ctrl+p", "ban_ip(True)", "Nuke IP"),
]
async def action_quit_app(self):
self.mpvw.shutdown()
await self.app.action_quit()
def action_refresh(self):
ftable = self.query_one("#ftable")
ftable.watch_query(None, None)
def action_filter_clear(self):
self.query_one("#filter_container").display = False
ftable = self.query_one("#ftable")
ftable.focus()
ftable.query = ftable.base_query
def action_filter(self, fcol: int, label: str):
self.query_one("#filter_label").update(label)
finput = self.query_one("#filter_input")
self.filter_col = fcol
self.query_one("#filter_container").display = True
finput.focus()
self._refresh_layout()
if self.current_file:
match fcol:
case 1: finput.value = ""
case 2: finput.value = self.current_file.addr
case 3: finput.value = self.current_file.mime
case 4: finput.value = self.current_file.ext
def on_input_submitted(self, message: Input.Submitted) -> None:
self.query_one("#filter_container").display = False
ftable = self.query_one("#ftable")
ftable.focus()
if len(message.value):
match self.filter_col:
case 1:
try: ftable.query = ftable.base_query.filter(File.id == su.debase(message.value))
except ValueError: pass
case 2: ftable.query = ftable.base_query.filter(File.addr == message.value)
case 3: ftable.query = ftable.base_query.filter(File.mime.like(message.value))
case 4: ftable.query = ftable.base_query.filter(File.ext.like(message.value))
else:
ftable.query = ftable.base_query
def action_remove_file(self, permanent: bool) -> None:
if self.current_file:
self.current_file.delete(permanent)
db.session.commit()
self.mount(Notification(f"{'Banned' if permanent else 'Removed'} file {self.current_file.getname()}"))
self.action_refresh()
def action_ban_ip(self, nuke: bool) -> None:
if self.current_file:
if not fhost_app.config["FHOST_UPLOAD_BLACKLIST"]:
self.mount(Notification("Failed: FHOST_UPLOAD_BLACKLIST not set!"))
return
else:
if in_upload_bl(self.current_file.addr):
txt = f"{self.current_file.addr} is already banned"
else:
with fhost_app.open_instance_resource(fhost_app.config["FHOST_UPLOAD_BLACKLIST"], "a") as bl:
print(self.current_file.addr.lstrip("::ffff:"), file=bl)
txt = f"Banned {self.current_file.addr}"
if nuke:
tsize = 0
trm = 0
for f in File.query.filter(File.addr == self.current_file.addr):
if f.getpath().is_file():
tsize += f.size or f.getpath().stat().st_size
trm += 1
f.delete(True)
db.session.commit()
txt += f", removed {trm} {'files' if trm != 1 else 'file'} totaling {do_filesizeformat(tsize, True)}"
self.mount(Notification(txt))
self._refresh_layout()
ftable = self.query_one("#ftable")
ftable.watch_query(None, None)
def on_update(self) -> None:
stdout.write("\033[?25l")
stdout.flush()
def compose(self) -> ComposeResult:
yield Header()
yield Horizontal(
FileTable(id="ftable", zebra_stripes=True),
Vertical(
DataTable(id="finfo", show_header=False),
MpvWidget(id="mpv"),
TextLog(id="ftextlog"),
id="infopane"))
yield Horizontal(Static("Filter:", id="filter_label"), Input(id="filter_input"), id="filter_container")
yield Footer()
def on_mount(self) -> None:
self.current_file = None
self.ftable = self.query_one("#ftable")
self.ftable.focus()
self.finfo = self.query_one("#finfo")
self.finfo.add_columns("key", "value")
self.mpvw = self.query_one("#mpv")
self.ftlog = self.query_one("#ftextlog")
self.mimehandler = mime.MIMEHandler()
self.mimehandler.register(mime.MIMECategory.Archive, self.handle_libarchive)
self.mimehandler.register(mime.MIMECategory.Text, self.handle_text)
self.mimehandler.register(mime.MIMECategory.AV, self.handle_mpv)
self.mimehandler.register(mime.MIMECategory.Document, self.handle_mupdf)
self.mimehandler.register(mime.MIMECategory.Fallback, self.handle_libarchive)
self.mimehandler.register(mime.MIMECategory.Fallback, self.handle_mpv)
self.mimehandler.register(mime.MIMECategory.Fallback, self.handle_raw)
def handle_libarchive(self, cat):
import libarchive
with libarchive.file_reader(str(self.current_file.getpath())) as a:
self.ftlog.write("\n".join(e.path for e in a))
return True
def handle_text(self, cat):
with open(self.current_file.getpath(), "r") as sf:
data = sf.read(1000000).replace("\033","")
self.ftlog.write(data)
return True
def handle_mupdf(self, cat):
import fitz
with fitz.open(self.current_file.getpath(),
filetype=self.current_file.ext.lstrip(".")) as doc:
p = doc.load_page(0)
pix = p.get_pixmap(dpi=72)
imgdata = pix.tobytes("ppm").hex()
self.mpvw.styles.height = "40%"
self.mpvw.start_mpv("hex://" + imgdata, 0)
self.ftlog.write(Text.from_markup(f"[bold]Pages:[/bold] {doc.page_count}"))
self.ftlog.write(Text.from_markup("[bold]Metadata:[/bold]"))
for k, v in doc.metadata.items():
self.ftlog.write(Text.from_markup(f" [bold]{k}:[/bold] {v}"))
toc = doc.get_toc()
if len(toc):
self.ftlog.write(Text.from_markup("[bold]TOC:[/bold]"))
for lvl, title, page in toc:
self.ftlog.write(f"{' ' * lvl} {page}: {title}")
return True
def handle_mpv(self, cat):
if cat == mime.MIMECategory.AV or self.current_file.nsfw_score >= 0:
self.mpvw.styles.height = "20%"
self.mpvw.start_mpv(str(self.current_file.getpath()), 0)
import av
with av.open(str(self.current_file.getpath())) as c:
self.ftlog.write(Text("Format:", style="bold"))
self.ftlog.write(f" {c.format.long_name}")
if len(c.metadata):
self.ftlog.write(Text("Metadata:", style="bold"))
for k, v in c.metadata.items():
self.ftlog.write(f" {k}: {v}")
for s in c.streams:
self.ftlog.write(Text(f"Stream {s.index}:", style="bold"))
self.ftlog.write(f" Type: {s.type}")
if s.base_rate:
self.ftlog.write(f" Frame rate: {s.base_rate}")
if len(s.metadata):
self.ftlog.write(Text(" Metadata:", style="bold"))
for k, v in s.metadata.items():
self.ftlog.write(f" {k}: {v}")
return True
return False
def handle_raw(self, cat):
def hexdump(binf, length):
def fmt(s):
if isinstance(s, str):
c = chr(int(s, 16))
else:
c = chr(s)
s = c
if c.isalpha(): return f"\0[chartreuse1]{s}\0[/chartreuse1]"
if c.isdigit(): return f"\0[gold1]{s}\0[/gold1]"
if not c.isprintable():
g = "grey50" if c == "\0" else "cadet_blue"
return f"\0[{g}]{s if len(s) == 2 else '.'}\0[/{g}]"
return s
return Text.from_markup("\n".join(f"{' '.join(map(fmt, map(''.join, zip(*[iter(c.hex())] * 2))))}"
f"{' ' * (16 - len(c))}"
f" {''.join(map(fmt, c))}"
for c in map(lambda x: bytes([n for n in x if n != None]),
zip_longest(*[iter(binf.read(min(length, 16 * 10)))] * 16))))
with open(self.current_file.getpath(), "rb") as binf:
self.ftlog.write(hexdump(binf, self.current_file.size))
if self.current_file.size > 16*10*2:
binf.seek(self.current_file.size-16*10)
self.ftlog.write(" [...] ".center(64, ''))
self.ftlog.write(hexdump(binf, self.current_file.size - binf.tell()))
return True
def on_file_table_selected(self, message: FileTable.Selected) -> None:
f = message.file
self.current_file = f
self.finfo.clear()
self.finfo.add_rows([
("ID:", str(f.id)),
("File name:", f.getname()),
("URL:", f.geturl() if fhost_app.config["SERVER_NAME"] else "⚠ Set SERVER_NAME in config.py to display"),
("File size:", do_filesizeformat(f.size, True)),
("MIME type:", f.mime),
("SHA256 checksum:", f.sha256),
("Uploaded by:", Text(f.addr)),
("Management token:", f.mgmt_token),
("Secret:", f.secret),
("Is NSFW:", ("Yes" if f.is_nsfw else "No") + f" (Score: {f.nsfw_score:0.4f})"),
("Is banned:", "Yes" if f.removed else "No"),
("Expires:", time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(File.get_expiration(f.expiration, f.size)/1000)))
])
self.mpvw.stop_mpv(True)
self.ftlog.remove()
self.query_one("#infopane").mount(TextLog(id="ftextlog"))
self.ftlog = self.query_one("#ftextlog")
if f.getpath().is_file():
self.mimehandler.handle(f.mime, f.ext)
self.ftlog.scroll_home(animate=False)
class NullptrModApp(App):
CSS_PATH = "mod.css"
def on_mount(self) -> None:
self.title = "0x0 File Moderation Interface"
self.main_screen = NullptrMod()
self.install_screen(self.main_screen, name="main")
self.push_screen("main")
if __name__ == "__main__":
app = NullptrModApp()
app.run()

BIN
modui.webp Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 339 KiB

3
modui/__init__.py Normal file
View file

@ -0,0 +1,3 @@
from .filetable import FileTable
from .notification import Notification
from .mpvwidget import MpvWidget

72
modui/filetable.py Normal file
View file

@ -0,0 +1,72 @@
from textual.widgets import DataTable, Static
from textual.reactive import Reactive
from textual.message import Message, MessageTarget
from textual import events, log
from jinja2.filters import do_filesizeformat
from fhost import File
from modui import mime
class FileTable(DataTable):
query = Reactive(None)
order_col = Reactive(0)
order_desc = Reactive(True)
limit = 10000
colmap = [File.id, File.removed, File.nsfw_score, None, File.ext, File.size, File.mime]
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.add_columns("#", "☣️", "🔞", "📂", "name", "size", "mime")
self.base_query = File.query.filter(File.size != None)
self.query = self.base_query
class Selected(Message):
def __init__(self, sender: MessageTarget, f: File) -> None:
self.file = f
super().__init__(sender)
def watch_order_col(self, old, value) -> None:
self.watch_query(None, None)
def watch_order_desc(self, old, value) -> None:
self.watch_query(None, None)
def watch_query(self, old, value) -> None:
def fmt_file(f: File) -> tuple:
return (
str(f.id),
"🔴" if f.removed else " ",
"🚩" if f.is_nsfw else " ",
"👻" if not f.getpath().is_file() else " ",
f.getname(),
do_filesizeformat(f.size, True),
f"{mime.mimemoji.get(f.mime.split('/')[0], mime.mimemoji.get(f.mime)) or ' '} " + f.mime,
)
if (self.query):
self.clear()
order = FileTable.colmap[self.order_col]
q = self.query
if order: q = q.order_by(order.desc() if self.order_desc else order, File.id)
self.add_rows(map(fmt_file, q.limit(self.limit)))
def _scroll_cursor_in_to_view(self, animate: bool = False) -> None:
region = self._get_cell_region(self.cursor_row, 0)
spacing = self._get_cell_border()
self.scroll_to_region(region, animate=animate, spacing=spacing)
async def watch_cursor_cell(self, old, value) -> None:
super().watch_cursor_cell(old, value)
if value[0] < len(self.data) and value[0] >= 0:
f = File.query.get(int(self.data[value[0]][0]))
await self.emit(self.Selected(self, f))
def on_click(self, event: events.Click) -> None:
super().on_click(event)
meta = self.get_style_at(event.x, event.y).meta
if meta:
if meta["row"] == -1:
qi = FileTable.colmap[meta["column"]]
if meta["column"] == self.order_col:
self.order_desc = not self.order_desc
self.order_col = meta["column"]

126
modui/mime.py Normal file
View file

@ -0,0 +1,126 @@
from enum import Enum
from textual import log
mimemoji = {
"audio" : "🔈",
"video" : "🎞",
"text" : "📄",
"image" : "🖼",
"application/zip" : "🗜️",
"application/x-zip-compressed" : "🗜️",
"application/x-tar" : "🗄",
"application/x-cpio" : "🗄",
"application/x-xz" : "🗜️",
"application/x-7z-compressed" : "🗜️",
"application/gzip" : "🗜️",
"application/zstd" : "🗜️",
"application/x-rar" : "🗜️",
"application/x-rar-compressed" : "🗜️",
"application/vnd.ms-cab-compressed" : "🗜️",
"application/x-bzip2" : "🗜️",
"application/x-lzip" : "🗜️",
"application/x-iso9660-image" : "💿",
"application/pdf" : "📕",
"application/epub+zip" : "📕",
"application/mxf" : "🎞",
"application/vnd.android.package-archive" : "📦",
"application/vnd.debian.binary-package" : "📦",
"application/x-rpm" : "📦",
"application/x-dosexec" : "",
"application/x-execuftable" : "",
"application/x-sharedlib" : "",
"application/java-archive" : "",
"application/x-qemu-disk" : "🖴",
"application/pgp-encrypted" : "🔏",
}
MIMECategory = Enum("MIMECategory",
["Archive", "Text", "AV", "Document", "Fallback"]
)
class MIMEHandler:
def __init__(self):
self.handlers = {
MIMECategory.Archive : [[
"application/zip",
"application/x-zip-compressed",
"application/x-tar",
"application/x-cpio",
"application/x-xz",
"application/x-7z-compressed",
"application/gzip",
"application/zstd",
"application/x-rar",
"application/x-rar-compressed",
"application/vnd.ms-cab-compressed",
"application/x-bzip2",
"application/x-lzip",
"application/x-iso9660-image",
"application/vnd.android.package-archive",
"application/vnd.debian.binary-package",
"application/x-rpm",
"application/java-archive",
"application/vnd.openxmlformats"
], []],
MIMECategory.Text : [[
"text",
"application/json",
"application/xml",
], []],
MIMECategory.AV : [[
"audio", "video", "image",
"application/mxf"
], []],
MIMECategory.Document : [[
"application/pdf",
"application/epub",
"application/x-mobipocket-ebook",
], []],
MIMECategory.Fallback : [[], []]
}
self.exceptions = {
MIMECategory.Archive : {
".cbz" : MIMECategory.Document,
".xps" : MIMECategory.Document,
".epub" : MIMECategory.Document,
},
MIMECategory.Text : {
".fb2" : MIMECategory.Document,
}
}
def register(self, category, handler):
self.handlers[category][1].append(handler)
def handle(self, mime, ext):
def getcat(s):
cat = MIMECategory.Fallback
for k, v in self.handlers.items():
s = s.split(";")[0]
if s in v[0] or s.split("/")[0] in v[0]:
cat = k
break
for x in v[0]:
if s.startswith(x):
cat = k
break
if cat in self.exceptions:
cat = self.exceptions[cat].get(ext) or cat
return cat
cat = getcat(mime)
for handler in self.handlers[cat][1]:
try:
if handler(cat): return
except: pass
for handler in self.handlers[MIMECategory.Fallback][1]:
try:
if handler(None): return
except: pass
raise RuntimeError(f"Unhandled MIME type category: {cat}")

88
modui/mpvwidget.py Normal file
View file

@ -0,0 +1,88 @@
import time
import fcntl, struct, termios
from sys import stdout
from textual import events, log
from textual.widgets import Static
from fhost import app as fhost_app
class MpvWidget(Static):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.mpv = None
self.vo = fhost_app.config.get("MOD_PREVIEW_PROTO")
if not self.vo in ["sixel", "kitty"]:
self.update("⚠ Previews not enabled. \n\nSet MOD_PREVIEW_PROTO to 'sixel' or 'kitty' in config.py,\nwhichever is supported by your terminal.")
else:
try:
import mpv
self.mpv = mpv.MPV()
self.mpv.profile = "sw-fast"
self.mpv["vo"] = self.vo
self.mpv[f"vo-{self.vo}-config-clear"] = False
self.mpv[f"vo-{self.vo}-alt-screen"] = False
self.mpv[f"vo-sixel-buffered"] = True
self.mpv["audio"] = False
self.mpv["loop-file"] = "inf"
self.mpv["image-display-duration"] = 0.5 if self.vo == "sixel" else "inf"
except Exception as e:
self.mpv = None
self.update(f"⚠ Previews require python-mpv with libmpv 0.36.0 or later \n\nError was:\n{type(e).__name__}: {e}")
def start_mpv(self, f: str|None = None, pos: float|str|None = None) -> None:
self.display = True
self.screen._refresh_layout()
if self.mpv:
if self.content_region.x:
r, c, w, h = struct.unpack('hhhh', fcntl.ioctl(0, termios.TIOCGWINSZ, '12345678'))
width = int((w / c) * self.content_region.width)
height = int((h / r) * (self.content_region.height + (1 if self.vo == "sixel" else 0)))
self.mpv[f"vo-{self.vo}-left"] = self.content_region.x + 1
self.mpv[f"vo-{self.vo}-top"] = self.content_region.y + 1
self.mpv[f"vo-{self.vo}-rows"] = self.content_region.height + (1 if self.vo == "sixel" else 0)
self.mpv[f"vo-{self.vo}-cols"] = self.content_region.width
self.mpv[f"vo-{self.vo}-width"] = width
self.mpv[f"vo-{self.vo}-height"] = height
if pos != None:
self.mpv["start"] = pos
if f:
self.mpv.loadfile(f)
else:
self.mpv.playlist_play_index(0)
def stop_mpv(self, wait: bool = False) -> None:
if self.mpv:
if not self.mpv.idle_active:
self.mpv.stop(True)
if wait:
time.sleep(0.1)
self.clear_mpv()
self.display = False
def on_resize(self, size) -> None:
if self.mpv:
if not self.mpv.idle_active:
t = self.mpv.time_pos
self.stop_mpv()
if t:
self.mpv["start"] = t
self.start_mpv()
def clear_mpv(self) -> None:
if self.vo == "kitty":
stdout.write("\033_Ga=d;\033\\")
stdout.flush()
def shutdown(self) -> None:
if self.mpv:
self.mpv.stop()
del self.mpv
if self.vo == "kitty":
stdout.write("\033_Ga=d;\033\\\033[?25l")
stdout.flush()

8
modui/notification.py Normal file
View file

@ -0,0 +1,8 @@
from textual.widgets import Static
class Notification(Static):
def on_mount(self) -> None:
self.set_timer(3, self.remove)
def on_click(self) -> None:
self.remove()

View file

@ -1,11 +1,28 @@
click
Flask_Migrate
validators
alembic
requests
Jinja2
Flask
numpy
SQLAlchemy
requests
Flask_SQLAlchemy
validators
flask_migrate
flask_sqlalchemy
python_magic
# vscan
clamd
# nsfw detection
numpy
# mod ui
av
PyMuPDF
libarchive_c
textual
python-mpv
# dev
pytest
# fileserve
uwsgi