Jaroslav Beneš
ddad585e4b
The button cannot do the work, and that is the whole design. The service runs as an unprivileged account, cannot restart itself, and should not be able to: a web application that can restart its own service is one whose worst day is much worse. So /admin/updates writes a file, and an opt-in systemd .path unit runs deploy/update.sh as root. Three properties hold it up, and each is a thing that could have been got wrong. The request file carries nothing that reaches a command line -- no branch, no ref, no arguments -- because the branch is baked into the unit at install time, so pressing the button is always "deploy the branch this host was configured with" and can never be "deploy something else". It is off unless somebody passes INSTALL_UPDATE_HELPER=1, and re-running the installer without it removes both units and the marker. And without the helper the page says so and prints the manual command rather than writing a file nothing is watching, which would be a button that reports success and does nothing. The card that says all of this is rendered whether or not there is anything to apply. It was inside the "there is an update" branch first, so an administrator could not discover the helper was missing until the day they needed it, which is the worst possible moment. Opening the page makes no network request; Check is the one thing that fetches. And it shows the log between, not a count: "3 behind" is a number somebody has to go and look up, while the subjects are what decides whether this is worth restarting for right now. Docker is one stage, because there is nothing to build -- no Node, no compiled assets. It bakes no secret key (one in an image is one every copy shares, and rotating it makes stored API keys unreadable), no data, and no .git, so /admin/updates inside a container correctly reports that it was not installed from a checkout. Compose publishes on loopback and refuses to start without a key. TLS in front is a constraint rather than a recommendation: the service worker and the microphone both require HTTPS or localhost. The image was built and run before this was committed, which is how the missing COPY of LICENSE was found -- pyproject declares it and the build backend reads it, so the failure reads like a packaging problem and is one line. deploy/lxc-install.sh creates an unprivileged Debian container and runs the existing installer inside it. A wrapper, not a second install path: a parallel installer is two things to keep correct and one of them rots. /healthz opens the database rather than only proving the socket is listening -- a process that is up with a database it cannot open answers every page with a 500 -- and says nothing about what is here, being reachable without signing in. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
281 lines
11 KiB
Python
281 lines
11 KiB
Python
"""Application factory, lifespan and error handling."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from collections.abc import AsyncIterator
|
|
from contextlib import asynccontextmanager
|
|
|
|
from fastapi import FastAPI, Request, status
|
|
from fastapi.responses import JSONResponse, Response
|
|
from fastapi.staticfiles import StaticFiles
|
|
from starlette.exceptions import HTTPException as StarletteHTTPException
|
|
|
|
from lembas import __version__
|
|
from lembas.api import (
|
|
admin,
|
|
admin_agents,
|
|
admin_audio,
|
|
admin_branding,
|
|
admin_extraction,
|
|
admin_images,
|
|
admin_models,
|
|
admin_prompts,
|
|
admin_schedules,
|
|
admin_search,
|
|
admin_suggestions,
|
|
admin_tools,
|
|
admin_updates,
|
|
admin_users,
|
|
agents,
|
|
audio,
|
|
auth,
|
|
branding,
|
|
canvas,
|
|
chats,
|
|
files,
|
|
folders,
|
|
library,
|
|
messages,
|
|
pages,
|
|
preferences,
|
|
push,
|
|
reports,
|
|
schedules,
|
|
sharing,
|
|
terminal,
|
|
)
|
|
from lembas.api.deps import RedirectToLogin, is_htmx, login_redirect
|
|
from lembas.config import settings
|
|
from lembas.db.session import init_db
|
|
from lembas.services.library import indexing
|
|
from lembas.web.templating import STATIC_DIR, render
|
|
|
|
log = logging.getLogger("lembas")
|
|
|
|
|
|
def configure_logging() -> None:
|
|
logging.basicConfig(
|
|
level=settings.log_level.upper(),
|
|
format="%(asctime)s %(levelname)-7s %(name)s: %(message)s",
|
|
datefmt="%H:%M:%S",
|
|
)
|
|
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
|
configure_logging()
|
|
settings.ensure_dirs()
|
|
init_db()
|
|
|
|
if settings.secret_key_is_ephemeral:
|
|
log.warning(
|
|
"No LEMBAS_SECRET_KEY set, so a temporary one was generated. Every "
|
|
"restart will sign all users out and make stored API keys "
|
|
"unreadable. Generate a permanent key with:\n"
|
|
' python -c "import secrets; print(secrets.token_urlsafe(48))"'
|
|
)
|
|
|
|
# Files chosen in a composer that was never sent would otherwise sit on
|
|
# disk forever. Cheap, and startup is the natural moment for it.
|
|
try:
|
|
from lembas.db.session import session_scope
|
|
from lembas.services.chat import sweep_temporary
|
|
from lembas.services.files import sweep_orphans
|
|
from lembas.services.library.documents import sweep_unfiled
|
|
from lembas.services.library.indexing import sweep_orphans as sweep_chunks
|
|
from lembas.services.suggestions import seed_defaults as seed_suggestions
|
|
|
|
with session_scope() as db:
|
|
sweep_orphans(db)
|
|
# Documents that predate knowledge bases have nowhere to live until
|
|
# this runs; see services/library/documents.py.
|
|
sweep_unfiled(db)
|
|
# Temporary chats older than a day. Startup only, like the sweeps
|
|
# above it -- see services/chat.py:sweep_temporary.
|
|
sweep_temporary(db)
|
|
# Chunks whose record has gone. A backstop for a delete that
|
|
# happened with no event loop to schedule the tidy-up -- a CLI
|
|
# command, or a cascade from removing an account.
|
|
sweep_chunks(db)
|
|
# Three starting points on the empty screen, written once ever.
|
|
seed_suggestions(db)
|
|
except Exception: # noqa: BLE001 - housekeeping must never block startup
|
|
log.exception("orphaned upload sweep failed")
|
|
|
|
# Background jobs that were still running when we last stopped keep running
|
|
# on their own hosts; pick their watchers back up so the model is still
|
|
# woken when they finish. Best-effort, and inside the loop so its tasks land
|
|
# in this event loop.
|
|
try:
|
|
from lembas.services.agent.jobs import rehydrate as rehydrate_jobs
|
|
|
|
rehydrate_jobs()
|
|
except Exception: # noqa: BLE001 - a job that cannot be rehydrated is not fatal
|
|
log.exception("could not rehydrate background jobs")
|
|
|
|
# Schedules. `release_claims` first, because a firing interrupted by the
|
|
# last shutdown left a claim stamp that would otherwise read as permanently
|
|
# running. Then the ticker, started here rather than lazily like the
|
|
# terminal reaper: a schedule can be due at startup with nobody logged in,
|
|
# which is most of the point of having one. Inside the loop, so its tasks
|
|
# land in this event loop.
|
|
#
|
|
# Catching up on what was missed is deliberately NOT done here. It lives in
|
|
# the sweep, because a suspended laptop, a paused container and a long stall
|
|
# all reproduce "its time passed while nothing was running" with no restart
|
|
# for a startup hook to hang on.
|
|
try:
|
|
from lembas.services.schedule.ticker import release_claims
|
|
from lembas.services.schedule.ticker import start as start_ticker
|
|
|
|
released = release_claims()
|
|
if released:
|
|
log.info("released %s interrupted schedule claim(s)", released)
|
|
start_ticker()
|
|
except Exception: # noqa: BLE001 - scheduling failing must not block startup
|
|
log.exception("could not start the schedule ticker")
|
|
|
|
log.info("LLeMbas %s starting on http://%s:%s", __version__, settings.host, settings.port)
|
|
log.info("data directory: %s", settings.data_dir.resolve())
|
|
yield
|
|
|
|
# Replies still being written are cancelled and persisted with whatever
|
|
# they have, rather than left as permanently unfinished rows.
|
|
from lembas.services.agent.jobs import shutdown as stop_jobs
|
|
from lembas.services.agent.terminal import shutdown as stop_terminals
|
|
from lembas.services.generation import shutdown as stop_generations
|
|
from lembas.services.schedule.ticker import shutdown as stop_ticker
|
|
|
|
# Before the generations, so nothing new is fired into a chat whose reply is
|
|
# about to be cancelled and persisted.
|
|
await stop_ticker()
|
|
await stop_generations()
|
|
# Open shells have nothing to persist: whatever was running on the far side
|
|
# is cut off mid-command. Every deploy does this, and the panel is told why
|
|
# rather than left to guess -- see deploy/README.md.
|
|
await stop_terminals()
|
|
# Background jobs are the exception: cancelling a watcher does NOT stop the
|
|
# detached remote job, which keeps running and is rehydrated on the next
|
|
# start. Only the watching stops here.
|
|
await stop_jobs()
|
|
# A chunk set is written whole or not at all, so cancelling loses nothing
|
|
# a rebuild does not pick up again.
|
|
await indexing.shutdown()
|
|
log.info("LLeMbas stopped")
|
|
|
|
|
|
def create_app() -> FastAPI:
|
|
app = FastAPI(
|
|
title="LLeMbas",
|
|
version=__version__,
|
|
lifespan=lifespan,
|
|
# The API is an implementation detail of the UI, not a product surface.
|
|
docs_url="/api/docs" if settings.log_level == "debug" else None,
|
|
redoc_url=None,
|
|
)
|
|
|
|
app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
|
|
|
|
# One place that notices a library record changing, rather than a call in
|
|
# each of the ten writers that touch those tables. Idempotent, because the
|
|
# factory is called per test. See services/library/indexing.py:install.
|
|
indexing.install()
|
|
|
|
app.include_router(pages.router)
|
|
app.include_router(auth.router)
|
|
app.include_router(preferences.router)
|
|
app.include_router(chats.router)
|
|
app.include_router(canvas.router)
|
|
app.include_router(terminal.router)
|
|
app.include_router(audio.router)
|
|
app.include_router(files.router)
|
|
app.include_router(folders.router)
|
|
app.include_router(library.router)
|
|
app.include_router(messages.router)
|
|
app.include_router(reports.router)
|
|
app.include_router(schedules.router)
|
|
app.include_router(agents.router)
|
|
app.include_router(sharing.router)
|
|
app.include_router(admin.router)
|
|
app.include_router(admin_users.router)
|
|
app.include_router(admin_updates.router)
|
|
app.include_router(admin_models.router)
|
|
app.include_router(admin_audio.router)
|
|
app.include_router(admin_branding.router)
|
|
app.include_router(admin_extraction.router)
|
|
app.include_router(admin_search.router)
|
|
app.include_router(admin_schedules.router)
|
|
app.include_router(admin_images.router)
|
|
app.include_router(admin_prompts.router)
|
|
app.include_router(admin_suggestions.router)
|
|
app.include_router(admin_tools.router)
|
|
app.include_router(admin_agents.router)
|
|
app.include_router(push.router)
|
|
app.include_router(branding.router)
|
|
|
|
register_error_handlers(app)
|
|
return app
|
|
|
|
|
|
def register_error_handlers(app: FastAPI) -> None:
|
|
@app.exception_handler(RedirectToLogin)
|
|
async def _not_signed_in(request: Request, exc: RedirectToLogin) -> Response:
|
|
# An htmx request must not swap a login page into a fragment of the
|
|
# chat UI, so tell the browser to navigate instead.
|
|
if is_htmx(request):
|
|
response = Response(status_code=status.HTTP_204_NO_CONTENT)
|
|
response.headers["HX-Redirect"] = "/auth/login"
|
|
return response
|
|
return login_redirect(exc.next_url)
|
|
|
|
@app.exception_handler(StarletteHTTPException)
|
|
async def _http_error(request: Request, exc: StarletteHTTPException) -> Response:
|
|
# JSON callers and htmx fragments want the bare status; humans loading a
|
|
# page want a themed page they can navigate away from.
|
|
wants_page = "text/html" in request.headers.get("accept", "") and not is_htmx(request)
|
|
if not wants_page:
|
|
return JSONResponse({"detail": exc.detail}, status_code=exc.status_code)
|
|
|
|
return render(
|
|
request,
|
|
"error.html",
|
|
{
|
|
"status_code": exc.status_code,
|
|
"detail": exc.detail,
|
|
"flavour": error_flavour(exc.status_code),
|
|
},
|
|
status_code=exc.status_code,
|
|
)
|
|
|
|
@app.exception_handler(Exception)
|
|
async def _unhandled(request: Request, exc: Exception) -> Response:
|
|
log.exception("unhandled error at %s", request.url.path)
|
|
if is_htmx(request) or "text/html" not in request.headers.get("accept", ""):
|
|
return JSONResponse({"detail": "Internal server error"}, status_code=500)
|
|
return render(
|
|
request,
|
|
"error.html",
|
|
{"status_code": 500, "detail": "Something went wrong.",
|
|
"flavour": error_flavour(500)},
|
|
status_code=500,
|
|
)
|
|
|
|
|
|
# Flavour lives in error pages, empty states and theme names -- never in the
|
|
# functional UI. See CLAUDE.md.
|
|
#
|
|
# The three lines themselves moved into `services/branding.py` with the rest of
|
|
# what an administrator can replace. What is left here is the mapping from a
|
|
# status code to which of them, which is not something anybody would want to
|
|
# edit. `snapshot()` never raises, so an error page can still render its error
|
|
# on an instance whose database is the thing that broke.
|
|
def error_flavour(status_code: int) -> str:
|
|
from lembas.services import branding
|
|
|
|
text = branding.snapshot().text
|
|
return text.get(f"error_{status_code}") or text["error_500"]
|
|
|
|
|
|
app = create_app()
|