Working chat: auth, connections, streaming, folders
LLeMbas now runs end to end. Register, add an OpenAI-compatible connection, and hold a real streaming conversation organised into folders. Verified against the local llama-swap instance. Streaming is the one genuinely tricky part. Sending a message returns two HTML fragments -- the user bubble and an empty assistant bubble carrying an sse-connect -- and that attribute is the ONLY thing that starts a generation. Rendering an incomplete assistant message as a streaming shell falls out of the same template, which means loading a page whose last reply never finished simply picks it up again. Details worth knowing about, each commented where it matters: - SSE payloads are split across several data: lines. A raw newline in one data: line truncates the event, which shows up the first time a model emits a code block. - Markdown is rendered server-side by the same helper for both the page and the final streamed frame, so the two cannot disagree. The fence renderer is replaced outright rather than using markdown-it's highlight option, which re-wraps output in a second <pre>. - escape_text is html.escape, not nh3.clean_text: it escapes character by character, so escaping stream chunks separately equals escaping the whole string. - The stream opens its own session via session_scope(); it outlives the request handler and the dependency-scoped session may be closed. - Deleting a folder keeps the chats inside it (FK is SET NULL). Losing a conversation to a mis-clicked folder delete is unforgivable. - Login failures use one message for "no such account" and "wrong password" so the form cannot enumerate registered addresses. Also adds deploy/ for the gamebox install at https://chat.lan: system unit, nginx vhost with buffering off (buffering on turns streaming into one lump at the end), and install/update scripts following the same service-user and /srv bind-mount conventions as llama-swap and comfyui. 70 tests, ruff clean. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jaroslav Beneš
@@ -0,0 +1,225 @@
|
||||
"""Chat orchestration: building requests, streaming replies, naming chats."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session as DBSession
|
||||
|
||||
from lembas.db.models import (
|
||||
ROLE_ASSISTANT,
|
||||
ROLE_SYSTEM,
|
||||
ROLE_USER,
|
||||
Chat,
|
||||
Connection,
|
||||
Message,
|
||||
Model,
|
||||
)
|
||||
from lembas.services.llm.openai_client import Endpoint, LLMError, complete
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Sampling keys forwarded upstream. Anything else a user puts in params_json is
|
||||
# ignored rather than passed through, so a typo cannot produce a 400 from the
|
||||
# provider that looks like a LLeMbas bug.
|
||||
FORWARDED_PARAMS = frozenset(
|
||||
{"temperature", "top_p", "max_tokens", "presence_penalty", "frequency_penalty",
|
||||
"seed", "stop"}
|
||||
)
|
||||
|
||||
MAX_TITLE_LENGTH = 60
|
||||
|
||||
|
||||
def resolve_endpoint(db: DBSession, chat: Chat) -> tuple[Endpoint, str]:
|
||||
"""Find the connection and model a chat should use.
|
||||
|
||||
Chats store the model id as text rather than a foreign key so history
|
||||
survives an admin deleting a connection, which means the mapping back to a
|
||||
live connection has to be resolved at send time and can legitimately fail.
|
||||
"""
|
||||
if not chat.model_id:
|
||||
raise LLMError("This chat has no model selected.")
|
||||
|
||||
connection: Connection | None = None
|
||||
if chat.connection_id:
|
||||
connection = db.get(Connection, chat.connection_id)
|
||||
|
||||
if connection is None or not connection.enabled:
|
||||
# The original connection is gone or disabled. Any enabled connection
|
||||
# still offering this model id will do.
|
||||
model = db.scalar(
|
||||
select(Model)
|
||||
.join(Connection)
|
||||
.where(
|
||||
Model.model_id == chat.model_id,
|
||||
Model.enabled.is_(True),
|
||||
Connection.enabled.is_(True),
|
||||
)
|
||||
.order_by(Connection.position)
|
||||
)
|
||||
if model is None:
|
||||
raise LLMError(
|
||||
f"No enabled connection currently offers the model "
|
||||
f"'{chat.model_id}'. Pick another model for this chat."
|
||||
)
|
||||
connection = model.connection
|
||||
chat.connection_id = connection.id
|
||||
db.commit()
|
||||
|
||||
return Endpoint.from_connection(connection), chat.model_id
|
||||
|
||||
|
||||
def build_messages(db: DBSession, chat: Chat, *, upto: Message | None = None) -> list[dict]:
|
||||
"""Assemble the message list to send upstream.
|
||||
|
||||
`upto` excludes the placeholder assistant row being generated into, and
|
||||
everything after it.
|
||||
"""
|
||||
payload: list[dict[str, Any]] = []
|
||||
if chat.system_prompt.strip():
|
||||
payload.append({"role": ROLE_SYSTEM, "content": chat.system_prompt.strip()})
|
||||
|
||||
history = db.scalars(
|
||||
select(Message).where(Message.chat_id == chat.id).order_by(Message.created_at)
|
||||
).all()
|
||||
|
||||
for message in history:
|
||||
if upto is not None and message.id == upto.id:
|
||||
break
|
||||
# Skip turns that failed or produced nothing: sending an empty
|
||||
# assistant message upsets several providers.
|
||||
if message.error or not message.content.strip():
|
||||
continue
|
||||
payload.append({"role": message.role, "content": message.content})
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
def build_request(db: DBSession, chat: Chat, *, upto: Message | None = None) -> dict[str, Any]:
|
||||
params = {
|
||||
key: value
|
||||
for key, value in (chat.params_json or {}).items()
|
||||
if key in FORWARDED_PARAMS and value not in (None, "")
|
||||
}
|
||||
return {
|
||||
"model": chat.model_id,
|
||||
"messages": build_messages(db, chat, upto=upto),
|
||||
**params,
|
||||
}
|
||||
|
||||
|
||||
def default_model(db: DBSession) -> tuple[str, str] | None:
|
||||
"""First enabled model on the first enabled connection, or None."""
|
||||
model = db.scalar(
|
||||
select(Model)
|
||||
.join(Connection)
|
||||
.where(Model.enabled.is_(True), Connection.enabled.is_(True))
|
||||
.order_by(Connection.position, Model.model_id)
|
||||
)
|
||||
if model is None:
|
||||
return None
|
||||
return model.model_id, model.connection_id
|
||||
|
||||
|
||||
def available_models(db: DBSession) -> list[Model]:
|
||||
return list(
|
||||
db.scalars(
|
||||
select(Model)
|
||||
.join(Connection)
|
||||
.where(Model.enabled.is_(True), Connection.enabled.is_(True))
|
||||
.order_by(Connection.position, Model.model_id)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def fallback_title(text: str) -> str:
|
||||
"""Derive a chat title from the opening message, without calling a model."""
|
||||
cleaned = " ".join(text.split())
|
||||
if not cleaned:
|
||||
return "New chat"
|
||||
if len(cleaned) <= MAX_TITLE_LENGTH:
|
||||
return cleaned
|
||||
# Prefer a word boundary, but only if it does not cut the title in half.
|
||||
clipped = cleaned[:MAX_TITLE_LENGTH]
|
||||
space = clipped.rfind(" ")
|
||||
if space > MAX_TITLE_LENGTH * 0.6:
|
||||
clipped = clipped[:space]
|
||||
return clipped.rstrip(" ,.;:-") + "…"
|
||||
|
||||
|
||||
async def generate_title(endpoint: Endpoint, model_id: str, question: str, answer: str) -> str:
|
||||
"""Ask the model for a short chat title.
|
||||
|
||||
Best-effort by design: any failure falls back to trimming the first
|
||||
message. Naming a chat is never worth surfacing an error for.
|
||||
"""
|
||||
prompt = (
|
||||
"Summarise this exchange as a title of at most six words. "
|
||||
"Reply with the title alone: no quotes, no punctuation at the end, "
|
||||
"no preamble.\n\n"
|
||||
f"User: {question[:500]}\n\nAssistant: {answer[:500]}"
|
||||
)
|
||||
try:
|
||||
raw = await complete(
|
||||
endpoint,
|
||||
{
|
||||
"model": model_id,
|
||||
"messages": [{"role": ROLE_USER, "content": prompt}],
|
||||
"max_tokens": 24,
|
||||
"temperature": 0.2,
|
||||
},
|
||||
)
|
||||
except LLMError as exc:
|
||||
log.debug("auto-title failed, using fallback: %s", exc)
|
||||
return fallback_title(question)
|
||||
|
||||
title = " ".join(raw.split()).strip().strip('"“”\'')
|
||||
# Small models sometimes ignore the instruction and answer the question
|
||||
# instead; an over-long reply is a better signal of that than anything else.
|
||||
if not title or len(title) > MAX_TITLE_LENGTH * 1.5:
|
||||
return fallback_title(question)
|
||||
return title[:MAX_TITLE_LENGTH]
|
||||
|
||||
|
||||
def create_message(
|
||||
db: DBSession,
|
||||
chat: Chat,
|
||||
role: str,
|
||||
content: str = "",
|
||||
*,
|
||||
complete_: bool = True,
|
||||
model_id: str = "",
|
||||
) -> Message:
|
||||
message = Message(
|
||||
chat_id=chat.id,
|
||||
role=role,
|
||||
content=content,
|
||||
complete=complete_,
|
||||
model_id=model_id,
|
||||
)
|
||||
db.add(message)
|
||||
db.commit()
|
||||
return message
|
||||
|
||||
|
||||
def user_chats(db: DBSession, user_id: str, *, folder_id: str | None = None) -> list[Chat]:
|
||||
query = select(Chat).where(Chat.user_id == user_id, Chat.archived.is_(False))
|
||||
if folder_id is not None:
|
||||
query = query.where(Chat.folder_id == folder_id)
|
||||
return list(db.scalars(query.order_by(Chat.pinned.desc(), Chat.updated_at.desc())))
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ROLE_ASSISTANT",
|
||||
"ROLE_USER",
|
||||
"available_models",
|
||||
"build_request",
|
||||
"create_message",
|
||||
"default_model",
|
||||
"fallback_title",
|
||||
"generate_title",
|
||||
"resolve_endpoint",
|
||||
"user_chats",
|
||||
]
|
||||
Reference in New Issue
Block a user