Working chat: auth, connections, streaming, folders
LLeMbas now runs end to end. Register, add an OpenAI-compatible connection, and hold a real streaming conversation organised into folders. Verified against the local llama-swap instance. Streaming is the one genuinely tricky part. Sending a message returns two HTML fragments -- the user bubble and an empty assistant bubble carrying an sse-connect -- and that attribute is the ONLY thing that starts a generation. Rendering an incomplete assistant message as a streaming shell falls out of the same template, which means loading a page whose last reply never finished simply picks it up again. Details worth knowing about, each commented where it matters: - SSE payloads are split across several data: lines. A raw newline in one data: line truncates the event, which shows up the first time a model emits a code block. - Markdown is rendered server-side by the same helper for both the page and the final streamed frame, so the two cannot disagree. The fence renderer is replaced outright rather than using markdown-it's highlight option, which re-wraps output in a second <pre>. - escape_text is html.escape, not nh3.clean_text: it escapes character by character, so escaping stream chunks separately equals escaping the whole string. - The stream opens its own session via session_scope(); it outlives the request handler and the dependency-scoped session may be closed. - Deleting a folder keeps the chats inside it (FK is SET NULL). Losing a conversation to a mis-clicked folder delete is unforgivable. - Login failures use one message for "no such account" and "wrong password" so the form cannot enumerate registered addresses. Also adds deploy/ for the gamebox install at https://chat.lan: system unit, nginx vhost with buffering off (buffering on turns streaming into one lump at the end), and install/update scripts following the same service-user and /srv bind-mount conventions as llama-swap and comfyui. 70 tests, ruff clean. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jaroslav Beneš
@@ -0,0 +1,225 @@
|
||||
"""Chat orchestration: building requests, streaming replies, naming chats."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session as DBSession
|
||||
|
||||
from lembas.db.models import (
|
||||
ROLE_ASSISTANT,
|
||||
ROLE_SYSTEM,
|
||||
ROLE_USER,
|
||||
Chat,
|
||||
Connection,
|
||||
Message,
|
||||
Model,
|
||||
)
|
||||
from lembas.services.llm.openai_client import Endpoint, LLMError, complete
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Sampling keys forwarded upstream. Anything else a user puts in params_json is
|
||||
# ignored rather than passed through, so a typo cannot produce a 400 from the
|
||||
# provider that looks like a LLeMbas bug.
|
||||
FORWARDED_PARAMS = frozenset(
|
||||
{"temperature", "top_p", "max_tokens", "presence_penalty", "frequency_penalty",
|
||||
"seed", "stop"}
|
||||
)
|
||||
|
||||
MAX_TITLE_LENGTH = 60
|
||||
|
||||
|
||||
def resolve_endpoint(db: DBSession, chat: Chat) -> tuple[Endpoint, str]:
|
||||
"""Find the connection and model a chat should use.
|
||||
|
||||
Chats store the model id as text rather than a foreign key so history
|
||||
survives an admin deleting a connection, which means the mapping back to a
|
||||
live connection has to be resolved at send time and can legitimately fail.
|
||||
"""
|
||||
if not chat.model_id:
|
||||
raise LLMError("This chat has no model selected.")
|
||||
|
||||
connection: Connection | None = None
|
||||
if chat.connection_id:
|
||||
connection = db.get(Connection, chat.connection_id)
|
||||
|
||||
if connection is None or not connection.enabled:
|
||||
# The original connection is gone or disabled. Any enabled connection
|
||||
# still offering this model id will do.
|
||||
model = db.scalar(
|
||||
select(Model)
|
||||
.join(Connection)
|
||||
.where(
|
||||
Model.model_id == chat.model_id,
|
||||
Model.enabled.is_(True),
|
||||
Connection.enabled.is_(True),
|
||||
)
|
||||
.order_by(Connection.position)
|
||||
)
|
||||
if model is None:
|
||||
raise LLMError(
|
||||
f"No enabled connection currently offers the model "
|
||||
f"'{chat.model_id}'. Pick another model for this chat."
|
||||
)
|
||||
connection = model.connection
|
||||
chat.connection_id = connection.id
|
||||
db.commit()
|
||||
|
||||
return Endpoint.from_connection(connection), chat.model_id
|
||||
|
||||
|
||||
def build_messages(db: DBSession, chat: Chat, *, upto: Message | None = None) -> list[dict]:
|
||||
"""Assemble the message list to send upstream.
|
||||
|
||||
`upto` excludes the placeholder assistant row being generated into, and
|
||||
everything after it.
|
||||
"""
|
||||
payload: list[dict[str, Any]] = []
|
||||
if chat.system_prompt.strip():
|
||||
payload.append({"role": ROLE_SYSTEM, "content": chat.system_prompt.strip()})
|
||||
|
||||
history = db.scalars(
|
||||
select(Message).where(Message.chat_id == chat.id).order_by(Message.created_at)
|
||||
).all()
|
||||
|
||||
for message in history:
|
||||
if upto is not None and message.id == upto.id:
|
||||
break
|
||||
# Skip turns that failed or produced nothing: sending an empty
|
||||
# assistant message upsets several providers.
|
||||
if message.error or not message.content.strip():
|
||||
continue
|
||||
payload.append({"role": message.role, "content": message.content})
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
def build_request(db: DBSession, chat: Chat, *, upto: Message | None = None) -> dict[str, Any]:
|
||||
params = {
|
||||
key: value
|
||||
for key, value in (chat.params_json or {}).items()
|
||||
if key in FORWARDED_PARAMS and value not in (None, "")
|
||||
}
|
||||
return {
|
||||
"model": chat.model_id,
|
||||
"messages": build_messages(db, chat, upto=upto),
|
||||
**params,
|
||||
}
|
||||
|
||||
|
||||
def default_model(db: DBSession) -> tuple[str, str] | None:
|
||||
"""First enabled model on the first enabled connection, or None."""
|
||||
model = db.scalar(
|
||||
select(Model)
|
||||
.join(Connection)
|
||||
.where(Model.enabled.is_(True), Connection.enabled.is_(True))
|
||||
.order_by(Connection.position, Model.model_id)
|
||||
)
|
||||
if model is None:
|
||||
return None
|
||||
return model.model_id, model.connection_id
|
||||
|
||||
|
||||
def available_models(db: DBSession) -> list[Model]:
|
||||
return list(
|
||||
db.scalars(
|
||||
select(Model)
|
||||
.join(Connection)
|
||||
.where(Model.enabled.is_(True), Connection.enabled.is_(True))
|
||||
.order_by(Connection.position, Model.model_id)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def fallback_title(text: str) -> str:
|
||||
"""Derive a chat title from the opening message, without calling a model."""
|
||||
cleaned = " ".join(text.split())
|
||||
if not cleaned:
|
||||
return "New chat"
|
||||
if len(cleaned) <= MAX_TITLE_LENGTH:
|
||||
return cleaned
|
||||
# Prefer a word boundary, but only if it does not cut the title in half.
|
||||
clipped = cleaned[:MAX_TITLE_LENGTH]
|
||||
space = clipped.rfind(" ")
|
||||
if space > MAX_TITLE_LENGTH * 0.6:
|
||||
clipped = clipped[:space]
|
||||
return clipped.rstrip(" ,.;:-") + "…"
|
||||
|
||||
|
||||
async def generate_title(endpoint: Endpoint, model_id: str, question: str, answer: str) -> str:
|
||||
"""Ask the model for a short chat title.
|
||||
|
||||
Best-effort by design: any failure falls back to trimming the first
|
||||
message. Naming a chat is never worth surfacing an error for.
|
||||
"""
|
||||
prompt = (
|
||||
"Summarise this exchange as a title of at most six words. "
|
||||
"Reply with the title alone: no quotes, no punctuation at the end, "
|
||||
"no preamble.\n\n"
|
||||
f"User: {question[:500]}\n\nAssistant: {answer[:500]}"
|
||||
)
|
||||
try:
|
||||
raw = await complete(
|
||||
endpoint,
|
||||
{
|
||||
"model": model_id,
|
||||
"messages": [{"role": ROLE_USER, "content": prompt}],
|
||||
"max_tokens": 24,
|
||||
"temperature": 0.2,
|
||||
},
|
||||
)
|
||||
except LLMError as exc:
|
||||
log.debug("auto-title failed, using fallback: %s", exc)
|
||||
return fallback_title(question)
|
||||
|
||||
title = " ".join(raw.split()).strip().strip('"“”\'')
|
||||
# Small models sometimes ignore the instruction and answer the question
|
||||
# instead; an over-long reply is a better signal of that than anything else.
|
||||
if not title or len(title) > MAX_TITLE_LENGTH * 1.5:
|
||||
return fallback_title(question)
|
||||
return title[:MAX_TITLE_LENGTH]
|
||||
|
||||
|
||||
def create_message(
|
||||
db: DBSession,
|
||||
chat: Chat,
|
||||
role: str,
|
||||
content: str = "",
|
||||
*,
|
||||
complete_: bool = True,
|
||||
model_id: str = "",
|
||||
) -> Message:
|
||||
message = Message(
|
||||
chat_id=chat.id,
|
||||
role=role,
|
||||
content=content,
|
||||
complete=complete_,
|
||||
model_id=model_id,
|
||||
)
|
||||
db.add(message)
|
||||
db.commit()
|
||||
return message
|
||||
|
||||
|
||||
def user_chats(db: DBSession, user_id: str, *, folder_id: str | None = None) -> list[Chat]:
|
||||
query = select(Chat).where(Chat.user_id == user_id, Chat.archived.is_(False))
|
||||
if folder_id is not None:
|
||||
query = query.where(Chat.folder_id == folder_id)
|
||||
return list(db.scalars(query.order_by(Chat.pinned.desc(), Chat.updated_at.desc())))
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ROLE_ASSISTANT",
|
||||
"ROLE_USER",
|
||||
"available_models",
|
||||
"build_request",
|
||||
"create_message",
|
||||
"default_model",
|
||||
"fallback_title",
|
||||
"generate_title",
|
||||
"resolve_endpoint",
|
||||
"user_chats",
|
||||
]
|
||||
@@ -0,0 +1,245 @@
|
||||
"""Client for OpenAI-compatible chat endpoints.
|
||||
|
||||
Deliberately plain httpx rather than the official SDK. The target is not just
|
||||
api.openai.com but LM Studio, vLLM, llama.cpp, Ollama's compatibility layer,
|
||||
OpenRouter and anything else exposing /v1 -- and they differ in small ways. A
|
||||
thin client passes request parameters through untouched and is tolerant about
|
||||
what comes back, which is exactly what talking to all of them requires.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from lembas.config import settings
|
||||
from lembas.db.models import Connection
|
||||
from lembas.services.crypto import decrypt
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LLMError(Exception):
|
||||
"""An upstream failure with a message fit to show a user.
|
||||
|
||||
Every failure path in this module raises this rather than letting an httpx
|
||||
or JSON exception escape, so callers have exactly one thing to catch and
|
||||
the chat UI always has something intelligible to display.
|
||||
"""
|
||||
|
||||
def __init__(self, message: str, *, status_code: int | None = None) -> None:
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Endpoint:
|
||||
"""Everything needed to call a connection, with the key already decrypted.
|
||||
|
||||
A frozen snapshot rather than the ORM object because streaming outlives the
|
||||
request that started it, and a detached SQLAlchemy instance is a trap.
|
||||
"""
|
||||
|
||||
base_url: str
|
||||
api_key: str
|
||||
extra_headers: dict[str, str]
|
||||
name: str = ""
|
||||
|
||||
@classmethod
|
||||
def from_connection(cls, connection: Connection) -> Endpoint:
|
||||
return cls(
|
||||
base_url=connection.base_url.rstrip("/"),
|
||||
api_key=decrypt(connection.api_key_encrypted),
|
||||
extra_headers=dict(connection.extra_headers_json or {}),
|
||||
name=connection.name,
|
||||
)
|
||||
|
||||
def url(self, path: str) -> str:
|
||||
# Accept both "http://host:1234" and "http://host:1234/v1" so users do
|
||||
# not have to guess which form this expects.
|
||||
base = self.base_url
|
||||
if not base.endswith("/v1") and "/v1/" not in base:
|
||||
base = f"{base}/v1"
|
||||
return f"{base}/{path.lstrip('/')}"
|
||||
|
||||
def headers(self) -> dict[str, str]:
|
||||
headers = {"Content-Type": "application/json", **self.extra_headers}
|
||||
# Local endpoints frequently need no key at all; sending an empty
|
||||
# bearer token makes some of them reject the request outright.
|
||||
if self.api_key:
|
||||
headers["Authorization"] = f"Bearer {self.api_key}"
|
||||
return headers
|
||||
|
||||
|
||||
def _describe_http_error(exc: httpx.HTTPStatusError) -> str:
|
||||
"""Turn an upstream error response into something worth reading.
|
||||
|
||||
Providers put the useful part in wildly different places, so try the common
|
||||
shapes before falling back to the raw body.
|
||||
"""
|
||||
status = exc.response.status_code
|
||||
detail = ""
|
||||
try:
|
||||
payload = exc.response.json()
|
||||
if isinstance(payload, dict):
|
||||
error = payload.get("error")
|
||||
if isinstance(error, dict):
|
||||
detail = error.get("message", "")
|
||||
elif isinstance(error, str):
|
||||
detail = error
|
||||
detail = detail or payload.get("message", "") or payload.get("detail", "")
|
||||
except (ValueError, json.JSONDecodeError):
|
||||
detail = exc.response.text[:300]
|
||||
|
||||
friendly = {
|
||||
401: "The API key was rejected.",
|
||||
403: "The API key is not permitted to use this model.",
|
||||
404: "The endpoint or model was not found.",
|
||||
429: "Rate limited by the provider.",
|
||||
}.get(status)
|
||||
|
||||
if friendly and detail:
|
||||
return f"{friendly} {detail}"
|
||||
return friendly or detail or f"The endpoint returned HTTP {status}."
|
||||
|
||||
|
||||
def _wrap_transport_error(exc: httpx.RequestError, endpoint: Endpoint) -> LLMError:
|
||||
if isinstance(exc, httpx.ConnectError):
|
||||
return LLMError(
|
||||
f"Could not reach {endpoint.base_url}. Is the endpoint running and "
|
||||
f"the URL correct?"
|
||||
)
|
||||
if isinstance(exc, httpx.TimeoutException):
|
||||
return LLMError(
|
||||
f"{endpoint.base_url} did not respond within "
|
||||
f"{settings.request_timeout:.0f}s."
|
||||
)
|
||||
return LLMError(f"Could not reach {endpoint.base_url}: {exc}")
|
||||
|
||||
|
||||
async def list_models(endpoint: Endpoint) -> list[dict[str, Any]]:
|
||||
"""Fetch the models a connection advertises via GET /v1/models."""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
response = await client.get(endpoint.url("models"), headers=endpoint.headers())
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
except httpx.HTTPStatusError as exc:
|
||||
raise LLMError(_describe_http_error(exc), status_code=exc.response.status_code) from exc
|
||||
except httpx.RequestError as exc:
|
||||
raise _wrap_transport_error(exc, endpoint) from exc
|
||||
except ValueError as exc:
|
||||
raise LLMError("The endpoint returned a response that was not JSON.") from exc
|
||||
|
||||
# The spec says {"data": [...]}, but some servers return a bare list.
|
||||
entries = payload.get("data", payload) if isinstance(payload, dict) else payload
|
||||
if not isinstance(entries, list):
|
||||
raise LLMError("The endpoint's model list was not in the expected format.")
|
||||
|
||||
models = []
|
||||
for entry in entries:
|
||||
if isinstance(entry, dict) and entry.get("id"):
|
||||
models.append(entry)
|
||||
elif isinstance(entry, str):
|
||||
models.append({"id": entry})
|
||||
return models
|
||||
|
||||
|
||||
async def stream_chat(
|
||||
endpoint: Endpoint,
|
||||
payload: dict[str, Any],
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Stream a chat completion, yielding each parsed SSE data object.
|
||||
|
||||
Yields the raw upstream chunks; interpreting them is the caller's job. The
|
||||
terminating "[DONE]" sentinel is consumed here and not yielded.
|
||||
"""
|
||||
body = {**payload, "stream": True}
|
||||
|
||||
try:
|
||||
async with (
|
||||
httpx.AsyncClient(timeout=settings.request_timeout) as client,
|
||||
client.stream(
|
||||
"POST",
|
||||
endpoint.url("chat/completions"),
|
||||
headers=endpoint.headers(),
|
||||
json=body,
|
||||
) as response,
|
||||
):
|
||||
if response.status_code >= 400:
|
||||
# The body has not been read yet on a streaming response, and
|
||||
# the error detail is in it.
|
||||
await response.aread()
|
||||
response.raise_for_status()
|
||||
|
||||
async for line in response.aiter_lines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith(":"):
|
||||
continue # keep-alive comment
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
data = line[5:].strip()
|
||||
if data == "[DONE]":
|
||||
return
|
||||
try:
|
||||
yield json.loads(data)
|
||||
except json.JSONDecodeError:
|
||||
# A malformed frame is not worth killing a reply over.
|
||||
log.warning("skipping unparseable SSE frame: %.120s", data)
|
||||
continue
|
||||
|
||||
except httpx.HTTPStatusError as exc:
|
||||
raise LLMError(_describe_http_error(exc), status_code=exc.response.status_code) from exc
|
||||
except httpx.RequestError as exc:
|
||||
raise _wrap_transport_error(exc, endpoint) from exc
|
||||
|
||||
|
||||
async def complete(endpoint: Endpoint, payload: dict[str, Any]) -> str:
|
||||
"""Non-streaming completion. Used for short internal calls like auto-titling."""
|
||||
body = {**payload, "stream": False}
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
response = await client.post(
|
||||
endpoint.url("chat/completions"), headers=endpoint.headers(), json=body
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
except httpx.HTTPStatusError as exc:
|
||||
raise LLMError(_describe_http_error(exc), status_code=exc.response.status_code) from exc
|
||||
except httpx.RequestError as exc:
|
||||
raise _wrap_transport_error(exc, endpoint) from exc
|
||||
except ValueError as exc:
|
||||
raise LLMError("The endpoint returned a response that was not JSON.") from exc
|
||||
|
||||
try:
|
||||
return data["choices"][0]["message"]["content"] or ""
|
||||
except (KeyError, IndexError, TypeError) as exc:
|
||||
raise LLMError("The endpoint returned no completion.") from exc
|
||||
|
||||
|
||||
def delta_text(chunk: dict[str, Any]) -> str:
|
||||
"""Pull the text out of one streamed chunk, tolerating provider variation."""
|
||||
try:
|
||||
choices = chunk.get("choices") or []
|
||||
if not choices:
|
||||
return ""
|
||||
delta = choices[0].get("delta") or {}
|
||||
content = delta.get("content")
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
# Some providers send content as a list of typed parts even in deltas.
|
||||
if isinstance(content, list):
|
||||
return "".join(
|
||||
part.get("text", "")
|
||||
for part in content
|
||||
if isinstance(part, dict) and part.get("type") == "text"
|
||||
)
|
||||
return ""
|
||||
except (AttributeError, TypeError):
|
||||
return ""
|
||||
@@ -0,0 +1,134 @@
|
||||
"""Render assistant messages from Markdown to sanitised HTML.
|
||||
|
||||
Rendering happens on the server, in Python, so there is no JavaScript Markdown
|
||||
library to vendor and the streamed and final views cannot disagree about how
|
||||
something should look.
|
||||
|
||||
The output is sanitised with nh3 (Rust ammonia). Model output is untrusted
|
||||
input: it routinely contains HTML, and a model can be talked into emitting a
|
||||
script tag, so this is a real boundary and not a formality.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import html
|
||||
|
||||
import nh3
|
||||
from markdown_it import MarkdownIt
|
||||
from pygments import highlight
|
||||
from pygments.formatters import HtmlFormatter
|
||||
from pygments.lexers import get_lexer_by_name, guess_lexer
|
||||
from pygments.util import ClassNotFound
|
||||
|
||||
# Class-based highlighting; the colours come from theme tokens in chat.css, so
|
||||
# code blocks follow the active theme instead of carrying their own palette.
|
||||
_FORMATTER = HtmlFormatter(nowrap=True, classprefix="pg-")
|
||||
|
||||
ALLOWED_TAGS = {
|
||||
"p", "br", "hr", "div", "span",
|
||||
"strong", "em", "del", "sub", "sup", "mark",
|
||||
"h1", "h2", "h3", "h4", "h5", "h6",
|
||||
"ul", "ol", "li",
|
||||
"blockquote", "pre", "code",
|
||||
"table", "thead", "tbody", "tr", "th", "td",
|
||||
"a", "img",
|
||||
}
|
||||
|
||||
ALLOWED_ATTRIBUTES = {
|
||||
# "rel" is intentionally absent: nh3 rejects it here when link_rel is set,
|
||||
# because link_rel below is what writes it.
|
||||
"a": {"href", "title", "target"},
|
||||
"img": {"src", "alt", "title"},
|
||||
"code": {"class"},
|
||||
"pre": {"class"},
|
||||
"span": {"class"},
|
||||
"div": {"class"},
|
||||
"td": {"align"},
|
||||
"th": {"align"},
|
||||
}
|
||||
|
||||
# javascript: and data: URLs are the obvious injection route through a link.
|
||||
ALLOWED_URL_SCHEMES = {"http", "https", "mailto"}
|
||||
|
||||
|
||||
def _render_fence(tokens, idx, _options, _env) -> str:
|
||||
"""Render a fenced code block.
|
||||
|
||||
This replaces the renderer's `fence` rule outright rather than using
|
||||
markdown-it's `highlight` option, because that option re-wraps whatever it
|
||||
is given in <pre><code> unless the string already starts with "<pre" --
|
||||
which would nest a second <pre> inside the wrapper this returns.
|
||||
"""
|
||||
token = tokens[idx]
|
||||
code = token.content
|
||||
language = (token.info or "").strip().split()[0] if token.info else ""
|
||||
|
||||
lexer = None
|
||||
if language:
|
||||
try:
|
||||
lexer = get_lexer_by_name(language, stripall=False)
|
||||
except (ClassNotFound, ValueError):
|
||||
lexer = None
|
||||
elif code.strip():
|
||||
# Guessing is only worth it for a decent sample; on two lines of text
|
||||
# Pygments guesses confidently and wrongly.
|
||||
try:
|
||||
lexer = guess_lexer(code) if len(code) > 80 else None
|
||||
except (ClassNotFound, ValueError):
|
||||
lexer = None
|
||||
|
||||
if lexer is None:
|
||||
body = nh3.clean_text(code)
|
||||
label = language
|
||||
else:
|
||||
body = highlight(code, lexer, _FORMATTER)
|
||||
label = language or (lexer.aliases[0] if lexer.aliases else "")
|
||||
|
||||
label_html = (
|
||||
f'<div class="code-block__label">{nh3.clean_text(label)}</div>' if label else ""
|
||||
)
|
||||
return (
|
||||
f'<div class="code-block">{label_html}'
|
||||
f'<pre class="code-block__pre"><code>{body}</code></pre></div>'
|
||||
)
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _parser() -> MarkdownIt:
|
||||
md = MarkdownIt("commonmark", {"linkify": True, "typographer": False})
|
||||
md.enable(["table", "strikethrough", "linkify"])
|
||||
md.renderer.rules["fence"] = _render_fence
|
||||
return md
|
||||
|
||||
|
||||
def render_markdown(text: str) -> str:
|
||||
"""Markdown to safe HTML, ready to drop into a message bubble."""
|
||||
if not text:
|
||||
return ""
|
||||
|
||||
html = _parser().render(text)
|
||||
return nh3.clean(
|
||||
html,
|
||||
tags=ALLOWED_TAGS,
|
||||
attributes=ALLOWED_ATTRIBUTES,
|
||||
url_schemes=ALLOWED_URL_SCHEMES,
|
||||
# Anything opened from a model's output is untrusted; noopener stops it
|
||||
# reaching back through window.opener.
|
||||
link_rel="nofollow noopener noreferrer",
|
||||
)
|
||||
|
||||
|
||||
def escape_text(text: str) -> str:
|
||||
"""Escape a plain-text run for insertion as HTML element content.
|
||||
|
||||
Used for user messages and for partial assistant text mid-stream, where the
|
||||
content is not yet complete enough to parse as Markdown.
|
||||
|
||||
html.escape rather than nh3.clean_text: escaping the three structural
|
||||
characters is all that is needed for a text node, and it escapes character
|
||||
by character, so escaping a stream chunk-by-chunk gives the same result as
|
||||
escaping the whole string at once. nh3.clean_text also escapes spaces and
|
||||
slashes, which triples the size of a streamed token for no benefit.
|
||||
"""
|
||||
return html.escape(text, quote=False)
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Server-sent event framing.
|
||||
|
||||
Small, but worth isolating: getting the wire format subtly wrong is the usual
|
||||
cause of a stream that "works" until a model emits a newline.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# Every 15s of silence, so proxies that kill idle connections (nginx defaults
|
||||
# to 60s) do not drop a stream while a model is still thinking.
|
||||
KEEPALIVE = ": keepalive\n\n"
|
||||
|
||||
|
||||
def event(name: str, data: str) -> str:
|
||||
"""Frame one SSE event.
|
||||
|
||||
A payload containing newlines must be split across several `data:` lines;
|
||||
the browser rejoins them with "\\n". Sending a raw newline inside a single
|
||||
data line silently truncates the event, which is exactly what happens the
|
||||
first time a model emits a code block.
|
||||
"""
|
||||
lines = data.split("\n")
|
||||
body = "".join(f"data: {line}\n" for line in lines)
|
||||
return f"event: {name}\n{body}\n"
|
||||
Reference in New Issue
Block a user