HomerandClaude Opus 5
b1dbca7db6
llama-server publishes the loaded model's Jinja chat template on /props, and
that template is the very thing that rejects a reasoning effort it does not
recognise -- so the accepted set is written down, authoritatively, in a place
this application can simply read. There is a button on the model's page that
does.
The parser handles both shapes a template uses: the values inline in the test
that rejects them (Bonsai), and a named list set elsewhere with nothing near
the mention spelling them out (gpt-oss). It is deliberately conservative,
because a wrong answer here silently removes a level somebody is entitled to:
only known efforts count, an unrelated list of quoted strings is ignored, and a
single match is read as a default -- `{%- set reasoning_effort = 'medium' %}`
-- rather than as a vocabulary of one.
An endpoint with no such route says so. OpenAI and vLLM do not publish a
template, and "this cannot tell us" must not be recorded as "this model accepts
nothing".
/props sits at the server root, beside the OpenAI-compatible surface rather
than inside it, so a base URL written as .../v1 needs the suffix stripped.
Getting that wrong is a silent 404 that looks like detection simply not
working, so there is a test on it.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
454 lines
17 KiB
Python
454 lines
17 KiB
Python
"""Client for OpenAI-compatible chat endpoints.
|
|
|
|
Deliberately plain httpx rather than the official SDK. The target is not just
|
|
api.openai.com but LM Studio, vLLM, llama.cpp, Ollama's compatibility layer,
|
|
OpenRouter and anything else exposing /v1 -- and they differ in small ways. A
|
|
thin client passes request parameters through untouched and is tolerant about
|
|
what comes back, which is exactly what talking to all of them requires.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
from collections.abc import AsyncIterator
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from lembas.config import settings
|
|
from lembas.db.models import Connection
|
|
from lembas.services.crypto import decrypt
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
class LLMError(Exception):
|
|
"""An upstream failure with a message fit to show a user.
|
|
|
|
Every failure path in this module raises this rather than letting an httpx
|
|
or JSON exception escape, so callers have exactly one thing to catch and
|
|
the chat UI always has something intelligible to display.
|
|
"""
|
|
|
|
def __init__(self, message: str, *, status_code: int | None = None) -> None:
|
|
super().__init__(message)
|
|
self.message = message
|
|
self.status_code = status_code
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Endpoint:
|
|
"""Everything needed to call a connection, with the key already decrypted.
|
|
|
|
A frozen snapshot rather than the ORM object because streaming outlives the
|
|
request that started it, and a detached SQLAlchemy instance is a trap.
|
|
"""
|
|
|
|
base_url: str
|
|
api_key: str
|
|
extra_headers: dict[str, str]
|
|
name: str = ""
|
|
|
|
@classmethod
|
|
def from_connection(cls, connection: Connection) -> Endpoint:
|
|
return cls(
|
|
base_url=connection.base_url.rstrip("/"),
|
|
api_key=decrypt(connection.api_key_encrypted),
|
|
extra_headers=dict(connection.extra_headers_json or {}),
|
|
name=connection.name,
|
|
)
|
|
|
|
def url(self, path: str) -> str:
|
|
# Accept both "http://host:1234" and "http://host:1234/v1" so users do
|
|
# not have to guess which form this expects.
|
|
base = self.base_url
|
|
if not base.endswith("/v1") and "/v1/" not in base:
|
|
base = f"{base}/v1"
|
|
return f"{base}/{path.lstrip('/')}"
|
|
|
|
def root_url(self, path: str) -> str:
|
|
"""A URL at the *server's* root rather than under `/v1`.
|
|
|
|
llama-server's own endpoints -- `/props` is the one that matters here --
|
|
sit beside the OpenAI-compatible surface, not inside it. A base URL may
|
|
be written either way (`http://host:8080` or `.../v1`), so the suffix is
|
|
stripped rather than assumed absent.
|
|
"""
|
|
base = self.base_url.rstrip("/")
|
|
if base.endswith("/v1"):
|
|
base = base[: -len("/v1")]
|
|
return f"{base}/{path.lstrip('/')}"
|
|
|
|
def headers(self) -> dict[str, str]:
|
|
headers = {"Content-Type": "application/json", **self.extra_headers}
|
|
# Local endpoints frequently need no key at all; sending an empty
|
|
# bearer token makes some of them reject the request outright.
|
|
if self.api_key:
|
|
headers["Authorization"] = f"Bearer {self.api_key}"
|
|
return headers
|
|
|
|
|
|
async def fetch_chat_template(endpoint: Endpoint) -> str:
|
|
"""The model's own Jinja chat template, from llama-server's `/props`.
|
|
|
|
The one place the truth about a model's accepted values is actually
|
|
written down: `/props` returns `chat_template` verbatim, and that template
|
|
is what raises when it meets a `reasoning_effort` it does not know.
|
|
|
|
Returns "" rather than raising for anything that is not a llama-server --
|
|
OpenAI, vLLM and the rest have no such route, and "this endpoint cannot
|
|
tell us" is a normal answer here, not a failure.
|
|
"""
|
|
try:
|
|
async with httpx.AsyncClient(timeout=10.0) as client:
|
|
response = await client.get(
|
|
endpoint.root_url("props"), headers=endpoint.headers()
|
|
)
|
|
response.raise_for_status()
|
|
payload = response.json()
|
|
except (httpx.HTTPError, ValueError, json.JSONDecodeError):
|
|
return ""
|
|
template = payload.get("chat_template") if isinstance(payload, dict) else ""
|
|
return template if isinstance(template, str) else ""
|
|
|
|
|
|
def describe_http_error(exc: httpx.HTTPStatusError) -> str:
|
|
"""Turn an upstream error response into something worth reading.
|
|
|
|
Public because the audio and search clients talk to the same class of
|
|
server and want the same translation; LLMError stays the one thing a
|
|
caller has to catch.
|
|
|
|
Providers put the useful part in wildly different places, so try the common
|
|
shapes before falling back to the raw body.
|
|
"""
|
|
status = exc.response.status_code
|
|
detail = ""
|
|
try:
|
|
payload = exc.response.json()
|
|
if isinstance(payload, dict):
|
|
error = payload.get("error")
|
|
if isinstance(error, dict):
|
|
detail = error.get("message", "")
|
|
elif isinstance(error, str):
|
|
detail = error
|
|
detail = detail or payload.get("message", "") or payload.get("detail", "")
|
|
except (ValueError, json.JSONDecodeError):
|
|
detail = exc.response.text[:300]
|
|
|
|
friendly = {
|
|
401: "The API key was rejected.",
|
|
403: "The API key is not permitted to use this model.",
|
|
404: "The endpoint or model was not found.",
|
|
429: "Rate limited by the provider.",
|
|
}.get(status)
|
|
|
|
if friendly and detail:
|
|
return f"{friendly} {detail}"
|
|
return friendly or detail or f"The endpoint returned HTTP {status}."
|
|
|
|
|
|
def wrap_transport_error(exc: httpx.RequestError, endpoint: Endpoint) -> LLMError:
|
|
if isinstance(exc, httpx.ConnectError):
|
|
return LLMError(
|
|
f"Could not reach {endpoint.base_url}. Is the endpoint running and "
|
|
f"the URL correct?"
|
|
)
|
|
if isinstance(exc, httpx.TimeoutException):
|
|
return LLMError(
|
|
f"{endpoint.base_url} did not respond within "
|
|
f"{settings.request_timeout:.0f}s."
|
|
)
|
|
return LLMError(f"Could not reach {endpoint.base_url}: {exc}")
|
|
|
|
|
|
async def list_models(endpoint: Endpoint) -> list[dict[str, Any]]:
|
|
"""Fetch the models a connection advertises via GET /v1/models."""
|
|
try:
|
|
async with httpx.AsyncClient(timeout=30.0) as client:
|
|
response = await client.get(endpoint.url("models"), headers=endpoint.headers())
|
|
response.raise_for_status()
|
|
payload = response.json()
|
|
except httpx.HTTPStatusError as exc:
|
|
raise LLMError(describe_http_error(exc), status_code=exc.response.status_code) from exc
|
|
except httpx.RequestError as exc:
|
|
raise wrap_transport_error(exc, endpoint) from exc
|
|
except ValueError as exc:
|
|
raise LLMError("The endpoint returned a response that was not JSON.") from exc
|
|
|
|
# The spec says {"data": [...]}, but some servers return a bare list.
|
|
entries = payload.get("data", payload) if isinstance(payload, dict) else payload
|
|
if not isinstance(entries, list):
|
|
raise LLMError("The endpoint's model list was not in the expected format.")
|
|
|
|
models = []
|
|
for entry in entries:
|
|
if isinstance(entry, dict) and entry.get("id"):
|
|
models.append(entry)
|
|
elif isinstance(entry, str):
|
|
models.append({"id": entry})
|
|
return models
|
|
|
|
|
|
# Where the runners that bother to advertise a context length put it. There is
|
|
# no standard field, so this is a list of what the common ones actually emit.
|
|
_CONTEXT_KEYS = ("context_length", "max_model_len", "context_window", "max_context_length")
|
|
|
|
# Below the first, the number is not a context length; above the second it is a
|
|
# typo or a different unit. Either way, better to record nothing than a wrong
|
|
# figure a percentage would then be computed from.
|
|
MIN_CONTEXT = 256
|
|
MAX_CONTEXT = 10_000_000
|
|
|
|
|
|
def context_from(entry: dict[str, Any]) -> int:
|
|
"""A model's context length as advertised by /v1/models, or 0 if it is not.
|
|
|
|
Strings are accepted because some servers quote the number, but only when
|
|
they are digits alone -- "8192 tokens" is a label, not a measurement.
|
|
"""
|
|
candidates = [entry.get(key) for key in _CONTEXT_KEYS]
|
|
meta = entry.get("meta")
|
|
if isinstance(meta, dict):
|
|
candidates += [meta.get("n_ctx"), *(meta.get(key) for key in _CONTEXT_KEYS)]
|
|
|
|
for value in candidates:
|
|
if isinstance(value, bool):
|
|
continue
|
|
if isinstance(value, str):
|
|
value = value.strip()
|
|
if not value.isdigit():
|
|
continue
|
|
value = int(value)
|
|
if isinstance(value, int) and MIN_CONTEXT <= value <= MAX_CONTEXT:
|
|
return value
|
|
return 0
|
|
|
|
|
|
# Endpoints that rejected `stream_options`, so it is asked for once per base URL
|
|
# per process and then never again. Not persisted: it is a property of whatever
|
|
# is running there now, and a restart is the right time to find out afresh.
|
|
_NO_STREAM_OPTIONS: set[str] = set()
|
|
|
|
|
|
async def stream_chat(
|
|
endpoint: Endpoint,
|
|
payload: dict[str, Any],
|
|
) -> AsyncIterator[dict[str, Any]]:
|
|
"""Stream a chat completion, yielding each parsed SSE data object.
|
|
|
|
Yields the raw upstream chunks; interpreting them is the caller's job. The
|
|
terminating "[DONE]" sentinel is consumed here and not yielded.
|
|
|
|
`stream_options` asks for the final usage chunk, which is the only way to
|
|
learn what a streamed reply actually cost. Not every server implements it,
|
|
and an unknown key is a 400 from some of them -- the same hazard as sending
|
|
a `tools` array to an endpoint without support. So it is asked for once,
|
|
and an endpoint that refuses is remembered and never asked again. Retrying
|
|
is safe because the status is checked before a single line is read: nothing
|
|
has been yielded, so there is nothing to duplicate.
|
|
"""
|
|
wants_usage = endpoint.base_url not in _NO_STREAM_OPTIONS
|
|
|
|
try:
|
|
async for chunk in _stream_once(endpoint, payload, usage=wants_usage):
|
|
yield chunk
|
|
except LLMError as exc:
|
|
if not wants_usage or exc.status_code not in (400, 422):
|
|
raise
|
|
_NO_STREAM_OPTIONS.add(endpoint.base_url)
|
|
log.info(
|
|
"%s rejected stream_options; token counts will be estimated there",
|
|
endpoint.base_url,
|
|
)
|
|
async for chunk in _stream_once(endpoint, payload, usage=False):
|
|
yield chunk
|
|
|
|
|
|
async def _stream_once(
|
|
endpoint: Endpoint,
|
|
payload: dict[str, Any],
|
|
*,
|
|
usage: bool,
|
|
) -> AsyncIterator[dict[str, Any]]:
|
|
body = {**payload, "stream": True}
|
|
if usage:
|
|
body["stream_options"] = {"include_usage": True}
|
|
|
|
try:
|
|
async with (
|
|
httpx.AsyncClient(timeout=settings.request_timeout) as client,
|
|
client.stream(
|
|
"POST",
|
|
endpoint.url("chat/completions"),
|
|
headers=endpoint.headers(),
|
|
json=body,
|
|
) as response,
|
|
):
|
|
if response.status_code >= 400:
|
|
# The body has not been read yet on a streaming response, and
|
|
# the error detail is in it.
|
|
await response.aread()
|
|
response.raise_for_status()
|
|
|
|
async for line in response.aiter_lines():
|
|
line = line.strip()
|
|
if not line or line.startswith(":"):
|
|
continue # keep-alive comment
|
|
if not line.startswith("data:"):
|
|
continue
|
|
data = line[5:].strip()
|
|
if data == "[DONE]":
|
|
return
|
|
try:
|
|
yield json.loads(data)
|
|
except json.JSONDecodeError:
|
|
# A malformed frame is not worth killing a reply over.
|
|
log.warning("skipping unparseable SSE frame: %.120s", data)
|
|
continue
|
|
|
|
except httpx.HTTPStatusError as exc:
|
|
raise LLMError(describe_http_error(exc), status_code=exc.response.status_code) from exc
|
|
except httpx.RequestError as exc:
|
|
raise wrap_transport_error(exc, endpoint) from exc
|
|
|
|
|
|
async def complete(endpoint: Endpoint, payload: dict[str, Any]) -> str:
|
|
"""Non-streaming completion. Used for short internal calls like auto-titling."""
|
|
body = {**payload, "stream": False}
|
|
try:
|
|
async with httpx.AsyncClient(timeout=60.0) as client:
|
|
response = await client.post(
|
|
endpoint.url("chat/completions"), headers=endpoint.headers(), json=body
|
|
)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
except httpx.HTTPStatusError as exc:
|
|
raise LLMError(describe_http_error(exc), status_code=exc.response.status_code) from exc
|
|
except httpx.RequestError as exc:
|
|
raise wrap_transport_error(exc, endpoint) from exc
|
|
except ValueError as exc:
|
|
raise LLMError("The endpoint returned a response that was not JSON.") from exc
|
|
|
|
try:
|
|
return data["choices"][0]["message"]["content"] or ""
|
|
except (KeyError, IndexError, TypeError) as exc:
|
|
raise LLMError("The endpoint returned no completion.") from exc
|
|
|
|
|
|
def delta_reasoning(chunk: dict[str, Any]) -> str:
|
|
"""Pull a reasoning delta out of one streamed chunk.
|
|
|
|
Providers disagree on the field name -- llama.cpp, llama-swap and vLLM use
|
|
``reasoning_content``, some others just ``reasoning`` -- so both are read.
|
|
Models that emit ``<think>`` tags inline in ``content`` instead are handled
|
|
by lembas.services.reasoning.
|
|
"""
|
|
try:
|
|
choices = chunk.get("choices") or []
|
|
if not choices:
|
|
return ""
|
|
delta = choices[0].get("delta") or {}
|
|
for field in ("reasoning_content", "reasoning"):
|
|
value = delta.get(field)
|
|
if isinstance(value, str) and value:
|
|
return value
|
|
return ""
|
|
except (AttributeError, TypeError):
|
|
return ""
|
|
|
|
|
|
def delta_tool_calls(chunk: dict[str, Any]) -> list[dict[str, Any]]:
|
|
"""Pull tool-call fragments out of one streamed chunk.
|
|
|
|
Each entry carries an ``index`` and, across chunks, a name that arrives
|
|
once and an ``arguments`` string that arrives in pieces. Reassembling them
|
|
is lembas.services.tools.ToolCallAccumulator's job; this only extracts.
|
|
"""
|
|
try:
|
|
choices = chunk.get("choices") or []
|
|
if not choices:
|
|
return []
|
|
calls = (choices[0].get("delta") or {}).get("tool_calls")
|
|
return calls if isinstance(calls, list) else []
|
|
except (AttributeError, TypeError):
|
|
return []
|
|
|
|
|
|
def finish_reason(chunk: dict[str, Any]) -> str:
|
|
"""Why the model stopped, when the chunk says so.
|
|
|
|
``tool_calls`` here is the signal that the reply is not an answer but a
|
|
request to run something and come back. Some servers send ``stop`` even
|
|
when they emitted tool calls, so the accumulator's contents are the real
|
|
authority and this is only a hint.
|
|
"""
|
|
try:
|
|
choices = chunk.get("choices") or []
|
|
if not choices:
|
|
return ""
|
|
return choices[0].get("finish_reason") or ""
|
|
except (AttributeError, TypeError):
|
|
return ""
|
|
|
|
|
|
def chunk_usage(chunk: dict[str, Any]) -> dict[str, int] | None:
|
|
"""Token counts from a usage chunk, or None if this is not one.
|
|
|
|
A usage chunk carries `choices: []`, which is exactly the shape delta_text,
|
|
delta_reasoning, delta_tool_calls and finish_reason all return early on --
|
|
they have always tolerated it, so nothing else needs to change to let one
|
|
through.
|
|
|
|
Fields are read defensively because "the endpoint returned something odd"
|
|
must never be the reason a reply fails; a bad shape simply means no counts.
|
|
"""
|
|
try:
|
|
raw = chunk.get("usage")
|
|
except AttributeError:
|
|
return None
|
|
if not isinstance(raw, dict):
|
|
return None
|
|
|
|
counts: dict[str, int] = {}
|
|
for key in ("prompt_tokens", "completion_tokens", "total_tokens"):
|
|
value = raw.get(key)
|
|
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
continue
|
|
if value >= 0:
|
|
counts[key] = int(value)
|
|
|
|
# Some servers send a usage object of zeros on every chunk and the real
|
|
# numbers only at the end. All-zero is indistinguishable from that, and
|
|
# treating it as an answer would freeze the count at nothing.
|
|
if not counts or not any(counts.values()):
|
|
return None
|
|
counts.setdefault(
|
|
"total_tokens", counts.get("prompt_tokens", 0) + counts.get("completion_tokens", 0)
|
|
)
|
|
return counts
|
|
|
|
|
|
def delta_text(chunk: dict[str, Any]) -> str:
|
|
"""Pull the text out of one streamed chunk, tolerating provider variation."""
|
|
try:
|
|
choices = chunk.get("choices") or []
|
|
if not choices:
|
|
return ""
|
|
delta = choices[0].get("delta") or {}
|
|
content = delta.get("content")
|
|
if isinstance(content, str):
|
|
return content
|
|
# Some providers send content as a list of typed parts even in deltas.
|
|
if isinstance(content, list):
|
|
return "".join(
|
|
part.get("text", "")
|
|
for part in content
|
|
if isinstance(part, dict) and part.get("type") == "text"
|
|
)
|
|
return ""
|
|
except (AttributeError, TypeError):
|
|
return ""
|