Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0ed7dd9fc8
|
||
|
|
b1dbca7db6
|
@@ -16,6 +16,34 @@ for 1.0.0 have something to be assembled from.
|
|||||||
|
|
||||||
## Unreleased
|
## Unreleased
|
||||||
|
|
||||||
|
## 1.3.1
|
||||||
|
|
||||||
|
- Fixed: **updating to 1.2.0 or later broke every page that lists models**, with
|
||||||
|
a 500 and nothing but the error page to show for it. The per-model reasoning
|
||||||
|
effort list added in 1.2.0 was the first list-shaped setting this application
|
||||||
|
had ever added to a table that already had rows in it, and the code that fills
|
||||||
|
in such a column on existing rows could not tell a list from a dictionary — so
|
||||||
|
it wrote the wrong kind of empty value into every model, and reading one back
|
||||||
|
raised rather than returning nothing.
|
||||||
|
|
||||||
|
A fresh install was never affected, which is exactly why it was not caught:
|
||||||
|
the column is only filled in that way on a database that already existed.
|
||||||
|
|
||||||
|
This release both stops it happening and **puts right the rows already
|
||||||
|
written**, on start, with nothing to run by hand. If your instance is showing
|
||||||
|
the error page, updating is the whole fix.
|
||||||
|
|
||||||
|
## 1.3.0
|
||||||
|
|
||||||
|
- **A model's reasoning efforts can now be detected rather than known.** There
|
||||||
|
is a button on the model's page that asks the endpoint what its chat template
|
||||||
|
actually accepts, and ticks those. llama.cpp publishes the loaded model's
|
||||||
|
template, and that template is the very thing that rejects an effort it does
|
||||||
|
not recognise — so the answer is read from the place that is authoritative
|
||||||
|
instead of guessed at, or discovered by a failed reply.
|
||||||
|
- Endpoints that do not publish a template — OpenAI, vLLM — say so plainly
|
||||||
|
rather than being recorded as accepting nothing.
|
||||||
|
|
||||||
## 1.2.0
|
## 1.2.0
|
||||||
|
|
||||||
- Fixed: **choosing a reasoning effort could kill the reply outright**, with a
|
- Fixed: **choosing a reasoning effort could kill the reply outright**, with a
|
||||||
|
|||||||
@@ -1,3 +1,3 @@
|
|||||||
"""LLeMbas - a Middle-earth themed web UI for OpenAI-compatible LLM endpoints."""
|
"""LLeMbas - a Middle-earth themed web UI for OpenAI-compatible LLM endpoints."""
|
||||||
|
|
||||||
__version__ = "1.2.0"
|
__version__ = "1.3.1"
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import contextlib
|
import contextlib
|
||||||
import logging
|
import logging
|
||||||
|
from urllib.parse import quote
|
||||||
|
|
||||||
from fastapi import APIRouter, File, Form, HTTPException, Request, Response, UploadFile, status
|
from fastapi import APIRouter, File, Form, HTTPException, Request, Response, UploadFile, status
|
||||||
from fastapi.responses import FileResponse, RedirectResponse
|
from fastapi.responses import FileResponse, RedirectResponse
|
||||||
@@ -162,7 +163,13 @@ async def models_page(
|
|||||||
|
|
||||||
@router.get("/admin/models/{model_id}/edit")
|
@router.get("/admin/models/{model_id}/edit")
|
||||||
async def model_detail(
|
async def model_detail(
|
||||||
request: Request, db: Db, user: AdminUser, model_id: str, saved: str = ""
|
request: Request,
|
||||||
|
db: Db,
|
||||||
|
user: AdminUser,
|
||||||
|
model_id: str,
|
||||||
|
saved: str = "",
|
||||||
|
detected: str = "",
|
||||||
|
message: str = "",
|
||||||
):
|
):
|
||||||
"""Everything about one model, on its own page."""
|
"""Everything about one model, on its own page."""
|
||||||
model = _model(db, model_id)
|
model = _model(db, model_id)
|
||||||
@@ -182,6 +189,11 @@ async def model_detail(
|
|||||||
# current answer, which is the common three until somebody says.
|
# current answer, which is the common three until somebody says.
|
||||||
"efforts": chat_service.EFFORTS,
|
"efforts": chat_service.EFFORTS,
|
||||||
"model_efforts": chat_service.efforts_for(model),
|
"model_efforts": chat_service.efforts_for(model),
|
||||||
|
# What `detect-efforts` found, if it has just run. Escaped by the
|
||||||
|
# template like every other value; it is prose the endpoint or this
|
||||||
|
# application wrote, not markup.
|
||||||
|
"detected": detected if detected in ("success", "warning") else "",
|
||||||
|
"detected_message": message[:400],
|
||||||
# Rows predating the split have no tool_* keys at all. Showing them
|
# Rows predating the split have no tool_* keys at all. Showing them
|
||||||
# unticked would be a lie: tools.enabled_tools treats absent as on
|
# unticked would be a lie: tools.enabled_tools treats absent as on
|
||||||
# when `tools` is on, so that an upgrade does not silently take web
|
# when `tools` is on, so that an upgrade does not silently take web
|
||||||
@@ -342,6 +354,55 @@ async def move_model(
|
|||||||
return RedirectResponse(back or "/admin/models", status_code=303)
|
return RedirectResponse(back or "/admin/models", status_code=303)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/admin/models/{model_id}/detect-efforts")
|
||||||
|
async def detect_efforts(db: Db, user: AdminUser, model_id: str) -> Response:
|
||||||
|
"""Ask the endpoint which reasoning efforts this model actually takes.
|
||||||
|
|
||||||
|
llama-server hands its loaded model's Jinja chat template over on `/props`,
|
||||||
|
and that template is the thing that rejects an effort it does not know -- so
|
||||||
|
the accepted set is written down in the one place that is authoritative,
|
||||||
|
rather than having to be guessed at or discovered by a failed reply.
|
||||||
|
|
||||||
|
Anything that is not a llama-server answers nothing here, and that is a
|
||||||
|
normal outcome: OpenAI and vLLM have no such route, and their models are
|
||||||
|
documented rather than introspectable. The result then says so instead of
|
||||||
|
claiming the model accepts nothing.
|
||||||
|
"""
|
||||||
|
from lembas.services.llm.openai_client import Endpoint, fetch_chat_template
|
||||||
|
|
||||||
|
model = _model(db, model_id)
|
||||||
|
connection = db.get(Connection, model.connection_id)
|
||||||
|
if connection is None:
|
||||||
|
raise HTTPException(status.HTTP_404_NOT_FOUND, "That connection no longer exists.")
|
||||||
|
|
||||||
|
template = await fetch_chat_template(Endpoint.from_connection(connection))
|
||||||
|
found = chat_service.efforts_from_chat_template(template)
|
||||||
|
|
||||||
|
if found:
|
||||||
|
model.reasoning_efforts = found
|
||||||
|
db.commit()
|
||||||
|
message = "This model's template accepts: " + ", ".join(found) + "."
|
||||||
|
kind = "success"
|
||||||
|
elif template:
|
||||||
|
message = (
|
||||||
|
"The endpoint gave up its chat template, but nothing in it names a "
|
||||||
|
"set of reasoning efforts. Either this model does not take one, or "
|
||||||
|
"it accepts anything and never checks."
|
||||||
|
)
|
||||||
|
kind = "warning"
|
||||||
|
else:
|
||||||
|
message = (
|
||||||
|
"This endpoint does not publish its chat template, so there is "
|
||||||
|
"nothing to read. llama.cpp does; OpenAI and vLLM do not."
|
||||||
|
)
|
||||||
|
kind = "warning"
|
||||||
|
|
||||||
|
return RedirectResponse(
|
||||||
|
f"/admin/models/{model.id}/edit?detected={kind}&message={quote(message)}",
|
||||||
|
status_code=status.HTTP_303_SEE_OTHER,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/admin/models/{model_id}/default")
|
@router.post("/admin/models/{model_id}/default")
|
||||||
async def set_default_model(
|
async def set_default_model(
|
||||||
db: Db, user: AdminUser, model_id: str, back: str = Form("")
|
db: Db, user: AdminUser, model_id: str, back: str = Form("")
|
||||||
|
|||||||
@@ -39,6 +39,29 @@ log = logging.getLogger(__name__)
|
|||||||
MANUAL_STEPS: list[str] = []
|
MANUAL_STEPS: list[str] = []
|
||||||
|
|
||||||
|
|
||||||
|
def _default_shape(column: Column) -> type | None:
|
||||||
|
"""`list` or `dict`, from the column's own Python-side default.
|
||||||
|
|
||||||
|
`default=list` and `default=dict` are how the two JSON flavours are
|
||||||
|
declared, and SQLAlchemy keeps the callable. Calling it is cheap and is the
|
||||||
|
only way to tell a MutableList column from a MutableDict one -- see the note
|
||||||
|
in `_literal_default`.
|
||||||
|
"""
|
||||||
|
default = column.default
|
||||||
|
if default is None or not getattr(default, "is_callable", False):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
# SQLAlchemy wraps a zero-argument callable to take a context.
|
||||||
|
produced = default.arg(None)
|
||||||
|
except Exception: # noqa: BLE001 - a default we cannot call tells us nothing
|
||||||
|
return None
|
||||||
|
if isinstance(produced, list):
|
||||||
|
return list
|
||||||
|
if isinstance(produced, dict):
|
||||||
|
return dict
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _literal_default(column: Column) -> str | None:
|
def _literal_default(column: Column) -> str | None:
|
||||||
"""A SQL literal to backfill an existing row's new column with.
|
"""A SQL literal to backfill an existing row's new column with.
|
||||||
|
|
||||||
@@ -63,8 +86,22 @@ def _literal_default(column: Column) -> str | None:
|
|||||||
if "JSON" in affinity:
|
if "JSON" in affinity:
|
||||||
# MutableList columns must start as [] and MutableDict as {}; guessing
|
# MutableList columns must start as [] and MutableDict as {}; guessing
|
||||||
# wrong makes the first read blow up rather than return empty.
|
# wrong makes the first read blow up rather than return empty.
|
||||||
python_type = getattr(column.type, "python_type", None)
|
#
|
||||||
return "'[]'" if python_type is list else "'{}'"
|
# 🚨 NOT `column.type.python_type`. `MutableList.as_mutable(JSON)`
|
||||||
|
# returns the *same* JSON type object with an event listener attached --
|
||||||
|
# it does not subclass or wrap it -- so the type cannot tell you which
|
||||||
|
# of the two it is, and `JSON.python_type` is `dict` for both. That read
|
||||||
|
# as "this is a dict column" for every list column, and the first one
|
||||||
|
# ever added by a migration (`Model.reasoning_efforts`, 1.2.0) arrived
|
||||||
|
# as `'{}'` on every existing row. `MutableList` refuses a dict, so the
|
||||||
|
# failure was not an empty list but a ValueError on *load* -- every page
|
||||||
|
# that lists models, 500, on an instance that had simply been updated.
|
||||||
|
#
|
||||||
|
# The Python-side default is the only honest signal: a JSONList column
|
||||||
|
# is declared `default=list` and a JSONDict one `default=dict`, and
|
||||||
|
# calling it says which. Anything that cannot be called or produces
|
||||||
|
# neither falls back to `{}`, which is what this always assumed.
|
||||||
|
return "'[]'" if _default_shape(column) is list else "'{}'"
|
||||||
if "BOOL" in affinity:
|
if "BOOL" in affinity:
|
||||||
return "0"
|
return "0"
|
||||||
if any(token in affinity for token in ("INT", "FLOAT", "NUMERIC", "DECIMAL")):
|
if any(token in affinity for token in ("INT", "FLOAT", "NUMERIC", "DECIMAL")):
|
||||||
@@ -190,6 +227,50 @@ def ensure_fts(engine: Engine) -> list[str]:
|
|||||||
return created
|
return created
|
||||||
|
|
||||||
|
|
||||||
|
def repair_json_shapes(engine: Engine) -> list[str]:
|
||||||
|
"""Put right any JSON column backfilled with the wrong empty value.
|
||||||
|
|
||||||
|
`_literal_default` used to read the shape off `column.type.python_type`,
|
||||||
|
which is `dict` for a MutableList column as well as a MutableDict one -- so
|
||||||
|
the first list-shaped JSON column ever added by a migration arrived as
|
||||||
|
`'{}'` on every row that already existed. `MutableList` refuses a dict, and
|
||||||
|
refuses it while *loading*, so the symptom was not an empty list but a
|
||||||
|
`ValueError` and a 500 on every page that touched the table.
|
||||||
|
|
||||||
|
Converges, like `ensure_fts` beside it: it runs on every start, it is
|
||||||
|
idempotent, and on a database that was never damaged it does nothing. Only
|
||||||
|
the exact wrong value is rewritten -- `'{}'` in a column whose default
|
||||||
|
produces a list -- because `{}` cannot be a legitimate value there, while
|
||||||
|
anything else in that column might be somebody's data.
|
||||||
|
"""
|
||||||
|
fixed: list[str] = []
|
||||||
|
inspector = inspect(engine)
|
||||||
|
known = set(inspector.get_table_names())
|
||||||
|
|
||||||
|
with engine.begin() as connection:
|
||||||
|
for table in Base.metadata.sorted_tables:
|
||||||
|
if table.name not in known:
|
||||||
|
continue
|
||||||
|
for column in table.columns:
|
||||||
|
if "JSON" not in column.type.__class__.__name__.upper():
|
||||||
|
continue
|
||||||
|
if _default_shape(column) is not list:
|
||||||
|
continue
|
||||||
|
result = connection.execute(
|
||||||
|
text(
|
||||||
|
f'UPDATE "{table.name}" SET "{column.name}" = \'[]\' '
|
||||||
|
f'WHERE "{column.name}" = \'{{}}\''
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if result.rowcount:
|
||||||
|
fixed.append(f"{table.name}.{column.name} ({result.rowcount} row(s))")
|
||||||
|
log.warning(
|
||||||
|
"repaired %s.%s on %d row(s): was '{}' in a list column",
|
||||||
|
table.name, column.name, result.rowcount,
|
||||||
|
)
|
||||||
|
return fixed
|
||||||
|
|
||||||
|
|
||||||
def sync_schema(engine: Engine) -> list[str]:
|
def sync_schema(engine: Engine) -> list[str]:
|
||||||
"""Bring the database up to the declared schema. Returns what it changed."""
|
"""Bring the database up to the declared schema. Returns what it changed."""
|
||||||
import lembas.db.models # noqa: F401 (registers every table on the metadata)
|
import lembas.db.models # noqa: F401 (registers every table on the metadata)
|
||||||
@@ -219,6 +300,14 @@ def sync_schema(engine: Engine) -> list[str]:
|
|||||||
changes.append(f"add column {table.name}.{column.name}")
|
changes.append(f"add column {table.name}.{column.name}")
|
||||||
log.info("schema: %s", statement)
|
log.info("schema: %s", statement)
|
||||||
|
|
||||||
|
# Before the search indexes, and before anything can try to load a row:
|
||||||
|
# a column left holding the wrong empty value makes the ORM raise on read.
|
||||||
|
try:
|
||||||
|
for repair in repair_json_shapes(engine):
|
||||||
|
changes.append(f"repair {repair}")
|
||||||
|
except Exception: # noqa: BLE001 - a repair that fails must not stop a start
|
||||||
|
log.exception("could not repair JSON column shapes")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
for index in ensure_fts(engine):
|
for index in ensure_fts(engine):
|
||||||
changes.append(f"create search index {index}")
|
changes.append(f"create search index {index}")
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import re
|
||||||
from datetime import UTC, datetime, timedelta
|
from datetime import UTC, datetime, timedelta
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
@@ -470,6 +471,66 @@ def resolved_effort(chat) -> str:
|
|||||||
return value if value in EFFORTS else ""
|
return value if value in EFFORTS else ""
|
||||||
|
|
||||||
|
|
||||||
|
def efforts_from_chat_template(template: str) -> list[str]:
|
||||||
|
"""Which efforts a model's Jinja chat template will actually accept.
|
||||||
|
|
||||||
|
The template is where the truth lives: the one on a Bonsai reads roughly
|
||||||
|
|
||||||
|
{%- if reasoning_effort not in ('xhigh', 'medium', 'low') %}
|
||||||
|
{{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ...
|
||||||
|
|
||||||
|
so the accepted set is written out beside the thing that rejects everything
|
||||||
|
else. `llama-server` hands the whole template over on `/props`, which makes
|
||||||
|
this readable rather than guessable.
|
||||||
|
|
||||||
|
Deliberately conservative, because a wrong answer here silently removes a
|
||||||
|
level somebody is entitled to:
|
||||||
|
|
||||||
|
- only quoted literals within a short window of a `reasoning_effort`
|
||||||
|
mention are considered, so an unrelated list elsewhere in a four-hundred
|
||||||
|
line template cannot contribute;
|
||||||
|
- the result is intersected with `EFFORTS`, so an unknown token is dropped
|
||||||
|
rather than stored;
|
||||||
|
- fewer than two survivors is treated as "the template did not say". One
|
||||||
|
match is far more likely to be a default assignment
|
||||||
|
(`{%- set reasoning_effort = 'medium' %}`) than a vocabulary.
|
||||||
|
|
||||||
|
Returns [] when nothing can be read, which every caller treats as "ask
|
||||||
|
somebody" rather than as "this model accepts nothing".
|
||||||
|
"""
|
||||||
|
if not template or "reasoning_effort" not in template:
|
||||||
|
return []
|
||||||
|
|
||||||
|
found: set[str] = set()
|
||||||
|
|
||||||
|
# Shape one: the values sit in the statement that tests them.
|
||||||
|
# {%- if reasoning_effort not in ('xhigh', 'medium', 'low') %}
|
||||||
|
for match in re.finditer(r"reasoning_effort", template):
|
||||||
|
window = template[match.start() : match.start() + 400]
|
||||||
|
# Stop at the end of the statement that mentions it, so a later,
|
||||||
|
# unrelated block cannot leak in.
|
||||||
|
window = window.split("%}")[0] if "%}" in window else window
|
||||||
|
for literal in re.findall(r"""['"]([a-z]{3,8})['"]""", window):
|
||||||
|
if literal in EFFORTS:
|
||||||
|
found.add(literal)
|
||||||
|
|
||||||
|
# Shape two: the values are a named list somewhere else, and the test says
|
||||||
|
# {%- if reasoning_effort not in valid_efforts %}
|
||||||
|
# so nothing near the mention names them. Any group of quoted literals in
|
||||||
|
# which *every* token is a known effort and there are at least two is taken
|
||||||
|
# -- that is a strong enough signal on its own, and a list of nothing but
|
||||||
|
# effort names that is not the effort vocabulary would be a strange thing
|
||||||
|
# for a chat template to contain.
|
||||||
|
for group in re.findall(r"[\[(]((?:\s*['\"][a-z]{3,8}['\"]\s*,?)+)[\])]", template):
|
||||||
|
literals = re.findall(r"""['"]([a-z]{3,8})['"]""", group)
|
||||||
|
if len(literals) >= 2 and all(value in EFFORTS for value in literals):
|
||||||
|
found.update(literals)
|
||||||
|
|
||||||
|
if len(found) < 2:
|
||||||
|
return []
|
||||||
|
return [effort for effort in EFFORTS if effort in found]
|
||||||
|
|
||||||
|
|
||||||
def apply_effort(
|
def apply_effort(
|
||||||
body: dict[str, Any], effort: str | None, supported: tuple[str, ...] | None = None
|
body: dict[str, Any], effort: str | None, supported: tuple[str, ...] | None = None
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|||||||
@@ -68,6 +68,19 @@ class Endpoint:
|
|||||||
base = f"{base}/v1"
|
base = f"{base}/v1"
|
||||||
return f"{base}/{path.lstrip('/')}"
|
return f"{base}/{path.lstrip('/')}"
|
||||||
|
|
||||||
|
def root_url(self, path: str) -> str:
|
||||||
|
"""A URL at the *server's* root rather than under `/v1`.
|
||||||
|
|
||||||
|
llama-server's own endpoints -- `/props` is the one that matters here --
|
||||||
|
sit beside the OpenAI-compatible surface, not inside it. A base URL may
|
||||||
|
be written either way (`http://host:8080` or `.../v1`), so the suffix is
|
||||||
|
stripped rather than assumed absent.
|
||||||
|
"""
|
||||||
|
base = self.base_url.rstrip("/")
|
||||||
|
if base.endswith("/v1"):
|
||||||
|
base = base[: -len("/v1")]
|
||||||
|
return f"{base}/{path.lstrip('/')}"
|
||||||
|
|
||||||
def headers(self) -> dict[str, str]:
|
def headers(self) -> dict[str, str]:
|
||||||
headers = {"Content-Type": "application/json", **self.extra_headers}
|
headers = {"Content-Type": "application/json", **self.extra_headers}
|
||||||
# Local endpoints frequently need no key at all; sending an empty
|
# Local endpoints frequently need no key at all; sending an empty
|
||||||
@@ -77,6 +90,30 @@ class Endpoint:
|
|||||||
return headers
|
return headers
|
||||||
|
|
||||||
|
|
||||||
|
async def fetch_chat_template(endpoint: Endpoint) -> str:
|
||||||
|
"""The model's own Jinja chat template, from llama-server's `/props`.
|
||||||
|
|
||||||
|
The one place the truth about a model's accepted values is actually
|
||||||
|
written down: `/props` returns `chat_template` verbatim, and that template
|
||||||
|
is what raises when it meets a `reasoning_effort` it does not know.
|
||||||
|
|
||||||
|
Returns "" rather than raising for anything that is not a llama-server --
|
||||||
|
OpenAI, vLLM and the rest have no such route, and "this endpoint cannot
|
||||||
|
tell us" is a normal answer here, not a failure.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||||
|
response = await client.get(
|
||||||
|
endpoint.root_url("props"), headers=endpoint.headers()
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
payload = response.json()
|
||||||
|
except (httpx.HTTPError, ValueError, json.JSONDecodeError):
|
||||||
|
return ""
|
||||||
|
template = payload.get("chat_template") if isinstance(payload, dict) else ""
|
||||||
|
return template if isinstance(template, str) else ""
|
||||||
|
|
||||||
|
|
||||||
def describe_http_error(exc: httpx.HTTPStatusError) -> str:
|
def describe_http_error(exc: httpx.HTTPStatusError) -> str:
|
||||||
"""Turn an upstream error response into something worth reading.
|
"""Turn an upstream error response into something worth reading.
|
||||||
|
|
||||||
|
|||||||
@@ -115,6 +115,31 @@
|
|||||||
</label>
|
</label>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
</div>
|
</div>
|
||||||
|
{% if detected %}
|
||||||
|
<div class="alert alert--{{ 'success' if detected == 'success' else 'warning' }}"
|
||||||
|
role="status">
|
||||||
|
{{ icon('sparkle' if detected == 'success' else 'warning', 'alert__icon') }}
|
||||||
|
<span>{{ detected_message }}</span>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{#
|
||||||
|
Reading the answer rather than asking somebody to know it. llama-server
|
||||||
|
publishes the loaded model's Jinja chat template on `/props`, and that
|
||||||
|
template is the thing that rejects an effort it does not recognise --
|
||||||
|
so the accepted set is written down in the one authoritative place.
|
||||||
|
Endpoints without that route (OpenAI, vLLM) say so rather than
|
||||||
|
pretending the model accepts nothing.
|
||||||
|
|
||||||
|
Its own form, because this page's main form is a PUT of everything and
|
||||||
|
a detect must not carry half-edited fields with it.
|
||||||
|
#}
|
||||||
|
<form method="post" action="/admin/models/{{ model.id }}/detect-efforts">
|
||||||
|
<button class="btn btn--sm" type="submit">
|
||||||
|
{{ icon('search', 'icon--sm') }} Detect from the endpoint
|
||||||
|
</button>
|
||||||
|
</form>
|
||||||
|
|
||||||
<p class="field__hint">
|
<p class="field__hint">
|
||||||
The vocabulary is <strong>not the same for every model</strong>, and
|
The vocabulary is <strong>not the same for every model</strong>, and
|
||||||
sending one a model does not know is not ignored — it is rendered into
|
sending one a model does not know is not ignored — it is rendered into
|
||||||
|
|||||||
@@ -456,3 +456,129 @@ def test_a_model_with_no_advertisement_simply_loses_the_refused_value():
|
|||||||
from lembas.services import generation
|
from lembas.services import generation
|
||||||
|
|
||||||
assert generation._advertised_efforts("Unexpected reasoning effort high.") == []
|
assert generation._advertised_efforts("Unexpected reasoning effort high.") == []
|
||||||
|
|
||||||
|
|
||||||
|
# --- Reading the answer instead of asking somebody to know it ----------------
|
||||||
|
#
|
||||||
|
# llama-server publishes the loaded model's Jinja chat template on /props, and
|
||||||
|
# that template is the thing that rejects an effort it does not know -- so the
|
||||||
|
# accepted set is written down in the one authoritative place.
|
||||||
|
BONSAI_TEMPLATE = (
|
||||||
|
"{%- if reasoning_effort not in ('xhigh', 'medium', 'low') %}"
|
||||||
|
"{{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ~ "
|
||||||
|
"'. Supported types are xhigh (default), medium, and low.') }}{%- endif %}"
|
||||||
|
)
|
||||||
|
GPT_OSS_TEMPLATE = (
|
||||||
|
'{%- set valid_efforts = ["low", "medium", "high"] %}'
|
||||||
|
"{%- if reasoning_effort not in valid_efforts %}"
|
||||||
|
"{{ raise_exception('bad effort') }}{% endif %}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_accepted_set_is_read_out_of_the_template():
|
||||||
|
from lembas.services import chat as chat_service
|
||||||
|
|
||||||
|
assert chat_service.efforts_from_chat_template(BONSAI_TEMPLATE) == [
|
||||||
|
"low", "medium", "xhigh",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_template_that_keeps_its_list_in_a_variable_is_read_too():
|
||||||
|
"""gpt-oss names the list rather than inlining it, so nothing near the
|
||||||
|
`reasoning_effort` mention spells the values out."""
|
||||||
|
from lembas.services import chat as chat_service
|
||||||
|
|
||||||
|
assert chat_service.efforts_from_chat_template(GPT_OSS_TEMPLATE) == [
|
||||||
|
"low", "medium", "high",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_unrelated_list_is_not_mistaken_for_a_vocabulary():
|
||||||
|
from lembas.services import chat as chat_service
|
||||||
|
|
||||||
|
template = '{%- set roles = ["user", "assistant", "system"] %}{{ messages }}'
|
||||||
|
assert chat_service.efforts_from_chat_template(template) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_single_mention_is_not_a_vocabulary():
|
||||||
|
"""`{%- set reasoning_effort = 'medium' %}` is a default, not a list, and
|
||||||
|
reading it as one would leave a model offering exactly one level."""
|
||||||
|
from lembas.services import chat as chat_service
|
||||||
|
|
||||||
|
assert chat_service.efforts_from_chat_template("{%- set reasoning_effort = 'medium' %}") == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_template_that_says_nothing_says_nothing():
|
||||||
|
from lembas.services import chat as chat_service
|
||||||
|
|
||||||
|
assert chat_service.efforts_from_chat_template("") == []
|
||||||
|
assert chat_service.efforts_from_chat_template("{{ messages }}") == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_props_lives_beside_the_openai_surface_not_inside_it():
|
||||||
|
"""`/props` is llama-server's own route, at the server root -- a base URL
|
||||||
|
written as `.../v1` would otherwise ask for `/v1/props`, which is a 404."""
|
||||||
|
from lembas.services.llm.openai_client import Endpoint
|
||||||
|
|
||||||
|
endpoint = Endpoint(base_url="http://host:8080/v1", api_key="", extra_headers={})
|
||||||
|
assert endpoint.root_url("props") == "http://host:8080/props"
|
||||||
|
|
||||||
|
bare = Endpoint(base_url="http://host:8080", api_key="", extra_headers={})
|
||||||
|
assert bare.root_url("props") == "http://host:8080/props"
|
||||||
|
# And the OpenAI surface is unchanged by any of this.
|
||||||
|
assert bare.url("chat/completions") == "http://host:8080/v1/chat/completions"
|
||||||
|
|
||||||
|
|
||||||
|
def test_detecting_from_the_endpoint_writes_the_list(client, db, registered, mock_http):
|
||||||
|
"""The whole path: a button, a GET to /props, the template parsed, the
|
||||||
|
model's list written."""
|
||||||
|
import httpx
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from lembas.db.models import Connection, Model
|
||||||
|
|
||||||
|
connection = Connection(name="local", base_url="http://127.0.0.1:1", api_key_encrypted="")
|
||||||
|
db.add(connection)
|
||||||
|
db.commit()
|
||||||
|
db.add(Model(connection_id=connection.id, model_id="bonsai"))
|
||||||
|
db.commit()
|
||||||
|
model = db.scalar(select(Model).where(Model.model_id == "bonsai"))
|
||||||
|
|
||||||
|
asked: list[str] = []
|
||||||
|
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
asked.append(str(request.url))
|
||||||
|
return httpx.Response(200, json={"chat_template": BONSAI_TEMPLATE})
|
||||||
|
|
||||||
|
mock_http(handler)
|
||||||
|
response = client.post(
|
||||||
|
f"/admin/models/{model.id}/detect-efforts", follow_redirects=False
|
||||||
|
)
|
||||||
|
assert response.status_code == 303
|
||||||
|
|
||||||
|
db.expire_all()
|
||||||
|
assert db.get(Model, model.id).reasoning_efforts == ["low", "medium", "xhigh"]
|
||||||
|
# At the server root, not under /v1.
|
||||||
|
assert asked and asked[0].endswith("/props")
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_endpoint_with_no_props_leaves_the_list_alone(client, db, registered, mock_http):
|
||||||
|
"""OpenAI and vLLM have no such route, and "this cannot tell us" must not
|
||||||
|
be recorded as "this model accepts nothing"."""
|
||||||
|
import httpx
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from lembas.db.models import Connection, Model
|
||||||
|
|
||||||
|
connection = Connection(name="hosted", base_url="http://127.0.0.1:2", api_key_encrypted="")
|
||||||
|
db.add(connection)
|
||||||
|
db.commit()
|
||||||
|
db.add(Model(connection_id=connection.id, model_id="gpt-x", reasoning_efforts=["low", "high"]))
|
||||||
|
db.commit()
|
||||||
|
model = db.scalar(select(Model).where(Model.model_id == "gpt-x"))
|
||||||
|
|
||||||
|
mock_http(lambda request: httpx.Response(404, json={"error": "not found"}))
|
||||||
|
client.post(f"/admin/models/{model.id}/detect-efforts", follow_redirects=False)
|
||||||
|
|
||||||
|
db.expire_all()
|
||||||
|
assert db.get(Model, model.id).reasoning_efforts == ["low", "high"]
|
||||||
|
|||||||
@@ -230,3 +230,124 @@ def test_each_new_table_is_usable_after_the_upgrade(db, table):
|
|||||||
|
|
||||||
declared = {column.name for column in Base.metadata.tables[table].c}
|
declared = {column.name for column in Base.metadata.tables[table].c}
|
||||||
assert _columns(engine, table) == declared
|
assert _columns(engine, table) == declared
|
||||||
|
|
||||||
|
|
||||||
|
# --- A list-shaped JSON column added to a database that already had rows -----
|
||||||
|
#
|
||||||
|
# Reported as a 500 on a live instance the moment it updated:
|
||||||
|
#
|
||||||
|
# ValueError: Attribute 'reasoning_efforts' does not accept objects
|
||||||
|
# of type <class 'dict'>
|
||||||
|
#
|
||||||
|
# `_literal_default` read the shape off `column.type.python_type`, and
|
||||||
|
# `MutableList.as_mutable(JSON)` returns the *same* JSON type object with a
|
||||||
|
# listener attached -- it does not subclass it -- so `python_type` is `dict` for
|
||||||
|
# both flavours. Every existing row got `'{}'` in a list column, and MutableList
|
||||||
|
# refuses a dict while *loading*, so every page that listed models raised.
|
||||||
|
#
|
||||||
|
# The suite never caught it because `conftest.py` builds a fresh database, where
|
||||||
|
# the column is created from the model rather than backfilled by a migration.
|
||||||
|
# These tests exercise the path that actually ran.
|
||||||
|
def test_a_list_column_is_backfilled_with_a_list():
|
||||||
|
from lembas.db.migrations import _default_shape, _literal_default
|
||||||
|
from lembas.db.models import Model
|
||||||
|
|
||||||
|
columns = {c.name: c for c in Model.__table__.columns}
|
||||||
|
assert _default_shape(columns["reasoning_efforts"]) is list
|
||||||
|
assert _literal_default(columns["reasoning_efforts"]) == "'[]'"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_dict_column_still_gets_a_dict():
|
||||||
|
from lembas.db.migrations import _literal_default
|
||||||
|
from lembas.db.models import Model
|
||||||
|
|
||||||
|
columns = {c.name: c for c in Model.__table__.columns}
|
||||||
|
assert _literal_default(columns["capabilities_json"]) == "'{}'"
|
||||||
|
assert _literal_default(columns["params_json"]) == "'{}'"
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_model(engine, **overrides):
|
||||||
|
"""A real row, made the way the application makes one.
|
||||||
|
|
||||||
|
Built through the ORM rather than a hand-written INSERT: the table has
|
||||||
|
several NOT NULL columns and a test that enumerates them is a test that
|
||||||
|
breaks every time one is added, for reasons having nothing to do with what
|
||||||
|
it is checking.
|
||||||
|
"""
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from lembas.db.models import Connection, Model
|
||||||
|
|
||||||
|
with Session(engine) as session:
|
||||||
|
connection = Connection(
|
||||||
|
name="local", base_url="http://127.0.0.1:1", api_key_encrypted=""
|
||||||
|
)
|
||||||
|
session.add(connection)
|
||||||
|
session.flush()
|
||||||
|
model = Model(connection_id=connection.id, model_id="bonsai", **overrides)
|
||||||
|
session.add(model)
|
||||||
|
session.commit()
|
||||||
|
return model.id
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_damage_already_written_is_repaired_on_start(tmp_path):
|
||||||
|
"""The fix to `_literal_default` helps the next instance. This is the one
|
||||||
|
that helps the instance that has already updated."""
|
||||||
|
from sqlalchemy import create_engine, text
|
||||||
|
|
||||||
|
from lembas.db.migrations import repair_json_shapes, sync_schema
|
||||||
|
|
||||||
|
engine = create_engine(f"sqlite:///{tmp_path}/repair.db")
|
||||||
|
sync_schema(engine)
|
||||||
|
model_id = _seed_model(engine)
|
||||||
|
|
||||||
|
# Exactly what the broken backfill left behind on a row that predated the
|
||||||
|
# column: the wrong empty value, in a column that refuses it on load.
|
||||||
|
with engine.begin() as connection:
|
||||||
|
connection.execute(
|
||||||
|
text("UPDATE models SET reasoning_efforts = '{}' WHERE id = :id"),
|
||||||
|
{"id": model_id},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert repair_json_shapes(engine)
|
||||||
|
|
||||||
|
with engine.begin() as connection:
|
||||||
|
stored = connection.execute(
|
||||||
|
text("SELECT reasoning_efforts FROM models WHERE id = :id"), {"id": model_id}
|
||||||
|
).scalar()
|
||||||
|
assert stored == "[]"
|
||||||
|
|
||||||
|
# And the row loads again, which is the whole point -- the failure was a
|
||||||
|
# ValueError while reading, not a wrong value sitting harmlessly.
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from lembas.db.models import Model
|
||||||
|
|
||||||
|
with Session(engine) as session:
|
||||||
|
assert session.get(Model, model_id).reasoning_efforts == []
|
||||||
|
|
||||||
|
# Converges: a second run finds nothing left to do.
|
||||||
|
assert repair_json_shapes(engine) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_repair_leaves_a_dict_column_alone(tmp_path):
|
||||||
|
"""`{}` is a legitimate value in a MutableDict column and must survive."""
|
||||||
|
from sqlalchemy import create_engine, text
|
||||||
|
|
||||||
|
from lembas.db.migrations import repair_json_shapes, sync_schema
|
||||||
|
|
||||||
|
engine = create_engine(f"sqlite:///{tmp_path}/keep.db")
|
||||||
|
sync_schema(engine)
|
||||||
|
model_id = _seed_model(engine)
|
||||||
|
|
||||||
|
with engine.begin() as connection:
|
||||||
|
connection.execute(
|
||||||
|
text("UPDATE models SET capabilities_json = '{}' WHERE id = :id"),
|
||||||
|
{"id": model_id},
|
||||||
|
)
|
||||||
|
repair_json_shapes(engine)
|
||||||
|
with engine.begin() as connection:
|
||||||
|
stored = connection.execute(
|
||||||
|
text("SELECT capabilities_json FROM models WHERE id = :id"), {"id": model_id}
|
||||||
|
).scalar()
|
||||||
|
assert stored == "{}"
|
||||||
|
|||||||
Reference in New Issue
Block a user