Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
32e2326d41
|
@@ -16,6 +16,30 @@ for 1.0.0 have something to be assembled from.
|
|||||||
|
|
||||||
## Unreleased
|
## Unreleased
|
||||||
|
|
||||||
|
## 1.2.0
|
||||||
|
|
||||||
|
- Fixed: **choosing a reasoning effort could kill the reply outright**, with a
|
||||||
|
Jinja traceback where the answer should have been. Reasoning effort is sent
|
||||||
|
two ways, and the second — `chat_template_kwargs` — is rendered into the
|
||||||
|
model's own chat template, which does not ignore a value it has never heard
|
||||||
|
of: it raises, and the whole request fails. The catch is that the vocabulary
|
||||||
|
is **not the same for every model**. gpt-oss takes `low/medium/high`; Bonsai
|
||||||
|
takes `low/medium/xhigh` and refuses `high`; OpenAI has added `minimal`,
|
||||||
|
`xhigh` and `max` at various points. This application offered the same three
|
||||||
|
to everything, so on some models the top setting was one the model would
|
||||||
|
throw for.
|
||||||
|
- **A model now has its own list of the efforts it accepts**, on its page under
|
||||||
|
Models, and the composer's picker and `/effort` offer only those. Tick none
|
||||||
|
and the familiar three are used, which is right for nearly everything.
|
||||||
|
- **And it corrects itself.** If an endpoint refuses an effort anyway — a model
|
||||||
|
swapped underneath a name, a runtime upgraded — that reply is retried once
|
||||||
|
without it instead of being lost, and the model's list is narrowed so the
|
||||||
|
menu stops offering something that does not work. Where the endpoint says
|
||||||
|
what it *does* take, that is what gets stored.
|
||||||
|
- `/effort` now reads the levels from the picker rather than from a second copy
|
||||||
|
of the list kept in the browser, so the two can no longer disagree about what
|
||||||
|
a valid effort is.
|
||||||
|
|
||||||
## 1.1.2
|
## 1.1.2
|
||||||
|
|
||||||
Two things a phone found that 1.1.0's phone pass had not.
|
Two things a phone found that 1.1.0's phone pass had not.
|
||||||
|
|||||||
@@ -1,3 +1,3 @@
|
|||||||
"""LLeMbas - a Middle-earth themed web UI for OpenAI-compatible LLM endpoints."""
|
"""LLeMbas - a Middle-earth themed web UI for OpenAI-compatible LLM endpoints."""
|
||||||
|
|
||||||
__version__ = "1.1.2"
|
__version__ = "1.2.0"
|
||||||
|
|||||||
@@ -177,7 +177,11 @@ async def model_detail(
|
|||||||
"groups": list(db.scalars(select(Group).order_by(Group.name))),
|
"groups": list(db.scalars(select(Group).order_by(Group.name))),
|
||||||
"capabilities": PROTOCOL_CAPABILITIES,
|
"capabilities": PROTOCOL_CAPABILITIES,
|
||||||
"tool_capabilities": TOOL_CAPABILITIES,
|
"tool_capabilities": TOOL_CAPABILITIES,
|
||||||
|
# Every effort this application understands, so an administrator
|
||||||
|
# can tick the ones their model actually takes -- and the model's
|
||||||
|
# current answer, which is the common three until somebody says.
|
||||||
"efforts": chat_service.EFFORTS,
|
"efforts": chat_service.EFFORTS,
|
||||||
|
"model_efforts": chat_service.efforts_for(model),
|
||||||
# Rows predating the split have no tool_* keys at all. Showing them
|
# Rows predating the split have no tool_* keys at all. Showing them
|
||||||
# unticked would be a lie: tools.enabled_tools treats absent as on
|
# unticked would be a lie: tools.enabled_tools treats absent as on
|
||||||
# when `tools` is on, so that an upgrade does not silently take web
|
# when `tools` is on, so that an upgrade does not silently take web
|
||||||
@@ -238,6 +242,7 @@ async def update_model(
|
|||||||
position: str = Form(""),
|
position: str = Form(""),
|
||||||
context_length: str = Form(""),
|
context_length: str = Form(""),
|
||||||
default_effort: str = Form(""),
|
default_effort: str = Form(""),
|
||||||
|
reasoning_efforts: list[str] = Form(default=[]),
|
||||||
group_ids: list[str] = Form(default=[]),
|
group_ids: list[str] = Form(default=[]),
|
||||||
capability: list[str] = Form(default=[]),
|
capability: list[str] = Form(default=[]),
|
||||||
) -> Response:
|
) -> Response:
|
||||||
@@ -260,9 +265,19 @@ async def update_model(
|
|||||||
# Merged rather than rebuilt, unlike the capabilities below: params_json
|
# Merged rather than rebuilt, unlike the capabilities below: params_json
|
||||||
# holds whatever sampling defaults an administrator has set and this form
|
# holds whatever sampling defaults an administrator has set and this form
|
||||||
# only carries one of them.
|
# only carries one of them.
|
||||||
|
# Which efforts this model takes at all. Submitted as a list of ticked
|
||||||
|
# values; empty means "nobody has said", and `chat.efforts_for` answers with
|
||||||
|
# the common three. Stored in the order `EFFORTS` declares rather than the
|
||||||
|
# order a browser happened to send.
|
||||||
|
chosen = [value for value in chat_service.EFFORTS if value in (reasoning_efforts or [])]
|
||||||
|
model.reasoning_efforts = chosen
|
||||||
|
|
||||||
params = dict(model.params_json or {})
|
params = dict(model.params_json or {})
|
||||||
wanted = default_effort.strip().lower()
|
wanted = default_effort.strip().lower()
|
||||||
if wanted in chat_service.EFFORTS:
|
# Checked against what this model takes, not against everything this
|
||||||
|
# application has heard of -- a default of `high` on a model whose template
|
||||||
|
# refuses it is a chat that fails on its first turn.
|
||||||
|
if wanted in chat_service.efforts_for(model):
|
||||||
params["reasoning_effort"] = wanted
|
params["reasoning_effort"] = wanted
|
||||||
else:
|
else:
|
||||||
params.pop("reasoning_effort", None)
|
params.pop("reasoning_effort", None)
|
||||||
|
|||||||
@@ -82,10 +82,12 @@ def _chat_context(db: DBSession, user: User, chat: Chat | None) -> dict:
|
|||||||
else []
|
else []
|
||||||
),
|
),
|
||||||
"attached_base_ids": [base.id for base in chat.knowledge_bases] if chat else [],
|
"attached_base_ids": [base.id for base in chat.knowledge_bases] if chat else [],
|
||||||
# The three a reasoning model understands. From the service so the
|
# What *this* model takes, not the three every model used to be assumed
|
||||||
# command, the control and the request builder cannot disagree about
|
# to take. The vocabulary is per model -- gpt-oss has no `xhigh` and
|
||||||
# what is a valid effort.
|
# Bonsai has no `high`, and sending the wrong one does not degrade, it
|
||||||
"efforts": chat_service.EFFORTS,
|
# raises inside the chat template and fails the reply. From the service
|
||||||
|
# so the command, the control and the request builder cannot disagree.
|
||||||
|
"efforts": chat_service.efforts_for(current) if current else chat_service.DEFAULT_EFFORTS,
|
||||||
# What the picker shows, and what `build_request` will send. One
|
# What the picker shows, and what `build_request` will send. One
|
||||||
# resolver so the two cannot disagree.
|
# resolver so the two cannot disagree.
|
||||||
"resolved_effort": chat_service.resolved_effort(chat) if chat else "",
|
"resolved_effort": chat_service.resolved_effort(chat) if chat else "",
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ from sqlalchemy import (
|
|||||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||||
|
|
||||||
from lembas.db.base import Base, Timestamps, UUIDPrimaryKey
|
from lembas.db.base import Base, Timestamps, UUIDPrimaryKey
|
||||||
from lembas.db.types import JSONDict
|
from lembas.db.types import JSONDict, JSONList
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
# Import only for the annotation; at runtime SQLAlchemy resolves the
|
# Import only for the annotation; at runtime SQLAlchemy resolves the
|
||||||
@@ -137,6 +137,20 @@ class Model(UUIDPrimaryKey, Timestamps, Base):
|
|||||||
# ticked anything.
|
# ticked anything.
|
||||||
context_length: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
context_length: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||||
|
|
||||||
|
# Which reasoning efforts this model actually accepts. Empty means "nobody
|
||||||
|
# has said", and `services/chat.efforts_for` answers with the common set.
|
||||||
|
#
|
||||||
|
# It has to be per model, because the vocabulary is: gpt-oss takes
|
||||||
|
# low/medium/high, Bonsai takes low/medium/xhigh and *raises* on high, and
|
||||||
|
# OpenAI's own list has grown minimal, xhigh and max at different times. A
|
||||||
|
# single global tuple is a guess that is wrong for somebody.
|
||||||
|
#
|
||||||
|
# ⚠ A column and not a key in `capabilities_json`, for exactly the reason
|
||||||
|
# `context_length` is one: that dict is rebuilt wholesale from the submitted
|
||||||
|
# checkboxes on every save, so anything in it that is not a checkbox is
|
||||||
|
# destroyed the next time an administrator ticks anything.
|
||||||
|
reasoning_efforts: Mapped[list[str]] = mapped_column(JSONList, default=list)
|
||||||
|
|
||||||
connection: Mapped[Connection] = relationship(back_populates="models")
|
connection: Mapped[Connection] = relationship(back_populates="models")
|
||||||
groups: Mapped[list[Group]] = relationship(
|
groups: Mapped[list[Group]] = relationship(
|
||||||
"Group", secondary=model_groups, back_populates="models"
|
"Group", secondary=model_groups, back_populates="models"
|
||||||
|
|||||||
@@ -387,7 +387,15 @@ def build_request(
|
|||||||
):
|
):
|
||||||
body["tool_choice"] = {"type": "function", "function": {"name": force_tool}}
|
body["tool_choice"] = {"type": "function", "function": {"name": force_tool}}
|
||||||
|
|
||||||
apply_effort(body, (chat.params_json or {}).get("reasoning_effort"))
|
# The model's own vocabulary, looked up here rather than passed in: every
|
||||||
|
# caller of `build_request` would otherwise have to remember, which is the
|
||||||
|
# trap `audio_service.template_flags` fell into.
|
||||||
|
chat_model = model_for(db, chat)
|
||||||
|
apply_effort(
|
||||||
|
body,
|
||||||
|
(chat.params_json or {}).get("reasoning_effort"),
|
||||||
|
efforts_for(chat_model) if chat_model is not None else None,
|
||||||
|
)
|
||||||
return body
|
return body
|
||||||
|
|
||||||
|
|
||||||
@@ -405,7 +413,42 @@ def build_request(
|
|||||||
# an effort on sends neither field and is byte-for-byte what it was. An endpoint
|
# an effort on sends neither field and is byte-for-byte what it was. An endpoint
|
||||||
# strict about unknown parameters will refuse the extra one -- but on a chat
|
# strict about unknown parameters will refuse the extra one -- but on a chat
|
||||||
# somebody deliberately set an effort on, not on every chat in the instance.
|
# somebody deliberately set an effort on, not on every chat in the instance.
|
||||||
EFFORTS = ("low", "medium", "high")
|
# Every reasoning effort this application understands, and the subset a model
|
||||||
|
# gets when nobody has said otherwise.
|
||||||
|
#
|
||||||
|
# 🚨 These are two different questions and conflating them is what broke a
|
||||||
|
# chat on Bonsai: `EFFORTS` was `("low", "medium", "high")` and was used both to
|
||||||
|
# validate what somebody chose *and* to decide what to offer, so a model whose
|
||||||
|
# vocabulary is low/medium/**xhigh** could not be given its own top setting,
|
||||||
|
# and the one it was given -- `high` -- made its chat template call
|
||||||
|
# `raise_exception` and took the whole reply with it.
|
||||||
|
#
|
||||||
|
# The known list is the union across providers, which have not agreed: OpenAI
|
||||||
|
# has added `minimal`, `xhigh` and `max` at different points; gpt-oss takes
|
||||||
|
# low/medium/high; Bonsai takes low/medium/xhigh and refuses high. `none` is
|
||||||
|
# deliberately absent -- this application already spells that `off`, and two
|
||||||
|
# spellings of off is the failure this codebase keeps cataloguing.
|
||||||
|
EFFORTS = ("minimal", "low", "medium", "high", "xhigh", "max")
|
||||||
|
|
||||||
|
# What a model is offered when its own list is empty. The three every reasoning
|
||||||
|
# model since the first one has understood.
|
||||||
|
DEFAULT_EFFORTS = ("low", "medium", "high")
|
||||||
|
|
||||||
|
|
||||||
|
def efforts_for(model) -> tuple[str, ...]:
|
||||||
|
"""The efforts this model accepts, in the order they should be offered.
|
||||||
|
|
||||||
|
A model's own list when an administrator has set one or the endpoint has
|
||||||
|
taught us one (see `generation._narrow_efforts`), and the common three
|
||||||
|
otherwise. Filtered against `EFFORTS` on the way out, so a value stored by
|
||||||
|
an older release -- or learned from an endpoint that advertised something
|
||||||
|
this application has never heard of -- cannot reach a request body.
|
||||||
|
"""
|
||||||
|
stored = list(getattr(model, "reasoning_efforts", None) or [])
|
||||||
|
chosen = [value for value in stored if value in EFFORTS]
|
||||||
|
if not chosen:
|
||||||
|
return DEFAULT_EFFORTS
|
||||||
|
return tuple(value for value in EFFORTS if value in chosen)
|
||||||
|
|
||||||
|
|
||||||
def resolved_effort(chat) -> str:
|
def resolved_effort(chat) -> str:
|
||||||
@@ -427,9 +470,19 @@ def resolved_effort(chat) -> str:
|
|||||||
return value if value in EFFORTS else ""
|
return value if value in EFFORTS else ""
|
||||||
|
|
||||||
|
|
||||||
def apply_effort(body: dict[str, Any], effort: str | None) -> None:
|
def apply_effort(
|
||||||
"""Put a chosen reasoning effort into a request body, in both forms."""
|
body: dict[str, Any], effort: str | None, supported: tuple[str, ...] | None = None
|
||||||
if not effort or effort not in EFFORTS:
|
) -> None:
|
||||||
|
"""Put a chosen reasoning effort into a request body, in both forms.
|
||||||
|
|
||||||
|
`supported` is the model's own vocabulary. An effort outside it is dropped
|
||||||
|
rather than sent, because the second form below is not advisory: it reaches
|
||||||
|
the model's Jinja chat template, and a template that does not know the value
|
||||||
|
raises rather than ignoring it -- which fails the whole request, not the
|
||||||
|
parameter.
|
||||||
|
"""
|
||||||
|
allowed = supported or DEFAULT_EFFORTS
|
||||||
|
if not effort or effort not in allowed:
|
||||||
return
|
return
|
||||||
body["reasoning_effort"] = effort
|
body["reasoning_effort"] = effort
|
||||||
kwargs = dict(body.get("chat_template_kwargs") or {})
|
kwargs = dict(body.get("chat_template_kwargs") or {})
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ import asyncio
|
|||||||
import contextlib
|
import contextlib
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
import re
|
||||||
import time
|
import time
|
||||||
import uuid
|
import uuid
|
||||||
from dataclasses import dataclass, field, replace
|
from dataclasses import dataclass, field, replace
|
||||||
@@ -454,6 +455,122 @@ def _narrower(instance: float, quota: int) -> float:
|
|||||||
return float(min(instance, quota))
|
return float(min(instance, quota))
|
||||||
|
|
||||||
|
|
||||||
|
# --- A reasoning effort the model will not take ------------------------------
|
||||||
|
#
|
||||||
|
# `chat_template_kwargs.reasoning_effort` is not advisory. It reaches the
|
||||||
|
# model's Jinja chat template, and a template that does not know the value does
|
||||||
|
# not ignore it -- gpt-oss and Bonsai both call `raise_exception`, which fails
|
||||||
|
# the whole request. The reader sees their reply die with a Jinja traceback in
|
||||||
|
# it, having chosen a perfectly ordinary-looking option from a menu this
|
||||||
|
# application drew.
|
||||||
|
#
|
||||||
|
# So the value is checked against the model's own vocabulary before it is sent
|
||||||
|
# (`chat.apply_effort`), and this is the second line: when it is refused anyway
|
||||||
|
# -- an endpoint upgraded underneath us, a model whose list nobody has set --
|
||||||
|
# the reply is retried once without it rather than lost, and the model's list is
|
||||||
|
# narrowed so the menu stops offering something that does not work.
|
||||||
|
|
||||||
|
|
||||||
|
def _effort_was_refused(message: str) -> bool:
|
||||||
|
"""Whether this error is the chat template refusing the effort we sent.
|
||||||
|
|
||||||
|
Deliberately narrow. Anything that merely mentions reasoning would also
|
||||||
|
match a model politely declining to think, and retrying *that* silently
|
||||||
|
would hide a real failure behind a second request.
|
||||||
|
"""
|
||||||
|
lowered = message.lower()
|
||||||
|
return "effort" in lowered and ("unexpected" in lowered or "supported" in lowered)
|
||||||
|
|
||||||
|
|
||||||
|
def _advertised_efforts(message: str) -> list[str]:
|
||||||
|
"""The efforts an error message says it will take, if it says.
|
||||||
|
|
||||||
|
Bonsai's is "Unexpected reasoning effort high. Supported types are xhigh
|
||||||
|
(default), medium, and low." -- which is the answer, written out, in the
|
||||||
|
failure. Read only from the part after "supported", so the *rejected* value
|
||||||
|
named in the first sentence is not collected as a supported one.
|
||||||
|
|
||||||
|
Best-effort by design: it only ever narrows what is offered, an
|
||||||
|
administrator can set the list by hand, and anything unrecognised is
|
||||||
|
dropped by `efforts_for` on the way out.
|
||||||
|
"""
|
||||||
|
lowered = message.lower()
|
||||||
|
if "supported" not in lowered:
|
||||||
|
return []
|
||||||
|
tail = lowered.split("supported", 1)[1]
|
||||||
|
# Whole words. `"high" in "xhigh"` is true, so a substring test reads
|
||||||
|
# Bonsai's "Supported types are xhigh (default), medium, and low" as
|
||||||
|
# advertising `high` -- the very value it has just refused -- and the list
|
||||||
|
# would learn the opposite of what the endpoint said.
|
||||||
|
words = set(re.findall(r"[a-z]+", tail))
|
||||||
|
return [effort for effort in chat_service.EFFORTS if effort in words]
|
||||||
|
|
||||||
|
|
||||||
|
def _learn_refused_effort(model_id: str, refused: str, message: str) -> None:
|
||||||
|
"""Write what the endpoint just taught us onto the model.
|
||||||
|
|
||||||
|
Its own session: this runs from inside a generation, which outlives the
|
||||||
|
request's session, and the whole point is that it survives to the next turn.
|
||||||
|
"""
|
||||||
|
from lembas.db.models import Model
|
||||||
|
|
||||||
|
if not model_id:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
with session_scope() as db:
|
||||||
|
models = list(db.scalars(select(Model).where(Model.model_id == model_id)))
|
||||||
|
for model in models:
|
||||||
|
advertised = _advertised_efforts(message)
|
||||||
|
current = list(model.reasoning_efforts or chat_service.DEFAULT_EFFORTS)
|
||||||
|
# What the endpoint advertised, when it did; otherwise simply
|
||||||
|
# the list it had, minus the one it has just refused.
|
||||||
|
wanted = advertised or [e for e in current if e != refused]
|
||||||
|
wanted = [e for e in wanted if e in chat_service.EFFORTS and e != refused]
|
||||||
|
if wanted and wanted != list(model.reasoning_efforts or []):
|
||||||
|
model.reasoning_efforts = wanted
|
||||||
|
log.info(
|
||||||
|
"model %s refused reasoning effort %r; efforts narrowed to %s",
|
||||||
|
model_id, refused, wanted,
|
||||||
|
)
|
||||||
|
except Exception: # noqa: BLE001 - never let bookkeeping fail a reply
|
||||||
|
log.exception("could not record the refused effort for model %s", model_id)
|
||||||
|
|
||||||
|
|
||||||
|
async def _stream_once(endpoint, payload, generation, model_id: str):
|
||||||
|
"""`stream_chat`, retried once without the reasoning effort if that is what
|
||||||
|
the endpoint objected to.
|
||||||
|
|
||||||
|
⚠ The retry is only safe because the template is rendered *before* any token
|
||||||
|
is produced, so a refusal arrives with nothing yet emitted. `sent` is the
|
||||||
|
guard that keeps it that way: once a single chunk has reached the caller,
|
||||||
|
the reply is under way and a second request would duplicate it.
|
||||||
|
"""
|
||||||
|
sent = False
|
||||||
|
try:
|
||||||
|
async for chunk in stream_chat(endpoint, payload):
|
||||||
|
sent = True
|
||||||
|
yield chunk
|
||||||
|
return
|
||||||
|
except LLMError as exc:
|
||||||
|
refused = str((payload.get("chat_template_kwargs") or {}).get("reasoning_effort") or "")
|
||||||
|
if sent or not refused or not _effort_was_refused(exc.message):
|
||||||
|
raise
|
||||||
|
log.info("retrying without reasoning effort %r: %s", refused, exc.message)
|
||||||
|
_learn_refused_effort(model_id, refused, exc.message)
|
||||||
|
|
||||||
|
retry = dict(payload)
|
||||||
|
retry.pop("reasoning_effort", None)
|
||||||
|
kwargs = dict(retry.get("chat_template_kwargs") or {})
|
||||||
|
kwargs.pop("reasoning_effort", None)
|
||||||
|
if kwargs:
|
||||||
|
retry["chat_template_kwargs"] = kwargs
|
||||||
|
else:
|
||||||
|
retry.pop("chat_template_kwargs", None)
|
||||||
|
|
||||||
|
async for chunk in stream_chat(endpoint, retry):
|
||||||
|
yield chunk
|
||||||
|
|
||||||
|
|
||||||
async def _run(generation: Generation) -> None:
|
async def _run(generation: Generation) -> None:
|
||||||
"""Produce one reply, then persist it. Never raises into the task.
|
"""Produce one reply, then persist it. Never raises into the task.
|
||||||
|
|
||||||
@@ -643,7 +760,7 @@ async def _run(generation: Generation) -> None:
|
|||||||
# round thinks at all -- plenty of rounds do not.
|
# round thinks at all -- plenty of rounds do not.
|
||||||
round_thinking: tuple[float, float] | None = None
|
round_thinking: tuple[float, float] | None = None
|
||||||
|
|
||||||
async for chunk in stream_chat(endpoint, payload):
|
async for chunk in _stream_once(endpoint, payload, generation, model_id):
|
||||||
counts = chunk_usage(chunk)
|
counts = chunk_usage(chunk)
|
||||||
if counts is not None:
|
if counts is not None:
|
||||||
generation.reported_usage = True
|
generation.reported_usage = True
|
||||||
|
|||||||
@@ -271,8 +271,21 @@
|
|||||||
|
|
||||||
/* --- Reasoning effort ---------------------------------------------------
|
/* --- Reasoning effort ---------------------------------------------------
|
||||||
The command drives the same select the composer shows, so there is one
|
The command drives the same select the composer shows, so there is one
|
||||||
piece of state and the control updates itself when the command is used. */
|
piece of state and the control updates itself when the command is used.
|
||||||
var EFFORTS = ["low", "medium", "high"];
|
|
||||||
|
Which efforts exist is read off that select's own options rather than
|
||||||
|
kept here. It used to be a second copy of `["low","medium","high"]`, which
|
||||||
|
was wrong the moment the vocabulary became per model: a Bonsai takes
|
||||||
|
`xhigh` and no `high`, so the list the server rendered and the list this
|
||||||
|
file believed in disagreed -- and the one that decides what `/effort xhigh`
|
||||||
|
does was this one. The select is the table; nothing else should hold it. */
|
||||||
|
function efforts() {
|
||||||
|
var select = el("[data-effort]");
|
||||||
|
if (!select) return [];
|
||||||
|
return Array.prototype.map
|
||||||
|
.call(select.options, function (option) { return option.value; })
|
||||||
|
.filter(function (value) { return value !== "off"; });
|
||||||
|
}
|
||||||
|
|
||||||
function setEffort(rest) {
|
function setEffort(rest) {
|
||||||
var select = el("[data-effort]");
|
var select = el("[data-effort]");
|
||||||
@@ -283,12 +296,14 @@
|
|||||||
"error"
|
"error"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
var available = efforts();
|
||||||
|
var listed = available.join(", ");
|
||||||
var wanted = (rest || "").trim().toLowerCase();
|
var wanted = (rest || "").trim().toLowerCase();
|
||||||
if (!wanted) {
|
if (!wanted) {
|
||||||
return note(
|
return note(
|
||||||
EFFORTS.indexOf(select.value) === -1
|
available.indexOf(select.value) === -1
|
||||||
? "No effort is being sent. Try low, medium or high."
|
? "No effort is being sent. Try " + listed + "."
|
||||||
: "Effort is " + select.value + ". /effort low, medium, high, or off."
|
: "Effort is " + select.value + ". /effort " + listed + ", or off."
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
/* "off" is the option's real value, not an empty string: the new-chat form
|
/* "off" is the option's real value, not an empty string: the new-chat form
|
||||||
@@ -296,8 +311,11 @@
|
|||||||
sentinel and this has to match it. "default" and "none" still work,
|
sentinel and this has to match it. "default" and "none" still work,
|
||||||
because somebody's fingers will type them. */
|
because somebody's fingers will type them. */
|
||||||
if (wanted === "default" || wanted === "none") wanted = "off";
|
if (wanted === "default" || wanted === "none") wanted = "off";
|
||||||
else if (wanted !== "off" && EFFORTS.indexOf(wanted) === -1) {
|
else if (wanted !== "off" && available.indexOf(wanted) === -1) {
|
||||||
return note("“" + wanted + "” is not an effort. Try low, medium, high or off.", "error");
|
return note(
|
||||||
|
"“" + wanted + "” is not an effort this model takes. Try " + listed + " or off.",
|
||||||
|
"error"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
select.value = wanted;
|
select.value = wanted;
|
||||||
select.dispatchEvent(new Event("change", { bubbles: true }));
|
select.dispatchEvent(new Event("change", { bubbles: true }));
|
||||||
|
|||||||
@@ -104,11 +104,39 @@
|
|||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div class="field">
|
||||||
|
<span class="field__label">Reasoning efforts this model accepts</span>
|
||||||
|
<div class="btn-row">
|
||||||
|
{% for value in efforts %}
|
||||||
|
<label class="checkbox">
|
||||||
|
<input type="checkbox" name="reasoning_efforts" value="{{ value }}"
|
||||||
|
{{ 'checked' if value in model_efforts }}>
|
||||||
|
<span class="mono">{{ value }}</span>
|
||||||
|
</label>
|
||||||
|
{% endfor %}
|
||||||
|
</div>
|
||||||
|
<p class="field__hint">
|
||||||
|
The vocabulary is <strong>not the same for every model</strong>, and
|
||||||
|
sending one a model does not know is not ignored — it is rendered into
|
||||||
|
the model's chat template, which raises and fails the whole reply.
|
||||||
|
gpt-oss takes <span class="mono">low/medium/high</span>; Bonsai takes
|
||||||
|
<span class="mono">low/medium/xhigh</span> and refuses
|
||||||
|
<span class="mono">high</span>; OpenAI has added
|
||||||
|
<span class="mono">minimal</span>, <span class="mono">xhigh</span> and
|
||||||
|
<span class="mono">max</span> at various points.
|
||||||
|
<br>
|
||||||
|
Tick none and the common three are offered, which is right for almost
|
||||||
|
everything. If an endpoint ever refuses one anyway, that reply is
|
||||||
|
retried without it and this list corrects itself — so this is worth
|
||||||
|
setting by hand only to save that one round trip.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div class="field">
|
<div class="field">
|
||||||
<label class="field__label" for="default-effort">Default reasoning effort</label>
|
<label class="field__label" for="default-effort">Default reasoning effort</label>
|
||||||
<select class="select" id="default-effort" name="default_effort">
|
<select class="select" id="default-effort" name="default_effort">
|
||||||
<option value="">None — send nothing</option>
|
<option value="">None — send nothing</option>
|
||||||
{% for value in efforts %}
|
{% for value in model_efforts %}
|
||||||
<option value="{{ value }}"
|
<option value="{{ value }}"
|
||||||
{{ 'selected' if model.params_json.get('reasoning_effort') == value }}>
|
{{ 'selected' if model.params_json.get('reasoning_effort') == value }}>
|
||||||
{{ value }}
|
{{ value }}
|
||||||
|
|||||||
@@ -366,3 +366,93 @@ def test_the_picker_never_says_default(client: TestClient, db, registered):
|
|||||||
assert "Effort: default" not in html
|
assert "Effort: default" not in html
|
||||||
assert "Effort: off" in html
|
assert "Effort: off" in html
|
||||||
assert '<option value="medium" selected>' in html.replace("\n", "").replace(" ", "")
|
assert '<option value="medium" selected>' in html.replace("\n", "").replace(" ", "")
|
||||||
|
|
||||||
|
|
||||||
|
# --- A vocabulary that is not the same for every model -----------------------
|
||||||
|
#
|
||||||
|
# Reported from a real instance, on a model called Bonsai:
|
||||||
|
#
|
||||||
|
# Jinja Exception: Unexpected reasoning effort high. Supported types are
|
||||||
|
# xhigh (default), medium, and low.
|
||||||
|
#
|
||||||
|
# `chat_template_kwargs.reasoning_effort` is rendered into the model's own chat
|
||||||
|
# template, and a template that does not know the value calls `raise_exception`
|
||||||
|
# rather than ignoring it -- so the whole reply died, from an option this
|
||||||
|
# application had drawn in a menu.
|
||||||
|
BONSAI_ERROR = (
|
||||||
|
"Jinja Exception: Unexpected reasoning effort high. "
|
||||||
|
"Supported types are xhigh (default), medium, and low."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeModel:
|
||||||
|
def __init__(self, efforts=None):
|
||||||
|
self.reasoning_efforts = efforts or []
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_model_that_has_said_nothing_gets_the_common_three():
|
||||||
|
from lembas.services import chat as chat_service
|
||||||
|
|
||||||
|
assert chat_service.efforts_for(_FakeModel()) == ("low", "medium", "high")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_model_can_take_xhigh_and_not_high():
|
||||||
|
from lembas.services import chat as chat_service
|
||||||
|
|
||||||
|
bonsai = _FakeModel(["xhigh", "medium", "low"])
|
||||||
|
assert chat_service.efforts_for(bonsai) == ("low", "medium", "xhigh")
|
||||||
|
assert "high" not in chat_service.efforts_for(bonsai)
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_effort_the_model_refuses_is_never_sent():
|
||||||
|
"""The check that stops the crash happening at all."""
|
||||||
|
from lembas.services import chat as chat_service
|
||||||
|
|
||||||
|
supported = chat_service.efforts_for(_FakeModel(["xhigh", "medium", "low"]))
|
||||||
|
body: dict = {}
|
||||||
|
chat_service.apply_effort(body, "high", supported)
|
||||||
|
assert body == {}
|
||||||
|
|
||||||
|
chat_service.apply_effort(body, "xhigh", supported)
|
||||||
|
assert body["reasoning_effort"] == "xhigh"
|
||||||
|
assert body["chat_template_kwargs"]["reasoning_effort"] == "xhigh"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_value_this_application_never_heard_of_cannot_reach_a_request():
|
||||||
|
from lembas.services import chat as chat_service
|
||||||
|
|
||||||
|
assert chat_service.efforts_for(_FakeModel(["ludicrous"])) == ("low", "medium", "high")
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_refusal_is_recognised_and_the_supported_list_read_out_of_it():
|
||||||
|
from lembas.services import generation
|
||||||
|
|
||||||
|
assert generation._effort_was_refused(BONSAI_ERROR)
|
||||||
|
assert generation._advertised_efforts(BONSAI_ERROR) == ["low", "medium", "xhigh"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_rejected_value_is_not_collected_as_a_supported_one():
|
||||||
|
"""The message names the refused effort first and the supported ones after,
|
||||||
|
so anything reading the whole string would learn `high` from a sentence
|
||||||
|
saying `high` is the problem."""
|
||||||
|
from lembas.services import generation
|
||||||
|
|
||||||
|
assert "high" not in generation._advertised_efforts(BONSAI_ERROR)
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_ordinary_failure_is_not_retried_as_an_effort_problem():
|
||||||
|
"""Retrying a genuine failure would hide it behind a second request."""
|
||||||
|
from lembas.services import generation
|
||||||
|
|
||||||
|
for message in (
|
||||||
|
"Connection refused.",
|
||||||
|
"The model is still loading.",
|
||||||
|
"context length exceeded",
|
||||||
|
):
|
||||||
|
assert not generation._effort_was_refused(message)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_model_with_no_advertisement_simply_loses_the_refused_value():
|
||||||
|
from lembas.services import generation
|
||||||
|
|
||||||
|
assert generation._advertised_efforts("Unexpected reasoning effort high.") == []
|
||||||
|
|||||||
Reference in New Issue
Block a user