Reading the answer instead of asking somebody to know it
llama-server publishes the loaded model's Jinja chat template on /props, and
that template is the very thing that rejects a reasoning effort it does not
recognise -- so the accepted set is written down, authoritatively, in a place
this application can simply read. There is a button on the model's page that
does.
The parser handles both shapes a template uses: the values inline in the test
that rejects them (Bonsai), and a named list set elsewhere with nothing near
the mention spelling them out (gpt-oss). It is deliberately conservative,
because a wrong answer here silently removes a level somebody is entitled to:
only known efforts count, an unrelated list of quoted strings is ignored, and a
single match is read as a default -- `{%- set reasoning_effort = 'medium' %}`
-- rather than as a vocabulary of one.
An endpoint with no such route says so. OpenAI and vLLM do not publish a
template, and "this cannot tell us" must not be recorded as "this model accepts
nothing".
/props sits at the server root, beside the OpenAI-compatible surface rather
than inside it, so a base URL written as .../v1 needs the suffix stripped.
Getting that wrong is a silent 404 that looks like detection simply not
working, so there is a test on it.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -456,3 +456,129 @@ def test_a_model_with_no_advertisement_simply_loses_the_refused_value():
|
||||
from lembas.services import generation
|
||||
|
||||
assert generation._advertised_efforts("Unexpected reasoning effort high.") == []
|
||||
|
||||
|
||||
# --- Reading the answer instead of asking somebody to know it ----------------
|
||||
#
|
||||
# llama-server publishes the loaded model's Jinja chat template on /props, and
|
||||
# that template is the thing that rejects an effort it does not know -- so the
|
||||
# accepted set is written down in the one authoritative place.
|
||||
BONSAI_TEMPLATE = (
|
||||
"{%- if reasoning_effort not in ('xhigh', 'medium', 'low') %}"
|
||||
"{{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ~ "
|
||||
"'. Supported types are xhigh (default), medium, and low.') }}{%- endif %}"
|
||||
)
|
||||
GPT_OSS_TEMPLATE = (
|
||||
'{%- set valid_efforts = ["low", "medium", "high"] %}'
|
||||
"{%- if reasoning_effort not in valid_efforts %}"
|
||||
"{{ raise_exception('bad effort') }}{% endif %}"
|
||||
)
|
||||
|
||||
|
||||
def test_the_accepted_set_is_read_out_of_the_template():
|
||||
from lembas.services import chat as chat_service
|
||||
|
||||
assert chat_service.efforts_from_chat_template(BONSAI_TEMPLATE) == [
|
||||
"low", "medium", "xhigh",
|
||||
]
|
||||
|
||||
|
||||
def test_a_template_that_keeps_its_list_in_a_variable_is_read_too():
|
||||
"""gpt-oss names the list rather than inlining it, so nothing near the
|
||||
`reasoning_effort` mention spells the values out."""
|
||||
from lembas.services import chat as chat_service
|
||||
|
||||
assert chat_service.efforts_from_chat_template(GPT_OSS_TEMPLATE) == [
|
||||
"low", "medium", "high",
|
||||
]
|
||||
|
||||
|
||||
def test_an_unrelated_list_is_not_mistaken_for_a_vocabulary():
|
||||
from lembas.services import chat as chat_service
|
||||
|
||||
template = '{%- set roles = ["user", "assistant", "system"] %}{{ messages }}'
|
||||
assert chat_service.efforts_from_chat_template(template) == []
|
||||
|
||||
|
||||
def test_a_single_mention_is_not_a_vocabulary():
|
||||
"""`{%- set reasoning_effort = 'medium' %}` is a default, not a list, and
|
||||
reading it as one would leave a model offering exactly one level."""
|
||||
from lembas.services import chat as chat_service
|
||||
|
||||
assert chat_service.efforts_from_chat_template("{%- set reasoning_effort = 'medium' %}") == []
|
||||
|
||||
|
||||
def test_a_template_that_says_nothing_says_nothing():
|
||||
from lembas.services import chat as chat_service
|
||||
|
||||
assert chat_service.efforts_from_chat_template("") == []
|
||||
assert chat_service.efforts_from_chat_template("{{ messages }}") == []
|
||||
|
||||
|
||||
def test_props_lives_beside_the_openai_surface_not_inside_it():
|
||||
"""`/props` is llama-server's own route, at the server root -- a base URL
|
||||
written as `.../v1` would otherwise ask for `/v1/props`, which is a 404."""
|
||||
from lembas.services.llm.openai_client import Endpoint
|
||||
|
||||
endpoint = Endpoint(base_url="http://host:8080/v1", api_key="", extra_headers={})
|
||||
assert endpoint.root_url("props") == "http://host:8080/props"
|
||||
|
||||
bare = Endpoint(base_url="http://host:8080", api_key="", extra_headers={})
|
||||
assert bare.root_url("props") == "http://host:8080/props"
|
||||
# And the OpenAI surface is unchanged by any of this.
|
||||
assert bare.url("chat/completions") == "http://host:8080/v1/chat/completions"
|
||||
|
||||
|
||||
def test_detecting_from_the_endpoint_writes_the_list(client, db, registered, mock_http):
|
||||
"""The whole path: a button, a GET to /props, the template parsed, the
|
||||
model's list written."""
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
|
||||
from lembas.db.models import Connection, Model
|
||||
|
||||
connection = Connection(name="local", base_url="http://127.0.0.1:1", api_key_encrypted="")
|
||||
db.add(connection)
|
||||
db.commit()
|
||||
db.add(Model(connection_id=connection.id, model_id="bonsai"))
|
||||
db.commit()
|
||||
model = db.scalar(select(Model).where(Model.model_id == "bonsai"))
|
||||
|
||||
asked: list[str] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
asked.append(str(request.url))
|
||||
return httpx.Response(200, json={"chat_template": BONSAI_TEMPLATE})
|
||||
|
||||
mock_http(handler)
|
||||
response = client.post(
|
||||
f"/admin/models/{model.id}/detect-efforts", follow_redirects=False
|
||||
)
|
||||
assert response.status_code == 303
|
||||
|
||||
db.expire_all()
|
||||
assert db.get(Model, model.id).reasoning_efforts == ["low", "medium", "xhigh"]
|
||||
# At the server root, not under /v1.
|
||||
assert asked and asked[0].endswith("/props")
|
||||
|
||||
|
||||
def test_an_endpoint_with_no_props_leaves_the_list_alone(client, db, registered, mock_http):
|
||||
"""OpenAI and vLLM have no such route, and "this cannot tell us" must not
|
||||
be recorded as "this model accepts nothing"."""
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
|
||||
from lembas.db.models import Connection, Model
|
||||
|
||||
connection = Connection(name="hosted", base_url="http://127.0.0.1:2", api_key_encrypted="")
|
||||
db.add(connection)
|
||||
db.commit()
|
||||
db.add(Model(connection_id=connection.id, model_id="gpt-x", reasoning_efforts=["low", "high"]))
|
||||
db.commit()
|
||||
model = db.scalar(select(Model).where(Model.model_id == "gpt-x"))
|
||||
|
||||
mock_http(lambda request: httpx.Response(404, json={"error": "not found"}))
|
||||
client.post(f"/admin/models/{model.id}/detect-efforts", follow_redirects=False)
|
||||
|
||||
db.expire_all()
|
||||
assert db.get(Model, model.id).reasoning_efforts == ["low", "high"]
|
||||
|
||||
Reference in New Issue
Block a user