Models know how much context they hold

A column rather than a key in capabilities_json, which is rebuilt wholesale
from the submitted checkboxes on every save and would destroy a number
living in it.

0 means unknown, and unknown has to stay tellable from small: the context
percentage and automatic compaction both refuse to act on a figure nobody
supplied. Filled in from /v1/models where the runner advertises it --
OpenRouter, vLLM and llama.cpp each spell it differently, so context_from()
reads the four spellings actually in use, accepts a quoted number but not
"8192 tokens", and rejects anything outside 256..10,000,000. Applied on
discovery only when nothing is set: a refresh must never undo a correction,
since an administrator sets this precisely because the endpoint was wrong.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

src/lembas/api/admin.py

@@ -14,7 +14,7 @@ from lembas.api.deps import AdminUser, Db
 from lembas.db.models import Connection, Model, User
 from lembas.services import settings_store
 from lembas.services.crypto import UNCHANGED_SENTINEL, decrypt, encrypt, mask
-from lembas.services.llm.openai_client import Endpoint, LLMError, list_models
+from lembas.services.llm.openai_client import Endpoint, LLMError, context_from, list_models
 from lembas.web.templating import render
 
 log = logging.getLogger(__name__)
@@ -198,8 +198,21 @@ async def _refresh_models(db: DBSession, connection: Connection) -> tuple[int, s
         model_id = str(entry["id"])[:300]
         seen.add(model_id)
         if model_id in existing:
+            # A context length is filled in only when nobody has one yet. A
+            # refresh must never overwrite a number an administrator typed --
+            # they are usually correcting the endpoint.
+            model = existing[model_id]
+            if not model.context_length:
+                model.context_length = context_from(entry)
             continue
-        db.add(Model(connection_id=connection.id, model_id=model_id, position=next_position))
+        db.add(
+            Model(
+                connection_id=connection.id,
+                model_id=model_id,
+                position=next_position,
+                context_length=context_from(entry),
+            )
+        )
         next_position += 1
 
     # Models that vanished upstream are dropped, so the picker never offers

src/lembas/api/admin_models.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import contextlib
 import logging
 
 from fastapi import APIRouter, File, Form, HTTPException, Request, Response, UploadFile, status
@@ -12,6 +13,7 @@ from sqlalchemy.orm import Session as DBSession
 from lembas.api.deps import AdminUser, Db, RequiredUser
 from lembas.db.models import Connection, Group, Model
 from lembas.services import settings_store, uploads
+from lembas.services.llm.openai_client import MAX_CONTEXT
 from lembas.web.templating import render
 
 log = logging.getLogger(__name__)
@@ -213,6 +215,7 @@ async def update_model(
     pinned: bool = Form(False),
     public: bool = Form(False),
     position: str = Form(""),
+    context_length: str = Form(""),
     group_ids: list[str] = Form(default=[]),
     capability: list[str] = Form(default=[]),
 ) -> Response:
@@ -221,6 +224,13 @@ async def update_model(
     model.display_name = display_name.strip()[:300]
     model.description = description.strip()[:2000]
     model.system_prompt = system_prompt.strip()[:8000]
+    # A string, so an emptied field is distinguishable and junk can be ignored
+    # rather than becoming a 422 -- the same shape `position` uses below.
+    if context_length.strip():
+        with contextlib.suppress(ValueError):
+            model.context_length = min(max(int(context_length), 0), MAX_CONTEXT)
+    else:
+        model.context_length = 0
     model.enabled = enabled
     model.pinned = pinned
     model.public = public

src/lembas/db/models/connection.py

@@ -114,6 +114,17 @@ class Model(UUIDPrimaryKey, Timestamps, Base):
     # Default sampling params applied to new chats using this model.
     params_json: Mapped[dict[str, Any]] = mapped_column(JSONDict, default=dict)
 
+    # How many tokens this model can hold. 0 means unknown, which is what an
+    # endpoint that does not advertise it leaves behind -- and unknown has to
+    # stay tellable from "small", because the context percentage and automatic
+    # compaction both refuse to act on a number nobody supplied.
+    #
+    # A column rather than a key in capabilities_json: that dict is rebuilt
+    # wholesale from the submitted checkboxes on every save (api/admin_models.py),
+    # so a number living in it would be destroyed the next time an administrator
+    # ticked anything.
+    context_length: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+
     connection: Mapped[Connection] = relationship(back_populates="models")
     groups: Mapped[list[Group]] = relationship(
         "Group", secondary=model_groups, back_populates="models"

src/lembas/services/llm/openai_client.py

@@ -155,6 +155,41 @@ async def list_models(endpoint: Endpoint) -> list[dict[str, Any]]:
     return models
 
 
+# Where the runners that bother to advertise a context length put it. There is
+# no standard field, so this is a list of what the common ones actually emit.
+_CONTEXT_KEYS = ("context_length", "max_model_len", "context_window", "max_context_length")
+
+# Below the first, the number is not a context length; above the second it is a
+# typo or a different unit. Either way, better to record nothing than a wrong
+# figure a percentage would then be computed from.
+MIN_CONTEXT = 256
+MAX_CONTEXT = 10_000_000
+
+
+def context_from(entry: dict[str, Any]) -> int:
+    """A model's context length as advertised by /v1/models, or 0 if it is not.
+
+    Strings are accepted because some servers quote the number, but only when
+    they are digits alone -- "8192 tokens" is a label, not a measurement.
+    """
+    candidates = [entry.get(key) for key in _CONTEXT_KEYS]
+    meta = entry.get("meta")
+    if isinstance(meta, dict):
+        candidates += [meta.get("n_ctx"), *(meta.get(key) for key in _CONTEXT_KEYS)]
+
+    for value in candidates:
+        if isinstance(value, bool):
+            continue
+        if isinstance(value, str):
+            value = value.strip()
+            if not value.isdigit():
+                continue
+            value = int(value)
+        if isinstance(value, int) and MIN_CONTEXT <= value <= MAX_CONTEXT:
+            return value
+    return 0
+
+
 async def stream_chat(
     endpoint: Endpoint,
     payload: dict[str, Any],

src/lembas/web/templates/admin/model_detail.html

@@ -92,6 +92,18 @@
       </div>
     </div>
 
+    <div class="field">
+      <label class="field__label" for="context-length">Context length</label>
+      <input class="input" id="context-length" name="context_length" type="number"
+             min="0" step="1" placeholder="unknown"
+             value="{{ model.context_length or '' }}">
+      <p class="field__hint">
+        How many tokens this model can hold, filled in from the endpoint where
+        it says. Leave it empty if you do not know: the context percentage and
+        automatic compaction both stay off rather than working from a guess.
+      </p>
+    </div>
+
     <div class="field">
       <label class="field__label" for="description">Description</label>
       <textarea class="textarea" id="description" name="description" rows="2"

tests/test_metrics.py

@@ -0,0 +1,122 @@
+"""How full the context is, what a reply cost, and how fast it arrived."""
+
+from __future__ import annotations
+
+from fastapi.testclient import TestClient
+from sqlalchemy import select
+
+from lembas.db.models import Connection, Model
+from lembas.services.crypto import encrypt
+from lembas.services.llm.openai_client import context_from
+
+
+def _model(db, **kwargs) -> Model:
+    connection = Connection(
+        name="Test", base_url="http://127.0.0.1:1", api_key_encrypted=encrypt("")
+    )
+    db.add(connection)
+    db.commit()
+    model = Model(connection_id=connection.id, model_id="test-model", **kwargs)
+    db.add(model)
+    db.commit()
+    return model
+
+
+# --- Reading a context length off /v1/models ---------------------------------
+def test_context_length_is_read_from_any_of_the_spellings():
+    assert context_from({"id": "m", "context_length": 8192}) == 8192
+    assert context_from({"id": "m", "max_model_len": 32768}) == 32768
+    assert context_from({"id": "m", "context_window": 4096}) == 4096
+    assert context_from({"id": "m", "meta": {"n_ctx": 2048}}) == 2048
+
+
+def test_a_quoted_number_is_accepted_but_a_label_is_not():
+    """Some servers quote it. "8192 tokens" is a label, not a measurement."""
+    assert context_from({"id": "m", "context_length": "8192"}) == 8192
+    assert context_from({"id": "m", "context_length": "8192 tokens"}) == 0
+
+
+def test_an_absent_or_implausible_context_length_is_zero():
+    assert context_from({"id": "m"}) == 0
+    assert context_from({"id": "m", "context_length": 0}) == 0
+    assert context_from({"id": "m", "context_length": 64}) == 0
+    assert context_from({"id": "m", "context_length": 10**9}) == 0
+    # True is an int in Python, and it is not a context length.
+    assert context_from({"id": "m", "context_length": True}) == 0
+
+
+# --- Discovery ---------------------------------------------------------------
+async def test_discovery_fills_in_a_context_length(client: TestClient, db, registered, mock_http):
+    import httpx
+
+    mock_http(
+        lambda _r: httpx.Response(
+            200, json={"data": [{"id": "big-model", "context_length": 16384}]}
+        )
+    )
+    client.post(
+        "/admin/connections",
+        data={"name": "Local", "base_url": "http://x.test", "api_key": ""},
+        follow_redirects=False,
+    )
+    model = db.scalar(select(Model).where(Model.model_id == "big-model"))
+    assert model.context_length == 16384
+
+
+async def test_discovery_never_overwrites_a_number_an_admin_typed(
+    client: TestClient, db, registered, mock_http
+):
+    """A refresh must not undo a correction. Administrators set this precisely
+    because the endpoint was wrong or silent."""
+    import httpx
+
+    mock_http(
+        lambda _r: httpx.Response(200, json={"data": [{"id": "m", "context_length": 4096}]})
+    )
+    client.post(
+        "/admin/connections",
+        data={"name": "Local", "base_url": "http://x.test", "api_key": ""},
+        follow_redirects=False,
+    )
+    model = db.scalar(select(Model).where(Model.model_id == "m"))
+    model.context_length = 131072
+    db.commit()
+
+    connection = db.scalar(select(Connection))
+    client.post(f"/admin/connections/{connection.id}/refresh", follow_redirects=False)
+
+    db.refresh(model)
+    assert model.context_length == 131072
+
+
+# --- The admin field ---------------------------------------------------------
+def test_an_admin_can_set_and_clear_the_context_length(client: TestClient, db, registered):
+    model = _model(db)
+    client.post(
+        f"/admin/models/{model.id}",
+        data={"context_length": "8192", "position": ""},
+        follow_redirects=False,
+    )
+    db.refresh(model)
+    assert model.context_length == 8192
+
+    client.post(
+        f"/admin/models/{model.id}", data={"context_length": "", "position": ""},
+        follow_redirects=False,
+    )
+    db.refresh(model)
+    assert model.context_length == 0
+
+
+def test_junk_in_the_context_length_field_is_ignored_not_a_500(
+    client: TestClient, db, registered
+):
+    model = _model(db, context_length=4096)
+    response = client.post(
+        f"/admin/models/{model.id}",
+        data={"context_length": "eight thousand", "position": ""},
+        follow_redirects=False,
+    )
+    assert response.status_code == 303
+    db.refresh(model)
+    assert model.context_length == 4096