diff --git a/src/lembas/api/admin.py b/src/lembas/api/admin.py index a786531..e0ed952 100644 --- a/src/lembas/api/admin.py +++ b/src/lembas/api/admin.py @@ -14,7 +14,7 @@ from lembas.api.deps import AdminUser, Db from lembas.db.models import Connection, Model, User from lembas.services import settings_store from lembas.services.crypto import UNCHANGED_SENTINEL, decrypt, encrypt, mask -from lembas.services.llm.openai_client import Endpoint, LLMError, list_models +from lembas.services.llm.openai_client import Endpoint, LLMError, context_from, list_models from lembas.web.templating import render log = logging.getLogger(__name__) @@ -198,8 +198,21 @@ async def _refresh_models(db: DBSession, connection: Connection) -> tuple[int, s model_id = str(entry["id"])[:300] seen.add(model_id) if model_id in existing: + # A context length is filled in only when nobody has one yet. A + # refresh must never overwrite a number an administrator typed -- + # they are usually correcting the endpoint. + model = existing[model_id] + if not model.context_length: + model.context_length = context_from(entry) continue - db.add(Model(connection_id=connection.id, model_id=model_id, position=next_position)) + db.add( + Model( + connection_id=connection.id, + model_id=model_id, + position=next_position, + context_length=context_from(entry), + ) + ) next_position += 1 # Models that vanished upstream are dropped, so the picker never offers diff --git a/src/lembas/api/admin_models.py b/src/lembas/api/admin_models.py index 813b488..7a3175e 100644 --- a/src/lembas/api/admin_models.py +++ b/src/lembas/api/admin_models.py @@ -2,6 +2,7 @@ from __future__ import annotations +import contextlib import logging from fastapi import APIRouter, File, Form, HTTPException, Request, Response, UploadFile, status @@ -12,6 +13,7 @@ from sqlalchemy.orm import Session as DBSession from lembas.api.deps import AdminUser, Db, RequiredUser from lembas.db.models import Connection, Group, Model from lembas.services import settings_store, uploads +from lembas.services.llm.openai_client import MAX_CONTEXT from lembas.web.templating import render log = logging.getLogger(__name__) @@ -213,6 +215,7 @@ async def update_model( pinned: bool = Form(False), public: bool = Form(False), position: str = Form(""), + context_length: str = Form(""), group_ids: list[str] = Form(default=[]), capability: list[str] = Form(default=[]), ) -> Response: @@ -221,6 +224,13 @@ async def update_model( model.display_name = display_name.strip()[:300] model.description = description.strip()[:2000] model.system_prompt = system_prompt.strip()[:8000] + # A string, so an emptied field is distinguishable and junk can be ignored + # rather than becoming a 422 -- the same shape `position` uses below. + if context_length.strip(): + with contextlib.suppress(ValueError): + model.context_length = min(max(int(context_length), 0), MAX_CONTEXT) + else: + model.context_length = 0 model.enabled = enabled model.pinned = pinned model.public = public diff --git a/src/lembas/db/models/connection.py b/src/lembas/db/models/connection.py index 31b5189..0e0b7ab 100644 --- a/src/lembas/db/models/connection.py +++ b/src/lembas/db/models/connection.py @@ -114,6 +114,17 @@ class Model(UUIDPrimaryKey, Timestamps, Base): # Default sampling params applied to new chats using this model. params_json: Mapped[dict[str, Any]] = mapped_column(JSONDict, default=dict) + # How many tokens this model can hold. 0 means unknown, which is what an + # endpoint that does not advertise it leaves behind -- and unknown has to + # stay tellable from "small", because the context percentage and automatic + # compaction both refuse to act on a number nobody supplied. + # + # A column rather than a key in capabilities_json: that dict is rebuilt + # wholesale from the submitted checkboxes on every save (api/admin_models.py), + # so a number living in it would be destroyed the next time an administrator + # ticked anything. + context_length: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + connection: Mapped[Connection] = relationship(back_populates="models") groups: Mapped[list[Group]] = relationship( "Group", secondary=model_groups, back_populates="models" diff --git a/src/lembas/services/llm/openai_client.py b/src/lembas/services/llm/openai_client.py index 85445c7..f78878e 100644 --- a/src/lembas/services/llm/openai_client.py +++ b/src/lembas/services/llm/openai_client.py @@ -155,6 +155,41 @@ async def list_models(endpoint: Endpoint) -> list[dict[str, Any]]: return models +# Where the runners that bother to advertise a context length put it. There is +# no standard field, so this is a list of what the common ones actually emit. +_CONTEXT_KEYS = ("context_length", "max_model_len", "context_window", "max_context_length") + +# Below the first, the number is not a context length; above the second it is a +# typo or a different unit. Either way, better to record nothing than a wrong +# figure a percentage would then be computed from. +MIN_CONTEXT = 256 +MAX_CONTEXT = 10_000_000 + + +def context_from(entry: dict[str, Any]) -> int: + """A model's context length as advertised by /v1/models, or 0 if it is not. + + Strings are accepted because some servers quote the number, but only when + they are digits alone -- "8192 tokens" is a label, not a measurement. + """ + candidates = [entry.get(key) for key in _CONTEXT_KEYS] + meta = entry.get("meta") + if isinstance(meta, dict): + candidates += [meta.get("n_ctx"), *(meta.get(key) for key in _CONTEXT_KEYS)] + + for value in candidates: + if isinstance(value, bool): + continue + if isinstance(value, str): + value = value.strip() + if not value.isdigit(): + continue + value = int(value) + if isinstance(value, int) and MIN_CONTEXT <= value <= MAX_CONTEXT: + return value + return 0 + + async def stream_chat( endpoint: Endpoint, payload: dict[str, Any], diff --git a/src/lembas/web/templates/admin/model_detail.html b/src/lembas/web/templates/admin/model_detail.html index 9ab262a..2e56eb3 100644 --- a/src/lembas/web/templates/admin/model_detail.html +++ b/src/lembas/web/templates/admin/model_detail.html @@ -92,6 +92,18 @@ +
+ How many tokens this model can hold, filled in from the endpoint where + it says. Leave it empty if you do not know: the context percentage and + automatic compaction both stay off rather than working from a guess. +
+