Custom connections are gone: the CLI talks to the LLeMbas server it is logged in to
LLeMbas CLI 2.0 is LLeMbas-server-specific (CLI rule 2). Removed: the anthropic, gemini, ollama and responses clients, context discovery and llama-swap unloading (discover.ts), the Connection schema with its dialect, auth, key_cmd, discover, unload_url and one_model_at_a_time, the model `fallback` (it only ever moved to another server), effort_map and cache, and the multi-connection refs. Kept: openai-chat, http, sse, think, effort, tokens and all of src/lembas. - connections.yaml holds the login (`type: webui`, as lembas login writes it). A custom entry from 1.x is read past with a warning that says how to log in, so an old file still starts; with several logins left, the latest is used and the others named. - A model's ref is the id the server serves it under; the older forms (`<login>/<id>`, `<login>/<model>`) still resolve and config.yaml is rewritten once. Without a login there is no model, and the error (and the first-run screen, now the sign-in alone) says `lembas login <address>`. - A login to another server signs this machine out of the previous one. - The window is the server's word (no discovery); a capacity group comes from the server's `capacity.group` alone; embeddings use the server's /v1/embeddings. - `lembas models` lists the server's models; install.sh's starter files, the README and the JSON schemas say the same. Tests: every fixture is a login now (the fake server lists models, answers other GETs with 404, and can hand one model to another script, as a server with several backends); the dialect suites and the cross-connection tests are gone with their code. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
80 files changed
+587
-2575
No files matched your search
+1
-20
@@ -17,7 +17,6 @@ import { DEFAULT_RULES, toRules } from "./permission/evaluate.ts"
|
||||
import { hardlineRules } from "./permission/hardline.ts"
|
||||
import { defaultEffort } from "./provider/effort.ts"
|
||||
import { clientFor, familyOf, resolveModel } from "./provider/index.ts"
|
||||
import { discoverContext, unload } from "./provider/discover.ts"
|
||||
import { assembleSystem, instructionFiles, knowledgeScope } from "./prompt/assemble.ts"
|
||||
import { describeRepo, repoState } from "./git/repo.ts"
|
||||
import { findProject, persistProjectRule, trustOf, type Project } from "./project/root.ts"
|
||||
@@ -100,7 +99,7 @@ export interface App {
|
||||
/** The subagents the task tool offers (built-in, global, and a trusted project's), as of start. */
|
||||
agents: import("./project/agents.ts").AgentSpec[]
|
||||
store: Store | undefined
|
||||
/** Every configured `connection/model`, in config order. */
|
||||
/** Every model the server serves, by its id there, in its order. */
|
||||
modelRefs(): string[]
|
||||
/** A ref in any form it may be written in (the old `<login>/<model>` too) as the ref the
|
||||
* model has now; the ref itself when it names nothing. */
|
||||
@@ -497,7 +496,6 @@ export function createApp(o: AppOptions): App {
|
||||
...(at ? { root: at.root, cwd: at.cwd, projectDir: undefined } : {}),
|
||||
jobs: { registry, owner: () => ({ sessionId: childSessionId, job }) },
|
||||
},
|
||||
fallback: engine.o.fallback,
|
||||
steerMark: MARKS.to_subagent,
|
||||
system: (mode, current, plan) =>
|
||||
system(
|
||||
@@ -567,16 +565,6 @@ export function createApp(o: AppOptions): App {
|
||||
system: (m, current, plan) => system(m, current, undefined, undefined, plan),
|
||||
persistRule: trusted ? (rule) => persistProjectRule(project, rule) : undefined,
|
||||
spawn,
|
||||
fallback(current, tried) {
|
||||
for (const ref of current.spec.fallback ?? []) {
|
||||
if (tried.has(ref)) continue
|
||||
try {
|
||||
const m = resolveModel(loaded, ref)
|
||||
return { model: m, client: clientFor(m), effort: startEffort(m) }
|
||||
} catch {}
|
||||
}
|
||||
return undefined
|
||||
},
|
||||
})
|
||||
|
||||
// MCP tools join the engine's list as servers connect (the engine reads it at each prompt).
|
||||
@@ -647,18 +635,11 @@ export function createApp(o: AppOptions): App {
|
||||
const snapshots = o.snapshots === false ? undefined : new Snapshots(project.root, project.gitRoot)
|
||||
engine.busyInput = loaded.config.busy_input ?? "steer"
|
||||
const turns = new Turns(engine, snapshots, store)
|
||||
// After a reply, learn the window if nobody configured it (the model is loaded by then).
|
||||
bus.on((e) => {
|
||||
if (e.type === "done" && !engine.model.spec.context) void discoverContext(engine.model, engine.client).then((n) => n && engine.usage && bus.emit({ type: "usage", usage: engine.usage, context: n, used: engine.contextUsed() }))
|
||||
})
|
||||
// By provider for an instance's models (`deepseek/deepseek-flash`), src/provider/refs.ts.
|
||||
const modelRefs = () => loaded.refs.map((r) => r.ref)
|
||||
const switchModel = (ref: string) => {
|
||||
const m = resolveModel(loaded, ref)
|
||||
const prev = engine.model
|
||||
engine.setModel(m, clientFor(m), startEffort(m))
|
||||
// Moving to another connection frees the old one's memory (llama-swap /unload).
|
||||
if (prev.connectionName !== m.connectionName) void unload(prev)
|
||||
}
|
||||
const setIn = (key: string, value: unknown) => {
|
||||
const path = key.split(".")
|
||||
|
||||
+7
-21
@@ -12,7 +12,7 @@ import { renderJson, renderPlain, settleJobs, terminalAsker } from "./headless.t
|
||||
import { enterLocal, localFor, LocalError, localRefusal, localTmp, LOCAL_FLAGS } from "./project/local.ts"
|
||||
import { findProject, setTrust, trustOf } from "./project/root.ts"
|
||||
import { effortsFor } from "./provider/effort.ts"
|
||||
import { clientFor, ModelError, resolveModel } from "./provider/index.ts"
|
||||
import { clientFor, ModelError, NOT_LOGGED_IN, resolveModel } from "./provider/index.ts"
|
||||
import { Store } from "./session/store.ts"
|
||||
import { promptTitle } from "./session/title.ts"
|
||||
import { VERSION } from "./version.ts"
|
||||
@@ -23,7 +23,7 @@ Usage:
|
||||
lembas [-l] [-m model] [--mode M] [-c | -s id]
|
||||
the TUI (-c continues the last session here, -s resumes one)
|
||||
lembas run [options] "…" one prompt, headless
|
||||
lembas models [--discover] list configured models
|
||||
lembas models list the server's models
|
||||
lembas sessions list recent sessions
|
||||
lembas sessions delete <id>… delete sessions, here and in the web UI
|
||||
lembas sessions prune [--dry-run]
|
||||
@@ -110,7 +110,7 @@ async function loginCommand(cmd: "login" | "logout", args: string[]) {
|
||||
if (!name) fail(Object.keys(known).length ? `which one? lembas logout <${Object.keys(known).join("|")}>` : "not logged in to any instance", 2)
|
||||
const r = await logout(name)
|
||||
console.log(`Signed out of ${name}${r.revoked ? "" : " here (the instance could not be told — revoke it under User → Security → Devices there)"}; its connection is removed.`)
|
||||
if (r.modelLeft) console.log(`config.yaml still starts on ${r.modelLeft}: choose another with lembas config set model <connection/model>.`)
|
||||
if (r.modelLeft) console.log(`config.yaml still starts on ${r.modelLeft}: choose another with lembas config set model <id>.`)
|
||||
return
|
||||
}
|
||||
let address = positionals[0] ?? ""
|
||||
@@ -305,17 +305,11 @@ async function main() {
|
||||
|
||||
if (cmd === "models") {
|
||||
const loaded = loadConfig()
|
||||
const discover = argv.includes("--discover")
|
||||
if (!Object.keys(loaded.connections).length && !Object.keys(loaded.broken).length) console.log(NOT_LOGGED_IN)
|
||||
for (const [name, c] of Object.entries(loaded.connections)) {
|
||||
console.log(
|
||||
c.webui
|
||||
? `${name} (webui, ${c.webui.url}${c.webui.name ? ` — ${c.webui.name}` : ""}; models ${c.webui.fetched_at ? `read ${c.webui.fetched_at.slice(0, 16).replace("T", " ")}` : "not read yet"})`
|
||||
: `${name} (${c.dialect}, ${c.base_url})`,
|
||||
)
|
||||
console.log(`${name} (${c.webui?.url ?? c.base_url}${c.webui?.name ? ` — ${c.webui.name}` : ""}; models ${c.webui?.fetched_at ? `read ${c.webui.fetched_at.slice(0, 16).replace("T", " ")}` : "not read yet"})`)
|
||||
for (const [id, spec] of Object.entries(c.models)) {
|
||||
// Its ref: an instance's model by provider, anything else connection/model.
|
||||
const ref = loaded.refs.find((r) => r.connection === name && r.id === id)?.ref ?? `${name}/${id}`
|
||||
const m = resolveModel(loaded, ref)
|
||||
const m = resolveModel(loaded, id)
|
||||
const bits = [
|
||||
spec.context ? `ctx ${spec.context}` : "",
|
||||
spec.max_output ? `out ${spec.max_output}` : "",
|
||||
@@ -323,15 +317,7 @@ async function main() {
|
||||
spec.tools === false ? "no-tools" : "",
|
||||
spec.effort || spec.efforts ? `effort ${spec.effort ?? "off"} of ${effortsFor(m).join("/")}` : "",
|
||||
].filter(Boolean)
|
||||
console.log(` ${ref}${bits.length ? " — " + bits.join(", ") : ""}`)
|
||||
}
|
||||
if (discover) {
|
||||
try {
|
||||
const found = await clientFor(resolveModel(loaded, `${name}/${Object.keys(c.models)[0] ?? "?"}`)).listModels()
|
||||
for (const d of found) if (!c.models[d.id]) console.log(` ${name}/${d.id} (discovered${d.context ? `, ctx ${d.context}` : ""})`)
|
||||
} catch (e) {
|
||||
console.log(` (discovery failed: ${(e as Error).message})`)
|
||||
}
|
||||
console.log(` ${m.ref}${bits.length ? " — " + bits.join(", ") : ""}`)
|
||||
}
|
||||
}
|
||||
for (const [name, why] of Object.entries(loaded.broken)) console.log(`${name} ✗ ${why}`)
|
||||
|
||||
@@ -29,7 +29,7 @@ export function jsonSchemas(): Record<keyof typeof SCHEMA_FILES, Record<string,
|
||||
})
|
||||
return {
|
||||
config: make(Config, "LLeMbas CLI config.yaml", "~/.config/lembas/config.yaml, or a project's .agent/config.yaml. Strings may be {env:NAME} or {file:path}."),
|
||||
connections: make(ConnectionsFile, "LLeMbas CLI connections.yaml", "~/.config/lembas/connections.yaml: the LLM endpoints. Global only. Keys as {env:NAME}, {file:path} or key_cmd — never pasted."),
|
||||
connections: make(ConnectionsFile, "LLeMbas CLI connections.yaml", "~/.config/lembas/connections.yaml: the LLeMbas server this machine is logged in to, as lembas login writes it. Global only."),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+37
-36
@@ -3,8 +3,8 @@ import { join } from "node:path"
|
||||
import { isScalar, parse, parseDocument } from "yaml"
|
||||
import { z } from "zod"
|
||||
import { connectionFiles, isLocal, paths } from "./paths.ts"
|
||||
import { asMode, Config, Connection, ConnectionsFile, McpServer, PERSONALITIES, renamePermissionKeys, WebuiConnection, type PermissionConfig } from "./schema.ts"
|
||||
import { runKeyCmd, substituteDeep } from "./substitute.ts"
|
||||
import { asMode, Config, McpServer, PERSONALITIES, renamePermissionKeys, WebuiConnection, type Connection, type PermissionConfig } from "./schema.ts"
|
||||
import { substituteDeep } from "./substitute.ts"
|
||||
import { expandWebui, refreshFailures, type WebuiMark } from "../lembas/webui.ts"
|
||||
import { defaultLibrary, withInstancePersonalization } from "../lembas/personal.ts"
|
||||
import { instances } from "../lembas/login.ts"
|
||||
@@ -14,8 +14,8 @@ export class ConfigError extends Error {}
|
||||
|
||||
export interface Loaded {
|
||||
config: Config
|
||||
/** A `type: webui` entry is here as the openai-chat connection it is spoken to as, with the
|
||||
* instance's models and a `webui` mark. */
|
||||
/** The LLeMbas server this machine is logged in to — at most one: its `webui` entry as the
|
||||
* `/v1` it is spoken to at, with its models and a `webui` mark (custom connections are gone). */
|
||||
connections: Record<string, Connection & { webui?: WebuiMark }>
|
||||
/** Connections whose {env:}/{file:} substitution failed, with the reason. Unusable, not fatal. */
|
||||
broken: Record<string, string>
|
||||
@@ -270,41 +270,47 @@ export function loadConfig(opts: { projectConfigDir?: string; trusted?: boolean
|
||||
|
||||
const connections: Loaded["connections"] = {}
|
||||
const broken: Record<string, string> = {}
|
||||
// Validate each raw file, then substitute each connection on its own: a missing variable for one
|
||||
// connection must not stop the others from working. In local mode, the user's own file first
|
||||
// (read, never written), then the local one, whose entries win by name.
|
||||
// The login: a `webui` entry, as `lembas login` writes it. An entry of any other kind is a custom
|
||||
// connection from 1.x, gone in 2.0 (the CLI is LLeMbas-server-specific): said, and read past, so
|
||||
// an old file still starts. In local mode, the user's own file first (read, never written), then
|
||||
// the local one, whose entries win by name. Each entry is substituted on its own.
|
||||
const raw: Record<string, unknown> = {}
|
||||
for (const connFile of connectionFiles()) {
|
||||
files.push(connFile)
|
||||
const mode = statSync(connFile).mode & 0o777
|
||||
if (mode & 0o077) warnings.push(`${connFile} is mode ${mode.toString(8)}; chmod 600 it — it references credentials`)
|
||||
Object.assign(raw, validate(ConnectionsFile, readYaml(connFile), connFile).connections)
|
||||
const doc = readYaml(connFile)
|
||||
const entries = doc && typeof doc === "object" ? (doc as { connections?: unknown }).connections : undefined
|
||||
if (entries === undefined || entries === null) continue
|
||||
if (typeof entries !== "object" || Array.isArray(entries)) throw new ConfigError(`${connFile} is invalid:\n connections: expected the login as written by lembas login`)
|
||||
for (const [name, c] of Object.entries(entries as Record<string, unknown>)) {
|
||||
if ((c as { type?: string } | null)?.type === "webui") raw[name] = c
|
||||
else warnings.push(`${connFile}: ${name} is a custom connection, and LLeMbas CLI 2.0 has none — it talks only to the LLeMbas server it is logged in to (lembas login <address>). Ignored; delete it.`)
|
||||
}
|
||||
}
|
||||
for (const [name, c] of Object.entries(raw)) {
|
||||
// One server: the login made last, where several are left from 1.x.
|
||||
const logins = instances()
|
||||
const names = Object.keys(raw)
|
||||
const chosen = names.length > 1 ? [...names].sort((a, b) => (logins[b]?.logged_in_at ?? "").localeCompare(logins[a]?.logged_in_at ?? ""))[0]! : names[0]
|
||||
if (names.length > 1) warnings.push(`connections.yaml holds ${names.length} LLeMbas logins (${names.join(", ")}); LLeMbas CLI 2.0 uses one server — ${chosen}, the latest. lembas logout the others.`)
|
||||
if (chosen) {
|
||||
const name = chosen
|
||||
try {
|
||||
if ((c as { type?: string }).type === "webui") {
|
||||
const w = WebuiConnection.safeParse(substituteDeep(c, `connections.${name}`))
|
||||
if (w.success) {
|
||||
const c = (connections[name] = expandWebui(name, w.data))
|
||||
if (refreshFailures[name])
|
||||
warnings.push(
|
||||
`${name}: the instance could not be asked for its models (${refreshFailures[name]}); ${c.webui.fetched_at ? `using what it said at ${c.webui.fetched_at.slice(0, 16).replace("T", " ")}` : "none known yet"}`,
|
||||
)
|
||||
}
|
||||
else broken[name] = `once {env:}/{file:} are filled in: ${w.error.issues.map((i) => `${i.path.join(".")}: ${i.message}`).join("; ")}`
|
||||
continue
|
||||
}
|
||||
// Again once filled in: "{env:LLM_URL}" passed as a placeholder; what it became must be a URL.
|
||||
const r = Connection.safeParse(substituteDeep(c, `connections.${name}`))
|
||||
if (r.success) connections[name] = r.data
|
||||
else broken[name] = `once {env:}/{file:} are filled in: ${r.error.issues.map((i) => `${i.path.join(".")}: ${i.message}`).join("; ")}`
|
||||
const w = WebuiConnection.safeParse(substituteDeep(raw[name], `connections.${name}`))
|
||||
if (w.success) {
|
||||
const c = (connections[name] = expandWebui(name, w.data))
|
||||
if (refreshFailures[name])
|
||||
warnings.push(
|
||||
`${name}: the instance could not be asked for its models (${refreshFailures[name]}); ${c.webui.fetched_at ? `using what it said at ${c.webui.fetched_at.slice(0, 16).replace("T", " ")}` : "none known yet"}`,
|
||||
)
|
||||
} else broken[name] = `once {env:}/{file:} are filled in: ${w.error.issues.map((i) => `${i.path.join(".")}: ${i.message}`).join("; ")}`
|
||||
} catch (e) {
|
||||
broken[name] = (e as Error).message
|
||||
}
|
||||
}
|
||||
|
||||
/** The webui connection a voice half or the search means: the one named, or the only one. A
|
||||
* string says why there is none. */
|
||||
/** The server a voice half or the search goes through: the login (`connection:` may still name
|
||||
* it). A string says why there is none. */
|
||||
const webuiFor = (named?: string): { name: string; c: Loaded["connections"][string] } | string => {
|
||||
if (named) return connections[named]?.webui ? { name: named, c: connections[named]! } : broken[named] ? `${named} is unusable: ${broken[named]}` : `there is no webui connection named ${named}`
|
||||
const all = Object.entries(connections).filter(([, c]) => c.webui)
|
||||
@@ -408,10 +414,7 @@ export function loadConfig(opts: { projectConfigDir?: string; trusted?: boolean
|
||||
config = personal.config
|
||||
if (config.library === undefined && !isLocal() && defaultLibrary()) config = { ...config, library: "lembas" }
|
||||
|
||||
// An instance that speaks protocol 2 serves `<provider>/<model>` ids.
|
||||
const logins = instances()
|
||||
for (const [name, c] of Object.entries(connections)) if (c.webui && logins[name]?.protocols?.includes(2)) c.webui.v2 = true
|
||||
const refs = modelRefs(connections, Object.keys(logins))
|
||||
const refs = modelRefs(connections)
|
||||
// A config.yaml naming the model the way it was named before providers (`example/deepseek-flash`)
|
||||
// is rewritten to the new name, once (migrateRefs). Only that form: a pinned `<login>/<provider>/
|
||||
// <model>` means what it says, and `embedding` stays as written — its name keys the stored vectors.
|
||||
@@ -480,9 +483,7 @@ function localOnly(config: Config, connections: Loaded["connections"], warnings:
|
||||
return out
|
||||
}
|
||||
|
||||
/** The API key for a connection: `api_key` (already substituted) or the output of `key_cmd`. */
|
||||
export function resolveKey(name: string, c: Connection): string | undefined {
|
||||
if (c.api_key) return c.api_key
|
||||
if (c.key_cmd) return runKeyCmd(c.key_cmd, `connections.${name}.key_cmd`)
|
||||
return undefined
|
||||
/** The server's token: the login's `api_key`, already substituted from its key file. */
|
||||
export function resolveKey(_name: string, c: Pick<Connection, "api_key">): string | undefined {
|
||||
return c.api_key || undefined
|
||||
}
|
||||
+39
-40
@@ -1,6 +1,5 @@
|
||||
import { z } from "zod"
|
||||
|
||||
export const DIALECTS = ["openai-chat", "responses", "anthropic", "gemini", "ollama"] as const
|
||||
export const EFFORTS = ["minimal", "low", "medium", "high", "xhigh", "max"] as const
|
||||
export const MODES = ["manual", "edit", "auto", "plan"] as const
|
||||
/** The personality presets (harness/prompts/personality/, shared with the web UI), then custom. */
|
||||
@@ -54,46 +53,44 @@ export const ModelSpec = z.strictObject({
|
||||
top_p: z.number().min(0).max(1).optional().describe("Nucleus sampling; absent = the server's default."),
|
||||
efforts: z.array(z.enum(EFFORTS)).optional().describe("The model's own effort vocabulary, in offering order."),
|
||||
effort: z.union([z.enum(EFFORTS), z.literal("off")]).optional().describe("Default effort; `off` sends none."),
|
||||
effort_style: z.enum(["top", "kwargs", "both"]).optional().describe("openai-chat only: where the effort goes. llama.cpp drops top-level, so `both` is the default."),
|
||||
effort_map: z.partialRecord(z.enum(EFFORTS), z.number().int().positive()).optional().describe("anthropic/gemini: effort → thinking budget tokens."),
|
||||
effort_style: z.enum(["top", "kwargs", "both"]).optional().describe("Where the effort goes in a request: top-level, in chat_template_kwargs, or both (the default: llama.cpp drops top-level)."),
|
||||
vision: z.boolean().optional().describe("The model reads images: @image files, pasted paths and view_image."),
|
||||
tools: z.boolean().optional().describe("false: a model without tool calls — it can only talk."),
|
||||
cache: z.boolean().optional().describe("anthropic: prompt caching breakpoints."),
|
||||
headers: z.record(z.string(), z.string()).optional().describe("Extra HTTP headers for this model's requests."),
|
||||
body: z.record(z.string(), z.unknown()).optional().describe("Merged into every request body for this model."),
|
||||
notes: z.string().max(300).optional().describe("What this model is good at, in a line: the model in use is told which others there are, for handing work over (task) or switching."),
|
||||
fallback: z.array(z.string()).optional().describe("Other models (connection/model), in order, to switch to when this one's server cannot be reached at all — never once a reply has begun."),
|
||||
single_session: z.boolean().optional().describe("The server serves one request at a time: a subagent on this same model would take its only slot and push the session's cached prompt out, so it is refused."),
|
||||
group: z.string().optional().describe("Models sharing one server that holds one model at a time (llama-swap), named by any string: a subagent on another model of the same group would unload this one, so it is refused. One connection's one_model_at_a_time is the same rule for all its models; this is for models one connection serves from several such servers (a LLeMbas instance)."),
|
||||
group: z.string().optional().describe("Models sharing one GPU server that holds one model at a time (llama-swap), as the LLeMbas server says (capacity.group): a subagent on another model of the same group would unload this one, so it is refused."),
|
||||
})
|
||||
export type ModelSpec = z.infer<typeof ModelSpec>
|
||||
|
||||
export const Connection = z.strictObject({
|
||||
dialect: z.enum(DIALECTS).describe("The API it speaks: openai-chat (llama.cpp, vLLM, LM Studio, OpenAI…), responses (OpenAI Responses), anthropic, gemini, ollama (its native /api/chat)."),
|
||||
base_url: Url.describe("Where the API is, e.g. https://api.openai.com/v1 or http://localhost:8080/v1."),
|
||||
api_key: z.string().optional().describe("Never pasted: {env:NAME} or {file:~/path}. Or use key_cmd."),
|
||||
key_cmd: z.string().optional().describe("A command whose output is the key, e.g. pass show openai."),
|
||||
auth: z.enum(["bearer", "x-api-key", "x-goog-api-key", "none"]).optional().describe("How the key is sent. Default per dialect: anthropic x-api-key, gemini x-goog-api-key, others bearer."),
|
||||
headers: z.record(z.string(), z.string()).optional().describe("Extra HTTP headers for every request."),
|
||||
body: z.record(z.string(), z.unknown()).optional().describe("Merged into every request body."),
|
||||
tls: Tls.optional().describe("TLS for endpoints behind a private CA. `ca` is a PEM file added to the trusted roots; `insecure` skips verification entirely (last resort). The system store is always trusted."),
|
||||
timeout: z.number().positive().optional().describe("Seconds of silence allowed: waiting for the response, then between two pieces of a streamed reply. Not a limit on how long a reply may take. Default 600."),
|
||||
discover: z.boolean().optional().describe("Merge ids and context lengths from GET /models."),
|
||||
unload_url: Url.optional().describe("llama-swap style: frees VRAM when switching away."),
|
||||
one_model_at_a_time: z.boolean().optional().describe("The server holds one model at a time (llama-swap in front of one GPU): a subagent on another of its models would unload the session's, so it is refused. The session's own model may still be its own subagent."),
|
||||
unload_method: z.enum(["GET", "POST"]).optional().describe("How unload_url is called (default POST)."),
|
||||
quirks: z
|
||||
.strictObject({
|
||||
stream_usage: Tristate.optional().describe("Ask for token usage in the stream (stream_options); auto learns when a server refuses it."),
|
||||
prompt_progress: Tristate.optional().describe("Ask how far the server is through reading the prompt (llama.cpp's return_progress), shown while it reads; auto learns when a server refuses it, and one that ignores it shows nothing."),
|
||||
think_tags: Tristate.optional().describe("<think>…</think> in a reply is taken as reasoning; off leaves it as text."),
|
||||
max_tokens_field: z.enum(["max_tokens", "max_completion_tokens"]).optional().describe("OpenAI reasoning models refuse `max_tokens`; most local servers only know it."),
|
||||
})
|
||||
.optional()
|
||||
.describe("Server oddities; auto is right almost always."),
|
||||
models: z.record(z.string(), ModelSpec).default({}).describe("The models to offer, by the id the server knows them by."),
|
||||
})
|
||||
export type Connection = z.infer<typeof Connection>
|
||||
/** Server oddities; auto is right almost always. */
|
||||
export const Quirks = z
|
||||
.strictObject({
|
||||
stream_usage: Tristate.optional().describe("Ask for token usage in the stream (stream_options); auto learns when a server refuses it."),
|
||||
prompt_progress: Tristate.optional().describe("Ask how far the server is through reading the prompt (llama.cpp's return_progress), shown while it reads; auto learns when a server refuses it, and one that ignores it shows nothing."),
|
||||
think_tags: Tristate.optional().describe("<think>…</think> in a reply is taken as reasoning; off leaves it as text."),
|
||||
max_tokens_field: z.enum(["max_tokens", "max_completion_tokens"]).optional().describe("OpenAI reasoning models refuse `max_tokens`; most local servers only know it."),
|
||||
})
|
||||
.optional()
|
||||
.describe("Server oddities; auto is right almost always.")
|
||||
|
||||
/** The LLeMbas server as LLeMbas CLI speaks to it: its OpenAI-shaped `/v1` with this machine's
|
||||
* token, and the models it serves. Not something a person writes — custom connections were removed
|
||||
* in 2.0 (CLI rule 2: the CLI is LLeMbas-server-specific); `lembas login` writes the one `webui`
|
||||
* entry below, and the loader turns it into this (lembas/webui.ts, expandWebui). */
|
||||
export interface Connection {
|
||||
base_url: string
|
||||
api_key?: string
|
||||
tls?: z.infer<typeof Tls>
|
||||
/** Seconds of silence allowed in a reply (default 600). */
|
||||
timeout?: number
|
||||
quirks?: z.infer<typeof Quirks>
|
||||
/** Extra headers and body fields for every request (none from a login; tests use them). */
|
||||
headers?: Record<string, string>
|
||||
body?: Record<string, unknown>
|
||||
models: Record<string, ModelSpec>
|
||||
}
|
||||
|
||||
/** A LLeMbas instance: its address and this machine's key, and nothing else. The models,
|
||||
* their settings, and — where config.yaml says `provider: webui` — the voice and the web search come
|
||||
@@ -101,20 +98,22 @@ export type Connection = z.infer<typeof Connection>
|
||||
export const WebuiConnection = z.strictObject({
|
||||
type: z.literal("webui").describe("A LLeMbas instance: its models, their settings, its voice and its web search."),
|
||||
url: Url.describe("The instance's address, e.g. https://ai.example.org (no /v1)."),
|
||||
api_key: z.string().optional().describe("This machine's token, as login wrote it: {file:~/.config/lembas/lembas/<name>.key}. Or key_cmd."),
|
||||
key_cmd: z.string().optional(),
|
||||
api_key: z.string().optional().describe("This machine's token, as login wrote it: {file:~/.config/lembas/lembas/<name>.key}."),
|
||||
tls: Tls.optional().describe("For an instance behind a private CA: `ca`, the PEM file that signed its certificate."),
|
||||
timeout: z.number().positive().optional().describe("Seconds of silence allowed in a reply, as for any connection (default 600)."),
|
||||
quirks: Connection.shape.quirks,
|
||||
quirks: Quirks,
|
||||
models: z.record(z.string(), ModelSpec).optional().describe("Only to change what the instance says about a model, here: each entry goes over the instance's. Usually absent."),
|
||||
})
|
||||
export type WebuiConnection = z.infer<typeof WebuiConnection>
|
||||
|
||||
/** connections.yaml: the login, as `lembas login` writes it. An entry of any other kind — a custom
|
||||
* connection from 1.x — is read past with a warning (config/load.ts), not refused, so an old file
|
||||
* still starts. */
|
||||
export const ConnectionsFile = z.strictObject({
|
||||
connections: z
|
||||
.record(z.string(), z.union([WebuiConnection, Connection]))
|
||||
.record(z.string(), WebuiConnection)
|
||||
.default({})
|
||||
.describe("Each LLM endpoint by a name of your choosing; a model is then `name/model-id`. `type: webui` is a LLeMbas instance, whose models come from it."),
|
||||
.describe("The LLeMbas server this machine is logged in to, as `lembas login` writes it: its address and this machine's token. Its models come from it at every start."),
|
||||
})
|
||||
export type ConnectionsFile = z.infer<typeof ConnectionsFile>
|
||||
|
||||
@@ -231,8 +230,8 @@ export const Update = z.strictObject({
|
||||
export type Update = z.infer<typeof Update>
|
||||
|
||||
export const Config = z.strictObject({
|
||||
model: z.string().optional().describe("The model a session starts with: `connection/model`, a connection from connections.yaml and one of its models."),
|
||||
small_model: z.string().optional().describe("`connection/model` for small jobs — session titles, commit messages. Default: the session's model. On a server that holds one model at a time (llama-swap), another model here swaps."),
|
||||
model: z.string().optional().describe("The model a session starts with: one the LLeMbas server you are logged in to serves, by its id there (`deepseek/deepseek-flash`). Default: the server's default for your account."),
|
||||
small_model: z.string().optional().describe("The model for small jobs — session titles, commit messages, by its id on the server. Default: the session's model."),
|
||||
titles: z.enum(["model", "prompt"]).optional().describe("Session titles: model (default) — named from the first prompt, then by the model after its first reply; prompt — only from the first prompt, no extra request."),
|
||||
mode: z.enum(MODES).optional().describe("The permission mode a session starts in (default manual)."),
|
||||
theme: z.string().optional().describe("Colours: moria (default), shire, lembas, lembas-light, hermes, mono, or a themes/<name>.yaml of your own."),
|
||||
@@ -293,7 +292,7 @@ export const Config = z.strictObject({
|
||||
voice: Voice.optional().describe("Voice in and out. Global only — it carries endpoints and keys, like a connection."),
|
||||
mcp: z.record(z.string(), McpServer).optional().describe("MCP servers by name. `{env:}`/`{file:}` are filled in per server (a missing one disables that server only). A project's servers count only once the project is trusted."),
|
||||
knowledge: z.array(z.string()).optional().describe("The knowledge bases searched (lembas kb); absent: all of them. A project's own config narrows it to what that project needs."),
|
||||
embedding: z.string().optional().describe("Global config only. connection/model of an embedding model (OpenAI-shaped /embeddings, or Ollama): knowledge bases are then searched by meaning as well as by words. lembas kb reindex after changing it."),
|
||||
embedding: z.string().optional().describe("Global config only. An embedding model the LLeMbas server serves, by its id there: knowledge bases are then searched by meaning as well as by words. lembas kb reindex after changing it."),
|
||||
effort: z.union([z.enum(EFFORTS), z.literal("off")]).optional().describe("The effort a session starts with, when the model takes it; otherwise the model's own default."),
|
||||
settings_tool: z.enum(["ask", "allow", "off"]).optional().describe("Global config only. The agent's settings tool: ask (default) — each change is approved like any tool call; allow — changes go through without asking (a less strict mode still asks); off — no settings tool."),
|
||||
update: Update.optional().describe("Global config only. Updating LLeMbas CLI itself."),
|
||||
|
||||
@@ -19,7 +19,7 @@ export type Source = "default" | "global" | "project" | "session"
|
||||
|
||||
export interface SettingDef {
|
||||
key: string
|
||||
/** model: a configured `connection/model`; effort: one the model takes, or off; list: comma-separated. */
|
||||
/** model: a model the server serves, by its id; effort: one the model takes, or off; list: comma-separated. */
|
||||
kind: "enum" | "model" | "effort" | "number" | "boolean" | "list" | "string"
|
||||
values?: readonly string[]
|
||||
/** The agent may change it (through the settings tool, with approval). */
|
||||
|
||||
+11
-1
@@ -7,7 +7,8 @@
|
||||
// pasted into the YAML. The models are not written anywhere: a webui connection reads them from the
|
||||
// instance at every start (webui.ts). `lembas.json` remembers which connections a login made, so
|
||||
// `login` again refreshes them and `logout` removes exactly them. Without a login nothing here is
|
||||
// read, and LLeMbas CLI runs on its hand-written config as before.
|
||||
// read, and LLeMbas CLI has no model to talk to: 2.0 talks to the server it is logged in to, one at
|
||||
// a time, and nothing else.
|
||||
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"
|
||||
import { join, resolve } from "node:path"
|
||||
import { parseDocument } from "yaml"
|
||||
@@ -416,6 +417,15 @@ export async function login(address: string, o: LoginOptions): Promise<LoginResu
|
||||
changes.push("the previous login could not be signed out: revoke it under User → Security → Devices on the instance")
|
||||
}
|
||||
await handOver(replaced)
|
||||
// One server at a time (2.0: the CLI is LLeMbas-server-specific): a login to another one signs
|
||||
// this machine out of the one it was logged in to.
|
||||
for (const [name, old] of Object.entries(ownInstances()))
|
||||
if (name !== connection && old.base_url !== discovery.base_url) {
|
||||
const out = await logout(name).catch(() => undefined)
|
||||
if (!out) continue
|
||||
changes.push(`signed out of ${old.instance_name ?? old.base_url} (${name}): LLeMbas CLI uses one server at a time`)
|
||||
if (out.modelLeft) changes.push(`config.yaml still starts on ${out.modelLeft}, a model of that server: change it with lembas config set model <id>, or remove it to start on this server's default`)
|
||||
}
|
||||
// Only logins this mode made: in local mode, the user's own login to the same instance is not
|
||||
// signed out or replaced — it is not this project's.
|
||||
const known = ownInstances()
|
||||
|
||||
+2
-14
@@ -19,7 +19,7 @@ import { join } from "node:path"
|
||||
import { parse } from "yaml"
|
||||
import { connectionFiles, isLocal, paths } from "../config/paths.ts"
|
||||
import type { Connection, ModelSpec, WebuiConnection } from "../config/schema.ts"
|
||||
import { runKeyCmd, substituteDeep } from "../config/substitute.ts"
|
||||
import { substituteDeep } from "../config/substitute.ts"
|
||||
import { instanceInfo, LembasError, models as listModels, normalise, personalization, type InstanceInfo, type Personalization, type ServedModel, type Tls } from "./client.ts"
|
||||
|
||||
/** What a loaded webui connection carries besides an openai-chat connection's fields. */
|
||||
@@ -33,14 +33,9 @@ export interface WebuiMark {
|
||||
services?: InstanceInfo["services"]
|
||||
/** When the models were last read from the instance; absent: never. */
|
||||
fetched_at?: string
|
||||
/** Each served model's provider, where the instance said it (`provider`). */
|
||||
providers?: Record<string, string>
|
||||
/** The served ids of its embedding models, which are not chat models and so not in
|
||||
* `models`: `embedding: llama/nomic-embed` finds its connection by them (library/embed.ts). */
|
||||
embeddings?: string[]
|
||||
/** The instance speaks link protocol 2: its ids are `<provider>/<model>` even
|
||||
* where a model does not say its provider. Set by loadConfig from the login. */
|
||||
v2?: boolean
|
||||
}
|
||||
|
||||
export type WebuiLoaded = Connection & { webui: WebuiMark }
|
||||
@@ -134,18 +129,12 @@ export function expandWebui(name: string, w: WebuiConnection, cache = readCache(
|
||||
for (const m of ordered) models[m.id] = { ...specFor(m, name), ...(overrides.get(m.id) ?? {}) }
|
||||
// A model the entry names that the instance does not (yet) list: kept, as written.
|
||||
for (const [id, spec] of unmatched) models[id] ??= spec
|
||||
const providers = Object.fromEntries(served.filter((m) => m.provider && m.id.startsWith(`${m.provider}/`)).map((m) => [m.id, m.provider!]))
|
||||
return {
|
||||
dialect: "openai-chat",
|
||||
base_url: `${url}/v1`,
|
||||
...(w.api_key !== undefined ? { api_key: w.api_key } : {}),
|
||||
...(w.key_cmd !== undefined ? { key_cmd: w.key_cmd } : {}),
|
||||
...(w.tls ? { tls: w.tls } : {}),
|
||||
...(w.timeout ? { timeout: w.timeout } : {}),
|
||||
...(w.quirks ? { quirks: w.quirks } : {}),
|
||||
// The instance decides which models exist: one it serves that the cache does not know yet is
|
||||
// still a model, not an error.
|
||||
discover: true,
|
||||
models,
|
||||
webui: {
|
||||
url,
|
||||
@@ -153,7 +142,6 @@ export function expandWebui(name: string, w: WebuiConnection, cache = readCache(
|
||||
...(cache?.instance?.name ? { name: cache.instance.name } : {}),
|
||||
...(cache?.instance?.services ? { services: cache.instance.services } : {}),
|
||||
...(cache && cache.url === url ? { fetched_at: cache.fetched_at } : {}),
|
||||
...(Object.keys(providers).length ? { providers } : {}),
|
||||
...(embeddings.length ? { embeddings } : {}),
|
||||
},
|
||||
}
|
||||
@@ -200,7 +188,7 @@ export async function refreshWebui(o: { timeoutMs?: number; only?: string } = {}
|
||||
entries.map(async ([name, c]): Promise<Refreshed> => {
|
||||
try {
|
||||
const w = substituteDeep(c, `connections.${name}`) as WebuiConnection
|
||||
const token = w.api_key ?? (w.key_cmd ? runKeyCmd(w.key_cmd, `connections.${name}.key_cmd`) : undefined)
|
||||
const token = w.api_key
|
||||
if (!token) throw new LembasError("no key (lembas login)")
|
||||
const got = await fetchWebui(name, w.url, token.trim(), w.tls, o.timeoutMs ?? 4000)
|
||||
delete refreshFailures[name]
|
||||
|
||||
+19
-33
@@ -1,6 +1,6 @@
|
||||
// Embeddings for the library: `embedding: connection/model` in config.yaml, over the connection's
|
||||
// OpenAI-shaped /embeddings (llama.cpp with --embedding, vLLM, LM Studio, OpenAI…) or Ollama's
|
||||
// /api/embed. Vectors are made unit length here, so a dot product is the cosine.
|
||||
// Embeddings for the library: `embedding:` in config.yaml names an embedding model of the LLeMbas
|
||||
// server, reached at its OpenAI-shaped /v1/embeddings. Vectors are made unit length here, so a dot
|
||||
// product is the cosine.
|
||||
import { resolveKey, type Loaded } from "../config/load.ts"
|
||||
import { authHeaders, joinUrl, request, tlsFor } from "../provider/http.ts"
|
||||
import { resolveModel } from "../provider/index.ts"
|
||||
@@ -15,46 +15,32 @@ function unit(v: number[]): Float32Array {
|
||||
|
||||
export function embedderFor(loaded: Loaded, ref: string | undefined): Embedder | undefined {
|
||||
if (!ref) return undefined
|
||||
// Through the same resolver as every model ref: an instance's model by its provider
|
||||
// (`llama/nomic-embed`) or in the older form (`example/nomic-embed`). The name stored with the
|
||||
// vectors stays the ref as written, so a library indexed under it is still found.
|
||||
// An embedding model need not be listed on its connection (`connection/<any id>`, as before).
|
||||
let name: string
|
||||
// An embedding model of the server, by its served id (`llama/nomic-embed`) — not a chat model, so
|
||||
// not a model ref (lembas/webui.ts, isChatModel): found among the ids the server says it serves
|
||||
// for embeddings, or as a ref in any of its forms. The name stored with the vectors stays the ref
|
||||
// as written, so a library indexed under it is still found.
|
||||
const [name, c] = Object.entries(loaded.connections)[0] ?? []
|
||||
if (!name || !c) throw new Error("embedding: not logged in to a LLeMbas server (lembas login)")
|
||||
let id: string
|
||||
try {
|
||||
const resolved = resolveModel(loaded, ref)
|
||||
name = resolved.connectionName
|
||||
id = resolved.id
|
||||
} catch (e) {
|
||||
// An instance's embedding model by its served id (`llama/nomic-embed`): not a
|
||||
// model ref, since only chat models are (lembas/webui.ts, isChatModel), so found by the ids
|
||||
// its connection says it serves.
|
||||
const served = Object.entries(loaded.connections).find(([, c]) => (c as { webui?: { embeddings?: string[] } }).webui?.embeddings?.includes(ref))
|
||||
const slash = ref.indexOf("/")
|
||||
if (served) {
|
||||
name = served[0]
|
||||
id = ref
|
||||
} else {
|
||||
if (slash <= 0) throw new Error(`embedding: "${ref}" must be written connection/model`)
|
||||
name = ref.slice(0, slash)
|
||||
id = ref.slice(slash + 1)
|
||||
if (!loaded.connections[name]) throw new Error(`embedding: ${(e as Error).message}`)
|
||||
if (c.webui?.embeddings?.includes(ref)) id = ref
|
||||
else if (ref.startsWith(`${name}/`) && c.webui?.embeddings?.includes(ref.slice(name.length + 1))) id = ref.slice(name.length + 1)
|
||||
else {
|
||||
try {
|
||||
id = resolveModel(loaded, ref).id
|
||||
} catch (e) {
|
||||
throw new Error(`embedding: ${(e as Error).message}`)
|
||||
}
|
||||
}
|
||||
const c = loaded.connections[name]!
|
||||
if (c.dialect === "anthropic" || c.dialect === "gemini") throw new Error(`embedding: the ${c.dialect} dialect has no embeddings here; use an OpenAI-compatible or Ollama connection`)
|
||||
const ollama = c.dialect === "ollama"
|
||||
return {
|
||||
model: ref,
|
||||
async embed(texts, signal) {
|
||||
const res = await request(
|
||||
joinUrl(c.base_url, ollama ? "api/embed" : "embeddings"),
|
||||
joinUrl(c.base_url, "embeddings"),
|
||||
{ method: "POST", headers: authHeaders(c, resolveKey(name, c)), body: JSON.stringify({ model: id, input: texts }), signal, timeoutMs: (c.timeout ?? 120) * 1000, tls: tlsFor(c) },
|
||||
`${ref} (embeddings)`,
|
||||
)
|
||||
const j = (await res.json()) as { data?: { embedding: number[]; index?: number }[]; embeddings?: number[][] }
|
||||
const rows = ollama ? (j.embeddings ?? []) : [...(j.data ?? [])].sort((a, b) => (a.index ?? 0) - (b.index ?? 0)).map((d) => d.embedding)
|
||||
return rows.map(unit)
|
||||
const j = (await res.json()) as { data?: { embedding: number[]; index?: number }[] }
|
||||
return [...(j.data ?? [])].sort((a, b) => (a.index ?? 0) - (b.index ?? 0)).map((d) => unit(d.embedding))
|
||||
},
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -8,7 +8,7 @@ const CONFIG_TEMPLATE = `# LLeMbas CLI settings for this project. Merged over ~/
|
||||
# and read only while the project is trusted. Connections cannot be set here.
|
||||
#
|
||||
# mode: edit # manual | edit | auto | plan
|
||||
# model: connection/model
|
||||
# model: provider/model
|
||||
# permission:
|
||||
# bash:
|
||||
# "npm test *": allow
|
||||
|
||||
@@ -1,228 +0,0 @@
|
||||
// Anthropic Messages (`POST {base}/messages`) — api.anthropic.com, and servers that imitate it
|
||||
// (vLLM, llama-server). Content blocks both ways; thinking with signatures that must be sent back
|
||||
// on the next turn; optional prompt-caching breakpoints.
|
||||
import type { Effort } from "../config/schema.ts"
|
||||
import { resolveKey } from "../config/load.ts"
|
||||
import { Parts, record, recordFailure, retrying, statusOf, SwapBanner } from "./common.ts"
|
||||
import { authHeaders, joinUrl, request, tlsFor } from "./http.ts"
|
||||
import { sseJson } from "./sse.ts"
|
||||
import { estimateTokens, estimateValue } from "./tokens.ts"
|
||||
import { ProviderError, type ChatRequest, type Client, type DiscoveredModel, type Message, type ResolvedModel, type StreamEvent, type ToolCallPart } from "./types.ts"
|
||||
|
||||
/** Thinking budgets by effort, when the model's config gives no effort_map. */
|
||||
export const DEFAULT_BUDGETS: Record<Effort, number> = { minimal: 1024, low: 2048, medium: 8192, high: 16384, xhigh: 32000, max: 48000 }
|
||||
|
||||
type Block = Record<string, unknown>
|
||||
type AMessage = { role: "user" | "assistant"; content: Block[] }
|
||||
|
||||
/** Anthropic ids must match ^[a-zA-Z0-9_-]+$; ids from other providers are mapped consistently. */
|
||||
export const safeId = (id: string) => id.replace(/[^a-zA-Z0-9_-]/g, "_") || "call"
|
||||
|
||||
export function toAnthropicMessages(messages: Message[], vision: boolean, thinking: boolean): AMessage[] {
|
||||
const out: AMessage[] = []
|
||||
const push = (role: AMessage["role"], blocks: Block[]) => {
|
||||
if (!blocks.length) return
|
||||
const last = out[out.length - 1]
|
||||
// Consecutive turns of one role are merged: tool results and a following note are one user turn.
|
||||
if (last && last.role === role) last.content.push(...blocks)
|
||||
else out.push({ role, content: blocks })
|
||||
}
|
||||
for (const m of messages) {
|
||||
if (m.role === "user") {
|
||||
push(
|
||||
"user",
|
||||
m.parts.map((p) =>
|
||||
p.type === "text"
|
||||
? { type: "text", text: p.text }
|
||||
: vision
|
||||
? { type: "image", source: { type: "base64", media_type: p.mime, data: p.data } }
|
||||
: { type: "text", text: "[image omitted: this model has no vision]" },
|
||||
),
|
||||
)
|
||||
} else if (m.role === "assistant") {
|
||||
const blocks: Block[] = []
|
||||
for (const p of m.parts) {
|
||||
if (p.type === "reasoning") {
|
||||
// A thinking block can only go back with its signature, and only while thinking is on.
|
||||
if (!thinking) continue
|
||||
const redacted = (p.opaque as { redacted?: string } | undefined)?.redacted
|
||||
if (redacted) blocks.push({ type: "redacted_thinking", data: redacted })
|
||||
else if (p.signature) blocks.push({ type: "thinking", thinking: p.text, signature: p.signature })
|
||||
} else if (p.type === "text") {
|
||||
if (p.text.trim()) blocks.push({ type: "text", text: p.text })
|
||||
} else {
|
||||
let input: unknown = {}
|
||||
try {
|
||||
input = p.args.trim() ? JSON.parse(p.args) : {}
|
||||
} catch {
|
||||
input = { _unparsed: p.args }
|
||||
}
|
||||
blocks.push({ type: "tool_use", id: safeId(p.id), name: p.name, input })
|
||||
}
|
||||
}
|
||||
push("assistant", blocks)
|
||||
} else {
|
||||
push("user", [{ type: "tool_result", tool_use_id: safeId(m.callId), content: m.content || "(empty)", ...(m.isError ? { is_error: true } : {}) }])
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
export class AnthropicClient implements Client {
|
||||
constructor(private m: ResolvedModel) {}
|
||||
|
||||
private headers() {
|
||||
return authHeaders(this.m.connection, resolveKey(this.m.connectionName, this.m.connection), { "anthropic-version": "2023-06-01", ...this.m.spec.headers })
|
||||
}
|
||||
|
||||
buildBody(req: ChatRequest): Record<string, unknown> {
|
||||
const { m } = this
|
||||
const budget = req.effort ? (m.spec.effort_map?.[req.effort] ?? DEFAULT_BUDGETS[req.effort]) : undefined
|
||||
let maxTokens = m.spec.max_output ?? 8192
|
||||
if (budget && maxTokens <= budget) maxTokens = budget + Math.min(8192, maxTokens)
|
||||
const cache = m.spec.cache === true
|
||||
const body: Record<string, unknown> = {
|
||||
model: m.id,
|
||||
max_tokens: maxTokens,
|
||||
stream: true,
|
||||
messages: toAnthropicMessages(req.messages, m.spec.vision === true, !!budget),
|
||||
...m.connection.body,
|
||||
...m.spec.body,
|
||||
}
|
||||
if (req.system) body.system = cache ? [{ type: "text", text: req.system, cache_control: { type: "ephemeral" } }] : req.system
|
||||
if (req.tools.length && m.spec.tools !== false) {
|
||||
const tools = req.tools.map((t) => ({ name: t.name, description: t.description, input_schema: t.parameters }))
|
||||
if (cache) (tools[tools.length - 1] as Block).cache_control = { type: "ephemeral" }
|
||||
body.tools = tools
|
||||
}
|
||||
if (cache) {
|
||||
const msgs = body.messages as AMessage[]
|
||||
const last = msgs[msgs.length - 1]?.content
|
||||
if (last?.length) last[last.length - 1] = { ...last[last.length - 1], cache_control: { type: "ephemeral" } }
|
||||
}
|
||||
if (budget) body.thinking = { type: "enabled", budget_tokens: budget }
|
||||
else {
|
||||
// Sampling parameters and extended thinking do not mix on this API.
|
||||
if (m.spec.temperature !== undefined) body.temperature = m.spec.temperature
|
||||
if (m.spec.top_p !== undefined) body.top_p = m.spec.top_p
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
async *stream(req: ChatRequest): AsyncGenerator<StreamEvent> {
|
||||
const body = this.buildBody(req)
|
||||
yield* retrying(() => this.attempt(req, body))
|
||||
}
|
||||
|
||||
private async *attempt(req: ChatRequest, body: Record<string, unknown>): AsyncGenerator<StreamEvent> {
|
||||
const { m } = this
|
||||
const res = await request(
|
||||
joinUrl(m.connection.base_url, "messages"),
|
||||
{ method: "POST", headers: this.headers(), body: JSON.stringify(body), signal: req.signal, timeoutMs: (m.connection.timeout ?? 600) * 1000, tls: tlsFor(m.connection) },
|
||||
m.ref,
|
||||
).catch((e) => {
|
||||
recordFailure(m.ref, body, e)
|
||||
throw e
|
||||
})
|
||||
if (!res.body) throw new ProviderError(`${m.ref}: empty response`)
|
||||
|
||||
const out = new Parts()
|
||||
const banner = new SwapBanner()
|
||||
type Open = { type: string; text: string; signature?: string; id?: string; name?: string; json: string; data?: string; index: number }
|
||||
const blocks = new Map<number, Open>()
|
||||
let toolIndex = 0
|
||||
let finish = "stop"
|
||||
let input = 0
|
||||
let output = 0
|
||||
let cached: number | undefined
|
||||
let sawUsage = false
|
||||
|
||||
for await (const { data } of sseJson(record(res.body, m.ref, body))) {
|
||||
const ev = data as any
|
||||
switch (ev?.type) {
|
||||
case "message_start": {
|
||||
const u = ev.message?.usage
|
||||
if (u) {
|
||||
sawUsage = true
|
||||
cached = u.cache_read_input_tokens || undefined
|
||||
input = (u.input_tokens ?? 0) + (u.cache_read_input_tokens ?? 0) + (u.cache_creation_input_tokens ?? 0)
|
||||
output = u.output_tokens ?? 0
|
||||
}
|
||||
break
|
||||
}
|
||||
case "content_block_start": {
|
||||
const b = ev.content_block ?? {}
|
||||
blocks.set(ev.index, { type: b.type, text: b.text ?? b.thinking ?? "", id: b.id, name: b.name, json: "", data: b.data, index: ev.index })
|
||||
if (b.type === "tool_use") yield { type: "tool_call_delta", index: toolIndex, name: b.name, argsDelta: "" }
|
||||
if (b.type === "text" && b.text) {
|
||||
out.add("text", b.text)
|
||||
yield { type: "text", text: b.text }
|
||||
}
|
||||
break
|
||||
}
|
||||
case "content_block_delta": {
|
||||
const blk = blocks.get(ev.index)
|
||||
const d = ev.delta ?? {}
|
||||
if (!blk) break
|
||||
if (d.type === "text_delta") {
|
||||
out.add("text", d.text)
|
||||
yield { type: "text", text: d.text }
|
||||
} else if (d.type === "thinking_delta") {
|
||||
const b = banner.feed(d.thinking ?? "")
|
||||
if (b.notice) yield { type: "notice", message: b.notice }
|
||||
if (b.text) {
|
||||
blk.text += b.text
|
||||
if (b.text.trim()) out.output = true
|
||||
yield { type: "reasoning", text: b.text }
|
||||
}
|
||||
} else if (d.type === "signature_delta") blk.signature = (blk.signature ?? "") + d.signature
|
||||
else if (d.type === "input_json_delta") {
|
||||
blk.json += d.partial_json ?? ""
|
||||
out.output = true
|
||||
yield { type: "tool_call_delta", index: toolIndex, argsDelta: d.partial_json ?? "" }
|
||||
}
|
||||
break
|
||||
}
|
||||
case "content_block_stop": {
|
||||
const blk = blocks.get(ev.index)
|
||||
if (!blk) break
|
||||
if (blk.type === "thinking" && blk.text) out.push({ type: "reasoning", text: blk.text, signature: blk.signature })
|
||||
else if (blk.type === "redacted_thinking") out.push({ type: "reasoning", text: "", opaque: { redacted: blk.data } })
|
||||
else if (blk.type === "tool_use") {
|
||||
const call: ToolCallPart = { type: "tool_call", id: blk.id || `toolu_${toolIndex}`, name: blk.name ?? "", args: blk.json || "{}" }
|
||||
out.push(call)
|
||||
toolIndex++
|
||||
}
|
||||
blocks.delete(ev.index)
|
||||
break
|
||||
}
|
||||
case "message_delta": {
|
||||
if (ev.delta?.stop_reason) finish = ev.delta.stop_reason
|
||||
if (ev.usage?.output_tokens !== undefined) {
|
||||
sawUsage = true
|
||||
output = ev.usage.output_tokens
|
||||
}
|
||||
break
|
||||
}
|
||||
case "error": {
|
||||
const err = ev.error ?? {}
|
||||
throw new ProviderError(`${m.ref}: ${err.message ?? JSON.stringify(err)}`, statusOf(err) ?? (err.type === "overloaded_error" ? 529 : undefined), undefined, !out.output)
|
||||
}
|
||||
default:
|
||||
if (ev?.error) throw new ProviderError(`${m.ref}: ${ev.error.message ?? JSON.stringify(ev.error)}`, statusOf(ev.error), undefined, !out.output)
|
||||
}
|
||||
}
|
||||
const message = out.message()
|
||||
const reason = finish === "max_tokens" ? "length" : finish === "tool_use" || message.parts.some((p) => p.type === "tool_call") ? "tool_calls" : "stop"
|
||||
yield sawUsage
|
||||
? { type: "usage", usage: { input, output, cached } }
|
||||
: { type: "usage", usage: { input: estimateValue(body.messages), output: message.parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.args : p.text), 0), estimated: true } }
|
||||
yield { type: "finish", reason, message }
|
||||
}
|
||||
|
||||
async listModels(): Promise<DiscoveredModel[]> {
|
||||
const res = await request(joinUrl(this.m.connection.base_url, "models"), { headers: this.headers(), timeoutMs: 15_000, tls: tlsFor(this.m.connection) }, this.m.connectionName)
|
||||
const j = (await res.json()) as any
|
||||
return (Array.isArray(j?.data) ? j.data : []).map((x: any) => ({ id: String(x.id), context: x.context_window ?? x.max_input_tokens })).filter((x: DiscoveredModel) => x.id)
|
||||
}
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
// A model's context window when the config does not give one. Config always wins; what is found
|
||||
// here is kept in learned.json. Tried in order: the server's model list; llama-server's /props
|
||||
// (directly, or through llama-swap's /upstream/<model>/ passthrough); Ollama's /api/show.
|
||||
import { learned, learnContext } from "./learned.ts"
|
||||
import { authHeaders, request, tlsFor } from "./http.ts"
|
||||
import { OllamaClient } from "./ollama.ts"
|
||||
import type { Client, ResolvedModel } from "./types.ts"
|
||||
import { resolveKey } from "../config/load.ts"
|
||||
|
||||
/** The window to plan with: configured, else learned, else undefined (unknown). */
|
||||
export function contextWindow(m: ResolvedModel): number | undefined {
|
||||
return m.spec.context || learned().contexts[m.ref] || undefined
|
||||
}
|
||||
|
||||
export async function discoverContext(m: ResolvedModel, client: Client): Promise<number | undefined> {
|
||||
if (m.spec.context) return m.spec.context
|
||||
const known = learned().contexts[m.ref]
|
||||
if (known) return known
|
||||
const found = (await fromList(m, client)) ?? (await fromProps(m)) ?? (m.connection.dialect === "ollama" ? await (client as OllamaClient).contextOf(m.id).catch(() => undefined) : undefined)
|
||||
if (found && found > 0) learnContext(m.ref, found)
|
||||
return found
|
||||
}
|
||||
|
||||
async function fromList(m: ResolvedModel, client: Client): Promise<number | undefined> {
|
||||
try {
|
||||
return (await client.listModels()).find((d) => d.id === m.id)?.context
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
async function fromProps(m: ResolvedModel): Promise<number | undefined> {
|
||||
const base = m.connection.base_url.replace(/\/+$/, "")
|
||||
const origin = base.replace(/\/v1$/, "")
|
||||
const headers = authHeaders(m.connection, resolveKey(m.connectionName, m.connection))
|
||||
for (const url of [`${origin}/upstream/${encodeURIComponent(m.id)}/props`, `${origin}/props`]) {
|
||||
try {
|
||||
const res = await request(url, { headers, timeoutMs: 15_000, tls: tlsFor(m.connection) }, m.ref)
|
||||
const j = (await res.json()) as any
|
||||
const n = Number(j?.default_generation_settings?.n_ctx ?? j?.n_ctx)
|
||||
if (Number.isFinite(n) && n > 0) return n
|
||||
} catch {}
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
/** llama-swap and friends: free the connection's memory when the session moves elsewhere. */
|
||||
export async function unload(m: ResolvedModel): Promise<void> {
|
||||
if (!m.connection.unload_url) return
|
||||
try {
|
||||
await request(m.connection.unload_url, { method: m.connection.unload_method ?? "POST", headers: authHeaders(m.connection, resolveKey(m.connectionName, m.connection)), timeoutMs: 10_000, tls: tlsFor(m.connection) }, m.ref)
|
||||
} catch {}
|
||||
}
|
||||
@@ -1,316 +0,0 @@
|
||||
// Google Gemini, native (`{base}/models/{id}:streamGenerateContent?alt=sse`), base
|
||||
// https://generativelanguage.googleapis.com/v1beta. UNTESTED: no Gemini endpoint has been
|
||||
// available to try it against (see the wiki's Working-notes).
|
||||
import type { Effort } from "../config/schema.ts"
|
||||
import { resolveKey } from "../config/load.ts"
|
||||
import { DEFAULT_BUDGETS } from "./anthropic.ts"
|
||||
import { Parts, record, recordFailure, retrying, statusOf } from "./common.ts"
|
||||
import { authHeaders, joinUrl, request, tlsFor } from "./http.ts"
|
||||
import { sseJson } from "./sse.ts"
|
||||
import { estimateTokens, estimateValue } from "./tokens.ts"
|
||||
import { ProviderError, type ChatRequest, type Client, type DiscoveredModel, type Message, type ResolvedModel, type StreamEvent, type ToolCallPart } from "./types.ts"
|
||||
|
||||
type Content = { role: "user" | "model"; parts: Record<string, unknown>[] }
|
||||
|
||||
// Checked against Hermes Agent (agent/gemini_native_adapter.py, agent/gemini_schema.py) and
|
||||
// OpenCode (provider/transform.ts). Still untested against a real Gemini endpoint.
|
||||
|
||||
/** Gemini's own id, without an aggregator prefix. */
|
||||
const bare = (id: string) => id.replace(/^(models|google|gemini)\//i, "")
|
||||
/** Gemini 3 and later: tool calls and results carry ids, and thinking is a level, not a budget. */
|
||||
export const isGemini3 = (id: string) => Number(/gemini-(\d+)/i.exec(bare(id))?.[1] ?? 0) >= 3
|
||||
const LEGACY = /gemini-(?:(?:flash|pro)-)?[12](?:[.-]|$)/i
|
||||
/** A call replayed without a signature (made by another model, or before signatures existed):
|
||||
* Google's documented sentinel, without which Gemini 3 refuses the whole history. */
|
||||
export const SKIP_SIGNATURE = "skip_thought_signature_validator"
|
||||
/** Thought tokens count against maxOutputTokens, and Gemini's own default is low. */
|
||||
export const DEFAULT_MAX_OUTPUT = 65535
|
||||
const PLACEHOLDER = "[The previous response was interrupted before it completed.]"
|
||||
|
||||
// ── tool schemas ──────────────────────────────────────────────────────────────────────────────
|
||||
// v1beta takes plain JSON Schema as `parametersJsonSchema`; only same-document $refs have to be
|
||||
// inlined. Other API versions take `parameters`, an OpenAPI subset where one unknown key 400s the
|
||||
// whole request, so that path drops everything outside the subset.
|
||||
|
||||
const ALLOWED = new Set(["type", "format", "title", "description", "nullable", "enum", "maxItems", "minItems", "properties", "required", "minProperties", "maxProperties", "minLength", "maxLength", "pattern", "example", "anyOf", "propertyOrdering", "default", "items", "minimum", "maximum"])
|
||||
const STRUCTURAL: Record<string, string[]> = { array: ["items", "minItems", "maxItems"], object: ["properties", "required", "minProperties", "maxProperties", "propertyOrdering"] }
|
||||
const isObj = (v: unknown): v is Record<string, unknown> => !!v && typeof v === "object" && !Array.isArray(v)
|
||||
|
||||
export function geminiSchema(s: unknown): Record<string, unknown> {
|
||||
if (!isObj(s)) return {}
|
||||
const out: Record<string, unknown> = {}
|
||||
for (const [k, v] of Object.entries(s)) {
|
||||
if (!ALLOWED.has(k)) continue
|
||||
if (k === "properties") {
|
||||
if (isObj(v)) out.properties = Object.fromEntries(Object.entries(v).map(([p, sub]) => [p, geminiSchema(sub)]))
|
||||
} else if (k === "items") out.items = geminiSchema(v)
|
||||
else if (k === "anyOf") {
|
||||
if (Array.isArray(v)) out.anyOf = v.filter(isObj).map(geminiSchema)
|
||||
} else out[k] = v
|
||||
}
|
||||
const types = out.type
|
||||
if (Array.isArray(types)) {
|
||||
delete out.type
|
||||
const real = types.filter((t): t is string => typeof t === "string" && t !== "null")
|
||||
if (!real.length) out.type = types.includes("null") ? "null" : "object"
|
||||
else if (real.length === 1) out.type = real[0]
|
||||
else {
|
||||
// Several types: one branch each, and the structure belongs on the branch it describes.
|
||||
const moved: Record<string, unknown> = {}
|
||||
for (const keys of Object.values(STRUCTURAL)) for (const k of keys) if (k in out) (moved[k] = out[k]), delete out[k]
|
||||
out.anyOf = real.map((t) => geminiSchema({ type: t, ...Object.fromEntries(Object.entries(moved).filter(([k]) => STRUCTURAL[t]?.includes(k))) }))
|
||||
}
|
||||
if (types.includes("null")) out.nullable = true
|
||||
}
|
||||
// Every enum entry must be a string, whatever the declared type.
|
||||
if (Array.isArray(out.enum) && (Array.isArray(types) || ["integer", "number", "boolean"].includes(out.type as string))) {
|
||||
const vals = [...new Set(out.enum.map((e) => (typeof e === "boolean" || (typeof e === "number" && Number.isFinite(e)) ? String(e) : typeof e === "string" ? e : undefined)).filter((e) => e !== undefined))]
|
||||
if (vals.length) out.enum = vals
|
||||
else delete out.enum
|
||||
}
|
||||
// `required` naming a property that is not there fails the entire request.
|
||||
if (Array.isArray(out.required)) {
|
||||
const names = isObj(out.properties) ? out.properties : {}
|
||||
const req = out.required.filter((r): r is string => typeof r === "string" && r in names)
|
||||
if (req.length) out.required = req
|
||||
else delete out.required
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** For `parameters`: the subset, and never an empty schema. */
|
||||
export function geminiParameters(s: unknown): Record<string, unknown> {
|
||||
const out = geminiSchema(s)
|
||||
return Object.keys(out).length ? out : { type: "object", properties: {} }
|
||||
}
|
||||
|
||||
function inlineRefs(node: unknown, root: Record<string, unknown>, budget: { n: number }, stack: string[] = []): unknown {
|
||||
if (Array.isArray(node)) return node.map((x) => inlineRefs(x, root, budget, stack))
|
||||
if (!isObj(node)) return node
|
||||
const ref = node.$ref
|
||||
if (typeof ref !== "string") return Object.fromEntries(Object.entries(node).map(([k, v]) => [k, inlineRefs(v, root, budget, stack)]))
|
||||
if (stack.includes(ref)) throw new Error(`circular $ref ${ref}`)
|
||||
if (--budget.n < 0) throw new Error("$ref budget exhausted")
|
||||
if (!ref.startsWith("#/")) throw new Error(`unresolvable $ref ${ref}`)
|
||||
let target: unknown = root
|
||||
for (const raw of ref.slice(2).split("/")) {
|
||||
const part = raw.replace(/~1/g, "/").replace(/~0/g, "~")
|
||||
target = isObj(target) ? target[part] : undefined
|
||||
}
|
||||
if (!isObj(target)) throw new Error(`unresolvable $ref ${ref}`)
|
||||
const inlined = inlineRefs(target, root, budget, [...stack, ref]) as Record<string, unknown>
|
||||
// Siblings of a $ref apply alongside it and win.
|
||||
const { $ref: _, ...siblings } = node
|
||||
return Object.keys(siblings).length ? { ...inlined, ...(inlineRefs(siblings, root, budget, stack) as object) } : inlined
|
||||
}
|
||||
|
||||
/** For `parametersJsonSchema`: root $schema dropped, local $refs inlined, an object root. A schema
|
||||
* whose refs cannot all be resolved goes as it is, so the provider names the real problem. */
|
||||
export function geminiJsonSchema(s: unknown): Record<string, unknown> {
|
||||
if (!isObj(s) || !Object.keys(s).length) return { type: "object", properties: {} }
|
||||
const { $schema: _, ...rest } = structuredClone(s)
|
||||
let out: Record<string, unknown>
|
||||
try {
|
||||
out = inlineRefs(rest, rest, { n: 256 }) as Record<string, unknown>
|
||||
} catch {
|
||||
return rest
|
||||
}
|
||||
delete out.$defs
|
||||
delete out.definitions
|
||||
if (!Object.keys(out).length) return { type: "object", properties: {} }
|
||||
if (out.type === "object" && !("properties" in out)) out.properties = {}
|
||||
return out
|
||||
}
|
||||
|
||||
/** `parametersJsonSchema` exists only on v1beta. */
|
||||
export const acceptsJsonSchema = (base: string) => /\/v1beta\/*$/i.test(base.trim())
|
||||
|
||||
// ── history ───────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/** A JSON Schema inside a function response is resolved by Gemini 3, and an unknown $ref 400s. */
|
||||
const hasRef = (v: unknown): boolean => (Array.isArray(v) ? v.some(hasRef) : isObj(v) ? Object.entries(v).some(([k, x]) => (k === "$ref" && typeof x === "string" && x.startsWith("#/")) || hasRef(x)) : false)
|
||||
|
||||
function functionResponse(content: string, isError: boolean | undefined): Record<string, unknown> {
|
||||
if (isError) return { error: content }
|
||||
// A JSON object result goes structured; anything else under "output", as Google documents.
|
||||
if (/^\s*\{/.test(content))
|
||||
try {
|
||||
const parsed = JSON.parse(content)
|
||||
if (isObj(parsed) && !hasRef(parsed)) return parsed
|
||||
} catch {}
|
||||
return { output: content }
|
||||
}
|
||||
|
||||
export function toGeminiContents(messages: Message[], vision: boolean, ids = false): Content[] {
|
||||
const out: Content[] = []
|
||||
const names = new Map<string, string>()
|
||||
const results = (c: Content) => c.parts.some((p) => "functionResponse" in p)
|
||||
const push = (role: Content["role"], parts: Record<string, unknown>[]) => {
|
||||
if (!parts.length) return
|
||||
const last = out[out.length - 1]
|
||||
// Same-role turns must merge (Gemini insists on alternation), except that a person's words are
|
||||
// never folded into a turn of tool results: Gemini 3 then reads them as part of the result and
|
||||
// answers with nothing. A placeholder model turn keeps the two apart.
|
||||
if (last && last.role === role && role === "user" && results(last) !== parts.some((p) => "functionResponse" in p)) out.push({ role: "model", parts: [{ text: PLACEHOLDER }] }, { role, parts })
|
||||
else if (last && last.role === role) last.parts.push(...parts)
|
||||
else out.push({ role, parts })
|
||||
}
|
||||
for (const m of messages) {
|
||||
if (m.role === "user")
|
||||
push(
|
||||
"user",
|
||||
m.parts.map((p) => (p.type === "text" ? { text: p.text } : vision ? { inlineData: { mimeType: p.mime, data: p.data } } : { text: "[image omitted: this model has no vision]" })),
|
||||
)
|
||||
else if (m.role === "assistant") {
|
||||
const parts: Record<string, unknown>[] = []
|
||||
for (const p of m.parts) {
|
||||
if (p.type === "text" && p.text.trim()) parts.push({ text: p.text })
|
||||
else if (p.type === "tool_call") {
|
||||
names.set(p.id, p.name)
|
||||
let args: unknown = {}
|
||||
try {
|
||||
args = p.args.trim() ? JSON.parse(p.args) : {}
|
||||
} catch {
|
||||
args = { _raw: p.args }
|
||||
}
|
||||
// Gemini 3 wants the thought signature back on the call it came with.
|
||||
parts.push({ functionCall: { name: p.name, args: isObj(args) ? args : { _value: args }, ...(ids ? { id: p.id } : {}) }, thoughtSignature: p.signature ?? SKIP_SIGNATURE })
|
||||
}
|
||||
}
|
||||
push("model", parts)
|
||||
} else push("user", [{ functionResponse: { name: names.get(m.callId) ?? m.name, response: functionResponse(m.content, m.isError), ...(ids ? { id: m.callId } : {}) } }])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ── thinking ──────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const LEVELS = ["minimal", "low", "medium", "high"] as const
|
||||
/** The levels a Gemini 3 model takes (OpenCode's table). */
|
||||
function levelsOf(id: string): readonly string[] {
|
||||
const s = id.toLowerCase()
|
||||
if (s.includes("gemma") || s.includes("flash-image")) return ["minimal", "high"]
|
||||
if (s.includes("pro-image")) return ["high"]
|
||||
if (s.includes("flash")) return LEVELS
|
||||
return ["low", "medium", "high"]
|
||||
}
|
||||
|
||||
export function thinkingConfig(id: string, effort: Effort, map?: Partial<Record<Effort, number>>): Record<string, unknown> {
|
||||
if (map?.[effort]) return { thinkingBudget: map[effort], includeThoughts: true }
|
||||
const b = bare(id)
|
||||
if (LEGACY.test(b)) {
|
||||
// 2.x takes a budget; 2.5 Pro goes to 32k, the rest to 24k.
|
||||
const max = /gemini-2[.-]5/i.test(b) && /pro/i.test(b) && !/flash/i.test(b) ? 32768 : 24576
|
||||
return { thinkingBudget: Math.min(DEFAULT_BUDGETS[effort], max), includeThoughts: true }
|
||||
}
|
||||
// 3 and later take a level: the one asked for, else the next one up the model has, else its top.
|
||||
const i = LEVELS.indexOf(effort as (typeof LEVELS)[number])
|
||||
const want = i < 0 ? LEVELS.length - 1 : i // xhigh and max: the top
|
||||
const have = levelsOf(b)
|
||||
const level = LEVELS.slice(want).find((l) => have.includes(l)) ?? have[have.length - 1]
|
||||
return { thinkingLevel: level, includeThoughts: true }
|
||||
}
|
||||
|
||||
/** Keys sorted, so a call sent again compares equal to itself (as Hermes does). */
|
||||
const sortedJson = (v: unknown) => JSON.stringify(v, (_, x) => (isObj(x) ? Object.fromEntries(Object.keys(x).sort().map((k) => [k, x[k]])) : x))
|
||||
|
||||
const FINISH: Record<string, string> = { STOP: "stop", MAX_TOKENS: "length", SAFETY: "content_filter", RECITATION: "content_filter", BLOCKLIST: "content_filter", PROHIBITED_CONTENT: "content_filter", SPII: "content_filter" }
|
||||
|
||||
export class GeminiClient implements Client {
|
||||
constructor(private m: ResolvedModel) {}
|
||||
|
||||
private headers() {
|
||||
return authHeaders(this.m.connection, resolveKey(this.m.connectionName, this.m.connection), this.m.spec.headers)
|
||||
}
|
||||
|
||||
buildBody(req: ChatRequest): Record<string, unknown> {
|
||||
const { m } = this
|
||||
const generationConfig: Record<string, unknown> = {}
|
||||
if (m.spec.temperature !== undefined) generationConfig.temperature = m.spec.temperature
|
||||
generationConfig.maxOutputTokens = m.spec.max_output ?? DEFAULT_MAX_OUTPUT
|
||||
if (m.spec.top_p !== undefined) generationConfig.topP = m.spec.top_p
|
||||
if (req.effort) generationConfig.thinkingConfig = thinkingConfig(m.id, req.effort, m.spec.effort_map)
|
||||
const body: Record<string, unknown> = { contents: toGeminiContents(req.messages, m.spec.vision === true, isGemini3(m.id)), generationConfig, ...m.connection.body, ...m.spec.body }
|
||||
if (req.system) body.systemInstruction = { parts: [{ text: req.system }] }
|
||||
if (req.tools.length && m.spec.tools !== false) {
|
||||
const json = acceptsJsonSchema(m.connection.base_url)
|
||||
body.tools = [{ functionDeclarations: req.tools.map((t) => ({ name: t.name, description: t.description, ...(json ? { parametersJsonSchema: geminiJsonSchema(t.parameters) } : { parameters: geminiParameters(t.parameters) }) })) }]
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
async *stream(req: ChatRequest): AsyncGenerator<StreamEvent> {
|
||||
const body = this.buildBody(req)
|
||||
yield* retrying(() => this.attempt(req, body))
|
||||
}
|
||||
|
||||
private async *attempt(req: ChatRequest, body: Record<string, unknown>): AsyncGenerator<StreamEvent> {
|
||||
const { m } = this
|
||||
const url = joinUrl(m.connection.base_url, `models/${encodeURIComponent(m.id)}:streamGenerateContent?alt=sse`)
|
||||
const res = await request(url, { method: "POST", headers: this.headers(), body: JSON.stringify(body), signal: req.signal, timeoutMs: (m.connection.timeout ?? 600) * 1000, tls: tlsFor(m.connection) }, m.ref).catch((e) => {
|
||||
recordFailure(m.ref, body, e)
|
||||
throw e
|
||||
})
|
||||
if (!res.body) throw new ProviderError(`${m.ref}: empty response`)
|
||||
const out = new Parts()
|
||||
let finish = "stop"
|
||||
let usage: StreamEvent | undefined
|
||||
// One slot per call. Gemini 3 ids its calls; 2.5 does not, restarts part numbering in every
|
||||
// event and may send a call again whole, so a payload repeating (or extending) a slot's arguments
|
||||
// is that call and anything else is a new one.
|
||||
const slots = new Map<string, { index: number; part: ToolCallPart }>()
|
||||
const slotFor = (fc: any, sig: string, at: number, args: string) => {
|
||||
if (typeof fc.id === "string" && fc.id) return `id:${fc.id}`
|
||||
const key = `${at}:${fc.name}:${sig}`
|
||||
const s = slots.get(key)
|
||||
if (!s || args.startsWith(s.part.args)) return key
|
||||
for (const [k, o] of slots) if (k.startsWith(`${key}#`) && args.startsWith(o.part.args)) return k
|
||||
return `${key}#${slots.size}`
|
||||
}
|
||||
for await (const { data } of sseJson(record(res.body, m.ref, body))) {
|
||||
const chunk = data as any
|
||||
if (chunk?.error) throw new ProviderError(`${m.ref}: ${chunk.error.message ?? JSON.stringify(chunk.error)}`, statusOf(chunk.error) ?? chunk.error.code, undefined, !out.output)
|
||||
const cand = chunk?.candidates?.[0]
|
||||
const parts: any[] = cand?.content?.parts ?? []
|
||||
for (const [at, p] of parts.entries()) {
|
||||
if (typeof p.text === "string" && p.text) {
|
||||
const kind = p.thought === true ? "reasoning" : "text"
|
||||
out.add(kind, p.text)
|
||||
yield { type: kind, text: p.text }
|
||||
}
|
||||
const fc = p.functionCall
|
||||
if (fc?.name) {
|
||||
const args = sortedJson(fc.args ?? {})
|
||||
const sig = typeof p.thoughtSignature === "string" ? p.thoughtSignature : ""
|
||||
const key = slotFor(fc, sig, at, args)
|
||||
let s = slots.get(key)
|
||||
if (!s) {
|
||||
s = { index: slots.size, part: { type: "tool_call", id: typeof fc.id === "string" && fc.id ? fc.id : `gemini_call_${slots.size}`, name: fc.name, args: "" } }
|
||||
slots.set(key, s)
|
||||
out.push(s.part)
|
||||
}
|
||||
const prev = s.part.args
|
||||
s.part.args = args
|
||||
if (sig) s.part.signature = sig
|
||||
yield { type: "tool_call_delta", index: s.index, name: fc.name, argsDelta: args.startsWith(prev) ? args.slice(prev.length) : args }
|
||||
}
|
||||
}
|
||||
if (cand?.finishReason) finish = FINISH[String(cand.finishReason).toUpperCase()] ?? "stop"
|
||||
const u = chunk?.usageMetadata
|
||||
if (u) usage = { type: "usage", usage: { input: u.promptTokenCount ?? 0, output: (u.candidatesTokenCount ?? 0) + (u.thoughtsTokenCount ?? 0), reasoning: u.thoughtsTokenCount, cached: u.cachedContentTokenCount } }
|
||||
}
|
||||
const message = out.message()
|
||||
if (slots.size && finish !== "length") finish = "tool_calls"
|
||||
yield usage ?? { type: "usage", usage: { input: estimateValue(body.contents), output: message.parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.args : p.text), 0), estimated: true } }
|
||||
yield { type: "finish", reason: finish, message }
|
||||
}
|
||||
|
||||
async listModels(): Promise<DiscoveredModel[]> {
|
||||
const res = await request(joinUrl(this.m.connection.base_url, "models?pageSize=200"), { headers: this.headers(), timeoutMs: 15_000, tls: tlsFor(this.m.connection) }, this.m.connectionName)
|
||||
const j = (await res.json()) as any
|
||||
return (j?.models ?? [])
|
||||
.filter((x: any) => !x.supportedGenerationMethods || x.supportedGenerationMethods.includes("generateContent"))
|
||||
.map((x: any) => ({ id: String(x.name ?? "").replace(/^models\//, ""), context: x.inputTokenLimit }))
|
||||
.filter((x: DiscoveredModel) => x.id)
|
||||
}
|
||||
}
|
||||
+3
-13
@@ -27,20 +27,10 @@ export function tlsFor(c: Pick<Connection, "tls">): { ca?: string; rejectUnautho
|
||||
return out
|
||||
}
|
||||
|
||||
const DEFAULT_AUTH: Record<Connection["dialect"], NonNullable<Connection["auth"]>> = {
|
||||
"openai-chat": "bearer",
|
||||
responses: "bearer",
|
||||
anthropic: "x-api-key",
|
||||
gemini: "x-goog-api-key",
|
||||
ollama: "bearer",
|
||||
}
|
||||
|
||||
/** Auth plus the connection's and model's own headers. */
|
||||
export function authHeaders(c: Connection, key: string | undefined, extra: Record<string, string> = {}): Record<string, string> {
|
||||
/** The server's token as a bearer, plus the connection's and model's own headers. */
|
||||
export function authHeaders(c: Pick<Connection, "headers">, key: string | undefined, extra: Record<string, string> = {}): Record<string, string> {
|
||||
const h: Record<string, string> = { "content-type": "application/json", ...extra }
|
||||
const style = c.auth ?? DEFAULT_AUTH[c.dialect]
|
||||
if (key && style === "bearer") h.authorization = `Bearer ${key}`
|
||||
else if (key && style !== "none") h[style] = key
|
||||
if (key) h.authorization = `Bearer ${key}`
|
||||
return { ...h, ...c.headers }
|
||||
}
|
||||
|
||||
|
||||
+28
-46
@@ -1,64 +1,40 @@
|
||||
import { findRef, modelRefs } from "./refs.ts"
|
||||
import type { Loaded } from "../config/load.ts"
|
||||
import { AnthropicClient } from "./anthropic.ts"
|
||||
import { OpenAIChatClient } from "./openai-chat.ts"
|
||||
import { ResponsesClient } from "./responses.ts"
|
||||
import { GeminiClient } from "./gemini.ts"
|
||||
import { OllamaClient } from "./ollama.ts"
|
||||
import { ProviderError, type Client, type ResolvedModel } from "./types.ts"
|
||||
import type { Client, ResolvedModel } from "./types.ts"
|
||||
|
||||
export class ModelError extends Error {}
|
||||
|
||||
/** `connection/model`. The model part may itself contain slashes (`openrouter/qwen/qwen3-coder`). */
|
||||
/** What a start without a login says: LLeMbas CLI 2.0 has no model of its own, only the server's. */
|
||||
export const NOT_LOGGED_IN = "Not logged in to a LLeMbas server. LLeMbas CLI uses the models of the server it is logged in to: lembas login <address> (e.g. lembas login ai.example.org)."
|
||||
|
||||
/** A model of the server this machine is logged in to, by its served id (`deepseek/deepseek-flash`)
|
||||
* or an older form of its name (refs.ts). Without a ref: config.yaml's `model`, else the server's
|
||||
* default for the account, else its first. */
|
||||
export function resolveModel(loaded: Loaded, ref: string | undefined): ResolvedModel {
|
||||
const refs = loaded.refs ?? modelRefs(loaded.connections)
|
||||
if (!ref) {
|
||||
// A LLeMbas instance says which model its account starts on; that, before anything else.
|
||||
const instance = Object.entries(loaded.connections).find(([, c]) => c.webui?.default && c.models[c.webui.default])
|
||||
if (instance) ref = refs.find((r) => r.connection === instance[0] && r.id === instance[1].webui!.default)?.ref ?? `${instance[0]}/${instance[1].webui!.default}`
|
||||
const [name, connection] = Object.entries(loaded.connections)[0] ?? []
|
||||
if (!name || !connection) {
|
||||
const why = Object.values(loaded.broken ?? {})[0]
|
||||
throw new ModelError(why ? `The LLeMbas login is unusable: ${why}` : NOT_LOGGED_IN)
|
||||
}
|
||||
if (!ref) {
|
||||
const first = refs[0]
|
||||
if (!first) throw new ModelError("No model configured. Add a connection to ~/.config/lembas/connections.yaml and set `model:` in config.yaml.")
|
||||
ref = first.ref
|
||||
}
|
||||
// Any form of the name (`deepseek/x`, or the old `example/x`): the model it names, under
|
||||
// the ref it has now.
|
||||
if (loaded.broken?.[name]) throw new ModelError(`The LLeMbas login ${name} is unusable: ${loaded.broken[name]}`)
|
||||
if (!ref) ref = connection.webui?.default && connection.models[connection.webui.default] ? connection.webui.default : refs[0]?.ref
|
||||
if (!ref) throw new ModelError(`${name} serves no model to this account yet. lembas login ${connection.webui?.url ?? ""} again once it does.`.replace(" ", " "))
|
||||
const known = findRef(loaded.connections, refs, ref)
|
||||
if (known) {
|
||||
const connection = loaded.connections[known.connection]!
|
||||
if (loaded.broken?.[known.connection]) throw new ModelError(`Connection "${known.connection}" is unusable: ${loaded.broken?.[known.connection]}`)
|
||||
return { ref: known.ref, connectionName: known.connection, connection, id: known.id, spec: connection.models[known.id] ?? {} }
|
||||
}
|
||||
const slash = ref.indexOf("/")
|
||||
if (slash <= 0) throw new ModelError(`Model "${ref}" must be written connection/model`)
|
||||
const connectionName = ref.slice(0, slash)
|
||||
const id = ref.slice(slash + 1)
|
||||
if (loaded.broken?.[connectionName]) throw new ModelError(`Connection "${connectionName}" is unusable: ${loaded.broken[connectionName]}`)
|
||||
const connection = loaded.connections[connectionName]
|
||||
if (!connection) throw new ModelError(`No connection named "${connectionName}" (have: ${Object.keys(loaded.connections).join(", ") || "none"})`)
|
||||
const spec = connection.models[id]
|
||||
if (!spec && !connection.discover)
|
||||
throw new ModelError(`Connection "${connectionName}" has no model "${id}" (have: ${Object.keys(connection.models).join(", ") || "none"}). Add it, or set discover: true.`)
|
||||
return { ref, connectionName, connection, id, spec: spec ?? {} }
|
||||
if (known) return { ref: known.ref, connectionName: name, connection, id: known.id, spec: connection.models[known.id] ?? {} }
|
||||
// A model the server serves that its list did not have when it was read: the server decides
|
||||
// which models exist, so it is asked for rather than refused here. The login's own name in front
|
||||
// of it (an older form) is taken off.
|
||||
const id = ref.startsWith(`${name}/`) ? ref.slice(name.length + 1) : ref
|
||||
return { ref: id, connectionName: name, connection, id, spec: {} }
|
||||
}
|
||||
|
||||
export function clientFor(m: ResolvedModel): Client {
|
||||
switch (m.connection.dialect) {
|
||||
case "openai-chat":
|
||||
return new OpenAIChatClient(m)
|
||||
case "anthropic":
|
||||
return new AnthropicClient(m)
|
||||
case "responses":
|
||||
return new ResponsesClient(m)
|
||||
case "gemini":
|
||||
return new GeminiClient(m)
|
||||
case "ollama":
|
||||
return new OllamaClient(m)
|
||||
}
|
||||
return new OpenAIChatClient(m)
|
||||
}
|
||||
|
||||
/** Which prompt family a model gets when the config does not say: anthropic, gpt, gemini, or
|
||||
/** Which prompt family a model gets when the server does not say: anthropic, gpt, gemini, or
|
||||
* local — every open-weight or unknown model, which gets Hermes' act-don't-describe guidance. */
|
||||
export function familyOf(m: ResolvedModel): string {
|
||||
if (m.spec.family) return m.spec.family
|
||||
@@ -68,3 +44,9 @@ export function familyOf(m: ResolvedModel): string {
|
||||
if (/\bgpt|^o[134]\b|^o[134]-|codex/.test(id)) return "gpt"
|
||||
return "local"
|
||||
}
|
||||
|
||||
/** The window to plan with, as the server gives it; undefined when it does not say (no percentage,
|
||||
* no automatic compaction). */
|
||||
export function contextWindow(m: ResolvedModel): number | undefined {
|
||||
return m.spec.context || undefined
|
||||
}
|
||||
+5
-10
@@ -6,14 +6,12 @@ import { paths } from "../config/paths.ts"
|
||||
// What endpoints have taught us, kept across runs in ~/.local/state/lembas/learned.json.
|
||||
// Only ever narrows behaviour; deleting the file is always safe.
|
||||
interface Learned {
|
||||
/** `connection/model` → the efforts it actually accepts. */
|
||||
/** A model ref → the efforts it actually accepts. */
|
||||
efforts: Record<string, Effort[]>
|
||||
/** base URLs that reject `stream_options`. */
|
||||
noStreamOptions: string[]
|
||||
/** base URLs that reject `return_progress`. */
|
||||
noProgress: string[]
|
||||
/** `connection/model` → context window found by discovery (config always wins). */
|
||||
contexts: Record<string, number>
|
||||
}
|
||||
|
||||
let cache: Learned | undefined
|
||||
@@ -25,9 +23,11 @@ function file() {
|
||||
export function learned(): Learned {
|
||||
if (cache) return cache
|
||||
try {
|
||||
cache = { efforts: {}, noStreamOptions: [], noProgress: [], contexts: {}, ...JSON.parse(readFileSync(file(), "utf8")) }
|
||||
// An older file's `contexts` (what discovery found, before 2.0 took the server's word) is dropped.
|
||||
const { contexts: _, ...kept } = JSON.parse(readFileSync(file(), "utf8")) as Record<string, unknown>
|
||||
cache = { efforts: {}, noStreamOptions: [], noProgress: [], ...kept } as Learned
|
||||
} catch {
|
||||
cache = { efforts: {}, noStreamOptions: [], noProgress: [], contexts: {} }
|
||||
cache = { efforts: {}, noStreamOptions: [], noProgress: [] }
|
||||
}
|
||||
return cache!
|
||||
}
|
||||
@@ -46,11 +46,6 @@ export function learnEfforts(ref: string, efforts: Effort[]) {
|
||||
save()
|
||||
}
|
||||
|
||||
export function learnContext(ref: string, n: number) {
|
||||
learned().contexts[ref] = n
|
||||
save()
|
||||
}
|
||||
|
||||
export function learnNoStreamOptions(baseUrl: string) {
|
||||
const l = learned()
|
||||
if (!l.noStreamOptions.includes(baseUrl)) l.noStreamOptions.push(baseUrl)
|
||||
|
||||
@@ -1,178 +0,0 @@
|
||||
// Ollama, native (`POST {base}/api/chat`, NDJSON), base e.g. http://localhost:11434. Its
|
||||
// OpenAI-compatible /v1 works through openai-chat too; the native API adds num_ctx (without it
|
||||
// Ollama silently cuts every conversation to its small default window) and `think` levels.
|
||||
// UNTESTED: no Ollama server has been available to try it against (see the wiki). Checked against
|
||||
// Hermes Agent's handling (agent/model_metadata.py, agent/reasoning_params.py): /api/show decides
|
||||
// the window to ask for and whether `think` may be sent at all.
|
||||
import { resolveKey } from "../config/load.ts"
|
||||
import { Parts, record, recordFailure, retrying, statusOf } from "./common.ts"
|
||||
import { authHeaders, joinUrl, request, tlsFor } from "./http.ts"
|
||||
import { estimateTokens, estimateValue } from "./tokens.ts"
|
||||
import { ProviderError, type ChatRequest, type Client, type DiscoveredModel, type Message, type ResolvedModel, type StreamEvent } from "./types.ts"
|
||||
|
||||
export function toOllamaMessages(system: string, messages: Message[], vision: boolean): Record<string, unknown>[] {
|
||||
const out: Record<string, unknown>[] = []
|
||||
if (system) out.push({ role: "system", content: system })
|
||||
for (const m of messages) {
|
||||
if (m.role === "user") {
|
||||
const images = vision ? m.parts.filter((p) => p.type === "image").map((p) => (p as { data: string }).data) : []
|
||||
const text = m.parts.map((p) => (p.type === "text" ? p.text : vision ? "" : "[image omitted: this model has no vision]")).filter(Boolean).join("\n")
|
||||
out.push({ role: "user", content: text, ...(images.length ? { images } : {}) })
|
||||
} else if (m.role === "assistant") {
|
||||
const text = m.parts.map((p) => (p.type === "text" ? p.text : "")).join("")
|
||||
const thinking = m.parts.map((p) => (p.type === "reasoning" ? p.text : "")).join("")
|
||||
const calls = m.parts.flatMap((p) => {
|
||||
if (p.type !== "tool_call") return []
|
||||
let args: unknown = {}
|
||||
try {
|
||||
args = p.args.trim() ? JSON.parse(p.args) : {}
|
||||
} catch {}
|
||||
return [{ function: { name: p.name, arguments: args } }]
|
||||
})
|
||||
out.push({ role: "assistant", content: text, ...(thinking ? { thinking } : {}), ...(calls.length ? { tool_calls: calls } : {}) })
|
||||
} else out.push({ role: "tool", content: m.content, tool_name: m.name })
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** NDJSON: one JSON object per line. */
|
||||
async function* ndjson(body: ReadableStream<Uint8Array>): AsyncGenerator<unknown> {
|
||||
const dec = new TextDecoder()
|
||||
let buf = ""
|
||||
for await (const chunk of body) {
|
||||
buf += dec.decode(chunk, { stream: true })
|
||||
let nl: number
|
||||
while ((nl = buf.indexOf("\n")) !== -1) {
|
||||
const line = buf.slice(0, nl).trim()
|
||||
buf = buf.slice(nl + 1)
|
||||
if (line)
|
||||
try {
|
||||
yield JSON.parse(line)
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
const last = (buf + dec.decode()).trim()
|
||||
if (last)
|
||||
try {
|
||||
yield JSON.parse(last)
|
||||
} catch {}
|
||||
}
|
||||
|
||||
/** What /api/show says about a model. */
|
||||
export interface OllamaShow {
|
||||
/** "completion", "tools", "thinking", "vision"…; absent before Ollama 0.6. */
|
||||
capabilities?: string[]
|
||||
/** The runtime window the Modelfile sets (`num_ctx`), which the owner of the server chose. */
|
||||
numCtx?: number
|
||||
/** The trained maximum (`model_info["<arch>.context_length"]`). */
|
||||
trained?: number
|
||||
}
|
||||
|
||||
export function parseShow(j: any): OllamaShow {
|
||||
const out: OllamaShow = {}
|
||||
if (Array.isArray(j?.capabilities)) out.capabilities = j.capabilities.map(String)
|
||||
for (const line of String(j?.parameters ?? "").split("\n")) {
|
||||
const f = line.trim().split(/\s+/)
|
||||
if (f[0] === "num_ctx" && Number(f[1]) > 0) out.numCtx = Number(f[1])
|
||||
}
|
||||
const info = j?.model_info ?? {}
|
||||
const key = Object.keys(info).find((k) => k.endsWith(".context_length"))
|
||||
if (key && Number(info[key]) > 0) out.trained = Number(info[key])
|
||||
return out
|
||||
}
|
||||
|
||||
export class OllamaClient implements Client {
|
||||
private shown?: Promise<OllamaShow | undefined>
|
||||
constructor(private m: ResolvedModel) {}
|
||||
|
||||
/** /api/show for this model, asked once; undefined when the server does not answer it. */
|
||||
show(id = this.m.id): Promise<OllamaShow | undefined> {
|
||||
const ask = async () => {
|
||||
const res = await request(joinUrl(this.m.connection.base_url, "api/show"), { method: "POST", headers: this.headers(), body: JSON.stringify({ model: id, name: id }), timeoutMs: 15_000, tls: tlsFor(this.m.connection) }, this.m.connectionName)
|
||||
return parseShow(await res.json())
|
||||
}
|
||||
if (id !== this.m.id) return ask().catch(() => undefined)
|
||||
return (this.shown ??= ask().catch(() => undefined))
|
||||
}
|
||||
|
||||
private headers() {
|
||||
return authHeaders(this.m.connection, resolveKey(this.m.connectionName, this.m.connection), this.m.spec.headers)
|
||||
}
|
||||
|
||||
buildBody(req: ChatRequest, show?: OllamaShow): Record<string, unknown> {
|
||||
const { m } = this
|
||||
const options: Record<string, unknown> = {}
|
||||
// Without num_ctx Ollama cuts every conversation to its small default window. Configured wins;
|
||||
// else the Modelfile's, else the trained maximum (set `context` to cap it to what memory holds).
|
||||
const ctx = m.spec.context ?? show?.numCtx ?? show?.trained
|
||||
if (ctx) options.num_ctx = ctx
|
||||
if (m.spec.max_output) options.num_predict = m.spec.max_output
|
||||
if (m.spec.temperature !== undefined) options.temperature = m.spec.temperature
|
||||
if (m.spec.top_p !== undefined) options.top_p = m.spec.top_p
|
||||
const body: Record<string, unknown> = { model: m.id, messages: toOllamaMessages(req.system, req.messages, m.spec.vision === true), stream: true, options, ...m.connection.body, ...m.spec.body }
|
||||
if (req.tools.length && m.spec.tools !== false)
|
||||
body.tools = req.tools.map((t) => ({ type: "function", function: { name: t.name, description: t.description, parameters: t.parameters } }))
|
||||
// gpt-oss takes a level; other thinking models take a switch. A model /api/show says cannot
|
||||
// think gets no `think` at all: Ollama refuses the request.
|
||||
const thinks = !show?.capabilities || show.capabilities.includes("thinking")
|
||||
if (thinks && req.effort) body.think = /gpt-oss/.test(m.id) ? (req.effort === "minimal" ? "low" : req.effort === "xhigh" || req.effort === "max" ? "high" : req.effort) : true
|
||||
else if (thinks && m.spec.efforts?.length) body.think = false
|
||||
return body
|
||||
}
|
||||
|
||||
async *stream(req: ChatRequest): AsyncGenerator<StreamEvent> {
|
||||
const body = this.buildBody(req, await this.show())
|
||||
yield* retrying(() => this.attempt(req, body))
|
||||
}
|
||||
|
||||
private async *attempt(req: ChatRequest, body: Record<string, unknown>): AsyncGenerator<StreamEvent> {
|
||||
const { m } = this
|
||||
const res = await request(joinUrl(m.connection.base_url, "api/chat"), { method: "POST", headers: this.headers(), body: JSON.stringify(body), signal: req.signal, timeoutMs: (m.connection.timeout ?? 600) * 1000, tls: tlsFor(m.connection) }, m.ref).catch((e) => {
|
||||
recordFailure(m.ref, body, e)
|
||||
throw e
|
||||
})
|
||||
if (!res.body) throw new ProviderError(`${m.ref}: empty response`)
|
||||
const out = new Parts()
|
||||
let finish = "stop"
|
||||
let usage: StreamEvent | undefined
|
||||
let calls = 0
|
||||
for await (const chunk of ndjson(record(res.body, m.ref, body)) as AsyncGenerator<any>) {
|
||||
if (chunk?.error) throw new ProviderError(`${m.ref}: ${chunk.error}`, statusOf(chunk) ?? 500, undefined, !out.output)
|
||||
const msg = chunk?.message ?? {}
|
||||
if (msg.thinking) {
|
||||
out.add("reasoning", msg.thinking)
|
||||
yield { type: "reasoning", text: msg.thinking }
|
||||
}
|
||||
if (msg.content) {
|
||||
out.add("text", msg.content)
|
||||
yield { type: "text", text: msg.content }
|
||||
}
|
||||
for (const tc of msg.tool_calls ?? []) {
|
||||
const args = JSON.stringify(tc.function?.arguments ?? {})
|
||||
yield { type: "tool_call_delta", index: calls, name: tc.function?.name, argsDelta: args }
|
||||
out.push({ type: "tool_call", id: tc.id ?? `ollama_call_${calls}`, name: tc.function?.name ?? "", args })
|
||||
calls++
|
||||
}
|
||||
if (chunk?.done) {
|
||||
finish = chunk.done_reason === "length" ? "length" : "stop"
|
||||
if (chunk.prompt_eval_count !== undefined || chunk.eval_count !== undefined) usage = { type: "usage", usage: { input: chunk.prompt_eval_count ?? 0, output: chunk.eval_count ?? 0 } }
|
||||
}
|
||||
}
|
||||
const message = out.message()
|
||||
if (calls && finish !== "length") finish = "tool_calls"
|
||||
yield usage ?? { type: "usage", usage: { input: estimateValue(body.messages), output: message.parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.args : p.text), 0), estimated: true } }
|
||||
yield { type: "finish", reason: finish, message }
|
||||
}
|
||||
|
||||
async listModels(): Promise<DiscoveredModel[]> {
|
||||
const res = await request(joinUrl(this.m.connection.base_url, "api/tags"), { headers: this.headers(), timeoutMs: 15_000, tls: tlsFor(this.m.connection) }, this.m.connectionName)
|
||||
const j = (await res.json()) as any
|
||||
return (j?.models ?? []).map((x: any) => ({ id: String(x.name ?? x.model ?? "") })).filter((x: DiscoveredModel) => x.id)
|
||||
}
|
||||
|
||||
/** The model's window: the Modelfile's num_ctx, else its trained maximum. */
|
||||
async contextOf(id: string): Promise<number | undefined> {
|
||||
const s = await this.show(id)
|
||||
return s?.numCtx ?? s?.trained
|
||||
}
|
||||
}
|
||||
+18
-59
@@ -1,69 +1,29 @@
|
||||
// How a model is named: by its provider, not by the login it came through. A LLeMbas
|
||||
// instance that speaks protocol 2 serves every model as `<provider>/<model>` — `deepseek/deepseek-flash`,
|
||||
// `llama/bonsai` — and that id is the model's ref here too: each provider is a connection of its
|
||||
// own as far as the person can see, backed by the instance (requests still go to its /v1, with the
|
||||
// served id). The older form of the ref was `<login connection>/<id>` (`example/deepseek-flash`).
|
||||
// How a model is named: by the id the LLeMbas server serves it under — `deepseek/deepseek-flash`,
|
||||
// `llama/bonsai` — which is its ref here too. LLeMbas CLI 2.0 talks to one server, the one it is
|
||||
// logged in to, so there is nothing to tell apart: no connection prefix, no second login.
|
||||
//
|
||||
// - An instance that does not speak protocol 2 serves bare ids: those stay `<login>/<id>`.
|
||||
// - A connection of the user's own with the provider's name (`deepseek:` in connections.yaml)
|
||||
// keeps `deepseek/x`; the instance's is then `<login>/deepseek/x`.
|
||||
// - Two instances serving the same provider: the one logged in to first keeps the short form, the
|
||||
// other's are `<login>/<provider>/<model>`.
|
||||
// - The old forms still resolve — `example/deepseek-flash` and `example/deepseek/deepseek-flash`
|
||||
// are `deepseek/deepseek-flash` — and a config.yaml that names one is rewritten to the new form.
|
||||
// The older forms still resolve, and a config.yaml that names one is rewritten to the id (load.ts):
|
||||
// `<login>/<id>` (`example/deepseek/deepseek-flash`) and, from before providers, `<login>/<model>`
|
||||
// for the one served `<provider>/<model>` of that name (`example/deepseek-flash`).
|
||||
import type { Connection } from "../config/schema.ts"
|
||||
|
||||
export interface ModelRef {
|
||||
/** What the person sees and writes. */
|
||||
/** What the person sees and writes: the served id. */
|
||||
ref: string
|
||||
/** The connection it is spoken to through, and the id sent there. */
|
||||
/** The login it is spoken to through, and the id sent there. */
|
||||
connection: string
|
||||
id: string
|
||||
/** An instance's model: its provider, when known — what the old `<login>/<model>` is matched by. */
|
||||
provider?: string
|
||||
}
|
||||
|
||||
type Conns = Record<string, Connection & { webui?: { providers?: Record<string, string>; v2?: boolean } }>
|
||||
type Conns = Record<string, Connection>
|
||||
|
||||
/** The provider of a served id: as the instance said it (`provider`), else — from an
|
||||
* instance that speaks protocol 2, whose ids are always `<provider>/<model>` — the part before the
|
||||
* first `/`. An id with a `/` from an older instance is just an id. */
|
||||
export function providerOf(c: Conns[string], id: string): string | undefined {
|
||||
if (!c.webui) return undefined
|
||||
const said = c.webui.providers?.[id]
|
||||
if (said) return said
|
||||
return c.webui.v2 && id.indexOf("/") > 0 ? id.slice(0, id.indexOf("/")) : undefined
|
||||
/** Every model the server serves, in its order, with its ref. */
|
||||
export function modelRefs(connections: Conns): ModelRef[] {
|
||||
return Object.entries(connections).flatMap(([name, c]) => Object.keys(c.models).map((id) => ({ ref: id, connection: name, id })))
|
||||
}
|
||||
|
||||
/** Every model, in config order, with its ref. `logins`: the webui connections in the order they
|
||||
* were logged in to (first keeps a shared provider's short form). */
|
||||
export function modelRefs(connections: Conns, logins: string[] = []): ModelRef[] {
|
||||
const names = new Set(Object.keys(connections))
|
||||
const webui = Object.keys(connections).filter((n) => connections[n]!.webui)
|
||||
const order = [...logins.filter((n) => webui.includes(n)), ...webui.filter((n) => !logins.includes(n))]
|
||||
// Which webui connection a provider's short form belongs to.
|
||||
const owner = new Map<string, string>()
|
||||
for (const w of order)
|
||||
for (const id of Object.keys(connections[w]!.models)) {
|
||||
const provider = providerOf(connections[w]!, id)
|
||||
if (!provider) continue
|
||||
// A connection of that name wins it — unless it is this login itself (`llama/bonsai` on a
|
||||
// login named llama is the same string either way).
|
||||
if ((names.has(provider) && provider !== w) || owner.has(provider)) continue
|
||||
owner.set(provider, w)
|
||||
}
|
||||
const out: ModelRef[] = []
|
||||
for (const [name, c] of Object.entries(connections))
|
||||
for (const id of Object.keys(c.models)) {
|
||||
const provider = providerOf(c, id)
|
||||
const short = provider && owner.get(provider) === name
|
||||
out.push({ ref: short ? id : `${name}/${id}`, connection: name, id, ...(provider ? { provider } : {}) })
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** A ref as written — any of the forms above — as the model it names, or undefined. `old`: it was
|
||||
* the form from before providers (`<login>/<model>` for a served `<provider>/<model>`). */
|
||||
/** A ref as written — the id, or an older form — as the model it names, or undefined. `old`: it
|
||||
* was an older form, which config.yaml is rewritten from. */
|
||||
export function findRef(connections: Conns, refs: ModelRef[], ref: string): (ModelRef & { old?: boolean }) | undefined {
|
||||
const exact = refs.find((r) => r.ref === ref)
|
||||
if (exact) return exact
|
||||
@@ -71,13 +31,12 @@ export function findRef(connections: Conns, refs: ModelRef[], ref: string): (Mod
|
||||
if (slash <= 0) return undefined
|
||||
const conn = ref.slice(0, slash)
|
||||
const id = ref.slice(slash + 1)
|
||||
const c = connections[conn]
|
||||
if (!c) return undefined
|
||||
if (!connections[conn]) return undefined
|
||||
const direct = refs.find((r) => r.connection === conn && r.id === id)
|
||||
if (direct) return direct
|
||||
if (direct) return { ...direct, old: true }
|
||||
// `<login>/<model>` from before providers: the one served model of that name, whatever its provider.
|
||||
if (c.webui && !id.includes("/")) {
|
||||
const by = refs.filter((r) => r.connection === conn && r.provider && r.id === `${r.provider}/${id}`)
|
||||
if (!id.includes("/")) {
|
||||
const by = refs.filter((r) => r.connection === conn && r.id.endsWith(`/${id}`) && r.id.indexOf("/") === r.id.length - id.length - 1)
|
||||
if (by.length === 1) return { ...by[0]!, old: true }
|
||||
}
|
||||
return undefined
|
||||
|
||||
@@ -1,174 +0,0 @@
|
||||
// OpenAI Responses (`POST {base}/responses`) — api.openai.com, and vLLM. Stateless (store: false):
|
||||
// the whole history is sent as input items, and reasoning items come back opaque (with their
|
||||
// encrypted content when the server provides it) so a reasoning model keeps its thread.
|
||||
import { resolveKey } from "../config/load.ts"
|
||||
import { historyArgs, Parts, record, recordFailure, retrying, statusOf, SwapBanner } from "./common.ts"
|
||||
import { authHeaders, joinUrl, request, tlsFor } from "./http.ts"
|
||||
import { contextFrom } from "./openai-chat.ts"
|
||||
import { sseJson } from "./sse.ts"
|
||||
import { estimateTokens, estimateValue } from "./tokens.ts"
|
||||
import { ProviderError, type ChatRequest, type Client, type DiscoveredModel, type Message, type ResolvedModel, type StreamEvent } from "./types.ts"
|
||||
|
||||
type Item = Record<string, unknown>
|
||||
|
||||
export function toResponsesInput(messages: Message[], vision: boolean): Item[] {
|
||||
const out: Item[] = []
|
||||
for (const m of messages) {
|
||||
if (m.role === "user") {
|
||||
out.push({
|
||||
role: "user",
|
||||
content: m.parts.map((p) =>
|
||||
p.type === "text"
|
||||
? { type: "input_text", text: p.text }
|
||||
: vision
|
||||
? { type: "input_image", image_url: `data:${p.mime};base64,${p.data}` }
|
||||
: { type: "input_text", text: "[image omitted: this model has no vision]" },
|
||||
),
|
||||
})
|
||||
} else if (m.role === "assistant") {
|
||||
let text: string[] = []
|
||||
const flush = () => {
|
||||
if (text.join("").trim()) out.push({ role: "assistant", content: [{ type: "output_text", text: text.join("") }] })
|
||||
text = []
|
||||
}
|
||||
for (const p of m.parts) {
|
||||
if (p.type === "text") text.push(p.text)
|
||||
else if (p.type === "reasoning") {
|
||||
// Only a reasoning item this dialect produced can go back; text from elsewhere cannot.
|
||||
const item = (p.opaque as { responses?: Item } | undefined)?.responses
|
||||
if (item) {
|
||||
flush()
|
||||
out.push(item)
|
||||
}
|
||||
} else {
|
||||
flush()
|
||||
out.push({ type: "function_call", call_id: p.id, name: p.name, arguments: historyArgs(p.args) })
|
||||
}
|
||||
}
|
||||
flush()
|
||||
} else out.push({ type: "function_call_output", call_id: m.callId, output: m.content })
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
export class ResponsesClient implements Client {
|
||||
constructor(private m: ResolvedModel) {}
|
||||
|
||||
private headers() {
|
||||
return authHeaders(this.m.connection, resolveKey(this.m.connectionName, this.m.connection), this.m.spec.headers)
|
||||
}
|
||||
|
||||
buildBody(req: ChatRequest): Record<string, unknown> {
|
||||
const { m } = this
|
||||
const body: Record<string, unknown> = {
|
||||
model: m.id,
|
||||
input: toResponsesInput(req.messages, m.spec.vision === true),
|
||||
stream: true,
|
||||
store: false,
|
||||
...m.connection.body,
|
||||
...m.spec.body,
|
||||
}
|
||||
if (req.system) body.instructions = req.system
|
||||
if (req.tools.length && m.spec.tools !== false)
|
||||
body.tools = req.tools.map((t) => ({ type: "function", name: t.name, description: t.description, parameters: t.parameters, strict: false }))
|
||||
if (m.spec.max_output) body.max_output_tokens = m.spec.max_output
|
||||
if (req.effort) {
|
||||
body.reasoning = { effort: req.effort, summary: "auto" }
|
||||
body.include = ["reasoning.encrypted_content"]
|
||||
} else {
|
||||
if (m.spec.temperature !== undefined) body.temperature = m.spec.temperature
|
||||
if (m.spec.top_p !== undefined) body.top_p = m.spec.top_p
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
async *stream(req: ChatRequest): AsyncGenerator<StreamEvent> {
|
||||
let body = this.buildBody(req)
|
||||
yield* retrying(
|
||||
() => this.attempt(req, body),
|
||||
(e) => {
|
||||
// Some servers do not know `include`; drop it once rather than fail.
|
||||
if (body.include && (e.status === 400 || e.status === 422) && /include/i.test(e.message)) {
|
||||
const { include: _, ...rest } = body
|
||||
body = rest
|
||||
return { retry: true }
|
||||
}
|
||||
return { retry: false }
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
private async *attempt(req: ChatRequest, body: Record<string, unknown>): AsyncGenerator<StreamEvent> {
|
||||
const { m } = this
|
||||
const res = await request(
|
||||
joinUrl(m.connection.base_url, "responses"),
|
||||
{ method: "POST", headers: this.headers(), body: JSON.stringify(body), signal: req.signal, timeoutMs: (m.connection.timeout ?? 600) * 1000, tls: tlsFor(m.connection) },
|
||||
m.ref,
|
||||
).catch((e) => {
|
||||
recordFailure(m.ref, body, e)
|
||||
throw e
|
||||
})
|
||||
if (!res.body) throw new ProviderError(`${m.ref}: empty response`)
|
||||
|
||||
const out = new Parts()
|
||||
const banner = new SwapBanner()
|
||||
const reasoning = new Map<string, string>()
|
||||
const callIndex = new Map<string, number>()
|
||||
let finish = "stop"
|
||||
let usage: StreamEvent | undefined
|
||||
|
||||
for await (const { data } of sseJson(record(res.body, m.ref, body))) {
|
||||
const ev = data as any
|
||||
const t: string = ev?.type ?? ""
|
||||
if (t === "response.output_text.delta") {
|
||||
out.add("text", ev.delta ?? "")
|
||||
yield { type: "text", text: ev.delta ?? "" }
|
||||
} else if (t === "response.reasoning_summary_text.delta" || t === "response.reasoning_text.delta") {
|
||||
const b = banner.feed(ev.delta ?? "")
|
||||
if (b.notice) yield { type: "notice", message: b.notice }
|
||||
if (!b.text) continue
|
||||
reasoning.set(ev.item_id ?? "", (reasoning.get(ev.item_id ?? "") ?? "") + b.text)
|
||||
if (b.text.trim()) out.output = true
|
||||
yield { type: "reasoning", text: b.text }
|
||||
} else if (t === "response.output_item.added" && ev.item?.type === "function_call") {
|
||||
const index = callIndex.size
|
||||
callIndex.set(ev.item.id ?? ev.item.call_id, index)
|
||||
yield { type: "tool_call_delta", index, name: ev.item.name, argsDelta: "" }
|
||||
} else if (t === "response.function_call_arguments.delta") {
|
||||
out.output = true
|
||||
yield { type: "tool_call_delta", index: callIndex.get(ev.item_id) ?? 0, argsDelta: ev.delta ?? "" }
|
||||
} else if (t === "response.output_item.done") {
|
||||
const item = ev.item ?? {}
|
||||
if (item.type === "function_call") out.push({ type: "tool_call", id: item.call_id ?? item.id, name: item.name, args: item.arguments || "{}" })
|
||||
else if (item.type === "reasoning") {
|
||||
const summary = Array.isArray(item.summary) ? item.summary.map((s: any) => s.text ?? "").join("\n") : ""
|
||||
const text = reasoning.get(item.id ?? "") || summary
|
||||
// vLLM puts the raw reasoning in `content`, which it will not accept back; keep what can return.
|
||||
const { content: _c, ...back } = item
|
||||
if (text || item.encrypted_content) out.push({ type: "reasoning", text, opaque: item.encrypted_content ? { responses: back } : undefined })
|
||||
}
|
||||
} else if (t === "response.completed" || t === "response.incomplete") {
|
||||
const r = ev.response ?? {}
|
||||
if (t === "response.incomplete" || r.status === "incomplete") finish = "length"
|
||||
const u = r.usage
|
||||
if (u) usage = { type: "usage", usage: { input: u.input_tokens ?? 0, output: u.output_tokens ?? 0, reasoning: u.output_tokens_details?.reasoning_tokens, cached: u.input_tokens_details?.cached_tokens || undefined } }
|
||||
} else if (t === "response.failed" || t === "error" || ev?.error) {
|
||||
const err = ev.response?.error ?? ev.error ?? ev
|
||||
throw new ProviderError(`${m.ref}: ${err.message ?? JSON.stringify(err)}`, statusOf(err), undefined, !out.output)
|
||||
}
|
||||
}
|
||||
const message = out.message()
|
||||
if (finish !== "length" && message.parts.some((p) => p.type === "tool_call")) finish = "tool_calls"
|
||||
yield usage ?? {
|
||||
type: "usage",
|
||||
usage: { input: estimateValue(body.input), output: message.parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.args : p.text), 0), estimated: true },
|
||||
}
|
||||
yield { type: "finish", reason: finish, message }
|
||||
}
|
||||
|
||||
async listModels(): Promise<DiscoveredModel[]> {
|
||||
const res = await request(joinUrl(this.m.connection.base_url, "models"), { headers: this.headers(), timeoutMs: 15_000, tls: tlsFor(this.m.connection) }, this.m.connectionName)
|
||||
const j = (await res.json()) as any
|
||||
return (Array.isArray(j?.data) ? j.data : []).map((x: any) => ({ id: String(x.id), context: contextFrom(x) })).filter((x: DiscoveredModel) => x.id)
|
||||
}
|
||||
}
|
||||
@@ -56,7 +56,7 @@ export interface ChatRequest {
|
||||
}
|
||||
|
||||
export interface ResolvedModel {
|
||||
/** `connection/model` */
|
||||
/** The served id (`deepseek/deepseek-flash`). */
|
||||
ref: string
|
||||
connectionName: string
|
||||
connection: Connection
|
||||
|
||||
@@ -19,8 +19,8 @@ export function capacityRefusal(main: ResolvedModel, helper: ResolvedModel): str
|
||||
}
|
||||
const shared = groupOf(main)
|
||||
if (shared !== undefined && shared === groupOf(helper)) {
|
||||
const where = main.spec.group !== undefined ? `${main.ref}'s server (group ${main.spec.group})` : main.connectionName
|
||||
return `${where} holds one model at a time (${main.spec.group !== undefined ? "group" : "one_model_at_a_time"}), so a subagent on ${helper.ref} would unload ${main.ref}, and this session's cached prompt with it, while its turn waits.`
|
||||
const where = `${main.ref}'s server (group ${main.spec.group})`
|
||||
return `${where} holds one model at a time, so a subagent on ${helper.ref} would unload ${main.ref}, and this session's cached prompt with it, while its turn waits.`
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -35,15 +35,14 @@ export function capacityBeside(running: ResolvedModel, helper: ResolvedModel): s
|
||||
}
|
||||
const shared = groupOf(running)
|
||||
if (shared !== undefined && shared === groupOf(helper)) {
|
||||
const where = running.spec.group !== undefined ? `${running.ref}'s server (group ${running.spec.group})` : running.connectionName
|
||||
const where = `${running.ref}'s server (group ${running.spec.group})`
|
||||
return `${where} holds one model at a time, and a subagent is already running on ${running.ref} there; one on ${helper.ref} would unload it.`
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
/** Which one-model-at-a-time server a model is on: its own `group`, else its connection when the
|
||||
* connection sets one_model_at_a_time, else none. The spec's capacity.group. */
|
||||
/** Which one-model-at-a-time server a model is on, as the LLeMbas server says (`capacity.group`,
|
||||
* the spec's), or none. */
|
||||
function groupOf(m: ResolvedModel): string | undefined {
|
||||
if (m.spec.group !== undefined) return `group:${m.spec.group}`
|
||||
return m.connection.one_model_at_a_time ? `connection:${m.connectionName}` : undefined
|
||||
return m.spec.group !== undefined ? `group:${m.spec.group}` : undefined
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
// /context: where the window goes. Estimates (≈4 characters a token) per part, scaled so they add
|
||||
// up to what the server last reported when it reported anything.
|
||||
import type { Engine } from "./engine.ts"
|
||||
import { contextWindow } from "../provider/discover.ts"
|
||||
import { contextWindow } from "../provider/index.ts"
|
||||
import { estimateTokens, imageTokens } from "../provider/tokens.ts"
|
||||
import { toSpec, type Tool } from "../tool/tool.ts"
|
||||
|
||||
|
||||
+2
-19
@@ -14,7 +14,7 @@ import { fillPrompt } from "../prompt/fill.ts"
|
||||
import { splitPurpose, toSpec, ToolError, type Tool, type ToolContext, type ToolResult } from "../tool/tool.ts"
|
||||
import { fill, promptText } from "../prompt/assemble.ts"
|
||||
import { resolveCall } from "../tool/names.ts"
|
||||
import { contextWindow } from "../provider/discover.ts"
|
||||
import { contextWindow } from "../provider/index.ts"
|
||||
import { summaryTurns, type Store } from "./store.ts"
|
||||
import { estimateTokens, estimateValue } from "../provider/tokens.ts"
|
||||
import { duration } from "../duration.ts"
|
||||
@@ -43,9 +43,6 @@ export interface EngineOptions {
|
||||
compaction?: { autoAt?: number }
|
||||
/** Starts a subagent for the task tool; absent in a subagent. */
|
||||
spawn?: ToolContext["spawn"]
|
||||
/** The model to switch to when this one's server cannot be reached (its `fallback` list), or
|
||||
* undefined when there is none left to try. */
|
||||
fallback?: (current: ResolvedModel, tried: Set<string>) => { model: ResolvedModel; client: Client; effort: Effort | null } | undefined
|
||||
/** What a message sent mid-turn starts with: the user's words for a session, the main agent's for
|
||||
* a subagent (loop.json jobs.marks to_subagent). */
|
||||
steerMark?: string
|
||||
@@ -264,7 +261,7 @@ export class Engine {
|
||||
}
|
||||
|
||||
private droppedThinking(): number {
|
||||
if (!this.usage || this.model.connection.dialect !== "openai-chat") return 0
|
||||
if (!this.usage) return 0
|
||||
const reply = this.messages[this.usageAt - 1]
|
||||
if (reply?.role !== "assistant") return 0
|
||||
const estimate = reply.parts.reduce((n, p) => n + (p.type === "reasoning" ? estimateTokens(p.text) : 0), 0)
|
||||
@@ -468,7 +465,6 @@ export class Engine {
|
||||
let ranOut: { limit: string; said: string } | undefined
|
||||
let nudges = 0
|
||||
let midRetries = 0
|
||||
const tried = new Set<string>()
|
||||
|
||||
for (let step = 1; ; step++) {
|
||||
// Checked between steps, never mid-reply: a reply cut off mid-sentence is worth nothing.
|
||||
@@ -519,19 +515,6 @@ export class Engine {
|
||||
bus.emit({ type: "done", reason: "cancelled" })
|
||||
return "cancelled"
|
||||
}
|
||||
// The server is not there at all: the model's fallback, if it has one, for the rest of the
|
||||
// session — said, and shown in the status bar.
|
||||
if (e instanceof ProviderError && e.unreachable && this.o.fallback) {
|
||||
tried.add(this.model.ref)
|
||||
const next = this.o.fallback(this.model, tried)
|
||||
if (next) {
|
||||
bus.emit({ type: "notice", message: `${e.message.split("\n")[0]!.slice(0, 160)} — switching to ${next.model.ref}, its fallback` })
|
||||
this.setModel(next.model, next.client, next.effort)
|
||||
bus.emit({ type: "setting", key: "model", value: next.model.ref })
|
||||
step--
|
||||
continue
|
||||
}
|
||||
}
|
||||
// A server failure part-way through a reply (llama.cpp failing to parse gpt-oss's own
|
||||
// output is the one seen) is worth two more goes. What was already shown is discarded.
|
||||
if (e instanceof ProviderError && !e.beforeOutput && (e.status ?? 0) >= 500 && midRetries < 2) {
|
||||
|
||||
+2
-2
@@ -85,7 +85,7 @@ const pendingCodes = new Map<string, (text: string) => void>()
|
||||
|
||||
const AGENT_TEMPLATE = (name: string) => `---
|
||||
description: what ${name} is for — the model reads this when it picks a subagent
|
||||
# model: connection/model # run it on another model (default: the session's)
|
||||
# model: provider/model # run it on another of the server's models (default: the session's)
|
||||
# mode: plan # plan: read-only; edit; manual (asks you) — never looser than yours
|
||||
# tools: [read, glob, grep, list, bash, web_search, web_fetch] # default: every tool a subagent may have
|
||||
# isolation: worktree # work in a git checkout and branch of its own
|
||||
@@ -153,7 +153,7 @@ export const COMMANDS: Command[] = [
|
||||
},
|
||||
{
|
||||
name: "model",
|
||||
args: "[connection/model]",
|
||||
args: "[model]",
|
||||
description: "switch model",
|
||||
run(c, arg) {
|
||||
const refs = c.app.modelRefs()
|
||||
|
||||
@@ -1,18 +1,9 @@
|
||||
import { TextAttributes } from "@opentui/core"
|
||||
import { useKeyboard } from "@opentui/solid"
|
||||
import { join } from "node:path"
|
||||
import { createSignal, Show } from "solid-js"
|
||||
import { paths } from "../../config/paths.ts"
|
||||
import { ModelError } from "../../provider/index.ts"
|
||||
import { ModelError, NOT_LOGGED_IN } from "../../provider/index.ts"
|
||||
import { useTui } from "../context.ts"
|
||||
|
||||
const EXAMPLE = `connections:
|
||||
local: # any name; llama.cpp, LM Studio, vLLM, Ollama's /v1…
|
||||
dialect: openai-chat
|
||||
base_url: http://127.0.0.1:8080/v1
|
||||
models:
|
||||
qwen3: { context: 131072 }`
|
||||
|
||||
/** Before there is a session: no model to talk to (a fresh install), or a config file that cannot
|
||||
* be read. Says what to do and where; r tries again once the files are fixed, q or ctrl+c leaves.
|
||||
* Not an exception: thrown with the terminal still the TUI's, mouse reporting would stay on, and
|
||||
@@ -38,7 +29,7 @@ export function Setup(p: { error: Error; onRetry: () => void; onQuit: () => void
|
||||
try {
|
||||
const { codeInstructions, login } = await import("../../lembas/login.ts")
|
||||
const r = await login(target, { onCode: (start, d) => setStatus(codeInstructions(start, d)) })
|
||||
setStatus(`Signed in: ${r.models.length} model${r.models.length === 1 ? "" : "s"} as ${r.connection}/…${r.instanceDefault ? `, starting on ${r.connection}/${r.instanceDefault}` : ""}.`)
|
||||
setStatus(`Signed in: ${r.models.length} model${r.models.length === 1 ? "" : "s"}${r.instanceDefault ? `, starting on ${r.instanceDefault}` : ""}.`)
|
||||
p.onRetry()
|
||||
} catch (e) {
|
||||
setStatus((e as Error).message)
|
||||
@@ -65,11 +56,11 @@ export function Setup(p: { error: Error; onRetry: () => void; onQuit: () => void
|
||||
return (
|
||||
<box flexDirection="column" width="100%" height="100%" paddingLeft={2} paddingTop={1} paddingRight={2} gap={1}>
|
||||
<text fg={c().title} attributes={TextAttributes.BOLD}>
|
||||
{theme().branding.agentName} — {noModel() ? "no model to talk to yet" : "the configuration has a mistake"}
|
||||
{theme().branding.agentName} — {p.error.message === NOT_LOGGED_IN ? "not logged in to a LLeMbas server" : noModel() ? "no model to talk to" : "the configuration has a mistake"}
|
||||
</text>
|
||||
{/* "No model configured" is what the rest of the screen explains; anything else (a model
|
||||
that does not exist, a file that does not parse) is shown as it is. */}
|
||||
<Show when={!p.error.message.startsWith("No model configured")}>
|
||||
{/* Not being logged in is what the rest of the screen explains; anything else (a model the
|
||||
server does not serve, a file that does not parse) is shown as it is. */}
|
||||
<Show when={p.error.message !== NOT_LOGGED_IN}>
|
||||
<text fg={noModel() ? c().text : c().error} wrapMode="word">
|
||||
{p.error.message}
|
||||
</text>
|
||||
@@ -80,7 +71,7 @@ export function Setup(p: { error: Error; onRetry: () => void; onQuit: () => void
|
||||
when={typing()}
|
||||
fallback={
|
||||
<text fg={c().muted} wrapMode="word">
|
||||
{"Sign in to a LLeMbas instance and use its models: press l (or, from a shell, lembas login <address>)."}
|
||||
{"LLeMbas CLI uses the models of the LLeMbas server it is logged in to. Log in: press l (or, from a shell, lembas login <address>)."}
|
||||
</text>
|
||||
}
|
||||
>
|
||||
@@ -96,24 +87,9 @@ export function Setup(p: { error: Error; onRetry: () => void; onQuit: () => void
|
||||
{status()}
|
||||
</text>
|
||||
</Show>
|
||||
<text fg={c().muted} wrapMode="word">
|
||||
{`Or, without an instance — nothing is predefined: add the server you use to ${join(paths.config, "connections.yaml")}, for example`}
|
||||
</text>
|
||||
<box paddingLeft={2}>
|
||||
<text fg={c().text}>{EXAMPLE}</text>
|
||||
</box>
|
||||
<text fg={c().muted} wrapMode="word">
|
||||
{`and pick the model in ${join(paths.config, "config.yaml")}:`}
|
||||
</text>
|
||||
<box paddingLeft={2}>
|
||||
<text fg={c().text}>model: local/qwen3</text>
|
||||
</box>
|
||||
<text fg={c().muted} wrapMode="word">
|
||||
A key is never pasted in: api_key: "{"{env:NAME}"}" or "{"{file:~/path}"}". https://llembas.eu/docs/ has the other dialects (Anthropic, OpenAI, Gemini, Ollama).
|
||||
</text>
|
||||
</Show>
|
||||
<text fg={c().muted} wrapMode="word">
|
||||
From a shell, lembas config check reads the files and says what is wrong, and lembas models lists the models it found.
|
||||
From a shell, lembas config check reads the files and says what is wrong, and lembas models lists the server's models.
|
||||
</text>
|
||||
<text fg={c().label}>{typing() ? "enter sign in · esc back · ctrl+r try again · ctrl+c quit" : `${noModel() ? "l log in · " : ""}r try again · q quit`}</text>
|
||||
</box>
|
||||
|
||||
Reference in new issue
Block a user