Custom connections are gone: the CLI talks to the LLeMbas server it is logged in to
LLeMbas CLI 2.0 is LLeMbas-server-specific (CLI rule 2). Removed: the anthropic, gemini, ollama and responses clients, context discovery and llama-swap unloading (discover.ts), the Connection schema with its dialect, auth, key_cmd, discover, unload_url and one_model_at_a_time, the model `fallback` (it only ever moved to another server), effort_map and cache, and the multi-connection refs. Kept: openai-chat, http, sse, think, effort, tokens and all of src/lembas. - connections.yaml holds the login (`type: webui`, as lembas login writes it). A custom entry from 1.x is read past with a warning that says how to log in, so an old file still starts; with several logins left, the latest is used and the others named. - A model's ref is the id the server serves it under; the older forms (`<login>/<id>`, `<login>/<model>`) still resolve and config.yaml is rewritten once. Without a login there is no model, and the error (and the first-run screen, now the sign-in alone) says `lembas login <address>`. - A login to another server signs this machine out of the previous one. - The window is the server's word (no discovery); a capacity group comes from the server's `capacity.group` alone; embeddings use the server's /v1/embeddings. - `lembas models` lists the server's models; install.sh's starter files, the README and the JSON schemas say the same. Tests: every fixture is a login now (the fake server lists models, answers other GETs with 404, and can hand one model to another script, as a server with several backends); the dialect suites and the cross-connection tests are gone with their code. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
80 files changed
+587
-2575
No files matched your search
@@ -1,228 +0,0 @@
|
||||
// Anthropic Messages (`POST {base}/messages`) — api.anthropic.com, and servers that imitate it
|
||||
// (vLLM, llama-server). Content blocks both ways; thinking with signatures that must be sent back
|
||||
// on the next turn; optional prompt-caching breakpoints.
|
||||
import type { Effort } from "../config/schema.ts"
|
||||
import { resolveKey } from "../config/load.ts"
|
||||
import { Parts, record, recordFailure, retrying, statusOf, SwapBanner } from "./common.ts"
|
||||
import { authHeaders, joinUrl, request, tlsFor } from "./http.ts"
|
||||
import { sseJson } from "./sse.ts"
|
||||
import { estimateTokens, estimateValue } from "./tokens.ts"
|
||||
import { ProviderError, type ChatRequest, type Client, type DiscoveredModel, type Message, type ResolvedModel, type StreamEvent, type ToolCallPart } from "./types.ts"
|
||||
|
||||
/** Thinking budgets by effort, when the model's config gives no effort_map. */
|
||||
export const DEFAULT_BUDGETS: Record<Effort, number> = { minimal: 1024, low: 2048, medium: 8192, high: 16384, xhigh: 32000, max: 48000 }
|
||||
|
||||
type Block = Record<string, unknown>
|
||||
type AMessage = { role: "user" | "assistant"; content: Block[] }
|
||||
|
||||
/** Anthropic ids must match ^[a-zA-Z0-9_-]+$; ids from other providers are mapped consistently. */
|
||||
export const safeId = (id: string) => id.replace(/[^a-zA-Z0-9_-]/g, "_") || "call"
|
||||
|
||||
export function toAnthropicMessages(messages: Message[], vision: boolean, thinking: boolean): AMessage[] {
|
||||
const out: AMessage[] = []
|
||||
const push = (role: AMessage["role"], blocks: Block[]) => {
|
||||
if (!blocks.length) return
|
||||
const last = out[out.length - 1]
|
||||
// Consecutive turns of one role are merged: tool results and a following note are one user turn.
|
||||
if (last && last.role === role) last.content.push(...blocks)
|
||||
else out.push({ role, content: blocks })
|
||||
}
|
||||
for (const m of messages) {
|
||||
if (m.role === "user") {
|
||||
push(
|
||||
"user",
|
||||
m.parts.map((p) =>
|
||||
p.type === "text"
|
||||
? { type: "text", text: p.text }
|
||||
: vision
|
||||
? { type: "image", source: { type: "base64", media_type: p.mime, data: p.data } }
|
||||
: { type: "text", text: "[image omitted: this model has no vision]" },
|
||||
),
|
||||
)
|
||||
} else if (m.role === "assistant") {
|
||||
const blocks: Block[] = []
|
||||
for (const p of m.parts) {
|
||||
if (p.type === "reasoning") {
|
||||
// A thinking block can only go back with its signature, and only while thinking is on.
|
||||
if (!thinking) continue
|
||||
const redacted = (p.opaque as { redacted?: string } | undefined)?.redacted
|
||||
if (redacted) blocks.push({ type: "redacted_thinking", data: redacted })
|
||||
else if (p.signature) blocks.push({ type: "thinking", thinking: p.text, signature: p.signature })
|
||||
} else if (p.type === "text") {
|
||||
if (p.text.trim()) blocks.push({ type: "text", text: p.text })
|
||||
} else {
|
||||
let input: unknown = {}
|
||||
try {
|
||||
input = p.args.trim() ? JSON.parse(p.args) : {}
|
||||
} catch {
|
||||
input = { _unparsed: p.args }
|
||||
}
|
||||
blocks.push({ type: "tool_use", id: safeId(p.id), name: p.name, input })
|
||||
}
|
||||
}
|
||||
push("assistant", blocks)
|
||||
} else {
|
||||
push("user", [{ type: "tool_result", tool_use_id: safeId(m.callId), content: m.content || "(empty)", ...(m.isError ? { is_error: true } : {}) }])
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
export class AnthropicClient implements Client {
|
||||
constructor(private m: ResolvedModel) {}
|
||||
|
||||
private headers() {
|
||||
return authHeaders(this.m.connection, resolveKey(this.m.connectionName, this.m.connection), { "anthropic-version": "2023-06-01", ...this.m.spec.headers })
|
||||
}
|
||||
|
||||
buildBody(req: ChatRequest): Record<string, unknown> {
|
||||
const { m } = this
|
||||
const budget = req.effort ? (m.spec.effort_map?.[req.effort] ?? DEFAULT_BUDGETS[req.effort]) : undefined
|
||||
let maxTokens = m.spec.max_output ?? 8192
|
||||
if (budget && maxTokens <= budget) maxTokens = budget + Math.min(8192, maxTokens)
|
||||
const cache = m.spec.cache === true
|
||||
const body: Record<string, unknown> = {
|
||||
model: m.id,
|
||||
max_tokens: maxTokens,
|
||||
stream: true,
|
||||
messages: toAnthropicMessages(req.messages, m.spec.vision === true, !!budget),
|
||||
...m.connection.body,
|
||||
...m.spec.body,
|
||||
}
|
||||
if (req.system) body.system = cache ? [{ type: "text", text: req.system, cache_control: { type: "ephemeral" } }] : req.system
|
||||
if (req.tools.length && m.spec.tools !== false) {
|
||||
const tools = req.tools.map((t) => ({ name: t.name, description: t.description, input_schema: t.parameters }))
|
||||
if (cache) (tools[tools.length - 1] as Block).cache_control = { type: "ephemeral" }
|
||||
body.tools = tools
|
||||
}
|
||||
if (cache) {
|
||||
const msgs = body.messages as AMessage[]
|
||||
const last = msgs[msgs.length - 1]?.content
|
||||
if (last?.length) last[last.length - 1] = { ...last[last.length - 1], cache_control: { type: "ephemeral" } }
|
||||
}
|
||||
if (budget) body.thinking = { type: "enabled", budget_tokens: budget }
|
||||
else {
|
||||
// Sampling parameters and extended thinking do not mix on this API.
|
||||
if (m.spec.temperature !== undefined) body.temperature = m.spec.temperature
|
||||
if (m.spec.top_p !== undefined) body.top_p = m.spec.top_p
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
async *stream(req: ChatRequest): AsyncGenerator<StreamEvent> {
|
||||
const body = this.buildBody(req)
|
||||
yield* retrying(() => this.attempt(req, body))
|
||||
}
|
||||
|
||||
private async *attempt(req: ChatRequest, body: Record<string, unknown>): AsyncGenerator<StreamEvent> {
|
||||
const { m } = this
|
||||
const res = await request(
|
||||
joinUrl(m.connection.base_url, "messages"),
|
||||
{ method: "POST", headers: this.headers(), body: JSON.stringify(body), signal: req.signal, timeoutMs: (m.connection.timeout ?? 600) * 1000, tls: tlsFor(m.connection) },
|
||||
m.ref,
|
||||
).catch((e) => {
|
||||
recordFailure(m.ref, body, e)
|
||||
throw e
|
||||
})
|
||||
if (!res.body) throw new ProviderError(`${m.ref}: empty response`)
|
||||
|
||||
const out = new Parts()
|
||||
const banner = new SwapBanner()
|
||||
type Open = { type: string; text: string; signature?: string; id?: string; name?: string; json: string; data?: string; index: number }
|
||||
const blocks = new Map<number, Open>()
|
||||
let toolIndex = 0
|
||||
let finish = "stop"
|
||||
let input = 0
|
||||
let output = 0
|
||||
let cached: number | undefined
|
||||
let sawUsage = false
|
||||
|
||||
for await (const { data } of sseJson(record(res.body, m.ref, body))) {
|
||||
const ev = data as any
|
||||
switch (ev?.type) {
|
||||
case "message_start": {
|
||||
const u = ev.message?.usage
|
||||
if (u) {
|
||||
sawUsage = true
|
||||
cached = u.cache_read_input_tokens || undefined
|
||||
input = (u.input_tokens ?? 0) + (u.cache_read_input_tokens ?? 0) + (u.cache_creation_input_tokens ?? 0)
|
||||
output = u.output_tokens ?? 0
|
||||
}
|
||||
break
|
||||
}
|
||||
case "content_block_start": {
|
||||
const b = ev.content_block ?? {}
|
||||
blocks.set(ev.index, { type: b.type, text: b.text ?? b.thinking ?? "", id: b.id, name: b.name, json: "", data: b.data, index: ev.index })
|
||||
if (b.type === "tool_use") yield { type: "tool_call_delta", index: toolIndex, name: b.name, argsDelta: "" }
|
||||
if (b.type === "text" && b.text) {
|
||||
out.add("text", b.text)
|
||||
yield { type: "text", text: b.text }
|
||||
}
|
||||
break
|
||||
}
|
||||
case "content_block_delta": {
|
||||
const blk = blocks.get(ev.index)
|
||||
const d = ev.delta ?? {}
|
||||
if (!blk) break
|
||||
if (d.type === "text_delta") {
|
||||
out.add("text", d.text)
|
||||
yield { type: "text", text: d.text }
|
||||
} else if (d.type === "thinking_delta") {
|
||||
const b = banner.feed(d.thinking ?? "")
|
||||
if (b.notice) yield { type: "notice", message: b.notice }
|
||||
if (b.text) {
|
||||
blk.text += b.text
|
||||
if (b.text.trim()) out.output = true
|
||||
yield { type: "reasoning", text: b.text }
|
||||
}
|
||||
} else if (d.type === "signature_delta") blk.signature = (blk.signature ?? "") + d.signature
|
||||
else if (d.type === "input_json_delta") {
|
||||
blk.json += d.partial_json ?? ""
|
||||
out.output = true
|
||||
yield { type: "tool_call_delta", index: toolIndex, argsDelta: d.partial_json ?? "" }
|
||||
}
|
||||
break
|
||||
}
|
||||
case "content_block_stop": {
|
||||
const blk = blocks.get(ev.index)
|
||||
if (!blk) break
|
||||
if (blk.type === "thinking" && blk.text) out.push({ type: "reasoning", text: blk.text, signature: blk.signature })
|
||||
else if (blk.type === "redacted_thinking") out.push({ type: "reasoning", text: "", opaque: { redacted: blk.data } })
|
||||
else if (blk.type === "tool_use") {
|
||||
const call: ToolCallPart = { type: "tool_call", id: blk.id || `toolu_${toolIndex}`, name: blk.name ?? "", args: blk.json || "{}" }
|
||||
out.push(call)
|
||||
toolIndex++
|
||||
}
|
||||
blocks.delete(ev.index)
|
||||
break
|
||||
}
|
||||
case "message_delta": {
|
||||
if (ev.delta?.stop_reason) finish = ev.delta.stop_reason
|
||||
if (ev.usage?.output_tokens !== undefined) {
|
||||
sawUsage = true
|
||||
output = ev.usage.output_tokens
|
||||
}
|
||||
break
|
||||
}
|
||||
case "error": {
|
||||
const err = ev.error ?? {}
|
||||
throw new ProviderError(`${m.ref}: ${err.message ?? JSON.stringify(err)}`, statusOf(err) ?? (err.type === "overloaded_error" ? 529 : undefined), undefined, !out.output)
|
||||
}
|
||||
default:
|
||||
if (ev?.error) throw new ProviderError(`${m.ref}: ${ev.error.message ?? JSON.stringify(ev.error)}`, statusOf(ev.error), undefined, !out.output)
|
||||
}
|
||||
}
|
||||
const message = out.message()
|
||||
const reason = finish === "max_tokens" ? "length" : finish === "tool_use" || message.parts.some((p) => p.type === "tool_call") ? "tool_calls" : "stop"
|
||||
yield sawUsage
|
||||
? { type: "usage", usage: { input, output, cached } }
|
||||
: { type: "usage", usage: { input: estimateValue(body.messages), output: message.parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.args : p.text), 0), estimated: true } }
|
||||
yield { type: "finish", reason, message }
|
||||
}
|
||||
|
||||
async listModels(): Promise<DiscoveredModel[]> {
|
||||
const res = await request(joinUrl(this.m.connection.base_url, "models"), { headers: this.headers(), timeoutMs: 15_000, tls: tlsFor(this.m.connection) }, this.m.connectionName)
|
||||
const j = (await res.json()) as any
|
||||
return (Array.isArray(j?.data) ? j.data : []).map((x: any) => ({ id: String(x.id), context: x.context_window ?? x.max_input_tokens })).filter((x: DiscoveredModel) => x.id)
|
||||
}
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
// A model's context window when the config does not give one. Config always wins; what is found
|
||||
// here is kept in learned.json. Tried in order: the server's model list; llama-server's /props
|
||||
// (directly, or through llama-swap's /upstream/<model>/ passthrough); Ollama's /api/show.
|
||||
import { learned, learnContext } from "./learned.ts"
|
||||
import { authHeaders, request, tlsFor } from "./http.ts"
|
||||
import { OllamaClient } from "./ollama.ts"
|
||||
import type { Client, ResolvedModel } from "./types.ts"
|
||||
import { resolveKey } from "../config/load.ts"
|
||||
|
||||
/** The window to plan with: configured, else learned, else undefined (unknown). */
|
||||
export function contextWindow(m: ResolvedModel): number | undefined {
|
||||
return m.spec.context || learned().contexts[m.ref] || undefined
|
||||
}
|
||||
|
||||
export async function discoverContext(m: ResolvedModel, client: Client): Promise<number | undefined> {
|
||||
if (m.spec.context) return m.spec.context
|
||||
const known = learned().contexts[m.ref]
|
||||
if (known) return known
|
||||
const found = (await fromList(m, client)) ?? (await fromProps(m)) ?? (m.connection.dialect === "ollama" ? await (client as OllamaClient).contextOf(m.id).catch(() => undefined) : undefined)
|
||||
if (found && found > 0) learnContext(m.ref, found)
|
||||
return found
|
||||
}
|
||||
|
||||
async function fromList(m: ResolvedModel, client: Client): Promise<number | undefined> {
|
||||
try {
|
||||
return (await client.listModels()).find((d) => d.id === m.id)?.context
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
async function fromProps(m: ResolvedModel): Promise<number | undefined> {
|
||||
const base = m.connection.base_url.replace(/\/+$/, "")
|
||||
const origin = base.replace(/\/v1$/, "")
|
||||
const headers = authHeaders(m.connection, resolveKey(m.connectionName, m.connection))
|
||||
for (const url of [`${origin}/upstream/${encodeURIComponent(m.id)}/props`, `${origin}/props`]) {
|
||||
try {
|
||||
const res = await request(url, { headers, timeoutMs: 15_000, tls: tlsFor(m.connection) }, m.ref)
|
||||
const j = (await res.json()) as any
|
||||
const n = Number(j?.default_generation_settings?.n_ctx ?? j?.n_ctx)
|
||||
if (Number.isFinite(n) && n > 0) return n
|
||||
} catch {}
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
/** llama-swap and friends: free the connection's memory when the session moves elsewhere. */
|
||||
export async function unload(m: ResolvedModel): Promise<void> {
|
||||
if (!m.connection.unload_url) return
|
||||
try {
|
||||
await request(m.connection.unload_url, { method: m.connection.unload_method ?? "POST", headers: authHeaders(m.connection, resolveKey(m.connectionName, m.connection)), timeoutMs: 10_000, tls: tlsFor(m.connection) }, m.ref)
|
||||
} catch {}
|
||||
}
|
||||
@@ -1,316 +0,0 @@
|
||||
// Google Gemini, native (`{base}/models/{id}:streamGenerateContent?alt=sse`), base
|
||||
// https://generativelanguage.googleapis.com/v1beta. UNTESTED: no Gemini endpoint has been
|
||||
// available to try it against (see the wiki's Working-notes).
|
||||
import type { Effort } from "../config/schema.ts"
|
||||
import { resolveKey } from "../config/load.ts"
|
||||
import { DEFAULT_BUDGETS } from "./anthropic.ts"
|
||||
import { Parts, record, recordFailure, retrying, statusOf } from "./common.ts"
|
||||
import { authHeaders, joinUrl, request, tlsFor } from "./http.ts"
|
||||
import { sseJson } from "./sse.ts"
|
||||
import { estimateTokens, estimateValue } from "./tokens.ts"
|
||||
import { ProviderError, type ChatRequest, type Client, type DiscoveredModel, type Message, type ResolvedModel, type StreamEvent, type ToolCallPart } from "./types.ts"
|
||||
|
||||
type Content = { role: "user" | "model"; parts: Record<string, unknown>[] }
|
||||
|
||||
// Checked against Hermes Agent (agent/gemini_native_adapter.py, agent/gemini_schema.py) and
|
||||
// OpenCode (provider/transform.ts). Still untested against a real Gemini endpoint.
|
||||
|
||||
/** Gemini's own id, without an aggregator prefix. */
|
||||
const bare = (id: string) => id.replace(/^(models|google|gemini)\//i, "")
|
||||
/** Gemini 3 and later: tool calls and results carry ids, and thinking is a level, not a budget. */
|
||||
export const isGemini3 = (id: string) => Number(/gemini-(\d+)/i.exec(bare(id))?.[1] ?? 0) >= 3
|
||||
const LEGACY = /gemini-(?:(?:flash|pro)-)?[12](?:[.-]|$)/i
|
||||
/** A call replayed without a signature (made by another model, or before signatures existed):
|
||||
* Google's documented sentinel, without which Gemini 3 refuses the whole history. */
|
||||
export const SKIP_SIGNATURE = "skip_thought_signature_validator"
|
||||
/** Thought tokens count against maxOutputTokens, and Gemini's own default is low. */
|
||||
export const DEFAULT_MAX_OUTPUT = 65535
|
||||
const PLACEHOLDER = "[The previous response was interrupted before it completed.]"
|
||||
|
||||
// ── tool schemas ──────────────────────────────────────────────────────────────────────────────
|
||||
// v1beta takes plain JSON Schema as `parametersJsonSchema`; only same-document $refs have to be
|
||||
// inlined. Other API versions take `parameters`, an OpenAPI subset where one unknown key 400s the
|
||||
// whole request, so that path drops everything outside the subset.
|
||||
|
||||
const ALLOWED = new Set(["type", "format", "title", "description", "nullable", "enum", "maxItems", "minItems", "properties", "required", "minProperties", "maxProperties", "minLength", "maxLength", "pattern", "example", "anyOf", "propertyOrdering", "default", "items", "minimum", "maximum"])
|
||||
const STRUCTURAL: Record<string, string[]> = { array: ["items", "minItems", "maxItems"], object: ["properties", "required", "minProperties", "maxProperties", "propertyOrdering"] }
|
||||
const isObj = (v: unknown): v is Record<string, unknown> => !!v && typeof v === "object" && !Array.isArray(v)
|
||||
|
||||
export function geminiSchema(s: unknown): Record<string, unknown> {
|
||||
if (!isObj(s)) return {}
|
||||
const out: Record<string, unknown> = {}
|
||||
for (const [k, v] of Object.entries(s)) {
|
||||
if (!ALLOWED.has(k)) continue
|
||||
if (k === "properties") {
|
||||
if (isObj(v)) out.properties = Object.fromEntries(Object.entries(v).map(([p, sub]) => [p, geminiSchema(sub)]))
|
||||
} else if (k === "items") out.items = geminiSchema(v)
|
||||
else if (k === "anyOf") {
|
||||
if (Array.isArray(v)) out.anyOf = v.filter(isObj).map(geminiSchema)
|
||||
} else out[k] = v
|
||||
}
|
||||
const types = out.type
|
||||
if (Array.isArray(types)) {
|
||||
delete out.type
|
||||
const real = types.filter((t): t is string => typeof t === "string" && t !== "null")
|
||||
if (!real.length) out.type = types.includes("null") ? "null" : "object"
|
||||
else if (real.length === 1) out.type = real[0]
|
||||
else {
|
||||
// Several types: one branch each, and the structure belongs on the branch it describes.
|
||||
const moved: Record<string, unknown> = {}
|
||||
for (const keys of Object.values(STRUCTURAL)) for (const k of keys) if (k in out) (moved[k] = out[k]), delete out[k]
|
||||
out.anyOf = real.map((t) => geminiSchema({ type: t, ...Object.fromEntries(Object.entries(moved).filter(([k]) => STRUCTURAL[t]?.includes(k))) }))
|
||||
}
|
||||
if (types.includes("null")) out.nullable = true
|
||||
}
|
||||
// Every enum entry must be a string, whatever the declared type.
|
||||
if (Array.isArray(out.enum) && (Array.isArray(types) || ["integer", "number", "boolean"].includes(out.type as string))) {
|
||||
const vals = [...new Set(out.enum.map((e) => (typeof e === "boolean" || (typeof e === "number" && Number.isFinite(e)) ? String(e) : typeof e === "string" ? e : undefined)).filter((e) => e !== undefined))]
|
||||
if (vals.length) out.enum = vals
|
||||
else delete out.enum
|
||||
}
|
||||
// `required` naming a property that is not there fails the entire request.
|
||||
if (Array.isArray(out.required)) {
|
||||
const names = isObj(out.properties) ? out.properties : {}
|
||||
const req = out.required.filter((r): r is string => typeof r === "string" && r in names)
|
||||
if (req.length) out.required = req
|
||||
else delete out.required
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** For `parameters`: the subset, and never an empty schema. */
|
||||
export function geminiParameters(s: unknown): Record<string, unknown> {
|
||||
const out = geminiSchema(s)
|
||||
return Object.keys(out).length ? out : { type: "object", properties: {} }
|
||||
}
|
||||
|
||||
function inlineRefs(node: unknown, root: Record<string, unknown>, budget: { n: number }, stack: string[] = []): unknown {
|
||||
if (Array.isArray(node)) return node.map((x) => inlineRefs(x, root, budget, stack))
|
||||
if (!isObj(node)) return node
|
||||
const ref = node.$ref
|
||||
if (typeof ref !== "string") return Object.fromEntries(Object.entries(node).map(([k, v]) => [k, inlineRefs(v, root, budget, stack)]))
|
||||
if (stack.includes(ref)) throw new Error(`circular $ref ${ref}`)
|
||||
if (--budget.n < 0) throw new Error("$ref budget exhausted")
|
||||
if (!ref.startsWith("#/")) throw new Error(`unresolvable $ref ${ref}`)
|
||||
let target: unknown = root
|
||||
for (const raw of ref.slice(2).split("/")) {
|
||||
const part = raw.replace(/~1/g, "/").replace(/~0/g, "~")
|
||||
target = isObj(target) ? target[part] : undefined
|
||||
}
|
||||
if (!isObj(target)) throw new Error(`unresolvable $ref ${ref}`)
|
||||
const inlined = inlineRefs(target, root, budget, [...stack, ref]) as Record<string, unknown>
|
||||
// Siblings of a $ref apply alongside it and win.
|
||||
const { $ref: _, ...siblings } = node
|
||||
return Object.keys(siblings).length ? { ...inlined, ...(inlineRefs(siblings, root, budget, stack) as object) } : inlined
|
||||
}
|
||||
|
||||
/** For `parametersJsonSchema`: root $schema dropped, local $refs inlined, an object root. A schema
|
||||
* whose refs cannot all be resolved goes as it is, so the provider names the real problem. */
|
||||
export function geminiJsonSchema(s: unknown): Record<string, unknown> {
|
||||
if (!isObj(s) || !Object.keys(s).length) return { type: "object", properties: {} }
|
||||
const { $schema: _, ...rest } = structuredClone(s)
|
||||
let out: Record<string, unknown>
|
||||
try {
|
||||
out = inlineRefs(rest, rest, { n: 256 }) as Record<string, unknown>
|
||||
} catch {
|
||||
return rest
|
||||
}
|
||||
delete out.$defs
|
||||
delete out.definitions
|
||||
if (!Object.keys(out).length) return { type: "object", properties: {} }
|
||||
if (out.type === "object" && !("properties" in out)) out.properties = {}
|
||||
return out
|
||||
}
|
||||
|
||||
/** `parametersJsonSchema` exists only on v1beta. */
|
||||
export const acceptsJsonSchema = (base: string) => /\/v1beta\/*$/i.test(base.trim())
|
||||
|
||||
// ── history ───────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/** A JSON Schema inside a function response is resolved by Gemini 3, and an unknown $ref 400s. */
|
||||
const hasRef = (v: unknown): boolean => (Array.isArray(v) ? v.some(hasRef) : isObj(v) ? Object.entries(v).some(([k, x]) => (k === "$ref" && typeof x === "string" && x.startsWith("#/")) || hasRef(x)) : false)
|
||||
|
||||
function functionResponse(content: string, isError: boolean | undefined): Record<string, unknown> {
|
||||
if (isError) return { error: content }
|
||||
// A JSON object result goes structured; anything else under "output", as Google documents.
|
||||
if (/^\s*\{/.test(content))
|
||||
try {
|
||||
const parsed = JSON.parse(content)
|
||||
if (isObj(parsed) && !hasRef(parsed)) return parsed
|
||||
} catch {}
|
||||
return { output: content }
|
||||
}
|
||||
|
||||
export function toGeminiContents(messages: Message[], vision: boolean, ids = false): Content[] {
|
||||
const out: Content[] = []
|
||||
const names = new Map<string, string>()
|
||||
const results = (c: Content) => c.parts.some((p) => "functionResponse" in p)
|
||||
const push = (role: Content["role"], parts: Record<string, unknown>[]) => {
|
||||
if (!parts.length) return
|
||||
const last = out[out.length - 1]
|
||||
// Same-role turns must merge (Gemini insists on alternation), except that a person's words are
|
||||
// never folded into a turn of tool results: Gemini 3 then reads them as part of the result and
|
||||
// answers with nothing. A placeholder model turn keeps the two apart.
|
||||
if (last && last.role === role && role === "user" && results(last) !== parts.some((p) => "functionResponse" in p)) out.push({ role: "model", parts: [{ text: PLACEHOLDER }] }, { role, parts })
|
||||
else if (last && last.role === role) last.parts.push(...parts)
|
||||
else out.push({ role, parts })
|
||||
}
|
||||
for (const m of messages) {
|
||||
if (m.role === "user")
|
||||
push(
|
||||
"user",
|
||||
m.parts.map((p) => (p.type === "text" ? { text: p.text } : vision ? { inlineData: { mimeType: p.mime, data: p.data } } : { text: "[image omitted: this model has no vision]" })),
|
||||
)
|
||||
else if (m.role === "assistant") {
|
||||
const parts: Record<string, unknown>[] = []
|
||||
for (const p of m.parts) {
|
||||
if (p.type === "text" && p.text.trim()) parts.push({ text: p.text })
|
||||
else if (p.type === "tool_call") {
|
||||
names.set(p.id, p.name)
|
||||
let args: unknown = {}
|
||||
try {
|
||||
args = p.args.trim() ? JSON.parse(p.args) : {}
|
||||
} catch {
|
||||
args = { _raw: p.args }
|
||||
}
|
||||
// Gemini 3 wants the thought signature back on the call it came with.
|
||||
parts.push({ functionCall: { name: p.name, args: isObj(args) ? args : { _value: args }, ...(ids ? { id: p.id } : {}) }, thoughtSignature: p.signature ?? SKIP_SIGNATURE })
|
||||
}
|
||||
}
|
||||
push("model", parts)
|
||||
} else push("user", [{ functionResponse: { name: names.get(m.callId) ?? m.name, response: functionResponse(m.content, m.isError), ...(ids ? { id: m.callId } : {}) } }])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ── thinking ──────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const LEVELS = ["minimal", "low", "medium", "high"] as const
|
||||
/** The levels a Gemini 3 model takes (OpenCode's table). */
|
||||
function levelsOf(id: string): readonly string[] {
|
||||
const s = id.toLowerCase()
|
||||
if (s.includes("gemma") || s.includes("flash-image")) return ["minimal", "high"]
|
||||
if (s.includes("pro-image")) return ["high"]
|
||||
if (s.includes("flash")) return LEVELS
|
||||
return ["low", "medium", "high"]
|
||||
}
|
||||
|
||||
export function thinkingConfig(id: string, effort: Effort, map?: Partial<Record<Effort, number>>): Record<string, unknown> {
|
||||
if (map?.[effort]) return { thinkingBudget: map[effort], includeThoughts: true }
|
||||
const b = bare(id)
|
||||
if (LEGACY.test(b)) {
|
||||
// 2.x takes a budget; 2.5 Pro goes to 32k, the rest to 24k.
|
||||
const max = /gemini-2[.-]5/i.test(b) && /pro/i.test(b) && !/flash/i.test(b) ? 32768 : 24576
|
||||
return { thinkingBudget: Math.min(DEFAULT_BUDGETS[effort], max), includeThoughts: true }
|
||||
}
|
||||
// 3 and later take a level: the one asked for, else the next one up the model has, else its top.
|
||||
const i = LEVELS.indexOf(effort as (typeof LEVELS)[number])
|
||||
const want = i < 0 ? LEVELS.length - 1 : i // xhigh and max: the top
|
||||
const have = levelsOf(b)
|
||||
const level = LEVELS.slice(want).find((l) => have.includes(l)) ?? have[have.length - 1]
|
||||
return { thinkingLevel: level, includeThoughts: true }
|
||||
}
|
||||
|
||||
/** Keys sorted, so a call sent again compares equal to itself (as Hermes does). */
|
||||
const sortedJson = (v: unknown) => JSON.stringify(v, (_, x) => (isObj(x) ? Object.fromEntries(Object.keys(x).sort().map((k) => [k, x[k]])) : x))
|
||||
|
||||
const FINISH: Record<string, string> = { STOP: "stop", MAX_TOKENS: "length", SAFETY: "content_filter", RECITATION: "content_filter", BLOCKLIST: "content_filter", PROHIBITED_CONTENT: "content_filter", SPII: "content_filter" }
|
||||
|
||||
export class GeminiClient implements Client {
|
||||
constructor(private m: ResolvedModel) {}
|
||||
|
||||
private headers() {
|
||||
return authHeaders(this.m.connection, resolveKey(this.m.connectionName, this.m.connection), this.m.spec.headers)
|
||||
}
|
||||
|
||||
buildBody(req: ChatRequest): Record<string, unknown> {
|
||||
const { m } = this
|
||||
const generationConfig: Record<string, unknown> = {}
|
||||
if (m.spec.temperature !== undefined) generationConfig.temperature = m.spec.temperature
|
||||
generationConfig.maxOutputTokens = m.spec.max_output ?? DEFAULT_MAX_OUTPUT
|
||||
if (m.spec.top_p !== undefined) generationConfig.topP = m.spec.top_p
|
||||
if (req.effort) generationConfig.thinkingConfig = thinkingConfig(m.id, req.effort, m.spec.effort_map)
|
||||
const body: Record<string, unknown> = { contents: toGeminiContents(req.messages, m.spec.vision === true, isGemini3(m.id)), generationConfig, ...m.connection.body, ...m.spec.body }
|
||||
if (req.system) body.systemInstruction = { parts: [{ text: req.system }] }
|
||||
if (req.tools.length && m.spec.tools !== false) {
|
||||
const json = acceptsJsonSchema(m.connection.base_url)
|
||||
body.tools = [{ functionDeclarations: req.tools.map((t) => ({ name: t.name, description: t.description, ...(json ? { parametersJsonSchema: geminiJsonSchema(t.parameters) } : { parameters: geminiParameters(t.parameters) }) })) }]
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
async *stream(req: ChatRequest): AsyncGenerator<StreamEvent> {
|
||||
const body = this.buildBody(req)
|
||||
yield* retrying(() => this.attempt(req, body))
|
||||
}
|
||||
|
||||
private async *attempt(req: ChatRequest, body: Record<string, unknown>): AsyncGenerator<StreamEvent> {
|
||||
const { m } = this
|
||||
const url = joinUrl(m.connection.base_url, `models/${encodeURIComponent(m.id)}:streamGenerateContent?alt=sse`)
|
||||
const res = await request(url, { method: "POST", headers: this.headers(), body: JSON.stringify(body), signal: req.signal, timeoutMs: (m.connection.timeout ?? 600) * 1000, tls: tlsFor(m.connection) }, m.ref).catch((e) => {
|
||||
recordFailure(m.ref, body, e)
|
||||
throw e
|
||||
})
|
||||
if (!res.body) throw new ProviderError(`${m.ref}: empty response`)
|
||||
const out = new Parts()
|
||||
let finish = "stop"
|
||||
let usage: StreamEvent | undefined
|
||||
// One slot per call. Gemini 3 ids its calls; 2.5 does not, restarts part numbering in every
|
||||
// event and may send a call again whole, so a payload repeating (or extending) a slot's arguments
|
||||
// is that call and anything else is a new one.
|
||||
const slots = new Map<string, { index: number; part: ToolCallPart }>()
|
||||
const slotFor = (fc: any, sig: string, at: number, args: string) => {
|
||||
if (typeof fc.id === "string" && fc.id) return `id:${fc.id}`
|
||||
const key = `${at}:${fc.name}:${sig}`
|
||||
const s = slots.get(key)
|
||||
if (!s || args.startsWith(s.part.args)) return key
|
||||
for (const [k, o] of slots) if (k.startsWith(`${key}#`) && args.startsWith(o.part.args)) return k
|
||||
return `${key}#${slots.size}`
|
||||
}
|
||||
for await (const { data } of sseJson(record(res.body, m.ref, body))) {
|
||||
const chunk = data as any
|
||||
if (chunk?.error) throw new ProviderError(`${m.ref}: ${chunk.error.message ?? JSON.stringify(chunk.error)}`, statusOf(chunk.error) ?? chunk.error.code, undefined, !out.output)
|
||||
const cand = chunk?.candidates?.[0]
|
||||
const parts: any[] = cand?.content?.parts ?? []
|
||||
for (const [at, p] of parts.entries()) {
|
||||
if (typeof p.text === "string" && p.text) {
|
||||
const kind = p.thought === true ? "reasoning" : "text"
|
||||
out.add(kind, p.text)
|
||||
yield { type: kind, text: p.text }
|
||||
}
|
||||
const fc = p.functionCall
|
||||
if (fc?.name) {
|
||||
const args = sortedJson(fc.args ?? {})
|
||||
const sig = typeof p.thoughtSignature === "string" ? p.thoughtSignature : ""
|
||||
const key = slotFor(fc, sig, at, args)
|
||||
let s = slots.get(key)
|
||||
if (!s) {
|
||||
s = { index: slots.size, part: { type: "tool_call", id: typeof fc.id === "string" && fc.id ? fc.id : `gemini_call_${slots.size}`, name: fc.name, args: "" } }
|
||||
slots.set(key, s)
|
||||
out.push(s.part)
|
||||
}
|
||||
const prev = s.part.args
|
||||
s.part.args = args
|
||||
if (sig) s.part.signature = sig
|
||||
yield { type: "tool_call_delta", index: s.index, name: fc.name, argsDelta: args.startsWith(prev) ? args.slice(prev.length) : args }
|
||||
}
|
||||
}
|
||||
if (cand?.finishReason) finish = FINISH[String(cand.finishReason).toUpperCase()] ?? "stop"
|
||||
const u = chunk?.usageMetadata
|
||||
if (u) usage = { type: "usage", usage: { input: u.promptTokenCount ?? 0, output: (u.candidatesTokenCount ?? 0) + (u.thoughtsTokenCount ?? 0), reasoning: u.thoughtsTokenCount, cached: u.cachedContentTokenCount } }
|
||||
}
|
||||
const message = out.message()
|
||||
if (slots.size && finish !== "length") finish = "tool_calls"
|
||||
yield usage ?? { type: "usage", usage: { input: estimateValue(body.contents), output: message.parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.args : p.text), 0), estimated: true } }
|
||||
yield { type: "finish", reason: finish, message }
|
||||
}
|
||||
|
||||
async listModels(): Promise<DiscoveredModel[]> {
|
||||
const res = await request(joinUrl(this.m.connection.base_url, "models?pageSize=200"), { headers: this.headers(), timeoutMs: 15_000, tls: tlsFor(this.m.connection) }, this.m.connectionName)
|
||||
const j = (await res.json()) as any
|
||||
return (j?.models ?? [])
|
||||
.filter((x: any) => !x.supportedGenerationMethods || x.supportedGenerationMethods.includes("generateContent"))
|
||||
.map((x: any) => ({ id: String(x.name ?? "").replace(/^models\//, ""), context: x.inputTokenLimit }))
|
||||
.filter((x: DiscoveredModel) => x.id)
|
||||
}
|
||||
}
|
||||
+3
-13
@@ -27,20 +27,10 @@ export function tlsFor(c: Pick<Connection, "tls">): { ca?: string; rejectUnautho
|
||||
return out
|
||||
}
|
||||
|
||||
const DEFAULT_AUTH: Record<Connection["dialect"], NonNullable<Connection["auth"]>> = {
|
||||
"openai-chat": "bearer",
|
||||
responses: "bearer",
|
||||
anthropic: "x-api-key",
|
||||
gemini: "x-goog-api-key",
|
||||
ollama: "bearer",
|
||||
}
|
||||
|
||||
/** Auth plus the connection's and model's own headers. */
|
||||
export function authHeaders(c: Connection, key: string | undefined, extra: Record<string, string> = {}): Record<string, string> {
|
||||
/** The server's token as a bearer, plus the connection's and model's own headers. */
|
||||
export function authHeaders(c: Pick<Connection, "headers">, key: string | undefined, extra: Record<string, string> = {}): Record<string, string> {
|
||||
const h: Record<string, string> = { "content-type": "application/json", ...extra }
|
||||
const style = c.auth ?? DEFAULT_AUTH[c.dialect]
|
||||
if (key && style === "bearer") h.authorization = `Bearer ${key}`
|
||||
else if (key && style !== "none") h[style] = key
|
||||
if (key) h.authorization = `Bearer ${key}`
|
||||
return { ...h, ...c.headers }
|
||||
}
|
||||
|
||||
|
||||
+28
-46
@@ -1,64 +1,40 @@
|
||||
import { findRef, modelRefs } from "./refs.ts"
|
||||
import type { Loaded } from "../config/load.ts"
|
||||
import { AnthropicClient } from "./anthropic.ts"
|
||||
import { OpenAIChatClient } from "./openai-chat.ts"
|
||||
import { ResponsesClient } from "./responses.ts"
|
||||
import { GeminiClient } from "./gemini.ts"
|
||||
import { OllamaClient } from "./ollama.ts"
|
||||
import { ProviderError, type Client, type ResolvedModel } from "./types.ts"
|
||||
import type { Client, ResolvedModel } from "./types.ts"
|
||||
|
||||
export class ModelError extends Error {}
|
||||
|
||||
/** `connection/model`. The model part may itself contain slashes (`openrouter/qwen/qwen3-coder`). */
|
||||
/** What a start without a login says: LLeMbas CLI 2.0 has no model of its own, only the server's. */
|
||||
export const NOT_LOGGED_IN = "Not logged in to a LLeMbas server. LLeMbas CLI uses the models of the server it is logged in to: lembas login <address> (e.g. lembas login ai.example.org)."
|
||||
|
||||
/** A model of the server this machine is logged in to, by its served id (`deepseek/deepseek-flash`)
|
||||
* or an older form of its name (refs.ts). Without a ref: config.yaml's `model`, else the server's
|
||||
* default for the account, else its first. */
|
||||
export function resolveModel(loaded: Loaded, ref: string | undefined): ResolvedModel {
|
||||
const refs = loaded.refs ?? modelRefs(loaded.connections)
|
||||
if (!ref) {
|
||||
// A LLeMbas instance says which model its account starts on; that, before anything else.
|
||||
const instance = Object.entries(loaded.connections).find(([, c]) => c.webui?.default && c.models[c.webui.default])
|
||||
if (instance) ref = refs.find((r) => r.connection === instance[0] && r.id === instance[1].webui!.default)?.ref ?? `${instance[0]}/${instance[1].webui!.default}`
|
||||
const [name, connection] = Object.entries(loaded.connections)[0] ?? []
|
||||
if (!name || !connection) {
|
||||
const why = Object.values(loaded.broken ?? {})[0]
|
||||
throw new ModelError(why ? `The LLeMbas login is unusable: ${why}` : NOT_LOGGED_IN)
|
||||
}
|
||||
if (!ref) {
|
||||
const first = refs[0]
|
||||
if (!first) throw new ModelError("No model configured. Add a connection to ~/.config/lembas/connections.yaml and set `model:` in config.yaml.")
|
||||
ref = first.ref
|
||||
}
|
||||
// Any form of the name (`deepseek/x`, or the old `example/x`): the model it names, under
|
||||
// the ref it has now.
|
||||
if (loaded.broken?.[name]) throw new ModelError(`The LLeMbas login ${name} is unusable: ${loaded.broken[name]}`)
|
||||
if (!ref) ref = connection.webui?.default && connection.models[connection.webui.default] ? connection.webui.default : refs[0]?.ref
|
||||
if (!ref) throw new ModelError(`${name} serves no model to this account yet. lembas login ${connection.webui?.url ?? ""} again once it does.`.replace(" ", " "))
|
||||
const known = findRef(loaded.connections, refs, ref)
|
||||
if (known) {
|
||||
const connection = loaded.connections[known.connection]!
|
||||
if (loaded.broken?.[known.connection]) throw new ModelError(`Connection "${known.connection}" is unusable: ${loaded.broken?.[known.connection]}`)
|
||||
return { ref: known.ref, connectionName: known.connection, connection, id: known.id, spec: connection.models[known.id] ?? {} }
|
||||
}
|
||||
const slash = ref.indexOf("/")
|
||||
if (slash <= 0) throw new ModelError(`Model "${ref}" must be written connection/model`)
|
||||
const connectionName = ref.slice(0, slash)
|
||||
const id = ref.slice(slash + 1)
|
||||
if (loaded.broken?.[connectionName]) throw new ModelError(`Connection "${connectionName}" is unusable: ${loaded.broken[connectionName]}`)
|
||||
const connection = loaded.connections[connectionName]
|
||||
if (!connection) throw new ModelError(`No connection named "${connectionName}" (have: ${Object.keys(loaded.connections).join(", ") || "none"})`)
|
||||
const spec = connection.models[id]
|
||||
if (!spec && !connection.discover)
|
||||
throw new ModelError(`Connection "${connectionName}" has no model "${id}" (have: ${Object.keys(connection.models).join(", ") || "none"}). Add it, or set discover: true.`)
|
||||
return { ref, connectionName, connection, id, spec: spec ?? {} }
|
||||
if (known) return { ref: known.ref, connectionName: name, connection, id: known.id, spec: connection.models[known.id] ?? {} }
|
||||
// A model the server serves that its list did not have when it was read: the server decides
|
||||
// which models exist, so it is asked for rather than refused here. The login's own name in front
|
||||
// of it (an older form) is taken off.
|
||||
const id = ref.startsWith(`${name}/`) ? ref.slice(name.length + 1) : ref
|
||||
return { ref: id, connectionName: name, connection, id, spec: {} }
|
||||
}
|
||||
|
||||
export function clientFor(m: ResolvedModel): Client {
|
||||
switch (m.connection.dialect) {
|
||||
case "openai-chat":
|
||||
return new OpenAIChatClient(m)
|
||||
case "anthropic":
|
||||
return new AnthropicClient(m)
|
||||
case "responses":
|
||||
return new ResponsesClient(m)
|
||||
case "gemini":
|
||||
return new GeminiClient(m)
|
||||
case "ollama":
|
||||
return new OllamaClient(m)
|
||||
}
|
||||
return new OpenAIChatClient(m)
|
||||
}
|
||||
|
||||
/** Which prompt family a model gets when the config does not say: anthropic, gpt, gemini, or
|
||||
/** Which prompt family a model gets when the server does not say: anthropic, gpt, gemini, or
|
||||
* local — every open-weight or unknown model, which gets Hermes' act-don't-describe guidance. */
|
||||
export function familyOf(m: ResolvedModel): string {
|
||||
if (m.spec.family) return m.spec.family
|
||||
@@ -68,3 +44,9 @@ export function familyOf(m: ResolvedModel): string {
|
||||
if (/\bgpt|^o[134]\b|^o[134]-|codex/.test(id)) return "gpt"
|
||||
return "local"
|
||||
}
|
||||
|
||||
/** The window to plan with, as the server gives it; undefined when it does not say (no percentage,
|
||||
* no automatic compaction). */
|
||||
export function contextWindow(m: ResolvedModel): number | undefined {
|
||||
return m.spec.context || undefined
|
||||
}
|
||||
+5
-10
@@ -6,14 +6,12 @@ import { paths } from "../config/paths.ts"
|
||||
// What endpoints have taught us, kept across runs in ~/.local/state/lembas/learned.json.
|
||||
// Only ever narrows behaviour; deleting the file is always safe.
|
||||
interface Learned {
|
||||
/** `connection/model` → the efforts it actually accepts. */
|
||||
/** A model ref → the efforts it actually accepts. */
|
||||
efforts: Record<string, Effort[]>
|
||||
/** base URLs that reject `stream_options`. */
|
||||
noStreamOptions: string[]
|
||||
/** base URLs that reject `return_progress`. */
|
||||
noProgress: string[]
|
||||
/** `connection/model` → context window found by discovery (config always wins). */
|
||||
contexts: Record<string, number>
|
||||
}
|
||||
|
||||
let cache: Learned | undefined
|
||||
@@ -25,9 +23,11 @@ function file() {
|
||||
export function learned(): Learned {
|
||||
if (cache) return cache
|
||||
try {
|
||||
cache = { efforts: {}, noStreamOptions: [], noProgress: [], contexts: {}, ...JSON.parse(readFileSync(file(), "utf8")) }
|
||||
// An older file's `contexts` (what discovery found, before 2.0 took the server's word) is dropped.
|
||||
const { contexts: _, ...kept } = JSON.parse(readFileSync(file(), "utf8")) as Record<string, unknown>
|
||||
cache = { efforts: {}, noStreamOptions: [], noProgress: [], ...kept } as Learned
|
||||
} catch {
|
||||
cache = { efforts: {}, noStreamOptions: [], noProgress: [], contexts: {} }
|
||||
cache = { efforts: {}, noStreamOptions: [], noProgress: [] }
|
||||
}
|
||||
return cache!
|
||||
}
|
||||
@@ -46,11 +46,6 @@ export function learnEfforts(ref: string, efforts: Effort[]) {
|
||||
save()
|
||||
}
|
||||
|
||||
export function learnContext(ref: string, n: number) {
|
||||
learned().contexts[ref] = n
|
||||
save()
|
||||
}
|
||||
|
||||
export function learnNoStreamOptions(baseUrl: string) {
|
||||
const l = learned()
|
||||
if (!l.noStreamOptions.includes(baseUrl)) l.noStreamOptions.push(baseUrl)
|
||||
|
||||
@@ -1,178 +0,0 @@
|
||||
// Ollama, native (`POST {base}/api/chat`, NDJSON), base e.g. http://localhost:11434. Its
|
||||
// OpenAI-compatible /v1 works through openai-chat too; the native API adds num_ctx (without it
|
||||
// Ollama silently cuts every conversation to its small default window) and `think` levels.
|
||||
// UNTESTED: no Ollama server has been available to try it against (see the wiki). Checked against
|
||||
// Hermes Agent's handling (agent/model_metadata.py, agent/reasoning_params.py): /api/show decides
|
||||
// the window to ask for and whether `think` may be sent at all.
|
||||
import { resolveKey } from "../config/load.ts"
|
||||
import { Parts, record, recordFailure, retrying, statusOf } from "./common.ts"
|
||||
import { authHeaders, joinUrl, request, tlsFor } from "./http.ts"
|
||||
import { estimateTokens, estimateValue } from "./tokens.ts"
|
||||
import { ProviderError, type ChatRequest, type Client, type DiscoveredModel, type Message, type ResolvedModel, type StreamEvent } from "./types.ts"
|
||||
|
||||
export function toOllamaMessages(system: string, messages: Message[], vision: boolean): Record<string, unknown>[] {
|
||||
const out: Record<string, unknown>[] = []
|
||||
if (system) out.push({ role: "system", content: system })
|
||||
for (const m of messages) {
|
||||
if (m.role === "user") {
|
||||
const images = vision ? m.parts.filter((p) => p.type === "image").map((p) => (p as { data: string }).data) : []
|
||||
const text = m.parts.map((p) => (p.type === "text" ? p.text : vision ? "" : "[image omitted: this model has no vision]")).filter(Boolean).join("\n")
|
||||
out.push({ role: "user", content: text, ...(images.length ? { images } : {}) })
|
||||
} else if (m.role === "assistant") {
|
||||
const text = m.parts.map((p) => (p.type === "text" ? p.text : "")).join("")
|
||||
const thinking = m.parts.map((p) => (p.type === "reasoning" ? p.text : "")).join("")
|
||||
const calls = m.parts.flatMap((p) => {
|
||||
if (p.type !== "tool_call") return []
|
||||
let args: unknown = {}
|
||||
try {
|
||||
args = p.args.trim() ? JSON.parse(p.args) : {}
|
||||
} catch {}
|
||||
return [{ function: { name: p.name, arguments: args } }]
|
||||
})
|
||||
out.push({ role: "assistant", content: text, ...(thinking ? { thinking } : {}), ...(calls.length ? { tool_calls: calls } : {}) })
|
||||
} else out.push({ role: "tool", content: m.content, tool_name: m.name })
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** NDJSON: one JSON object per line. */
|
||||
async function* ndjson(body: ReadableStream<Uint8Array>): AsyncGenerator<unknown> {
|
||||
const dec = new TextDecoder()
|
||||
let buf = ""
|
||||
for await (const chunk of body) {
|
||||
buf += dec.decode(chunk, { stream: true })
|
||||
let nl: number
|
||||
while ((nl = buf.indexOf("\n")) !== -1) {
|
||||
const line = buf.slice(0, nl).trim()
|
||||
buf = buf.slice(nl + 1)
|
||||
if (line)
|
||||
try {
|
||||
yield JSON.parse(line)
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
const last = (buf + dec.decode()).trim()
|
||||
if (last)
|
||||
try {
|
||||
yield JSON.parse(last)
|
||||
} catch {}
|
||||
}
|
||||
|
||||
/** What /api/show says about a model. */
|
||||
export interface OllamaShow {
|
||||
/** "completion", "tools", "thinking", "vision"…; absent before Ollama 0.6. */
|
||||
capabilities?: string[]
|
||||
/** The runtime window the Modelfile sets (`num_ctx`), which the owner of the server chose. */
|
||||
numCtx?: number
|
||||
/** The trained maximum (`model_info["<arch>.context_length"]`). */
|
||||
trained?: number
|
||||
}
|
||||
|
||||
export function parseShow(j: any): OllamaShow {
|
||||
const out: OllamaShow = {}
|
||||
if (Array.isArray(j?.capabilities)) out.capabilities = j.capabilities.map(String)
|
||||
for (const line of String(j?.parameters ?? "").split("\n")) {
|
||||
const f = line.trim().split(/\s+/)
|
||||
if (f[0] === "num_ctx" && Number(f[1]) > 0) out.numCtx = Number(f[1])
|
||||
}
|
||||
const info = j?.model_info ?? {}
|
||||
const key = Object.keys(info).find((k) => k.endsWith(".context_length"))
|
||||
if (key && Number(info[key]) > 0) out.trained = Number(info[key])
|
||||
return out
|
||||
}
|
||||
|
||||
export class OllamaClient implements Client {
|
||||
private shown?: Promise<OllamaShow | undefined>
|
||||
constructor(private m: ResolvedModel) {}
|
||||
|
||||
/** /api/show for this model, asked once; undefined when the server does not answer it. */
|
||||
show(id = this.m.id): Promise<OllamaShow | undefined> {
|
||||
const ask = async () => {
|
||||
const res = await request(joinUrl(this.m.connection.base_url, "api/show"), { method: "POST", headers: this.headers(), body: JSON.stringify({ model: id, name: id }), timeoutMs: 15_000, tls: tlsFor(this.m.connection) }, this.m.connectionName)
|
||||
return parseShow(await res.json())
|
||||
}
|
||||
if (id !== this.m.id) return ask().catch(() => undefined)
|
||||
return (this.shown ??= ask().catch(() => undefined))
|
||||
}
|
||||
|
||||
private headers() {
|
||||
return authHeaders(this.m.connection, resolveKey(this.m.connectionName, this.m.connection), this.m.spec.headers)
|
||||
}
|
||||
|
||||
buildBody(req: ChatRequest, show?: OllamaShow): Record<string, unknown> {
|
||||
const { m } = this
|
||||
const options: Record<string, unknown> = {}
|
||||
// Without num_ctx Ollama cuts every conversation to its small default window. Configured wins;
|
||||
// else the Modelfile's, else the trained maximum (set `context` to cap it to what memory holds).
|
||||
const ctx = m.spec.context ?? show?.numCtx ?? show?.trained
|
||||
if (ctx) options.num_ctx = ctx
|
||||
if (m.spec.max_output) options.num_predict = m.spec.max_output
|
||||
if (m.spec.temperature !== undefined) options.temperature = m.spec.temperature
|
||||
if (m.spec.top_p !== undefined) options.top_p = m.spec.top_p
|
||||
const body: Record<string, unknown> = { model: m.id, messages: toOllamaMessages(req.system, req.messages, m.spec.vision === true), stream: true, options, ...m.connection.body, ...m.spec.body }
|
||||
if (req.tools.length && m.spec.tools !== false)
|
||||
body.tools = req.tools.map((t) => ({ type: "function", function: { name: t.name, description: t.description, parameters: t.parameters } }))
|
||||
// gpt-oss takes a level; other thinking models take a switch. A model /api/show says cannot
|
||||
// think gets no `think` at all: Ollama refuses the request.
|
||||
const thinks = !show?.capabilities || show.capabilities.includes("thinking")
|
||||
if (thinks && req.effort) body.think = /gpt-oss/.test(m.id) ? (req.effort === "minimal" ? "low" : req.effort === "xhigh" || req.effort === "max" ? "high" : req.effort) : true
|
||||
else if (thinks && m.spec.efforts?.length) body.think = false
|
||||
return body
|
||||
}
|
||||
|
||||
async *stream(req: ChatRequest): AsyncGenerator<StreamEvent> {
|
||||
const body = this.buildBody(req, await this.show())
|
||||
yield* retrying(() => this.attempt(req, body))
|
||||
}
|
||||
|
||||
private async *attempt(req: ChatRequest, body: Record<string, unknown>): AsyncGenerator<StreamEvent> {
|
||||
const { m } = this
|
||||
const res = await request(joinUrl(m.connection.base_url, "api/chat"), { method: "POST", headers: this.headers(), body: JSON.stringify(body), signal: req.signal, timeoutMs: (m.connection.timeout ?? 600) * 1000, tls: tlsFor(m.connection) }, m.ref).catch((e) => {
|
||||
recordFailure(m.ref, body, e)
|
||||
throw e
|
||||
})
|
||||
if (!res.body) throw new ProviderError(`${m.ref}: empty response`)
|
||||
const out = new Parts()
|
||||
let finish = "stop"
|
||||
let usage: StreamEvent | undefined
|
||||
let calls = 0
|
||||
for await (const chunk of ndjson(record(res.body, m.ref, body)) as AsyncGenerator<any>) {
|
||||
if (chunk?.error) throw new ProviderError(`${m.ref}: ${chunk.error}`, statusOf(chunk) ?? 500, undefined, !out.output)
|
||||
const msg = chunk?.message ?? {}
|
||||
if (msg.thinking) {
|
||||
out.add("reasoning", msg.thinking)
|
||||
yield { type: "reasoning", text: msg.thinking }
|
||||
}
|
||||
if (msg.content) {
|
||||
out.add("text", msg.content)
|
||||
yield { type: "text", text: msg.content }
|
||||
}
|
||||
for (const tc of msg.tool_calls ?? []) {
|
||||
const args = JSON.stringify(tc.function?.arguments ?? {})
|
||||
yield { type: "tool_call_delta", index: calls, name: tc.function?.name, argsDelta: args }
|
||||
out.push({ type: "tool_call", id: tc.id ?? `ollama_call_${calls}`, name: tc.function?.name ?? "", args })
|
||||
calls++
|
||||
}
|
||||
if (chunk?.done) {
|
||||
finish = chunk.done_reason === "length" ? "length" : "stop"
|
||||
if (chunk.prompt_eval_count !== undefined || chunk.eval_count !== undefined) usage = { type: "usage", usage: { input: chunk.prompt_eval_count ?? 0, output: chunk.eval_count ?? 0 } }
|
||||
}
|
||||
}
|
||||
const message = out.message()
|
||||
if (calls && finish !== "length") finish = "tool_calls"
|
||||
yield usage ?? { type: "usage", usage: { input: estimateValue(body.messages), output: message.parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.args : p.text), 0), estimated: true } }
|
||||
yield { type: "finish", reason: finish, message }
|
||||
}
|
||||
|
||||
async listModels(): Promise<DiscoveredModel[]> {
|
||||
const res = await request(joinUrl(this.m.connection.base_url, "api/tags"), { headers: this.headers(), timeoutMs: 15_000, tls: tlsFor(this.m.connection) }, this.m.connectionName)
|
||||
const j = (await res.json()) as any
|
||||
return (j?.models ?? []).map((x: any) => ({ id: String(x.name ?? x.model ?? "") })).filter((x: DiscoveredModel) => x.id)
|
||||
}
|
||||
|
||||
/** The model's window: the Modelfile's num_ctx, else its trained maximum. */
|
||||
async contextOf(id: string): Promise<number | undefined> {
|
||||
const s = await this.show(id)
|
||||
return s?.numCtx ?? s?.trained
|
||||
}
|
||||
}
|
||||
+18
-59
@@ -1,69 +1,29 @@
|
||||
// How a model is named: by its provider, not by the login it came through. A LLeMbas
|
||||
// instance that speaks protocol 2 serves every model as `<provider>/<model>` — `deepseek/deepseek-flash`,
|
||||
// `llama/bonsai` — and that id is the model's ref here too: each provider is a connection of its
|
||||
// own as far as the person can see, backed by the instance (requests still go to its /v1, with the
|
||||
// served id). The older form of the ref was `<login connection>/<id>` (`example/deepseek-flash`).
|
||||
// How a model is named: by the id the LLeMbas server serves it under — `deepseek/deepseek-flash`,
|
||||
// `llama/bonsai` — which is its ref here too. LLeMbas CLI 2.0 talks to one server, the one it is
|
||||
// logged in to, so there is nothing to tell apart: no connection prefix, no second login.
|
||||
//
|
||||
// - An instance that does not speak protocol 2 serves bare ids: those stay `<login>/<id>`.
|
||||
// - A connection of the user's own with the provider's name (`deepseek:` in connections.yaml)
|
||||
// keeps `deepseek/x`; the instance's is then `<login>/deepseek/x`.
|
||||
// - Two instances serving the same provider: the one logged in to first keeps the short form, the
|
||||
// other's are `<login>/<provider>/<model>`.
|
||||
// - The old forms still resolve — `example/deepseek-flash` and `example/deepseek/deepseek-flash`
|
||||
// are `deepseek/deepseek-flash` — and a config.yaml that names one is rewritten to the new form.
|
||||
// The older forms still resolve, and a config.yaml that names one is rewritten to the id (load.ts):
|
||||
// `<login>/<id>` (`example/deepseek/deepseek-flash`) and, from before providers, `<login>/<model>`
|
||||
// for the one served `<provider>/<model>` of that name (`example/deepseek-flash`).
|
||||
import type { Connection } from "../config/schema.ts"
|
||||
|
||||
export interface ModelRef {
|
||||
/** What the person sees and writes. */
|
||||
/** What the person sees and writes: the served id. */
|
||||
ref: string
|
||||
/** The connection it is spoken to through, and the id sent there. */
|
||||
/** The login it is spoken to through, and the id sent there. */
|
||||
connection: string
|
||||
id: string
|
||||
/** An instance's model: its provider, when known — what the old `<login>/<model>` is matched by. */
|
||||
provider?: string
|
||||
}
|
||||
|
||||
type Conns = Record<string, Connection & { webui?: { providers?: Record<string, string>; v2?: boolean } }>
|
||||
type Conns = Record<string, Connection>
|
||||
|
||||
/** The provider of a served id: as the instance said it (`provider`), else — from an
|
||||
* instance that speaks protocol 2, whose ids are always `<provider>/<model>` — the part before the
|
||||
* first `/`. An id with a `/` from an older instance is just an id. */
|
||||
export function providerOf(c: Conns[string], id: string): string | undefined {
|
||||
if (!c.webui) return undefined
|
||||
const said = c.webui.providers?.[id]
|
||||
if (said) return said
|
||||
return c.webui.v2 && id.indexOf("/") > 0 ? id.slice(0, id.indexOf("/")) : undefined
|
||||
/** Every model the server serves, in its order, with its ref. */
|
||||
export function modelRefs(connections: Conns): ModelRef[] {
|
||||
return Object.entries(connections).flatMap(([name, c]) => Object.keys(c.models).map((id) => ({ ref: id, connection: name, id })))
|
||||
}
|
||||
|
||||
/** Every model, in config order, with its ref. `logins`: the webui connections in the order they
|
||||
* were logged in to (first keeps a shared provider's short form). */
|
||||
export function modelRefs(connections: Conns, logins: string[] = []): ModelRef[] {
|
||||
const names = new Set(Object.keys(connections))
|
||||
const webui = Object.keys(connections).filter((n) => connections[n]!.webui)
|
||||
const order = [...logins.filter((n) => webui.includes(n)), ...webui.filter((n) => !logins.includes(n))]
|
||||
// Which webui connection a provider's short form belongs to.
|
||||
const owner = new Map<string, string>()
|
||||
for (const w of order)
|
||||
for (const id of Object.keys(connections[w]!.models)) {
|
||||
const provider = providerOf(connections[w]!, id)
|
||||
if (!provider) continue
|
||||
// A connection of that name wins it — unless it is this login itself (`llama/bonsai` on a
|
||||
// login named llama is the same string either way).
|
||||
if ((names.has(provider) && provider !== w) || owner.has(provider)) continue
|
||||
owner.set(provider, w)
|
||||
}
|
||||
const out: ModelRef[] = []
|
||||
for (const [name, c] of Object.entries(connections))
|
||||
for (const id of Object.keys(c.models)) {
|
||||
const provider = providerOf(c, id)
|
||||
const short = provider && owner.get(provider) === name
|
||||
out.push({ ref: short ? id : `${name}/${id}`, connection: name, id, ...(provider ? { provider } : {}) })
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** A ref as written — any of the forms above — as the model it names, or undefined. `old`: it was
|
||||
* the form from before providers (`<login>/<model>` for a served `<provider>/<model>`). */
|
||||
/** A ref as written — the id, or an older form — as the model it names, or undefined. `old`: it
|
||||
* was an older form, which config.yaml is rewritten from. */
|
||||
export function findRef(connections: Conns, refs: ModelRef[], ref: string): (ModelRef & { old?: boolean }) | undefined {
|
||||
const exact = refs.find((r) => r.ref === ref)
|
||||
if (exact) return exact
|
||||
@@ -71,13 +31,12 @@ export function findRef(connections: Conns, refs: ModelRef[], ref: string): (Mod
|
||||
if (slash <= 0) return undefined
|
||||
const conn = ref.slice(0, slash)
|
||||
const id = ref.slice(slash + 1)
|
||||
const c = connections[conn]
|
||||
if (!c) return undefined
|
||||
if (!connections[conn]) return undefined
|
||||
const direct = refs.find((r) => r.connection === conn && r.id === id)
|
||||
if (direct) return direct
|
||||
if (direct) return { ...direct, old: true }
|
||||
// `<login>/<model>` from before providers: the one served model of that name, whatever its provider.
|
||||
if (c.webui && !id.includes("/")) {
|
||||
const by = refs.filter((r) => r.connection === conn && r.provider && r.id === `${r.provider}/${id}`)
|
||||
if (!id.includes("/")) {
|
||||
const by = refs.filter((r) => r.connection === conn && r.id.endsWith(`/${id}`) && r.id.indexOf("/") === r.id.length - id.length - 1)
|
||||
if (by.length === 1) return { ...by[0]!, old: true }
|
||||
}
|
||||
return undefined
|
||||
|
||||
@@ -1,174 +0,0 @@
|
||||
// OpenAI Responses (`POST {base}/responses`) — api.openai.com, and vLLM. Stateless (store: false):
|
||||
// the whole history is sent as input items, and reasoning items come back opaque (with their
|
||||
// encrypted content when the server provides it) so a reasoning model keeps its thread.
|
||||
import { resolveKey } from "../config/load.ts"
|
||||
import { historyArgs, Parts, record, recordFailure, retrying, statusOf, SwapBanner } from "./common.ts"
|
||||
import { authHeaders, joinUrl, request, tlsFor } from "./http.ts"
|
||||
import { contextFrom } from "./openai-chat.ts"
|
||||
import { sseJson } from "./sse.ts"
|
||||
import { estimateTokens, estimateValue } from "./tokens.ts"
|
||||
import { ProviderError, type ChatRequest, type Client, type DiscoveredModel, type Message, type ResolvedModel, type StreamEvent } from "./types.ts"
|
||||
|
||||
type Item = Record<string, unknown>
|
||||
|
||||
export function toResponsesInput(messages: Message[], vision: boolean): Item[] {
|
||||
const out: Item[] = []
|
||||
for (const m of messages) {
|
||||
if (m.role === "user") {
|
||||
out.push({
|
||||
role: "user",
|
||||
content: m.parts.map((p) =>
|
||||
p.type === "text"
|
||||
? { type: "input_text", text: p.text }
|
||||
: vision
|
||||
? { type: "input_image", image_url: `data:${p.mime};base64,${p.data}` }
|
||||
: { type: "input_text", text: "[image omitted: this model has no vision]" },
|
||||
),
|
||||
})
|
||||
} else if (m.role === "assistant") {
|
||||
let text: string[] = []
|
||||
const flush = () => {
|
||||
if (text.join("").trim()) out.push({ role: "assistant", content: [{ type: "output_text", text: text.join("") }] })
|
||||
text = []
|
||||
}
|
||||
for (const p of m.parts) {
|
||||
if (p.type === "text") text.push(p.text)
|
||||
else if (p.type === "reasoning") {
|
||||
// Only a reasoning item this dialect produced can go back; text from elsewhere cannot.
|
||||
const item = (p.opaque as { responses?: Item } | undefined)?.responses
|
||||
if (item) {
|
||||
flush()
|
||||
out.push(item)
|
||||
}
|
||||
} else {
|
||||
flush()
|
||||
out.push({ type: "function_call", call_id: p.id, name: p.name, arguments: historyArgs(p.args) })
|
||||
}
|
||||
}
|
||||
flush()
|
||||
} else out.push({ type: "function_call_output", call_id: m.callId, output: m.content })
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
export class ResponsesClient implements Client {
|
||||
constructor(private m: ResolvedModel) {}
|
||||
|
||||
private headers() {
|
||||
return authHeaders(this.m.connection, resolveKey(this.m.connectionName, this.m.connection), this.m.spec.headers)
|
||||
}
|
||||
|
||||
buildBody(req: ChatRequest): Record<string, unknown> {
|
||||
const { m } = this
|
||||
const body: Record<string, unknown> = {
|
||||
model: m.id,
|
||||
input: toResponsesInput(req.messages, m.spec.vision === true),
|
||||
stream: true,
|
||||
store: false,
|
||||
...m.connection.body,
|
||||
...m.spec.body,
|
||||
}
|
||||
if (req.system) body.instructions = req.system
|
||||
if (req.tools.length && m.spec.tools !== false)
|
||||
body.tools = req.tools.map((t) => ({ type: "function", name: t.name, description: t.description, parameters: t.parameters, strict: false }))
|
||||
if (m.spec.max_output) body.max_output_tokens = m.spec.max_output
|
||||
if (req.effort) {
|
||||
body.reasoning = { effort: req.effort, summary: "auto" }
|
||||
body.include = ["reasoning.encrypted_content"]
|
||||
} else {
|
||||
if (m.spec.temperature !== undefined) body.temperature = m.spec.temperature
|
||||
if (m.spec.top_p !== undefined) body.top_p = m.spec.top_p
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
async *stream(req: ChatRequest): AsyncGenerator<StreamEvent> {
|
||||
let body = this.buildBody(req)
|
||||
yield* retrying(
|
||||
() => this.attempt(req, body),
|
||||
(e) => {
|
||||
// Some servers do not know `include`; drop it once rather than fail.
|
||||
if (body.include && (e.status === 400 || e.status === 422) && /include/i.test(e.message)) {
|
||||
const { include: _, ...rest } = body
|
||||
body = rest
|
||||
return { retry: true }
|
||||
}
|
||||
return { retry: false }
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
private async *attempt(req: ChatRequest, body: Record<string, unknown>): AsyncGenerator<StreamEvent> {
|
||||
const { m } = this
|
||||
const res = await request(
|
||||
joinUrl(m.connection.base_url, "responses"),
|
||||
{ method: "POST", headers: this.headers(), body: JSON.stringify(body), signal: req.signal, timeoutMs: (m.connection.timeout ?? 600) * 1000, tls: tlsFor(m.connection) },
|
||||
m.ref,
|
||||
).catch((e) => {
|
||||
recordFailure(m.ref, body, e)
|
||||
throw e
|
||||
})
|
||||
if (!res.body) throw new ProviderError(`${m.ref}: empty response`)
|
||||
|
||||
const out = new Parts()
|
||||
const banner = new SwapBanner()
|
||||
const reasoning = new Map<string, string>()
|
||||
const callIndex = new Map<string, number>()
|
||||
let finish = "stop"
|
||||
let usage: StreamEvent | undefined
|
||||
|
||||
for await (const { data } of sseJson(record(res.body, m.ref, body))) {
|
||||
const ev = data as any
|
||||
const t: string = ev?.type ?? ""
|
||||
if (t === "response.output_text.delta") {
|
||||
out.add("text", ev.delta ?? "")
|
||||
yield { type: "text", text: ev.delta ?? "" }
|
||||
} else if (t === "response.reasoning_summary_text.delta" || t === "response.reasoning_text.delta") {
|
||||
const b = banner.feed(ev.delta ?? "")
|
||||
if (b.notice) yield { type: "notice", message: b.notice }
|
||||
if (!b.text) continue
|
||||
reasoning.set(ev.item_id ?? "", (reasoning.get(ev.item_id ?? "") ?? "") + b.text)
|
||||
if (b.text.trim()) out.output = true
|
||||
yield { type: "reasoning", text: b.text }
|
||||
} else if (t === "response.output_item.added" && ev.item?.type === "function_call") {
|
||||
const index = callIndex.size
|
||||
callIndex.set(ev.item.id ?? ev.item.call_id, index)
|
||||
yield { type: "tool_call_delta", index, name: ev.item.name, argsDelta: "" }
|
||||
} else if (t === "response.function_call_arguments.delta") {
|
||||
out.output = true
|
||||
yield { type: "tool_call_delta", index: callIndex.get(ev.item_id) ?? 0, argsDelta: ev.delta ?? "" }
|
||||
} else if (t === "response.output_item.done") {
|
||||
const item = ev.item ?? {}
|
||||
if (item.type === "function_call") out.push({ type: "tool_call", id: item.call_id ?? item.id, name: item.name, args: item.arguments || "{}" })
|
||||
else if (item.type === "reasoning") {
|
||||
const summary = Array.isArray(item.summary) ? item.summary.map((s: any) => s.text ?? "").join("\n") : ""
|
||||
const text = reasoning.get(item.id ?? "") || summary
|
||||
// vLLM puts the raw reasoning in `content`, which it will not accept back; keep what can return.
|
||||
const { content: _c, ...back } = item
|
||||
if (text || item.encrypted_content) out.push({ type: "reasoning", text, opaque: item.encrypted_content ? { responses: back } : undefined })
|
||||
}
|
||||
} else if (t === "response.completed" || t === "response.incomplete") {
|
||||
const r = ev.response ?? {}
|
||||
if (t === "response.incomplete" || r.status === "incomplete") finish = "length"
|
||||
const u = r.usage
|
||||
if (u) usage = { type: "usage", usage: { input: u.input_tokens ?? 0, output: u.output_tokens ?? 0, reasoning: u.output_tokens_details?.reasoning_tokens, cached: u.input_tokens_details?.cached_tokens || undefined } }
|
||||
} else if (t === "response.failed" || t === "error" || ev?.error) {
|
||||
const err = ev.response?.error ?? ev.error ?? ev
|
||||
throw new ProviderError(`${m.ref}: ${err.message ?? JSON.stringify(err)}`, statusOf(err), undefined, !out.output)
|
||||
}
|
||||
}
|
||||
const message = out.message()
|
||||
if (finish !== "length" && message.parts.some((p) => p.type === "tool_call")) finish = "tool_calls"
|
||||
yield usage ?? {
|
||||
type: "usage",
|
||||
usage: { input: estimateValue(body.input), output: message.parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.args : p.text), 0), estimated: true },
|
||||
}
|
||||
yield { type: "finish", reason: finish, message }
|
||||
}
|
||||
|
||||
async listModels(): Promise<DiscoveredModel[]> {
|
||||
const res = await request(joinUrl(this.m.connection.base_url, "models"), { headers: this.headers(), timeoutMs: 15_000, tls: tlsFor(this.m.connection) }, this.m.connectionName)
|
||||
const j = (await res.json()) as any
|
||||
return (Array.isArray(j?.data) ? j.data : []).map((x: any) => ({ id: String(x.id), context: contextFrom(x) })).filter((x: DiscoveredModel) => x.id)
|
||||
}
|
||||
}
|
||||
@@ -56,7 +56,7 @@ export interface ChatRequest {
|
||||
}
|
||||
|
||||
export interface ResolvedModel {
|
||||
/** `connection/model` */
|
||||
/** The served id (`deepseek/deepseek-flash`). */
|
||||
ref: string
|
||||
connectionName: string
|
||||
connection: Connection
|
||||
|
||||
Reference in new issue
Block a user