HomerandClaude Opus 5.5
28ff205ed0
LLeMbas CLI 2.0 is LLeMbas-server-specific (CLI rule 2). Removed: the anthropic, gemini, ollama and responses clients, context discovery and llama-swap unloading (discover.ts), the Connection schema with its dialect, auth, key_cmd, discover, unload_url and one_model_at_a_time, the model `fallback` (it only ever moved to another server), effort_map and cache, and the multi-connection refs. Kept: openai-chat, http, sse, think, effort, tokens and all of src/lembas. - connections.yaml holds the login (`type: webui`, as lembas login writes it). A custom entry from 1.x is read past with a warning that says how to log in, so an old file still starts; with several logins left, the latest is used and the others named. - A model's ref is the id the server serves it under; the older forms (`<login>/<id>`, `<login>/<model>`) still resolve and config.yaml is rewritten once. Without a login there is no model, and the error (and the first-run screen, now the sign-in alone) says `lembas login <address>`. - A login to another server signs this machine out of the previous one. - The window is the server's word (no discovery); a capacity group comes from the server's `capacity.group` alone; embeddings use the server's /v1/embeddings. - `lembas models` lists the server's models; install.sh's starter files, the README and the JSON schemas say the same. Tests: every fixture is a login now (the fake server lists models, answers other GETs with 404, and can hand one model to another script, as a server with several backends); the dialect suites and the cross-connection tests are gone with their code. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
68 lines
3.4 KiB
TypeScript
68 lines
3.4 KiB
TypeScript
// Budgets for one prompt (harness spec loop.json, from LLeMbas's agent limits): past one, the
|
|
// tools are withdrawn and the model answers from what it has — checked between steps, never
|
|
// mid-reply, and with time spent waiting for the user not counted.
|
|
import { afterEach, expect, test } from "bun:test"
|
|
import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"
|
|
import { tmpdir } from "node:os"
|
|
import { join } from "node:path"
|
|
import { createApp } from "../src/app.ts"
|
|
import type { AskReply, Event } from "../src/bus/index.ts"
|
|
import { paths } from "../src/config/paths.ts"
|
|
import { delta, fakeProvider, toolCall, usage, type Fake } from "./fake-provider.ts"
|
|
|
|
let fake: Fake | undefined
|
|
afterEach(() => fake?.stop())
|
|
|
|
function app(limits: string, script: Parameters<typeof fakeProvider>[0], ask: () => Promise<AskReply> = async () => ({ kind: "once" })) {
|
|
fake = fakeProvider(script)
|
|
mkdirSync(paths.config, { recursive: true })
|
|
writeFileSync(join(paths.config, "connections.yaml"), `connections:\n f:\n type: webui\n url: ${fake.url}\n models: { f/m: {} }\n`, { mode: 0o600 })
|
|
writeFileSync(join(paths.config, "config.yaml"), `model: f/m\ntitles: prompt\nlimits: { ${limits} }\n`)
|
|
const cwd = mkdtempSync(join(tmpdir(), "ph-budget-"))
|
|
writeFileSync(join(cwd, "big.txt"), "x".repeat(5000) + "\n")
|
|
return createApp({ cwd, mode: "edit", store: false, snapshots: false, asker: { ask } })
|
|
}
|
|
|
|
const tools = (r: any) => (r.tools ?? []).map((t: any) => t.function.name)
|
|
|
|
test("past the output budget the tools are withdrawn, the model is told, and it answers", async () => {
|
|
const a = app("output_bytes: 1000", [
|
|
{ chunks: [toolCall(0, "r1", "read", '{"path":"big.txt"}')] },
|
|
{ chunks: [delta({ content: "It is a file of x." }, "stop")] },
|
|
])
|
|
const notices: string[] = []
|
|
a.bus.on((e: Event) => e.type === "notice" && notices.push(e.message))
|
|
expect(await a.engine.prompt("what is in big.txt?")).toBe("budget")
|
|
expect(tools(fake!.requests[0])).toContain("read")
|
|
expect(tools(fake!.requests[1])).toEqual([])
|
|
expect(JSON.stringify(fake!.requests[1].messages.at(-1))).toContain("The tool output budget for this prompt has been reached")
|
|
expect(notices.join("\n")).toContain("with too much tool output to read")
|
|
})
|
|
|
|
test("the token budget counts what the model wrote this prompt", async () => {
|
|
const a = app("completion_tokens: 50", [
|
|
{ chunks: [toolCall(0, "r1", "read", '{"path":"big.txt"}'), usage(10, 80)] },
|
|
{ chunks: [delta({ content: "done" }, "stop")] },
|
|
])
|
|
await a.engine.prompt("go")
|
|
expect(tools(fake!.requests[1])).toEqual([])
|
|
})
|
|
|
|
test("time spent waiting for an approval is not spent from the wall-clock budget", async () => {
|
|
// bash asks in edit mode; the user takes 1.5 s, the budget is 1 s, and the model still gets its tools.
|
|
const a = app(
|
|
"wall_seconds: 1",
|
|
[{ chunks: [toolCall(0, "b1", "bash", '{"command":"echo hi"}')] }, { chunks: [toolCall(0, "r1", "read", '{"path":"big.txt"}')] }, { chunks: [delta({ content: "ok" }, "stop")] }],
|
|
async () => (await Bun.sleep(1500), { kind: "once" }),
|
|
)
|
|
await a.engine.prompt("go")
|
|
expect(tools(fake!.requests[1])).toContain("read")
|
|
expect(tools(fake!.requests[2])).toContain("read")
|
|
})
|
|
|
|
test("no budget set: nothing is withdrawn", async () => {
|
|
const a = app("steps: 200", [{ chunks: [toolCall(0, "r1", "read", '{"path":"big.txt"}')] }, { chunks: [delta({ content: "ok" }, "stop")] }])
|
|
await a.engine.prompt("go")
|
|
expect(tools(fake!.requests[1])).toContain("read")
|
|
})
|