HomerandClaude Opus 5.5
8e06590d57
ci / check (push) Canceled after 0s
The system prompt is made once per session (start, compaction, model switch), kept in session_meta and resent byte for byte; it holds no date. What changes mid-session -- the date, a mode the person chose, a plan or memory or instruction file they edited, a personality, MCP instructions that arrived later -- is a <session-note> on the next user message, stored with it. The tool list is fixed for the session: plan_submit is always offered and answers unavailable outside plan mode, the wrap-up keeps the tools and runs no call, a withdrawn tool stays listed and answers its refusal; the first request waits up to mcp_wait_ms for MCP servers. Compaction asks for its summary as the next turn of the same conversation. Cached input tokens are parsed (incl. llama.cpp timings), stored per request and shown in /context. Spec: context.md §7/§10, loop.json context.note (+ plan_ended, plan_own, personal, empty, mcp_wait_ms), modes.json plan.refused, compact.md without the transcript heading. Conformance area prefix. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
68 lines
3.6 KiB
TypeScript
68 lines
3.6 KiB
TypeScript
// Budgets for one prompt (harness spec loop.json, from LLeMbas's agent limits): past one, the
|
|
// tools are withdrawn and the model answers from what it has — checked between steps, never
|
|
// mid-reply, and with time spent waiting for the user not counted.
|
|
import { afterEach, expect, test } from "bun:test"
|
|
import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"
|
|
import { tmpdir } from "node:os"
|
|
import { join } from "node:path"
|
|
import { createApp } from "../src/app.ts"
|
|
import type { AskReply, Event } from "../src/bus/index.ts"
|
|
import { paths } from "../src/config/paths.ts"
|
|
import { delta, fakeProvider, toolCall, usage, type Fake } from "./fake-provider.ts"
|
|
|
|
let fake: Fake | undefined
|
|
afterEach(() => fake?.stop())
|
|
|
|
function app(limits: string, script: Parameters<typeof fakeProvider>[0], ask: () => Promise<AskReply> = async () => ({ kind: "once" })) {
|
|
fake = fakeProvider(script)
|
|
mkdirSync(paths.config, { recursive: true })
|
|
writeFileSync(join(paths.config, "connections.yaml"), `connections:\n f:\n type: webui\n url: ${fake.url}\n models: { f/m: {} }\n`, { mode: 0o600 })
|
|
writeFileSync(join(paths.config, "config.yaml"), `model: f/m\ntitles: prompt\nlimits: { ${limits} }\n`)
|
|
const cwd = mkdtempSync(join(tmpdir(), "ph-budget-"))
|
|
writeFileSync(join(cwd, "big.txt"), "x".repeat(5000) + "\n")
|
|
return createApp({ cwd, mode: "edit", store: false, snapshots: false, asker: { ask } })
|
|
}
|
|
|
|
const tools = (r: any) => (r.tools ?? []).map((t: any) => t.function.name)
|
|
|
|
test("past the output budget the model is told the tools are withdrawn, and it answers; the list stays the session's", async () => {
|
|
const a = app("output_bytes: 1000", [
|
|
{ chunks: [toolCall(0, "r1", "read", '{"path":"big.txt"}')] },
|
|
{ chunks: [delta({ content: "It is a file of x." }, "stop")] },
|
|
])
|
|
const notices: string[] = []
|
|
a.bus.on((e: Event) => e.type === "notice" && notices.push(e.message))
|
|
expect(await a.engine.prompt("what is in big.txt?")).toBe("budget")
|
|
expect(tools(fake!.requests[0])).toContain("read")
|
|
expect(tools(fake!.requests[1])).toEqual(tools(fake!.requests[0]))
|
|
expect(JSON.stringify(fake!.requests[1].messages.at(-1))).toContain("The tool output budget for this prompt has been reached")
|
|
expect(notices.join("\n")).toContain("with too much tool output to read")
|
|
})
|
|
|
|
test("the token budget counts what the model wrote this prompt", async () => {
|
|
const a = app("completion_tokens: 50", [
|
|
{ chunks: [toolCall(0, "r1", "read", '{"path":"big.txt"}'), usage(10, 80)] },
|
|
{ chunks: [delta({ content: "done" }, "stop")] },
|
|
])
|
|
expect(await a.engine.prompt("go")).toBe("budget")
|
|
expect(JSON.stringify(fake!.requests[1].messages.at(-1))).toContain("The output budget for this prompt has been reached")
|
|
})
|
|
|
|
test("time spent waiting for an approval is not spent from the wall-clock budget", async () => {
|
|
// bash asks in edit mode; the user takes 1.5 s, the budget is 1 s, and the model still gets its tools.
|
|
const a = app(
|
|
"wall_seconds: 1",
|
|
[{ chunks: [toolCall(0, "b1", "bash", '{"command":"echo hi"}')] }, { chunks: [toolCall(0, "r1", "read", '{"path":"big.txt"}')] }, { chunks: [delta({ content: "ok" }, "stop")] }],
|
|
async () => (await Bun.sleep(1500), { kind: "once" }),
|
|
)
|
|
await a.engine.prompt("go")
|
|
expect(tools(fake!.requests[1])).toContain("read")
|
|
expect(tools(fake!.requests[2])).toContain("read")
|
|
})
|
|
|
|
test("no budget set: nothing is withdrawn", async () => {
|
|
const a = app("steps: 200", [{ chunks: [toolCall(0, "r1", "read", '{"path":"big.txt"}')] }, { chunks: [delta({ content: "ok" }, "stop")] }])
|
|
await a.engine.prompt("go")
|
|
expect(tools(fake!.requests[1])).toContain("read")
|
|
})
|