The system prompt is made once per session (start, compaction, model switch), kept in session_meta and resent byte for byte; it holds no date. What changes mid-session -- the date, a mode the person chose, a plan or memory or instruction file they edited, a personality, MCP instructions that arrived later -- is a <session-note> on the next user message, stored with it. The tool list is fixed for the session: plan_submit is always offered and answers unavailable outside plan mode, the wrap-up keeps the tools and runs no call, a withdrawn tool stays listed and answers its refusal; the first request waits up to mcp_wait_ms for MCP servers. Compaction asks for its summary as the next turn of the same conversation. Cached input tokens are parsed (incl. llama.cpp timings), stored per request and shown in /context. Spec: context.md §7/§10, loop.json context.note (+ plan_ended, plan_own, personal, empty, mcp_wait_ms), modes.json plan.refused, compact.md without the transcript heading. Conformance area prefix. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
36 files changed
+1533
-169
No files matched your search
+13
-6
@@ -360,14 +360,21 @@ test("the context reaches the prompt: the instance's instructions, its personali
|
||||
// names nothing to the model — nothing unfilled reaches it either.
|
||||
expect(system()).not.toContain("Manuals")
|
||||
expect(system()).not.toContain("{{")
|
||||
// A later prompt replaces the context: personalization back to the device's, nothing from the instance.
|
||||
// A later prompt replaces the context: personalization back to the device's, nothing from the
|
||||
// instance. The system prompt stays as it was made (harness context.md §10); the change goes as a
|
||||
// note on that prompt's message.
|
||||
const first = system()
|
||||
await c.request("session/prompt", { sessionId: s.sessionId, prompt: [{ type: "text", text: "again" }], _meta: { lembas: { turnId: "T2", system_prompt: "", personalization: null, prototypes: { offered: false, sources: "" } } } })
|
||||
expect(system()).not.toContain("Instructions from example")
|
||||
expect(system()).toContain("local words")
|
||||
expect(system()).not.toContain("prototype-chat")
|
||||
// A prompt that says only its turn id changes nothing.
|
||||
expect(system()).toBe(first)
|
||||
const said = () => fake!.requests.at(-1)!.messages.at(-1).content as string
|
||||
expect(said()).toStartWith("again\n<session-note>\n")
|
||||
expect(said()).toContain("the person's own instructions changed; it now reads:\nlocal words")
|
||||
expect(said()).toContain("Instructions from example changed; it now reads:\n(none)")
|
||||
expect(said()).toContain("The person chose a different personality:\nBe brief.")
|
||||
// A prompt that says only its turn id changes nothing, and so says nothing.
|
||||
await c.request("session/prompt", { sessionId: s.sessionId, prompt: [{ type: "text", text: "more" }], _meta: { lembas: { turnId: "T3" } } })
|
||||
expect(system()).toContain("local words")
|
||||
expect(system()).toBe(first)
|
||||
expect(said()).toBe("more")
|
||||
})
|
||||
|
||||
test("the plan is sent after a session is opened from the store, and cleared with plan: null", async () => {
|
||||
|
||||
@@ -25,7 +25,7 @@ function app(limits: string, script: Parameters<typeof fakeProvider>[0], ask: ()
|
||||
|
||||
const tools = (r: any) => (r.tools ?? []).map((t: any) => t.function.name)
|
||||
|
||||
test("past the output budget the tools are withdrawn, the model is told, and it answers", async () => {
|
||||
test("past the output budget the model is told the tools are withdrawn, and it answers; the list stays the session's", async () => {
|
||||
const a = app("output_bytes: 1000", [
|
||||
{ chunks: [toolCall(0, "r1", "read", '{"path":"big.txt"}')] },
|
||||
{ chunks: [delta({ content: "It is a file of x." }, "stop")] },
|
||||
@@ -34,7 +34,7 @@ test("past the output budget the tools are withdrawn, the model is told, and it
|
||||
a.bus.on((e: Event) => e.type === "notice" && notices.push(e.message))
|
||||
expect(await a.engine.prompt("what is in big.txt?")).toBe("budget")
|
||||
expect(tools(fake!.requests[0])).toContain("read")
|
||||
expect(tools(fake!.requests[1])).toEqual([])
|
||||
expect(tools(fake!.requests[1])).toEqual(tools(fake!.requests[0]))
|
||||
expect(JSON.stringify(fake!.requests[1].messages.at(-1))).toContain("The tool output budget for this prompt has been reached")
|
||||
expect(notices.join("\n")).toContain("with too much tool output to read")
|
||||
})
|
||||
@@ -44,8 +44,8 @@ test("the token budget counts what the model wrote this prompt", async () => {
|
||||
{ chunks: [toolCall(0, "r1", "read", '{"path":"big.txt"}'), usage(10, 80)] },
|
||||
{ chunks: [delta({ content: "done" }, "stop")] },
|
||||
])
|
||||
await a.engine.prompt("go")
|
||||
expect(tools(fake!.requests[1])).toEqual([])
|
||||
expect(await a.engine.prompt("go")).toBe("budget")
|
||||
expect(JSON.stringify(fake!.requests[1].messages.at(-1))).toContain("The output budget for this prompt has been reached")
|
||||
})
|
||||
|
||||
test("time spent waiting for an approval is not spent from the wall-clock budget", async () => {
|
||||
|
||||
@@ -23,6 +23,9 @@ import { CONTEXT, hunks, lineCount, memoryResults, presentIds, ResultContext, sh
|
||||
import { readBody } from "../src/tool/read.ts"
|
||||
import { applyOps, type Op, type Target } from "../src/memory/store.ts"
|
||||
import { jobNote, type NoteCase } from "../src/jobs/note.ts"
|
||||
import { compactAsk, noteFor, offeredFor, toldAt, unavailable, type Now, type Requirement, type Told } from "../src/session/prefix.ts"
|
||||
import { promptText } from "../src/prompt/assemble.ts"
|
||||
import { TOOL_SPECS } from "../src/harness.ts"
|
||||
|
||||
type Case = Record<string, unknown>
|
||||
|
||||
@@ -150,6 +153,58 @@ export const AREAS: Record<string, (c: Case) => unknown> = {
|
||||
return text
|
||||
})
|
||||
},
|
||||
// The prefix (context.md §10): one session's requests, step by step — which prompt each sends (a
|
||||
// new number each time one is made), which tools it offers, and the note on a user message.
|
||||
prefix: (c) => {
|
||||
const start = c.start as Partial<Now> & { could?: Requirement[]; met?: Requirement[] }
|
||||
const could = new Set<Requirement>(start.could ?? [])
|
||||
let met = new Set<Requirement>(start.met ?? could)
|
||||
const now: Now = { date: start.date!, mode: start.mode ?? "manual", plan: start.plan ?? "", memory: start.memory ?? "", personality: start.personality ?? "", files: start.files ?? {}, mcp: start.mcp ?? {} }
|
||||
const modeText = (m: Mode) => fillPrompt(promptText(`modes/${m}.md`), { plan_dir: (c.plan_dir as string | undefined) ?? ".agent/plans" })
|
||||
const requires = (t: string) => TOOL_SPECS[t]?.requires as Requirement[] | undefined
|
||||
let made: { system: number; tools: string[]; told: Told } | undefined
|
||||
let prompts = 0
|
||||
let carried: string | undefined
|
||||
const make = () => {
|
||||
if (made) return made
|
||||
made = { system: ++prompts, tools: (c.tools as string[]).filter((t) => offeredFor(requires(t), could)), told: { ...toldAt(now), ...(carried ? { date: carried } : {}) } }
|
||||
carried = undefined
|
||||
return made
|
||||
}
|
||||
const fields = ["date", "mode", "plan", "memory", "personality", "files", "mcp"] as const
|
||||
return (c.steps as Record<string, any>[]).map((st) => {
|
||||
if (st.op === "set") {
|
||||
for (const f of fields) if (f in st) (now as unknown as Record<string, unknown>)[f] = structuredClone(st[f])
|
||||
if (st.met) met = new Set(st.met)
|
||||
// The model's own tool changed it: its result told the model.
|
||||
if (st.by === "model" && made) for (const f of fields) if (f in st && f !== "date") (made.told as unknown as Record<string, unknown>)[f] = structuredClone(st[f])
|
||||
return null
|
||||
}
|
||||
if (st.op === "call") {
|
||||
const here = new Set(met)
|
||||
if (now.mode === "plan") here.add("plan")
|
||||
else here.delete("plan")
|
||||
return unavailable(st.tool, requires(st.tool), here) ?? null
|
||||
}
|
||||
if (st.op === "model") {
|
||||
carried = made?.told.date
|
||||
made = undefined
|
||||
return null
|
||||
}
|
||||
const k = make()
|
||||
if (st.op === "user") {
|
||||
const { note, told } = noteFor(k.told, now, modeText)
|
||||
if (note) k.told = told
|
||||
return { system: k.system, tools: k.tools, note: note || null }
|
||||
}
|
||||
if (st.op === "tool" || st.op === "wrapup") return { system: k.system, tools: k.tools, note: null }
|
||||
if (st.op === "compact") {
|
||||
made = undefined
|
||||
return { system: k.system, tools: k.tools, ask: compactAsk() }
|
||||
}
|
||||
throw new Error(`prefix: unknown op ${String(st.op)}`)
|
||||
})
|
||||
},
|
||||
library_scope: (c) => resolveBase(c.bases as { id: string; name: string }[], c.base as string | null),
|
||||
toolcall_index: (c) => {
|
||||
// The deltas as the OpenAI stream carries them: `function.name` and `function.arguments`.
|
||||
|
||||
+10
-2
@@ -66,7 +66,15 @@ describe("context", () => {
|
||||
a.bus.on((e: Event) => e.type === "notice" && notices.push(e.message))
|
||||
await a.engine.prompt("go")
|
||||
expect(notices.some((n) => n.includes("compacting the conversation"))).toBe(true)
|
||||
expect(fake!.requests[1].messages[0].content).toContain("## Transcript")
|
||||
// The summary is the next turn of the same conversation (context.md §7, spec 3.1.0): the same
|
||||
// system prompt and tools, every message as it was, then the compaction text — no transcript.
|
||||
const [first, compaction] = [fake!.requests[0], fake!.requests[1]]
|
||||
expect(compaction.messages[0]).toEqual(first.messages[0])
|
||||
expect(compaction.tools).toEqual(first.tools)
|
||||
expect(compaction.messages.slice(0, first.messages.length)).toEqual(first.messages)
|
||||
expect(compaction.messages.at(-2).role).toBe("tool")
|
||||
expect(compaction.messages.at(-1).content).toStartWith("Summarise the conversation")
|
||||
expect(compaction.messages.at(-1).content).not.toContain("yyyy")
|
||||
const last = fake!.requests[2].messages
|
||||
expect(JSON.stringify(last[1])).toContain("A long task.")
|
||||
// the original request comes back verbatim, with the note
|
||||
@@ -77,7 +85,7 @@ describe("context", () => {
|
||||
test("/context: parts of the system prompt, tools, the conversation, against the window", async () => {
|
||||
const a = app(`connections:\n f:\n type: webui\n url: URL\n models: { f/m: { context: 32768 } }\n`, [{ chunks: [toolCall(0, "c1", "read", '{"path":"big.txt"}'), usage(1500, 20)] }, { chunks: [delta({ content: "ok" }), usage(3100, 5)] }])
|
||||
await a.engine.prompt("read it")
|
||||
const system = a.engine.o.system(a.engine.mode, a.engine.model)
|
||||
const system = a.engine.compose(a.engine.mode)
|
||||
const text = renderBreakdown(contextBreakdown(a.engine, system, [], undefined, a.engine.o.tools))
|
||||
expect(text).toContain("window 32.8k")
|
||||
expect(text).toContain("system prompt")
|
||||
|
||||
+18
-11
@@ -53,7 +53,7 @@ describe("end to end", () => {
|
||||
expect(fake!.requests[0].messages[0].content).toContain("Permission mode: edit")
|
||||
// knowledge_search/get are offered once any knowledge base has documents, and the suite shares
|
||||
// one LEMBAS_HOME: another file's base must not decide this list (found in the full run).
|
||||
expect(fake!.requests[0].tools.map((t: any) => t.function.name).filter((n: string) => !n.startsWith("knowledge_"))).toEqual(["read", "write", "edit", "multiedit", "apply_patch", "glob", "grep", "list", "bash", "job_output", "job_list", "job_stop", "job_send", "web_search", "web_fetch", "task", "plan_write", "plan_update", "ask_user", "memory", "session_search", "notes_search", "note_view", "note_manage", "skills_list", "skill_view", "skill_manage", "settings", "recall"])
|
||||
expect(fake!.requests[0].tools.map((t: any) => t.function.name).filter((n: string) => !n.startsWith("knowledge_"))).toEqual(["read", "write", "edit", "multiedit", "apply_patch", "glob", "grep", "list", "bash", "job_output", "job_list", "job_stop", "job_send", "web_search", "web_fetch", "task", "plan_write", "plan_update", "ask_user", "plan_submit", "memory", "session_search", "notes_search", "note_view", "note_manage", "skills_list", "skill_view", "skill_manage", "settings", "recall"])
|
||||
const ends = events.filter((e) => e.type === "tool_end")
|
||||
expect(ends.map((e) => e.type === "tool_end" && e.result.title)).toEqual(["hello.txt · 1–1 of 1", "hello.txt +1 −1"])
|
||||
const sid = store.sessions(1)[0]!.id
|
||||
@@ -109,16 +109,20 @@ describe("end to end", () => {
|
||||
expect(Bun.YAML.parse(saved)).toEqual({ mode: "manual", permission: { bash: { "echo *": "allow" } } })
|
||||
})
|
||||
|
||||
test("step ceiling withdraws tools and asks for an answer", async () => {
|
||||
test("step ceiling: the same tools are listed, the model is asked for an answer, and a call in it is not run", async () => {
|
||||
const dir = setup([
|
||||
{ chunks: [toolCall(0, "c1", "list", "{}")] },
|
||||
{ chunks: [delta({ content: "summary" })] },
|
||||
{ chunks: [delta({ content: "summary" }), toolCall(0, "c2", "list", "{}")] },
|
||||
])
|
||||
writeFileSync(join(paths.config, "config.yaml"), "model: fake/m\nlimits: { steps: 1 }\n")
|
||||
const app = createApp({ cwd: dir, mode: "edit", asker: noAsk, store: false })
|
||||
expect(await app.engine.prompt("go")).toBe("steps")
|
||||
expect(fake!.requests[1].tools).toBeUndefined()
|
||||
// The tool list is the session's, the wrap-up's request included (harness context.md §10).
|
||||
expect(fake!.requests[1].tools).toEqual(fake!.requests[0].tools)
|
||||
expect(fake!.requests[1].messages.at(-1).content).toContain("step limit")
|
||||
expect(fake!.requests).toHaveLength(2)
|
||||
const last = app.engine.messages.at(-1)!
|
||||
expect(last).toEqual({ role: "assistant", parts: [{ type: "text", text: "summary" }] })
|
||||
})
|
||||
|
||||
test("the CLI: lembas run", async () => {
|
||||
@@ -137,9 +141,11 @@ describe("end to end", () => {
|
||||
})
|
||||
|
||||
describe("nobody to approve", () => {
|
||||
test("a final denial withdraws every tool under that permission, and the prompt says so up front", async () => {
|
||||
test("a final denial withdraws every tool under that permission: still listed, each further call answers the refusal", async () => {
|
||||
const asked: string[] = []
|
||||
const dir = setup([
|
||||
{ chunks: [toolCall(0, "c1", "read", '{"path":"hello.txt"}'), toolCall(1, "c2", "edit", JSON.stringify({ path: "hello.txt", old: "world", new: "x" }))] },
|
||||
{ chunks: [toolCall(0, "c3", "write", JSON.stringify({ path: "hello.txt", content: "x" }))] },
|
||||
{ chunks: [delta({ content: "I would change world to x." })] },
|
||||
])
|
||||
const app = createApp({
|
||||
@@ -147,16 +153,17 @@ describe("nobody to approve", () => {
|
||||
mode: "manual",
|
||||
store: false,
|
||||
unattended: true,
|
||||
asker: { ask: async ({ tool }) => ({ kind: "deny", final: true, feedback: `nobody is present to approve ${tool}.` }) },
|
||||
asker: { ask: async ({ tool }) => (asked.push(tool), { kind: "deny", final: true, feedback: `nobody is present to approve ${tool}.` }) },
|
||||
})
|
||||
await app.engine.prompt("go")
|
||||
expect(fake!.requests[0].messages[0].content).toContain("nobody can approve anything")
|
||||
expect(fake!.requests[0].messages[0].content).toContain("any file change")
|
||||
const names = fake!.requests[1].tools.map((t: any) => t.function.name)
|
||||
expect(names).not.toContain("edit")
|
||||
expect(names).not.toContain("write")
|
||||
expect(names).toContain("read")
|
||||
expect(fake!.requests[1].messages.at(-1).content).toContain("no longer offered")
|
||||
expect(fake!.requests[1].tools).toEqual(fake!.requests[0].tools)
|
||||
const refusal = fake!.requests[1].messages.at(-1).content as string
|
||||
expect(refusal).toContain("every further call to write, edit, multiedit, apply_patch answers this same refusal")
|
||||
// The write is not put to anybody: its answer is that same refusal.
|
||||
expect(asked).toEqual(["edit"])
|
||||
expect(fake!.requests[2].messages.at(-1).content).toBe(refusal.replace(/^\[#0002\]/, "[#0003]"))
|
||||
})
|
||||
|
||||
test("a 5xx before any output is retried once, then given up", async () => {
|
||||
|
||||
@@ -206,7 +206,7 @@ test("every {{variable}} in a description is declared, and none reaches a model"
|
||||
const plan = normalise({ title: "T", phases: [{ title: "P", tasks: ["a"] }] }) as Plan
|
||||
for (const mode of MODES) {
|
||||
for (const p of [undefined, plan]) {
|
||||
const system = app.engine.o.system(mode, app.engine.model, p)
|
||||
const system = app.engine.compose(mode, p)
|
||||
expect(system).not.toContain("{{")
|
||||
expect(system).toContain("Use the instructions below and the tools you have to help with software engineering and with running the project.")
|
||||
expect(system).toContain("- Your output is shown in a terminal and rendered as GitHub-flavoured markdown in a monospace font. Be concise")
|
||||
@@ -221,7 +221,7 @@ test("every {{variable}} in a description is declared, and none reaches a model"
|
||||
}
|
||||
}
|
||||
app.setContext({ instructions: "Be brief.", prototypes: { offered: true, sources: "" } }, "example")
|
||||
const withContext = app.engine.o.system("manual", app.engine.model, plan)
|
||||
const withContext = app.engine.compose("manual", plan)
|
||||
expect(withContext).not.toContain("{{")
|
||||
expect(withContext).toContain("## Instructions from example\n\nBe brief.")
|
||||
expect(withContext).toContain("prototype-page")
|
||||
|
||||
@@ -59,6 +59,8 @@ describe("images", () => {
|
||||
const atts = att("see @shot.png", app.engine.o.toolCtx, true)
|
||||
await app.engine.prompt("see @shot.png", atts.flatMap((a) => (a.image ? [a.text, a.image] : [a.text])))
|
||||
const content = fake!.requests[0].messages.at(-1).content
|
||||
expect(content.map((p: any) => p.type)).toEqual(["text", "text", "image_url"])
|
||||
// The session's first user message also carries the date, as a note after everything else.
|
||||
expect(content.map((p: any) => p.type)).toEqual(["text", "text", "image_url", "text"])
|
||||
expect(content.at(-1).text).toStartWith("<session-note>\nToday is ")
|
||||
})
|
||||
})
|
||||
@@ -82,8 +82,8 @@ test("memory is the server's; notes, knowledge and skills stay this machine's",
|
||||
// The entries for the prompt: read from the resource as the session started.
|
||||
for (let i = 0; i < 50 && !lembas.calls.some((c) => c.uri); i++) await Bun.sleep(10)
|
||||
expect(lembas.calls.find((c) => c.uri)).toEqual({ auth: "Bearer lmb_lib", uri: "lembas://memory" })
|
||||
for (let i = 0; i < 50 && !app.engine.o.system("manual", app.engine.model).includes("Prefers tabs."); i++) await Bun.sleep(10)
|
||||
expect(app.engine.o.system("manual", app.engine.model)).toContain("MEMORY (the account's)\nPrefers tabs.")
|
||||
for (let i = 0; i < 50 && !app.engine.compose("manual").includes("Prefers tabs."); i++) await Bun.sleep(10)
|
||||
expect(app.engine.compose("manual")).toContain("MEMORY (the account's)\nPrefers tabs.")
|
||||
await app.close()
|
||||
})
|
||||
|
||||
@@ -106,7 +106,7 @@ test("a session the client opened sends its id on every memory call and on the r
|
||||
for (let i = 0; i < 50 && !lembas.calls.some((c) => c.uri && c.meta); i++) await Bun.sleep(10)
|
||||
expect(lembas.calls.find((c) => c.uri && c.meta)).toEqual({ auth: "Bearer lmb_lib", uri: "lembas://memory", meta: { scope: app.engine.sessionId } })
|
||||
// Knowledge is this machine's: nothing the server holds is named to the model.
|
||||
expect(app.engine.o.system("manual", app.engine.model)).not.toContain("Knowledge searches here cover only")
|
||||
expect(app.engine.compose("manual")).not.toContain("Knowledge searches here cover only")
|
||||
})
|
||||
|
||||
test("without a login it says so and keeps this machine's memory", () => {
|
||||
|
||||
@@ -155,10 +155,11 @@ test("the tools: knowledge_search is offered only with something to search; a pr
|
||||
lib.putDocument("private", { title: "Diary", source: "d", text: "The pump primes in my dreams." })
|
||||
await app.engine.prompt("anything")
|
||||
expect(offered()).not.toContain("knowledge_search")
|
||||
// A document in the project's base: offered, and only that base is searched.
|
||||
// A document in the project's base: offered from the next session — the tool list is the
|
||||
// session's, decided as it starts (harness context.md §10) — and only that base is searched.
|
||||
const pump = lib.putDocument("manuals", { title: "Pump", source: "p", text: "The pump primes in thirty seconds." })
|
||||
fake.requests.length = 0
|
||||
app.engine.messages = []
|
||||
app.newSession()
|
||||
await app.engine.prompt("how long does the pump take to prime?")
|
||||
expect(offered()).toContain("knowledge_search")
|
||||
const results = app.engine.messages.filter((m) => m.role === "tool").map((m) => (m.role === "tool" ? m.content : ""))
|
||||
|
||||
+18
-7
@@ -7,6 +7,7 @@ import type { Asker, AskReply } from "../src/bus/index.ts"
|
||||
import { paths } from "../src/config/paths.ts"
|
||||
import type { PlanReply, PlanRequest } from "../src/tool/plan_submit.ts"
|
||||
import { delta, fakeProvider, toolCall, type Fake } from "./fake-provider.ts"
|
||||
import { TOOL_SPECS } from "../src/harness.ts"
|
||||
|
||||
let fake: Fake | undefined
|
||||
afterEach(() => fake?.stop())
|
||||
@@ -29,7 +30,7 @@ const exitPlan = () => toolCall(0, "x1", "plan_submit", JSON.stringify({ path: "
|
||||
const names = (r: any) => r.tools.map((t: any) => t.function.name)
|
||||
|
||||
describe("plan mode", () => {
|
||||
test("plan_write, plan_submit, approve → edit mode in the same turn, and plan_submit is no longer offered", async () => {
|
||||
test("plan_write, plan_submit, approve → edit mode in the same turn; the prompt and tools stay as they were", async () => {
|
||||
const seen: PlanRequest[] = []
|
||||
const a = app([{ chunks: [writePlan()] }, { chunks: [exitPlan()] }, { chunks: [delta({ content: "Implementing." })] }], async (r) => {
|
||||
seen.push(r)
|
||||
@@ -48,8 +49,11 @@ describe("plan mode", () => {
|
||||
expect(names(fake!.requests[0])).toContain("plan_submit")
|
||||
expect(names(fake!.requests[0])).toContain("plan_write")
|
||||
expect(fake!.requests[0].messages[0].content).toContain("Permission mode: plan")
|
||||
expect(fake!.requests[2].messages[0].content).toContain("Permission mode: edit")
|
||||
expect(names(fake!.requests[2])).not.toContain("plan_submit")
|
||||
// Spec 3.1.0 (context.md §10): the system prompt and the tool list are the session's, byte for
|
||||
// byte; the approval's own result tells the model the mode, so no note says it again.
|
||||
expect(fake!.requests[2].messages[0]).toEqual(fake!.requests[0].messages[0])
|
||||
expect(fake!.requests[2].tools).toEqual(fake!.requests[0].tools)
|
||||
expect(names(fake!.requests[2])).toContain("plan_submit")
|
||||
// plan_write and plan_update are offered in every mode: the plan is kept current while implementing.
|
||||
expect(names(fake!.requests[2])).toContain("plan_update")
|
||||
expect(fake!.requests[2].messages.at(-1).content).toContain("The user approved the plan")
|
||||
@@ -69,13 +73,20 @@ describe("plan mode", () => {
|
||||
expect(b.engine.mode).toBe("plan")
|
||||
})
|
||||
|
||||
test("plan_submit with no plan written is an error naming plan_write; edit mode does not offer it", async () => {
|
||||
const a = app([{ chunks: [exitPlan()] }, { chunks: [delta({ content: "ok" })] }], async () => ({ kind: "approve", mode: "edit" }))
|
||||
test("plan_submit with no plan written is an error naming plan_write; outside plan mode it is offered and answers unavailable", async () => {
|
||||
const a = app([{ chunks: [exitPlan()] }, { chunks: [delta({ content: "ok" })] }, { chunks: [exitPlan()] }, { chunks: [delta({ content: "carrying on" })] }], async () => ({ kind: "approve", mode: "edit" }))
|
||||
await a.engine.prompt("go")
|
||||
expect(fake!.requests[1].messages.at(-1).content).toContain("Write it with plan_write")
|
||||
expect(a.engine.mode).toBe("plan")
|
||||
a.engine.mode = "edit"
|
||||
await a.engine.prompt("again").catch(() => {})
|
||||
expect(names(fake!.requests.at(-1))).not.toContain("plan_submit")
|
||||
await a.engine.prompt("again")
|
||||
const [again, after] = fake!.requests.slice(-2)
|
||||
expect(names(again)).toContain("plan_submit")
|
||||
expect(again.tools).toEqual(fake!.requests[0].tools)
|
||||
expect(again.messages[0]).toEqual(fake!.requests[0].messages[0])
|
||||
// The person's mode change comes as a note on their next message, the mode's own text with it.
|
||||
expect(again.messages.at(-1).content).toStartWith("again\n<session-note>\nThe permission mode is now edit:\nPermission mode: edit.")
|
||||
expect(after.messages.at(-1).content).toBe(`[#0002]\n${TOOL_SPECS.plan_submit!.unavailable}`)
|
||||
expect(a.engine.mode).toBe("edit")
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,232 @@
|
||||
// The prefix (harness spec 3.1.0, context.md §10): the system prompt and the tool list are made once
|
||||
// per session and resent byte for byte; what changes during it goes as a note on the next user
|
||||
// message; compaction asks for its summary as the next turn of the same conversation; every
|
||||
// request's cached input tokens are kept. These drive the real engine against a scripted server.
|
||||
import { afterEach, expect, test } from "bun:test"
|
||||
import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"
|
||||
import { tmpdir } from "node:os"
|
||||
import { join } from "node:path"
|
||||
import { createApp } from "../src/app.ts"
|
||||
import type { AskReply } from "../src/bus/index.ts"
|
||||
import { paths } from "../src/config/paths.ts"
|
||||
import { TOOL_SPECS } from "../src/harness.ts"
|
||||
import { Store } from "../src/session/store.ts"
|
||||
import { isNote, NOTE, noteFor, PREFIX_META, summaryOf, toldAt, type Kept } from "../src/session/prefix.ts"
|
||||
import { delta, fakeProvider, toolCall, usage, type Fake } from "./fake-provider.ts"
|
||||
|
||||
let fake: Fake | undefined
|
||||
afterEach(() => fake?.stop())
|
||||
|
||||
const once = { ask: async (): Promise<AskReply> => ({ kind: "once" }) }
|
||||
|
||||
function setup(script: Parameters<typeof fakeProvider>[0], extra = "") {
|
||||
fake = fakeProvider(script)
|
||||
mkdirSync(paths.config, { recursive: true })
|
||||
writeFileSync(join(paths.config, "connections.yaml"), `connections:\n fake:\n type: webui\n url: ${fake.url}\n models:\n fake/m: { context: 32768 }\n fake/n: { context: 32768 }\n`, { mode: 0o600 })
|
||||
writeFileSync(join(paths.config, "config.yaml"), `model: fake/m\ntitles: prompt\n${extra}`)
|
||||
const dir = mkdtempSync(join(tmpdir(), "lembas-prefix-"))
|
||||
Bun.spawnSync(["git", "init", "-q", dir])
|
||||
writeFileSync(join(dir, "AGENTS.md"), "# Demo\nBuild with make.\n")
|
||||
return dir
|
||||
}
|
||||
|
||||
const reply = (text: string) => ({ chunks: [delta({ content: text }, "stop")] })
|
||||
const system = (i: number) => fake!.requests.at(i)!.messages[0].content as string
|
||||
const lastUser = (i: number) => {
|
||||
const m = fake!.requests.at(i)!.messages
|
||||
return [...m].reverse().find((x: { role: string }) => x.role === "user").content as string
|
||||
}
|
||||
const names = (i: number) => (fake!.requests.at(i)!.tools as { function: { name: string } }[]).map((t) => t.function.name)
|
||||
|
||||
test("no clock in the prompt: two requests across a date change send the same system bytes, and the date goes as a note", async () => {
|
||||
const dir = setup([reply("one"), reply("two"), reply("three")])
|
||||
const app = createApp({ cwd: dir, asker: once, store: false, snapshots: false })
|
||||
let day = "2026-11-04"
|
||||
app.engine.o.today = () => day
|
||||
await app.engine.prompt("first")
|
||||
await app.engine.prompt("second")
|
||||
day = "2026-11-05"
|
||||
await app.engine.prompt("third")
|
||||
expect(system(0)).not.toContain("Today")
|
||||
expect(system(0)).not.toContain("2026-11-0")
|
||||
expect(system(1)).toBe(system(0))
|
||||
expect(system(2)).toBe(system(0))
|
||||
expect(fake!.requests[2].tools).toEqual(fake!.requests[0].tools)
|
||||
expect(lastUser(0)).toBe(`first\n${NOTE.open}\nToday is 2026-11-04.\n${NOTE.close}`)
|
||||
expect(lastUser(1)).toBe("second")
|
||||
expect(lastUser(2)).toBe(`third\n${NOTE.open}\nToday is 2026-11-05.\n${NOTE.close}`)
|
||||
// A note is part of its message from then on: the next request sends the first one as it was.
|
||||
expect(fake!.requests[2].messages[1].content).toBe(lastUser(0))
|
||||
})
|
||||
|
||||
test("a tool step carries no note; the note waits for the next user message", async () => {
|
||||
const dir = setup([{ chunks: [toolCall(0, "c1", "read", '{"path":"AGENTS.md"}')] }, reply("read it"), reply("ok")])
|
||||
const app = createApp({ cwd: dir, asker: once, store: false, snapshots: false })
|
||||
let day = "2026-11-04"
|
||||
app.engine.o.today = () => day
|
||||
app.bus.on((e) => {
|
||||
// The clock passes midnight while the tool runs.
|
||||
if (e.type === "tool_start") day = "2026-11-05"
|
||||
})
|
||||
await app.engine.prompt("look")
|
||||
expect(fake!.requests[1].messages.at(-1).role).toBe("tool")
|
||||
expect(system(1)).toBe(system(0))
|
||||
await app.engine.prompt("and now")
|
||||
expect(lastUser(2)).toBe(`and now\n${NOTE.open}\nToday is 2026-11-05.\n${NOTE.close}`)
|
||||
})
|
||||
|
||||
test("the person's changes come as one note in the spec's order; the model's own do not", async () => {
|
||||
const dir = setup([
|
||||
{ chunks: [toolCall(0, "w1", "plan_write", JSON.stringify({ title: "Fix", phases: [{ title: "P", tasks: ["a"] }] }))] },
|
||||
{ chunks: [toolCall(0, "e1", "edit", JSON.stringify({ path: "AGENTS.md", old: "Build with make.", new: "Build with make; test with make test." }))] },
|
||||
reply("done"),
|
||||
reply("ok"),
|
||||
reply("fine"),
|
||||
])
|
||||
const app = createApp({ cwd: dir, asker: once, mode: "edit", store: false, snapshots: false })
|
||||
app.engine.o.today = () => "2026-11-05"
|
||||
// The model reads AGENTS.md before it may edit it: read-before-edit is the tool's rule.
|
||||
app.engine.o.toolCtx.readFiles.add(join(dir, "AGENTS.md"))
|
||||
await app.engine.prompt("plan and fix")
|
||||
// Its own plan and its own edit: their results told it.
|
||||
await app.engine.prompt("thanks")
|
||||
expect(lastUser(3)).toBe("thanks")
|
||||
// Between turns the person changes the mode, the plan, an instruction file, and the personality.
|
||||
app.engine.mode = "manual"
|
||||
app.engine.setPlan(null)
|
||||
writeFileSync(join(dir, "AGENTS.md"), "# Demo\nBuild with ninja.\n")
|
||||
app.settings.set("personality", "concise", "session")
|
||||
await app.engine.prompt("again")
|
||||
const note = lastUser(4).slice("again\n".length)
|
||||
expect(isNote(note)).toBe(true)
|
||||
const order = ["The permission mode is now manual:\nPermission mode: manual.", "The person changed the plan; it is now:\n(none)", `${join(dir, "AGENTS.md")} changed; it now reads:\n# Demo\nBuild with ninja.`, "The person chose a different personality:\nBe brief."]
|
||||
for (const line of order) expect(note).toContain(line)
|
||||
expect(order.map((l) => note.indexOf(l))).toEqual([...order.map((l) => note.indexOf(l))].sort((a, b) => a - b))
|
||||
expect(system(4)).toBe(system(0))
|
||||
})
|
||||
|
||||
test("plan_submit is in the list in every mode, and answers its unavailable text outside plan mode", async () => {
|
||||
const dir = setup([{ chunks: [toolCall(0, "x1", "plan_submit", "{}")] }, reply("carrying on")])
|
||||
const app = createApp({ cwd: dir, asker: once, mode: "manual", store: false, snapshots: false })
|
||||
await app.engine.prompt("submit it")
|
||||
expect(names(0)).toContain("plan_submit")
|
||||
expect(fake!.requests[1].messages.at(-1).content).toBe(`[#0001]\n${TOOL_SPECS.plan_submit!.unavailable}`)
|
||||
// A terminal session can never show a file: show is not in its list at all.
|
||||
expect(names(0)).not.toContain("show")
|
||||
})
|
||||
|
||||
test("compaction: the same system and tools, every message, then the compaction text; a call in the reply is not run, and the request is no turn", async () => {
|
||||
const dir = setup([
|
||||
reply("The answer is 42."),
|
||||
{ chunks: [delta({ content: "## What we are doing\nFinding the answer." }), toolCall(0, "c1", "bash", '{"command":"rm -rf x","description":"x"}'), usage(900, 40)] },
|
||||
reply("ok"),
|
||||
])
|
||||
const store = new Store(":memory:")
|
||||
const app = createApp({ cwd: dir, asker: { ask: async () => { throw new Error("nothing may ask") } }, store, snapshots: false })
|
||||
await app.engine.prompt("what is the answer")
|
||||
const sid = app.engine.sessionId!
|
||||
const summary = await app.engine.compact()
|
||||
const [first, compaction] = [fake!.requests[0], fake!.requests[1]]
|
||||
expect(compaction.messages[0]).toEqual(first.messages[0])
|
||||
expect(compaction.tools).toEqual(first.tools)
|
||||
expect(compaction.messages.slice(1, -1).map((m: { content: string }) => m.content)).toEqual([first.messages[1].content, "The answer is 42."])
|
||||
expect(compaction.messages.at(-1).content).toStartWith("Summarise the conversation")
|
||||
expect(compaction.messages.at(-1).content).not.toContain("User: what is the answer")
|
||||
// The text before the call is the summary; the call was not run.
|
||||
expect(summary).toStartWith("## What we are doing\nFinding the answer.")
|
||||
expect(store.messages(sid).map((m) => (m.role === "tool" ? "tool" : m.parts.map((p) => (p.type === "text" ? p.text : p.type)).join("|")))).not.toContainEqual(expect.stringContaining("Summarise"))
|
||||
expect(store.messages(sid).some((m) => m.role === "tool")).toBe(false)
|
||||
// Its usage is kept, as the compaction's.
|
||||
expect(store.requests(sid).map((r) => r.kind)).toEqual(["step", "compaction"])
|
||||
// After it the prompt is made again, and the next user message says the date again.
|
||||
await app.engine.prompt("go on")
|
||||
expect(lastUser(2)).toMatch(/^go on\n<session-note>\nToday is \d{4}-\d{2}-\d{2}\.\n<\/session-note>$/)
|
||||
})
|
||||
|
||||
test("a resumed session sends the prompt it kept, byte for byte; what changed meanwhile is a note", async () => {
|
||||
const dir = setup([reply("one"), reply("two")])
|
||||
const store = new Store(":memory:")
|
||||
const a = createApp({ cwd: dir, asker: once, store, snapshots: false })
|
||||
await a.engine.prompt("first")
|
||||
const sid = a.engine.sessionId!
|
||||
const kept = store.meta<Kept>(sid, PREFIX_META)!
|
||||
expect(kept.system).toBe(system(0))
|
||||
await a.close()
|
||||
writeFileSync(join(dir, "AGENTS.md"), "# Demo\nBuild with ninja.\n")
|
||||
const b = createApp({ cwd: dir, asker: once, store, snapshots: false, resuming: sid })
|
||||
b.resume(sid)
|
||||
await b.engine.prompt("second")
|
||||
expect(system(1)).toBe(system(0))
|
||||
expect(system(1)).toContain("Build with make.")
|
||||
expect(lastUser(1)).toContain(`${join(dir, "AGENTS.md")} changed; it now reads:\n# Demo\nBuild with ninja.`)
|
||||
// Shown as typed: the note is not the person's words.
|
||||
expect(b.engine.messages.filter((m) => m.role === "user").length).toBe(2)
|
||||
await b.close()
|
||||
})
|
||||
|
||||
test("a model switch makes the prompt again for the new model; the date already said holds", async () => {
|
||||
const dir = setup([reply("one"), reply("two")])
|
||||
const app = createApp({ cwd: dir, asker: once, store: false, snapshots: false })
|
||||
await app.engine.prompt("first")
|
||||
app.switchModel("fake/n")
|
||||
await app.engine.prompt("second")
|
||||
expect(system(0)).toContain("Model: fake/m")
|
||||
expect(system(1)).toContain("Model: fake/n")
|
||||
expect(lastUser(1)).toBe("second")
|
||||
})
|
||||
|
||||
test("cached input tokens are kept with every request's usage and shown by /context", async () => {
|
||||
const dir = setup([
|
||||
{ chunks: [delta({ content: "one" }), { choices: [], usage: { prompt_tokens: 1000, completion_tokens: 5, prompt_tokens_details: { cached_tokens: 0 } } }] },
|
||||
// llama.cpp says it in its timings when the usage does not.
|
||||
{ chunks: [delta({ content: "two" }), usage(1100, 5), { choices: [], timings: { cache_n: 1000, prompt_n: 100, predicted_n: 5 } }] },
|
||||
])
|
||||
const store = new Store(":memory:")
|
||||
const app = createApp({ cwd: dir, asker: once, store, snapshots: false })
|
||||
await app.engine.prompt("first")
|
||||
await app.engine.prompt("second")
|
||||
expect(store.requests(app.engine.sessionId!).map((r) => [r.kind, r.input, r.cached])).toEqual([
|
||||
["step", 1000, 0],
|
||||
["step", 1100, 1000],
|
||||
])
|
||||
const { contextBreakdown, renderBreakdown } = await import("../src/session/context.ts")
|
||||
expect(renderBreakdown(contextBreakdown(app.engine, app.engine.systemPrompt(), [], undefined, app.engine.o.tools))).toContain("last request: 1.1k in, 1.0k of it cached (91%)")
|
||||
await app.close()
|
||||
})
|
||||
|
||||
test("noteFor: nothing changed says nothing; an MCP server that connected later is said by its own instructions", () => {
|
||||
const now = { date: "2026-11-05", mode: "manual" as const, plan: "", memory: "", personality: "", files: {}, mcp: { docs: "Search the docs." } }
|
||||
const told = { ...toldAt(now), date: "2026-11-05" }
|
||||
expect(noteFor(told, now, () => "").note).toBe("")
|
||||
const later = { ...now, mcp: { ...now.mcp, tickets: "Name the ticket." } }
|
||||
expect(noteFor(told, later, () => "").note).toBe(`${NOTE.open}\nInstructions from an MCP server that connected since the session began:\n<server name="tickets">\nName the ticket.\n</server>\n${NOTE.close}`)
|
||||
expect(summaryOf([{ type: "reasoning", text: "x" }, { type: "text", text: " S " }, { type: "tool_call" }, { type: "text", text: "after" }])).toBe("S")
|
||||
})
|
||||
|
||||
test("the first request waits for the configured MCP servers (up to loop.json's mcp_wait_ms): their tools and instructions are in the first list and prompt", async () => {
|
||||
const BUN = process.execPath
|
||||
const SERVER = join(import.meta.dir, "fixtures", "mcp", "server.ts")
|
||||
const dir = setup([reply("one")], `mcp:\n fix:\n command: ["${BUN}", "${SERVER}"]\n`)
|
||||
const app = createApp({ cwd: dir, asker: once, store: false, snapshots: false })
|
||||
// Not waiting for app.mcpReady here: the engine does, before it makes the prompt.
|
||||
await app.engine.prompt("first")
|
||||
expect(names(0)).toContain("mcp__fix__echo")
|
||||
expect(system(0)).toContain("<mcp_instructions>")
|
||||
await app.close()
|
||||
writeFileSync(join(paths.config, "config.yaml"), "")
|
||||
})
|
||||
|
||||
test("an empty value in a note reads loop.json's context.note.empty", () => {
|
||||
const now = { date: "2026-11-05", mode: "manual" as const, plan: "", memory: "", personality: "", files: {}, mcp: {} }
|
||||
const told = { ...toldAt({ ...now, memory: "Prefers tabs." }), date: "2026-11-05" }
|
||||
expect(noteFor(told, now, () => "").note).toBe(`${NOTE.open}\nThe person changed what is remembered; it is now:\n${NOTE.empty}\n${NOTE.close}`)
|
||||
})
|
||||
|
||||
test("compaction's ask has no previous summary and no transcript heading", async () => {
|
||||
const { compactAsk } = await import("../src/session/prefix.ts")
|
||||
const ask = compactAsk()
|
||||
expect(ask).not.toContain("## Transcript")
|
||||
expect(ask).not.toContain("Summary of even earlier turns")
|
||||
expect(ask).not.toMatch(/\n{3,}/)
|
||||
expect(ask).toEndWith("stay under 600 words.")
|
||||
})
|
||||
@@ -63,7 +63,8 @@ describe("subagents", () => {
|
||||
const child = fake!.requests[1]
|
||||
expect(child.messages[0].content).toContain('You are the "explore" subagent')
|
||||
expect(child.messages[0].content).not.toContain("Permission mode: plan")
|
||||
expect(child.messages.at(-1).content).toBe("Find why average is wrong in calc.py")
|
||||
// Its prompt has no date either: its first message says it (harness context.md §10).
|
||||
expect(child.messages.at(-1).content).toMatch(/^Find why average is wrong in calc\.py\n<session-note>\nToday is \d{4}-\d{2}-\d{2}\.\n<\/session-note>$/)
|
||||
// its own job tools for commands it starts, and report for a word to the main agent
|
||||
expect(names(child)).toEqual(["read", "glob", "grep", "list", "bash", "job_output", "job_list", "job_stop", "web_search", "web_fetch", "report"])
|
||||
// the preamble is the spec's text; a subagent is told nothing of plan_write or ask_user
|
||||
|
||||
@@ -47,5 +47,5 @@ test("an untrusted project's task board is neither in the prompt nor writable",
|
||||
setTrust(root, "readonly")
|
||||
const app = createApp({ cwd: root, store: false, snapshots: false, asker: { ask: async () => ({ kind: "once" }) } })
|
||||
expect(app.engine.o.toolCtx.projectDir).toBeUndefined()
|
||||
expect(app.engine.o.system("plan", app.engine.model)).not.toContain("IGNORE PREVIOUS INSTRUCTIONS")
|
||||
expect(app.engine.compose("plan")).not.toContain("IGNORE PREVIOUS INSTRUCTIONS")
|
||||
})
|
||||
@@ -104,7 +104,11 @@ test("/compact replaces the conversation with a summary", async () => {
|
||||
await t.input.typeText("/compact")
|
||||
t.input.pressEnter()
|
||||
expect(await t.until("compacted")).toBe(true)
|
||||
expect(t.fake.requests[1].messages[0].content).toContain("## Transcript")
|
||||
// Asked as the next turn of the same conversation: the same system prompt, the messages, then the
|
||||
// compaction text (harness context.md §7).
|
||||
expect(t.fake.requests[1].messages[0]).toEqual(t.fake.requests[0].messages[0])
|
||||
expect(t.fake.requests[1].messages.at(-2).content).toBe("The answer is 42.")
|
||||
expect(t.fake.requests[1].messages.at(-1).content).toStartWith("Summarise the conversation")
|
||||
expect(t.app.engine.messages[0]).toMatchObject({ role: "user" })
|
||||
expect(JSON.stringify(t.app.engine.messages[0])).toContain("It is 42.")
|
||||
})
|
||||
@@ -429,8 +433,10 @@ test("/review sends the session's diff with the review brief in plan mode, then
|
||||
t.input.pressEnter()
|
||||
expect(await t.until("No findings.")).toBe(true)
|
||||
const req = t.fake.requests.at(-1)
|
||||
expect(req.messages[0].content).toContain("Permission mode: plan")
|
||||
// The session's prompt as it was made; plan mode comes as a note on the review's message.
|
||||
expect(req.messages[0]).toEqual(t.fake.requests[0].messages[0])
|
||||
const user = req.messages.at(-1).content as string
|
||||
expect(user).toContain("<session-note>\nThe permission mode is now plan:\nPermission mode: plan.")
|
||||
expect(user).toContain("Findings first, ordered by severity")
|
||||
expect(user).toContain("+hello there")
|
||||
expect(t.frame()).toContain("┃ /review")
|
||||
|
||||
Reference in new issue
Block a user