diff --git a/schema/config.schema.json b/schema/config.schema.json index 28f7bce..3d69f09 100644 --- a/schema/config.schema.json +++ b/schema/config.schema.json @@ -241,16 +241,13 @@ "additionalProperties": false }, "compaction": { - "description": "Automatic context management: past auto_at of the window, old tool outputs are pruned, then the conversation is compacted. auto_at: 1 turns it off.", + "description": "Automatic context management: past auto_at of the window, the conversation is compacted. auto_at: 1 turns it off. (Old tool outputs are never cut afterwards: that would make the server read the whole conversation again.)", "type": "object", "properties": { "auto_at": { "type": "number", "minimum": 0.1, "maximum": 1 - }, - "prune": { - "type": "boolean" } }, "additionalProperties": false diff --git a/src/app.ts b/src/app.ts index 3074629..debb526 100644 --- a/src/app.ts +++ b/src/app.ts @@ -331,7 +331,7 @@ export function createApp(o: AppOptions): App { const prototypesBlock = () => (context.prototypes?.offered || (!context.prototypes && loaded.config.prototypes === true) ? { sources: context.prototypes?.sources ?? "" } : undefined) /** The three personal keys this session uses: the client's for it, else the config's. */ const personal = () => (context.personalization ? personalConfig({ enabled: true, available: true, ...context.personalization }) : loaded.config) - const system = (m: Mode, current: ResolvedModel, subagent?: { name: string; instructions: string; report: boolean; skills: boolean }, at?: { cwd: string; root: string }, plan?: Plan) => + const system = (m: Mode, current: ResolvedModel, subagent?: { name: string; instructions: string; report: boolean; skills: boolean; recall: boolean }, at?: { cwd: string; root: string }, plan?: Plan) => assembleSystem({ modelRef: current.ref, family: familyOf(current), @@ -360,6 +360,7 @@ export function createApp(o: AppOptions): App { ...(context.instructions ? { instance: { name: contextFrom, instructions: context.instructions } } : {}), library: libraryBlock(), prototypes: prototypesBlock(), + context: subagent ? subagent.recall : true, }) // The other configured models, for the task tool's `model`, or a switch. const roster = (current: string) => @@ -369,7 +370,7 @@ export function createApp(o: AppOptions): App { .slice(0, 15) .map((m) => `- ${m.ref}${m.spec.name ? ` (${m.spec.name})` : ""}${m.spec.notes ? ` — ${m.spec.notes}` : ""}`) .join("\n") - const compaction = { autoAt: loaded.config.compaction?.auto_at, prune: loaded.config.compaction?.prune } + const compaction = { autoAt: loaded.config.compaction?.auto_at } // Subagents (the task tool): a child engine with its own bus and read-tracker, the agent's // tools and mode, the same asker (permission prompts still reach the user), no task tool of its @@ -509,6 +510,7 @@ export function createApp(o: AppOptions): App { : spec.instructions, report: childTools.some((t) => t.name === "report"), skills: childTools.some((t) => t.name === "skill_view"), + recall: childTools.some((t) => t.name === "recall"), }, at, plan, @@ -699,7 +701,6 @@ export function createApp(o: AppOptions): App { else if (key === "limits.bash_timeout") engine.o.toolCtx.bashTimeoutMs = Number(v) * 1000 else if (key === "limits.wall_seconds" || key === "limits.output_bytes" || key === "limits.completion_tokens") engine.o.budget = { ...engine.o.budget, [key.slice("limits.".length)]: Number(v) } else if (key === "compaction.auto_at") compaction.autoAt = Number(v) - else if (key === "compaction.prune") compaction.prune = Boolean(v) else if (key === "knowledge") libraryCtx.bases = (v as string[]).length ? (v as string[]) : undefined if (key !== "model" && key !== "effort" && key !== "mode") setIn(key, v) bus.emit({ type: "setting", key, value: v }) diff --git a/src/config/load.ts b/src/config/load.ts index f6d74a1..070ae6c 100644 --- a/src/config/load.ts +++ b/src/config/load.ts @@ -121,6 +121,14 @@ function legacyKeys(all: Record, file: string, notes: string[]) if (!(typeof p === "string" && ["none", "default", "neutral"].includes(p.toLowerCase()))) notes.push(`${file}: personality "${String(p)}" is not a preset any more (${PERSONALITIES.join(", ")}); none is used`) } + // compaction.prune cut old tool outputs in place, which threw the prompt cache away; harness spec + // 3.0.0 never rewrites what was sent (context.md), so the key is gone. + const comp = all.compaction + if (comp && typeof comp === "object" && !Array.isArray(comp) && "prune" in comp) { + const { prune: _, ...rest } = comp as Record + all.compaction = rest + notes.push(`${file}: compaction.prune was removed — old tool outputs are no longer cut afterwards; delete it`) + } // One file written as a plain string, not a list: read as the list of that one, not dropped // without a word by the schema. if (typeof all.instruction_files === "string") { diff --git a/src/config/schema.ts b/src/config/schema.ts index ed9f79e..ab2d9da 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -276,7 +276,7 @@ export const Config = z.strictObject({ }) .optional(), question: z.strictObject({ unattended: z.enum(["recommended", "first", "fail"]).optional().describe("With nobody to answer (headless): take the recommended option, the first, or fail.") }).optional(), - compaction: z.strictObject({ auto_at: z.number().min(0.1).max(1).optional(), prune: z.boolean().optional() }).optional().describe("Automatic context management: past auto_at of the window, old tool outputs are pruned, then the conversation is compacted. auto_at: 1 turns it off."), + compaction: z.strictObject({ auto_at: z.number().min(0.1).max(1).optional() }).optional().describe("Automatic context management: past auto_at of the window, the conversation is compacted. auto_at: 1 turns it off. (Old tool outputs are never cut afterwards: that would make the server read the whole conversation again.)"), instructions: z.string().max(4000).optional().describe("Custom instructions: how you want to be helped, in your own words (at most 4000 characters). Appended near the end of the system prompt, under \"## How the person you are talking to wants to be helped\" — as in the LLeMbas web UI. (A list of files here, from an older config, is read as instruction_files.)"), instruction_files: z.array(z.string()).optional().describe("More instruction files for the system prompt, besides AGENTS.md / CLAUDE.md."), search: z diff --git a/src/config/settings.ts b/src/config/settings.ts index 289e1f3..c86d886 100644 --- a/src/config/settings.ts +++ b/src/config/settings.ts @@ -53,7 +53,6 @@ export const SETTINGS: SettingDef[] = [ { key: "icons", kind: "enum", values: ["emoji", "plain"], agent: true, scopes: USER, fallback: "emoji" }, { key: "mouse", kind: "boolean", agent: true, scopes: ["global"], reload: true, fallback: "true" }, { key: "compaction.auto_at", kind: "number", agent: true, scopes: ALL, fallback: "0.85" }, - { key: "compaction.prune", kind: "boolean", agent: true, scopes: ALL, fallback: "true" }, { key: "search.order", kind: "list", values: ["searxng", "firecrawl", "ddg"], agent: true, scopes: ALL, fallback: "those configured, then ddg" }, { key: "search.max_results", kind: "number", agent: true, scopes: ALL, fallback: "8" }, { key: "memory.enabled", kind: "boolean", agent: true, scopes: ALL, reload: true, fallback: "true" }, diff --git a/src/harness.ts b/src/harness.ts index 88a90f9..bffc7b1 100644 --- a/src/harness.ts +++ b/src/harness.ts @@ -111,6 +111,27 @@ export const LOOP: { running_max: { default: number } subagent_steps: { default: number } } + /** Context hygiene (spec 3.0.0, harness/context.md): the result ids, the texts of a read's + * header, a stub, the elided line, recall; the output cap; what is deduplicated; the table + * compaction adds. */ + context: { + ids: { digits: number } + read_header: string + read_header_empty: string + edit_header: string + header: string + unchanged: string + same_result: string + elided: string + created: string + no_result: string + recall_header: string + sha_digits: number + cap: { max_lines: number; max_chars: number; head_lines: number; head_chars: number; tail_lines: number; tail_chars: number } + dedupe: { by_hash: string[]; by_result: string[] } + compaction: { heading: string; line: string; rows: number } + reasoning: string + } } = LOOP_JSON /** The `purpose` argument (harness/purpose.json). */ diff --git a/src/prompt/assemble.ts b/src/prompt/assemble.ts index 94a748b..b4b850d 100644 --- a/src/prompt/assemble.ts +++ b/src/prompt/assemble.ts @@ -254,6 +254,8 @@ export interface PromptContext { /** The library block (blocks/library.md): set when knowledge tools are offered or the instance * library is in use; `scope` the sentence naming the bases in scope, or "" for all of them. */ library?: { scope: string } + /** recall is offered: the tool results carry ids and may be stubs (blocks/context.md, spec 3.0.0). */ + context?: boolean /** Prototypes are offered (blocks/prototypes.md); `sources` the sentence saying what a page * may load, or "" where nothing says. */ prototypes?: { sources: string } @@ -337,6 +339,8 @@ export function assembleSystem(ctx: PromptContext): string { else if (ctx.manageSkills && !ctx.subagent) sections.push(promptText("blocks/skills-empty.md")) // The library and prototypes (shared blocks): the library whenever its tools are offered, named // to the bases in scope; prototypes only where they are offered, with where the page is seen. + // How tool results are kept short (ids, stubs, recall): wherever recall is offered. + if (ctx.context) sections.push(promptText("blocks/context.md")) if (ctx.library && !ctx.subagent) sections.push(fillPrompt(promptText("blocks/library.md"), { knowledge_scope: ctx.library.scope })) if (ctx.prototypes && !ctx.subagent) sections.push(fillPrompt(promptText("blocks/prototypes.md"), { prototype_sources: ctx.prototypes.sources, prototype_surface: PROTOTYPE_SURFACE })) diff --git a/src/session/engine.ts b/src/session/engine.ts index c044977..afabfc9 100644 --- a/src/session/engine.ts +++ b/src/session/engine.ts @@ -18,6 +18,7 @@ import { contextWindow } from "../provider/discover.ts" import { summaryTurns, type Store } from "./store.ts" import { estimateTokens, estimateValue } from "../provider/tokens.ts" import { duration } from "../duration.ts" +import { memoryResults, presentIds, ResultContext } from "./results.ts" export interface EngineOptions { bus: Bus @@ -39,7 +40,7 @@ export interface EngineOptions { budget?: { wall_seconds?: number; output_bytes?: number; completion_tokens?: number } /** Rules added by "always allow (project)" are handed here to persist. */ persistRule?: (rule: Rule) => void - compaction?: { autoAt?: number; prune?: boolean } + compaction?: { autoAt?: number } /** Starts a subagent for the task tool; absent in a subagent. */ spawn?: ToolContext["spawn"] /** The model to switch to when this one's server cannot be reached (its `fallback` list), or @@ -59,7 +60,6 @@ export interface PromptMeta { } const MAX_PARALLEL = LOOP.parallel.max -const PRUNED = "[This output was removed to save context. Run the tool again if you need it.]" /** What a message sent mid-turn starts with. Said, not hidden: Hermes found that a model refuses an * unmarked one as an injection. A resumed session shows the message without it. */ /** What /continue (ctrl+g) sends: carry on after being stopped, or after an answer that ended @@ -139,6 +139,10 @@ export class Engine { /** The step's "background now" signal (ToolContext.backgroundNow), while its tools run. */ private bgNow: AbortController | undefined + /** The session's tool results: their ids, what each insertion says, what recall shows + * (session/results.ts, harness context.md). */ + results: ResultContext + constructor(readonly o: EngineOptions) { this.mode = o.mode this._effort = o.effort @@ -146,6 +150,13 @@ export class Engine { this.client = o.client this.sessionId = o.sessionId this.plan = this.storedPlan() + this.results = this.resultsFor(o.sessionId) + } + + /** A session's results: kept with it in the store, or in memory without one. A stub may point + * only at a result still in this conversation. */ + private resultsFor(sessionId: string | undefined): ResultContext { + return new ResultContext(this.o.store && sessionId ? this.o.store.results(sessionId) : memoryResults(), () => presentIds(this.messages)) } /** The plan the store holds for this session, if any. */ @@ -191,6 +202,7 @@ export class Engine { this.sessionRules = [] this.recentCalls = [] this.withdrawn.clear() + this.results = this.resultsFor(sessionId) // A new session's plan is whatever the store holds for it: none for a fresh one, the old one // on a resume. this.plan = this.storedPlan() @@ -230,6 +242,10 @@ export class Engine { for await (const ev of this.client.stream({ system: "", messages: [{ role: "user", parts: [{ type: "text", text: prompt }] }], tools: [], effort: null, signal })) if (ev.type === "finish") summary = ev.message.parts.filter((p) => p.type === "text").map((p) => (p as { text: string }).text).join("").trim() if (!summary) throw new Error("the model returned an empty summary") + // The harness's own table of the reads still in the conversation (context.md §7), after the + // summary; the conversation is then the summary alone, so no stub can point at a result before it. + const table = this.results.compactionTable() + if (table) summary = `${summary}\n\n${table}` if (this.o.store && this.sessionId) this.o.store.compaction(this.sessionId, summary) this.messages = summaryTurns(summary) this.usage = undefined @@ -255,42 +271,16 @@ export class Engine { return Math.min(this.usage.output, this.usage.reasoning ?? estimate) } - /** Replace the content of old tool results, keeping the most recent `protect` tokens whole. - * Returns the tokens saved. (OpenCode prunes the same way before it summarises.) */ - prune(protect: number): number { - let recent = 0 - let saved = 0 - for (let i = this.messages.length - 1; i >= 0; i--) { - const m = this.messages[i]! - const size = estimateValue(m) - if (recent < protect) { - recent += size - continue - } - if (m.role === "tool" && m.content.length > 400 && !m.content.startsWith(PRUNED)) { - saved += estimateTokens(m.content) - estimateTokens(PRUNED) - this.messages[i] = { ...m, content: PRUNED } - } - } - return saved - } - - /** Before a step: past compaction.auto_at of the window, prune; if that is not enough, compact. */ + /** Before a step: past compaction.auto_at of the window, compact. Old tool results are never + * pruned in place (harness context.md: nothing already sent is rewritten — it would throw the + * prompt cache away from that point); compaction is the one deliberate break. */ private async manageContext(signal: AbortSignal) { const window = contextWindow(this.model) const autoAt = this.o.compaction?.autoAt ?? 0.85 if (!window || autoAt >= 1) return const limit = window * autoAt - let used = this.contextUsed() + const used = this.contextUsed() if (used === undefined || used < limit) return - if (this.o.compaction?.prune !== false) { - const saved = this.prune(Math.min(40_000, Math.floor(window * 0.3))) - if (saved > 0) { - used -= saved - this.o.bus.emit({ type: "notice", message: `context at ${Math.round(((used + saved) / window) * 100)}%: pruned old tool outputs, ~${saved} tokens freed` }) - } - if (used < limit) return - } // Compacting again and again within one turn loses more each time: the window is simply too // small for the task. Say so once and stop, rather than summarising in circles. if (this.autoCompactions >= 2) { @@ -612,6 +602,8 @@ export class Engine { // Decide every call first — asking is sequential — then run what was allowed. const results = new Map() + /** Each call's arguments as its tool got them: a repeated listing or search is compared by them. */ + const argsOf = new Map() const runnable: { call: ToolCallPart; tool: Tool; args: unknown; paths: string[]; writes: boolean; edited?: string }[] = [] for (const call of calls) { if (signal.aborted) break @@ -619,7 +611,10 @@ export class Engine { if ("result" in decided) { results.set(call.id, decided.result) if (decided.denied) bus.emit({ type: "tool_denied", id: call.id, name: call.name, reason: decided.result.output }) - } else runnable.push({ call, ...decided }) + } else { + runnable.push({ call, ...decided }) + argsOf.set(call.id, decided.args) + } } const bgNow = new AbortController() this.bgNow = bgNow @@ -673,7 +668,7 @@ export class Engine { for (const call of calls) { const r = results.get(call.id) ?? { output: "Not run: the reply was cancelled.", isError: true } this.turnOutputBytes += Buffer.byteLength(r.output) - this.push({ role: "tool", callId: call.id, name: call.name, content: r.output, isError: r.isError }) + this.push({ role: "tool", callId: call.id, name: call.name, content: this.inserted(call.name, argsOf.get(call.id), r), isError: r.isError }) if (r.images) images.push(...r.images) } // No dialect takes an image inside a tool result the same way; a user turn right after the @@ -686,6 +681,16 @@ export class Engine { } } + /** What a result becomes in the conversation (harness context.md): its id first, a read's lines + * and hash, a stub for an unchanged re-read or a repeated listing, an edit's diff, a long output's + * head and tail. Decided now, once: nothing already sent is rewritten. */ + private inserted(tool: string, args: unknown, r: ToolResult): string { + const c = r.context + if (c?.kind === "read" && !r.isError) return this.results.insert({ ...c, body: r.output }) + if (c && c.kind !== "read" && c.kind !== "output" && !r.isError) return this.results.insert(c) + return this.results.insert({ kind: "output", tool, args: args ?? {}, text: r.output, ...(r.isError ? { error: true } : {}) }) + } + private async decide( call: ToolCallPart, toolMap: Map, diff --git a/src/session/results.ts b/src/session/results.ts new file mode 100644 index 0000000..dffd080 --- /dev/null +++ b/src/session/results.ts @@ -0,0 +1,262 @@ +// Context hygiene (harness spec 3.0.0, harness/context.md; texts and numbers in loop.json `context`; +// conformance `context`): what a tool result becomes when it is inserted into the conversation. +// Every result gets an id; a read says its lines and its content's hash; an unchanged re-read inside +// an earlier read, and a listing or search repeated with the same result, are stubs pointing at the +// full result; large output keeps its head and tail; edits answer with their diff; `recall` shows any +// stored result again; compaction adds a table of the reads. +// +// The one rule: every measure acts when a result is inserted, and nothing already sent is ever +// rewritten — changing earlier context throws the prompt cache away from that point. A stub only +// ever points at a result that is still in the conversation (`present`): after a compaction, an +// /undo or a rewind, the result it would point at is gone, so the read is answered in full. +import { createHash } from "node:crypto" +import { structuredPatch } from "diff" +import { LOOP } from "../harness.ts" + +export const CONTEXT = LOOP.context + +export type Cap = typeof CONTEXT.cap + +/** One result as the session keeps it: everything needed to answer `recall`, and to decide a + * later stub. `text` is the full result (before any cap; for a stub, the text it stands for). */ +export interface StoredResult { + id: string + kind: "read" | "edit" | "output" | "recall" + /** Not a stub: a stub never becomes the target of another. */ + full: boolean + tool?: string + /** read: the path as the call gave it, and what identifies the file (its absolute path). */ + path?: string + key?: string + /** read: the full SHA-256 of the file's content, and the lines shown. */ + sha?: string + from?: number + to?: number + of?: number + /** output: the call's tool and arguments (JSON, keys sorted), and the hash of the result. */ + args?: string + textSha?: string + text: string +} + +/** Where a session's results are kept: the session store, or memory without one. */ +export interface ResultStore { + /** How many ids this session has given: the next is one more. */ + count(): number + save(r: StoredResult): void + get(id: string): StoredResult | undefined + /** Every result's fields, oldest first (the text may be left out). */ + all(): StoredResult[] +} + +export function memoryResults(): ResultStore { + const rows: StoredResult[] = [] + return { + count: () => rows.length, + save: (r) => void rows.push(r), + get: (id) => rows.find((r) => r.id === id), + all: () => rows, + } +} + +/** What a tool's result is, for its insertion (ToolResult.context, filled in by the tool). */ +export type Insert = + /** A file read: `body` the read tool's lines (and its note when lines remain). */ + | { kind: "read"; path: string; key: string; sha: string; from: number; to: number; of: number; body: string } + /** One or more files changed: each with its new content's hash, and its hunks, or the number of + * lines of a file created, or deleted. `note` follows the diff (something the tool adds). */ + | { kind: "edit"; files: EditedFile[]; note?: string } + /** Anything else: `args` as the tool got them. */ + | { kind: "output"; tool: string; args: unknown; text: string; error?: boolean } + | { kind: "recall"; target: string; offset?: number; limit?: number } + +/** What a tool hands over in its result (ToolResult.context): a read without its body, which is the + * result's output. */ +export type ToolInsert = Exclude | Omit, "body"> + +export type EditedFile = { path: string; sha: string; hunks: string } | { path: string; sha: string; created: number } | { path: string; deleted: true } + +export const sha256 = (text: string | Uint8Array) => createHash("sha256").update(text).digest("hex") + +/** `{{name}}` filled from `vars` (loop.json's texts, which have no bullets to drop). */ +const fill = (text: string, vars: Record) => text.replace(/\{\{(\w+)\}\}/g, (m, k: string) => (k in vars ? String(vars[k]) : m)) +const short = (sha: string) => sha.slice(0, CONTEXT.sha_digits) + +/** JSON with every object's keys sorted: how two calls' arguments are compared. */ +export function stableJson(v: unknown): string { + const norm = (x: unknown): unknown => + Array.isArray(x) ? x.map(norm) : x && typeof x === "object" ? Object.fromEntries(Object.keys(x).sort().map((k) => [k, norm((x as Record)[k])])) : x + return JSON.stringify(norm(v ?? {})) +} + +/** The unified diff of a change, hunks only (the path is in the bracket), three lines of context. */ +export function hunks(before: string, after: string): string { + const p = structuredPatch("a", "b", before, after, "", "", { context: 3 }) + return p.hunks.map((h) => [`@@ -${h.oldStart},${h.oldLines} +${h.newStart},${h.newLines} @@`, ...h.lines].join("\n")).join("\n") +} + +/** How many lines a new file has, as the model wrote it (a final newline ends the last line). */ +export function lineCount(text: string): number { + if (!text) return 0 + const n = text.split("\n").length + return text.endsWith("\n") ? n - 1 : n +} + +/** A result over the cap's lines or characters: its head and its tail, with the elided line between + * (loop.json `cap`). The first line alone longer than the head is cut and marked. */ +export function capOutput(text: string, id: string, cap: Cap = CONTEXT.cap): string { + const lines = text.split("\n") + if (lines.length <= cap.max_lines && text.length <= cap.max_chars) return text + const head: string[] = [] + let chars = 0 + for (const l of lines) { + if (head.length >= cap.head_lines) break + const add = (head.length ? 1 : 0) + l.length + if (chars + add > cap.head_chars) break + head.push(l) + chars += add + } + let used = head.length + if (!head.length && lines.length) { + head.push(`${lines[0]!.slice(0, cap.head_chars)} […cut]`) + used = 1 + } + const tail: string[] = [] + chars = 0 + for (let i = lines.length - 1; i >= used; i--) { + if (tail.length >= cap.tail_lines) break + const l = lines[i]! + const add = (tail.length ? 1 : 0) + l.length + if (chars + add > cap.tail_chars) break + tail.unshift(l) + chars += add + } + const elided = lines.length - used - tail.length + return [...head, fill(CONTEXT.elided, { n: elided, id }), ...tail].join("\n") +} + +/** A session's results: the ids it gives, what it keeps, and what each insertion says. */ +export class ResultContext { + constructor( + private store: ResultStore, + /** The ids of the results still in the conversation (a stub may point only at one of them). */ + private present: () => Set, + private cap: Cap = CONTEXT.cap, + ) {} + + /** The next id: `#` and lowercase hex, at least ids.digits digits, never reused in the session. */ + nextId(): string { + return `#${(this.store.count() + 1).toString(16).padStart(CONTEXT.ids.digits, "0")}` + } + + /** The text the model is given for one result, which is stored in full. `id`: one given from + * outside (the conformance cases name theirs); otherwise the next. */ + insert(e: Insert, id = this.nextId()): string { + if (e.kind === "read") return this.read(id, e) + if (e.kind === "edit") return this.edit(id, e) + if (e.kind === "recall") return this.recall(id, e) + return this.output(id, e) + } + + private read(id: string, e: Extract): string { + const head = e.of === 0 ? fill(CONTEXT.read_header_empty, { id, path: e.path, sha: short(e.sha) }) : fill(CONTEXT.read_header, { id, path: e.path, from: e.from, to: e.to, of: e.of, sha: short(e.sha) }) + const base: StoredResult = { id, kind: "read", tool: "read", path: e.path, key: e.key, sha: e.sha, from: e.from, to: e.to, of: e.of, full: true, text: e.body } + // By hash: an earlier full read of this file, unchanged, whose lines hold these — the newest. + const earlier = e.of > 0 && CONTEXT.dedupe.by_hash.includes("read") ? this.table().filter((r) => r.kind === "read" && r.key === e.key && r.sha === e.sha && r.from! <= e.from && e.to <= r.to!).pop() : undefined + if (earlier) { + this.store.save({ ...base, full: false }) + return `${head} ${fill(CONTEXT.unchanged, { earlier: earlier.id, from: earlier.from!, to: earlier.to! })}` + } + this.store.save(base) + return `${head}\n${e.body}` + } + + private edit(id: string, e: Extract): string { + const one = (f: EditedFile): { bracket: string; body: string } => + "deleted" in f ? { bracket: `${f.path} deleted`, body: "" } : { bracket: `${f.path} sha ${short(f.sha)}`, body: "created" in f ? fill(CONTEXT.created, { n: f.created }) : f.hunks } + let text: string + if (e.files.length === 1) { + const { bracket, body } = one(e.files[0]!) + text = `[${id} ${bracket}]${body ? ("created" in e.files[0]! ? " " : "\n") + body : ""}` + } else { + // Several files (apply_patch): the id alone, then each file's bracket and its change. + text = [`[${id}]`, ...e.files.map((f) => { + const { bracket, body } = one(f) + return `[${bracket}]${body ? ("created" in f ? " " : "\n") + body : ""}` + })].join("\n") + } + if (e.note) text += e.note + this.store.save({ id, kind: "edit", full: true, text: text.slice(text.indexOf("]") + 1).replace(/^[ \n]/, "") }) + // The diff is capped like any output, the bracket kept. + const nl = text.indexOf("\n") + return nl < 0 ? text : `${text.slice(0, nl)}\n${capOutput(text.slice(nl + 1), id, this.cap)}` + } + + private output(id: string, e: Extract): string { + const args = stableJson(e.args) + const textSha = sha256(e.text) + const base: StoredResult = { id, kind: "output", tool: e.tool, args, textSha, full: true, text: e.text } + // By result: the same call, still in the table, whose result was this same text. + const dedupe = !e.error && CONTEXT.dedupe.by_result.includes(e.tool) + const earlier = dedupe ? this.table().filter((r) => r.kind === "output" && r.tool === e.tool && r.args === args).pop() : undefined + if (earlier && earlier.textSha === textSha) { + this.store.save({ ...base, full: false }) + return `${fill(CONTEXT.header, { id })} ${fill(CONTEXT.same_result, { earlier: earlier.id })}` + } + this.store.save(base) + return `${fill(CONTEXT.header, { id })}\n${capOutput(e.text, id, this.cap)}` + } + + private recall(id: string, e: Extract): string { + const raw = e.target.trim().toLowerCase() + const target = raw.startsWith("#") ? raw : `#${raw}` + const found = this.store.get(target) + if (!found) { + const text = fill(CONTEXT.no_result, { target }) + this.store.save({ id, kind: "recall", full: true, text }) + return `${fill(CONTEXT.header, { id })} ${text}` + } + const lines = found.text.split("\n") + const from = Math.max(1, e.offset ?? 1) + const to = Math.min(lines.length, from + (e.limit ?? 2000) - 1) + const shown = lines.slice(from - 1, to).join("\n") + this.store.save({ id, kind: "recall", full: true, text: shown }) + return `${fill(CONTEXT.recall_header, { id, target, from, to, of: lines.length })}\n${shown}` + } + + /** The dedupe table: full results still in the conversation, oldest first. */ + private table(): StoredResult[] { + const here = this.present() + return this.store.all().filter((r) => r.full && here.has(r.id)) + } + + /** The table compaction adds after its summary: the newest full read of each path still in the + * conversation, newest last, at most compaction.rows; "" with none. After it the conversation is + * the summary alone, so nothing is present and the dedupe table is empty by construction. */ + compactionTable(): string { + const newest = new Map() + for (const r of this.table()) if (r.kind === "read" && r.of! > 0) newest.delete(r.key!), newest.set(r.key!, r) + const rows = [...newest.values()].slice(-CONTEXT.compaction.rows) + if (!rows.length) return "" + return [ + CONTEXT.compaction.heading, + "", + CONTEXT.compaction.line, + "", + "| Path | Lines | sha | Result |", + "|---|---|---|---|", + ...rows.map((r) => `| ${r.path} | ${r.from}-${r.to} of ${r.of} | ${short(r.sha!)} | ${r.id} |`), + ].join("\n") + } +} + +/** The ids at the head of the tool results in a conversation: what is present for a stub. */ +export function presentIds(messages: readonly { role: string; content?: unknown }[]): Set { + const out = new Set() + for (const m of messages) { + if (m.role !== "tool" || typeof m.content !== "string") continue + const hit = /^\[(#[0-9a-f]+)[\] ]/.exec(m.content) + if (hit) out.add(hit[1]!) + } + return out +} diff --git a/src/session/store.ts b/src/session/store.ts index 525b2fd..2bf92f9 100644 --- a/src/session/store.ts +++ b/src/session/store.ts @@ -48,6 +48,11 @@ CREATE TABLE IF NOT EXISTS jobs ( PRIMARY KEY (session_id, id) ); CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(text, content='', contentless_delete=1); +CREATE TABLE IF NOT EXISTS results ( + session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, seq INTEGER NOT NULL, id TEXT NOT NULL, + json TEXT NOT NULL, text TEXT NOT NULL, + PRIMARY KEY (session_id, id) +); CREATE TABLE IF NOT EXISTS session_meta ( session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, key TEXT NOT NULL, value TEXT NOT NULL, PRIMARY KEY (session_id, key) @@ -195,6 +200,31 @@ export class Store { return last ? [...summaryTurns(JSON.parse(last.json).summary as string), ...msgs] : msgs } + /** A session's tool results, each in full (harness spec 3.0.0, context.md): what `recall` shows + * and what a stub is decided against. Kept as long as the session is. */ + results(sessionId: string): import("./results.ts").ResultStore { + const q = { + count: this.db.query("SELECT COUNT(*) AS n FROM results WHERE session_id = ?"), + save: this.db.query("INSERT OR REPLACE INTO results (session_id, seq, id, json, text) VALUES (?, ?, ?, ?, ?)"), + get: this.db.query("SELECT json, text FROM results WHERE session_id = ? AND id = ?"), + all: this.db.query("SELECT json FROM results WHERE session_id = ? ORDER BY seq"), + } + type R = import("./results.ts").StoredResult + return { + count: () => (q.count.get(sessionId) as { n: number }).n, + save: (r) => { + const { text, ...rest } = r + q.save.run(sessionId, (q.count.get(sessionId) as { n: number }).n + 1, r.id, JSON.stringify(rest), text) + }, + get: (id) => { + const row = q.get.get(sessionId, id) as { json: string; text: string } | null + return row ? ({ ...JSON.parse(row.json), text: row.text } as R) : undefined + }, + // Without their text: the table is decided by hashes and ranges. + all: () => (q.all.all(sessionId) as { json: string }[]).map((r) => ({ ...JSON.parse(r.json), text: "" }) as R), + } + } + /** Something a session keeps besides its messages, as JSON. */ meta(sessionId: string, key: string): T | undefined { const r = this.db.query("SELECT value FROM session_meta WHERE session_id = ? AND key = ?").get(sessionId, key) as { value: string } | null diff --git a/src/tool/apply_patch.ts b/src/tool/apply_patch.ts index 170e894..31ee852 100644 --- a/src/tool/apply_patch.ts +++ b/src/tool/apply_patch.ts @@ -12,6 +12,7 @@ import { straySkillNote } from "../skill/index.ts" import { absPath, defineTool, readText, showPath, ToolError, type ToolContext } from "./tool.ts" import { diffStat } from "./write.ts" import { described } from "../harness.ts" +import { hunks, lineCount, sha256, type EditedFile } from "../session/results.ts" function hunksOf(text: string): Hunk[] { try { @@ -35,7 +36,7 @@ function namedFiles(a: { patch: string; path?: string }): string[] { return hunksOf(a.patch).flatMap((h) => (h.type === "update" && h.movePath ? [h.path, h.movePath] : [h.path])) } -type Op = { kind: "write"; path: string; content: string; before: string } | { kind: "delete"; path: string; before: string } +type Op = { kind: "write"; path: string; content: string; before: string; created?: boolean } | { kind: "delete"; path: string; before: string } function opsOf(original: Map, current: Map): Op[] { const ops: Op[] = [] @@ -43,7 +44,7 @@ function opsOf(original: Map, current: Map showPath(o.path, ctx)).join(", ") - return { output: `Applied to ${names} (+${added} −${removed}).${straySkillNote(ops.filter((o) => o.kind === "write").map((o) => o.path), ctx)}`, title: `${names} +${added} −${removed}`, meta: { diff: all } } + const note = straySkillNote(ops.filter((o) => o.kind === "write").map((o) => o.path), ctx) + const files: EditedFile[] = ops.map((op) => { + const path = showPath(op.path, ctx) + if (op.kind === "delete") return { path, deleted: true } + return op.created ? { path, sha: sha256(op.content), created: lineCount(op.content) } : { path, sha: sha256(op.content), hunks: hunks(op.before, op.content) } + }) + return { output: `Applied to ${names} (+${added} −${removed}).${note}`, title: `${names} +${added} −${removed}`, meta: { diff: all }, context: { kind: "edit", files, ...(note ? { note } : {}) } } }, }) diff --git a/src/tool/edit.ts b/src/tool/edit.ts index 104a1d0..4b51daf 100644 --- a/src/tool/edit.ts +++ b/src/tool/edit.ts @@ -5,6 +5,7 @@ import { replace } from "./replace-text.ts" import { absPath, defineTool, readText, showPath, ToolError, type ToolContext } from "./tool.ts" import { diffStat, guardExisting } from "./write.ts" import { described } from "../harness.ts" +import { hunks, sha256 } from "../session/results.ts" async function planEdit(a: { path: string; old: string; new: string; all?: boolean }, ctx: ToolContext) { const abs = absPath(a.path, ctx) @@ -43,6 +44,7 @@ export const editTool = defineTool({ ctx.fileStamps.set(abs, (await stat(abs)).mtimeMs) const patch = createTwoFilesPatch(display, display, before, after, "", "", { context: 3 }) const { added, removed } = diffStat(patch) - return { output: `Edited ${display} (+${added} −${removed}).`, title: `${display} +${added} −${removed}`, meta: { diff: patch } } + const written = crlf ? after.replaceAll("\n", "\r\n") : after + return { output: `Edited ${display} (+${added} −${removed}).`, title: `${display} +${added} −${removed}`, meta: { diff: patch }, context: { kind: "edit", files: [{ path: a.path, sha: sha256(written), hunks: hunks(before, after) }] } } }, }) diff --git a/src/tool/multiedit.ts b/src/tool/multiedit.ts index a3107ae..e3df754 100644 --- a/src/tool/multiedit.ts +++ b/src/tool/multiedit.ts @@ -5,6 +5,7 @@ import { replace } from "./replace-text.ts" import { absPath, defineTool, readText, showPath, ToolError, type ToolContext } from "./tool.ts" import { diffStat, guardExisting } from "./write.ts" import { described } from "../harness.ts" +import { hunks, sha256 } from "../session/results.ts" async function planEdits(a: { path: string; edits: { old: string; new: string; all?: boolean }[] }, ctx: ToolContext) { const abs = absPath(a.path, ctx) @@ -45,6 +46,7 @@ export const multieditTool = defineTool({ ctx.fileStamps.set(abs, (await stat(abs)).mtimeMs) const patch = createTwoFilesPatch(display, display, before, after, "", "", { context: 3 }) const { added, removed } = diffStat(patch) - return { output: `Edited ${display}: ${a.edits.length} changes (+${added} −${removed}).`, title: `${display} +${added} −${removed}`, meta: { diff: patch } } + const written = crlf ? after.replaceAll("\n", "\r\n") : after + return { output: `Edited ${display}: ${a.edits.length} changes (+${added} −${removed}).`, title: `${display} +${added} −${removed}`, meta: { diff: patch }, context: { kind: "edit", files: [{ path: a.path, sha: sha256(written), hunks: hunks(before, after) }] } } }, }) diff --git a/src/tool/read.ts b/src/tool/read.ts index 7bf646a..fd7b42f 100644 --- a/src/tool/read.ts +++ b/src/tool/read.ts @@ -5,6 +5,7 @@ import { dirname, join } from "node:path" import { z } from "zod" import { absPath, defineTool, showPath, ToolError } from "./tool.ts" import { described } from "../harness.ts" +import { sha256 } from "../session/results.ts" const MAX_LINES = 2000 const MAX_LINE = 2000 @@ -37,23 +38,35 @@ export const readTool = defineTool({ if (bytes.includes(0)) throw new ToolError(`${a.path} is a binary file (${st.size} bytes)`) // Read whole, so only up to a size that fits in memory comfortably; past it, grep or bash. if (st.size > 50_000_000) throw new ToolError(`${a.path} is ${Math.round(st.size / 1e6)} MB — too large to read whole; search it with grep, or look at a part with bash (head, tail, sed -n)`) - const text = await file.text() - const all = text.split("\n") - if (all.length > 1 && all[all.length - 1] === "") all.pop() - const start = (a.offset ?? 1) - 1 - const end = Math.min(all.length, start + (a.limit ?? MAX_LINES)) - const body = all - .slice(start, end) - .map((l, i) => `${start + i + 1}: ${l.length > MAX_LINE ? l.slice(0, MAX_LINE) + " […cut]" : l}`) - .join("\n") + const raw = await file.bytes() + const text = new TextDecoder().decode(raw) + const { body, from, to, of } = readBody(text, a.offset, a.limit) ctx.readFiles.add(abs) ctx.fileStamps.set(abs, st.mtimeMs) - const more = end < all.length ? `\n\n(${all.length - end} more lines — read again with offset ${end + 1})` : "" - const empty = all.length === 0 ? "(empty file)" : "" - return { output: (empty || body) + more + nearbyInstructions(abs, ctx), title: `${showPath(abs, ctx)} · ${start + 1}–${end} of ${all.length}`, meta: { path: showPath(abs, ctx), from: start + 1, to: end, of: all.length } } + // For its insertion (session/results.ts): the lines shown and the content's hash, by which an + // unchanged re-read becomes a stub. + const context = { kind: "read" as const, path: a.path, key: abs, sha: sha256(raw), from, to, of } + return { output: body + nearbyInstructions(abs, ctx), title: `${showPath(abs, ctx)} · ${from}–${to} of ${of}`, meta: { path: showPath(abs, ctx), from, to, of }, context } }, }) +/** The read tool's body for a file's text: numbered lines from `offset` (1-indexed), at most + * `limit` (2000), a line longer than 2000 characters cut, and the note when lines remain; "(empty + * file)" for an empty one. `from`-`to` of `of`: the lines shown (conformance `context`). */ +export function readBody(text: string, offset?: number, limit?: number): { body: string; from: number; to: number; of: number } { + const all = text.split("\n") + if (all.length > 1 && all[all.length - 1] === "") all.pop() + if (all.length === 1 && all[0] === "") all.pop() + const start = (offset ?? 1) - 1 + const end = Math.min(all.length, start + (limit ?? MAX_LINES)) + const lines = all + .slice(start, end) + .map((l, i) => `${start + i + 1}: ${l.length > MAX_LINE ? l.slice(0, MAX_LINE) + " […cut]" : l}`) + .join("\n") + const more = end < all.length ? `\n\n(${all.length - end} more lines — read again with offset ${end + 1})` : "" + return { body: (all.length === 0 ? "(empty file)" : lines) + more, from: start + 1, to: end, of: all.length } +} + /** OpenCode's "nearby instructions": reading a file under a directory with its own AGENTS.md or * CLAUDE.md hands the model that file, once per session. The root's is already in the prompt. */ function nearbyInstructions(file: string, ctx: import("./tool.ts").ToolContext): string { diff --git a/src/tool/recall.ts b/src/tool/recall.ts new file mode 100644 index 0000000..6b7297b --- /dev/null +++ b/src/tool/recall.ts @@ -0,0 +1,19 @@ +// recall (harness spec 3.0.0, context.md): a stored result of this session shown again, in full or +// from a line on. The engine answers it from the session's results (session/results.ts); the tool +// only carries the call there, which is why its run says what it is. +import { z } from "zod" +import { described } from "../harness.ts" +import { defineTool } from "./tool.ts" + +export const recallTool = defineTool({ + ...described("recall"), + access: "recall", + purpose: false, + schema: z.object({ id: z.string(), offset: z.number().int().min(1).optional(), limit: z.number().int().min(1).optional() }), + permission: (a) => ({ permission: "recall", class: "read", patterns: [a.id] }), + async run(a) { + // The text is the session's insertion (`context` recall); this is what a transcript shows. + const target = a.id.trim().startsWith("#") ? a.id.trim() : `#${a.id.trim()}` + return { output: `recall ${target}`, title: target, context: { kind: "recall", target: a.id, ...(a.offset ? { offset: a.offset } : {}), ...(a.limit ? { limit: a.limit } : {}) } } + }, +}) diff --git a/src/tool/registry.ts b/src/tool/registry.ts index 52dc015..b0289b5 100644 --- a/src/tool/registry.ts +++ b/src/tool/registry.ts @@ -18,6 +18,7 @@ import { sessionSearchTool } from "./session_search.ts" import { skillManageTool, skillsListTool, skillViewTool } from "./skills.ts" import { settingsTool } from "./settings.ts" import { showTool } from "./show.ts" +import { recallTool } from "./recall.ts" import { knowledgeGetTool, knowledgeSearchTool, noteManageTool, notesSearchTool, noteViewTool } from "./library.ts" import type { Tool } from "./tool.ts" import { writeTool } from "./write.ts" @@ -58,6 +59,7 @@ export const BUILTIN_TOOLS: Tool[] = [ settingsTool, viewImageTool, showTool, + recallTool, // Offered to subagents only (the spec's `offered: subagent`; app.ts SUBAGENT_ONLY). reportTool, ] as Tool[] diff --git a/src/tool/tool.ts b/src/tool/tool.ts index 1e44111..1fde9ae 100644 --- a/src/tool/tool.ts +++ b/src/tool/tool.ts @@ -107,6 +107,9 @@ export interface ToolResult { meta?: Record /** Images for the model (view_image): sent in a user turn after the tool results. */ images?: import("../provider/types.ts").ImagePart[] + /** What the result is, for its insertion into the conversation (session/results.ts, harness + * context.md): a read's lines and hash, an edit's diff. Absent: an ordinary output. */ + context?: import("../session/results.ts").ToolInsert } export interface Tool { diff --git a/src/tool/write.ts b/src/tool/write.ts index 77bd004..08fcfde 100644 --- a/src/tool/write.ts +++ b/src/tool/write.ts @@ -5,6 +5,7 @@ import { z } from "zod" import { straySkillNote } from "../skill/index.ts" import { absPath, defineTool, showPath, ToolError, type ToolContext } from "./tool.ts" import { described } from "../harness.ts" +import { hunks, lineCount, sha256 } from "../session/results.ts" /** Refuse to overwrite a file the model has not read, or one changed on disk since it was read. */ export async function guardExisting(abs: string, display: string, ctx: ToolContext): Promise { @@ -57,10 +58,14 @@ export const writeTool = defineTool({ ctx.fileStamps.set(abs, st.mtimeMs) const patch = createTwoFilesPatch(display, display, before, a.content, "", "", { context: 3 }) const { added, removed } = diffStat(patch) + const note = straySkillNote([abs], ctx) + const sha = sha256(a.content) return { - output: (existed ? `Replaced ${display} (+${added} −${removed}).` : `Created ${display} (${a.content.split("\n").length} lines).`) + straySkillNote([abs], ctx), + output: (existed ? `Replaced ${display} (+${added} −${removed}).` : `Created ${display} (${a.content.split("\n").length} lines).`) + note, title: `${display} +${added} −${removed}`, meta: { diff: patch }, + // Over an existing file the diff; a new one is said in a line — the model wrote it and has it. + context: { kind: "edit", files: [existed ? { path: a.path, sha, hunks: hunks(before, a.content) } : { path: a.path, sha, created: lineCount(a.content) }], ...(note ? { note } : {}) }, } }, }) diff --git a/tests/approval.test.ts b/tests/approval.test.ts index 56e9b98..9ecb4ac 100644 --- a/tests/approval.test.ts +++ b/tests/approval.test.ts @@ -25,7 +25,7 @@ const result = () => fake!.requests[1].messages.at(-1).content as string test("the edited command runs, and the result says so first", async () => { const a = app("echo two") await a.engine.prompt("go") - expect(result()).toStartWith("The user changed the command before allowing it. What ran: echo two") + expect(result()).toStartWith("[#0001]\nThe user changed the command before allowing it. What ran: echo two") expect(result()).toContain("two") expect(result()).not.toContain("one\n") }) diff --git a/tests/conformance.ts b/tests/conformance.ts index b90b3f1..386b7e5 100644 --- a/tests/conformance.ts +++ b/tests/conformance.ts @@ -19,6 +19,8 @@ import { resolveBase } from "../src/library/store.ts" import { NarrowError, narrowParameters, type Narrow } from "../src/harness.ts" import { fillPrompt } from "../src/prompt/fill.ts" import { instructionChain } from "../src/prompt/assemble.ts" +import { CONTEXT, hunks, lineCount, memoryResults, presentIds, ResultContext, sha256 } from "../src/session/results.ts" +import { readBody } from "../src/tool/read.ts" import { applyOps, type Op, type Target } from "../src/memory/store.ts" import { jobNote, type NoteCase } from "../src/jobs/note.ts" @@ -115,6 +117,30 @@ export const AREAS: Record unknown> = { const files = new Set(c.files as string[]) return instructionChain(c.boundary as string, c.start as string, (p) => files.has(p) && readableUnder(c.grants as Record | undefined, p)) }, + // Context hygiene: one session's insertions, step by step. The conversation is the results + // inserted since the last compaction, which is what a stub may point at. + context: (c) => { + const messages: { role: string; content: string }[] = [] + const rc = new ResultContext(memoryResults(), () => presentIds(messages), { ...CONTEXT.cap, ...((c.limits ?? {}) as object) }) + return (c.steps as Record[]).map((st) => { + if (st.op === "compact") { + const table = rc.compactionTable() + messages.length = 0 + return table + } + let text: string + if (st.op === "read") { + const { body, from, to, of } = readBody(st.content, st.offset, st.limit) + text = rc.insert({ kind: "read", path: st.path, key: st.path, sha: sha256(st.content), from, to, of, body }, st.id) + } else if (st.op === "edit" || st.op === "write") { + const sha = sha256(st.after) + text = rc.insert({ kind: "edit", files: [st.before === undefined ? { path: st.path, sha, created: lineCount(st.after) } : { path: st.path, sha, hunks: hunks(st.before, st.after) }] }, st.id) + } else if (st.op === "recall") text = rc.insert({ kind: "recall", target: st.target, offset: st.offset, limit: st.limit }, st.id) + else text = rc.insert({ kind: "output", tool: st.tool, args: st.args, text: st.text }, st.id) + messages.push({ role: "tool", content: text }) + return text + }) + }, library_scope: (c) => resolveBase(c.bases as { id: string; name: string }[], c.base as string | null), toolcall_index: (c) => { // The deltas as the OpenAI stream carries them: `function.name` and `function.arguments`. diff --git a/tests/context.test.ts b/tests/context.test.ts index 2da3216..63cba8d 100644 --- a/tests/context.test.ts +++ b/tests/context.test.ts @@ -45,12 +45,14 @@ describe("context", () => { expect(fake!.calls.some((c) => c.path.endsWith("/unload"))).toBe(true) }) - test("past auto_at: the older of two large outputs is pruned, the recent one kept, no compaction", async () => { - // window 16000, auto_at 0.6 → limit 9600; the most recent 30% (4800 tokens) is protected. + test("past auto_at: nothing already sent is cut; the conversation is compacted, with a table of the reads", async () => { + // window 16000, auto_at 0.6 → limit 9600. Harness spec 3.0.0 (context.md): old tool outputs are + // never pruned in place — that would make the server read everything after them again. const a = app(`connections:\n f:\n dialect: openai-chat\n base_url: URL\n models: { m: { context: 16000 } }\n`, [ { chunks: [toolCall(0, "c1", "read", '{"path":"big.txt"}'), usage(1000, 10)] }, { chunks: [toolCall(0, "c2", "read", '{"path":"big2.txt"}'), usage(6600, 10)] }, - { chunks: [delta({ content: "done" }), usage(7000, 5)] }, + { chunks: [delta({ content: "## Summary\nRead two files." })] }, + { chunks: [delta({ content: "done" }), usage(2000, 5)] }, ], "compaction: { auto_at: 0.6 }\n") const lines = Array.from({ length: 1000 }, (_, i) => `line ${i} of a long file`).join("\n") writeFileSync(join(a.project.root, "big.txt"), lines) @@ -58,14 +60,21 @@ describe("context", () => { const notices: string[] = [] a.bus.on((e: Event) => e.type === "notice" && notices.push(e.message)) await a.engine.prompt("go") - expect(notices.some((n) => n.includes("pruned old tool outputs"))).toBe(true) - expect(notices.some((n) => n.includes("compacting"))).toBe(false) - const results = fake!.requests.at(-1).messages.filter((m: any) => m.role === "tool").map((m: any) => m.content as string) - expect(results[0]).toContain("removed to save context") - expect(results[1]).toContain("line 999 of a long file") + expect(notices.some((n) => n.includes("pruned"))).toBe(false) + expect(notices.some((n) => n.includes("compacting"))).toBe(true) + // The first read went out whole, and stayed whole until the summary replaced the conversation. + const sent = fake!.requests[1].messages.filter((m: any) => m.role === "tool").map((m: any) => m.content as string) + expect(sent[0]).toStartWith("[#0001 big.txt lines 1-1000 of 1000 sha ") + expect(sent[0]).toContain("line 999 of a long file") + // After it: the summary, the harness's table of what was read, and the request word for word. + const after = JSON.stringify(fake!.requests[3].messages) + expect(after).toContain("Read two files.") + expect(after).toContain("## Files read before this summary") + expect(after).toMatch(/\| big\.txt \| 1-1000 of 1000 \| [0-9a-f]{8} \| #0001 \|/) + expect(after).toMatch(/\| big2\.txt \| 1-1000 of 1000 \| [0-9a-f]{8} \| #0002 \|/) }) - test("still too full after pruning: compacted mid-task, then told to carry on", async () => { + test("too full: compacted mid-task, then told to carry on", async () => { const a = app(`connections:\n f:\n dialect: openai-chat\n base_url: URL\n models: { m: { context: 4000 } }\n`, [ { chunks: [delta({ content: "y".repeat(8000) }), toolCall(0, "c1", "list", "{}"), usage(3900, 2000)] }, { chunks: [delta({ content: "## What we are doing\nA long task." })] }, @@ -91,7 +100,7 @@ describe("context", () => { expect(text).toContain("window 32.8k") expect(text).toContain("system prompt") expect(text).toMatch(/tool definitions \(\d+\)/) - expect(text).toMatch(/tool results\s+503/) // one 6000-character line, cut at 2000 by read + expect(text).toMatch(/tool results\s+514/) // one 6000-character line, cut at 2000 by read, under its header expect(text).toMatch(/in use\s+3\.1k/) }) }) diff --git a/tests/e2e.test.ts b/tests/e2e.test.ts index b6791a5..e764a7c 100644 --- a/tests/e2e.test.ts +++ b/tests/e2e.test.ts @@ -45,12 +45,15 @@ describe("end to end", () => { // The tool result went back to the model as a tool message with the right id. expect(fake!.requests[1].messages.at(-1)).toMatchObject({ role: "tool", tool_call_id: "c1" }) expect(fake!.requests[1].messages.at(-1).content).toContain("1: hello world") + // Spec 3.0.0 (context.md): the read is headed by its id, lines and hash; the edit answers with its diff. + expect(fake!.requests[1].messages.at(-1).content).toMatch(/^\[#0001 hello\.txt lines 1-1 of 1 sha [0-9a-f]{8}\]\n1: hello world$/) + expect(fake!.requests[2].messages.at(-1).content).toMatch(/^\[#0002 hello\.txt sha [0-9a-f]{8}\]\n@@ -1,1 \+1,1 @@\n-hello world\n\+hello lembas$/) // System prompt carries env and the mode. expect(fake!.requests[0].messages[0].content).toContain(`Project root: ${dir}`) expect(fake!.requests[0].messages[0].content).toContain("Permission mode: edit") // knowledge_search/get are offered once any knowledge base has documents, and the suite shares // one LEMBAS_HOME: another file's base must not decide this list (found in the full run). - expect(fake!.requests[0].tools.map((t: any) => t.function.name).filter((n: string) => !n.startsWith("knowledge_"))).toEqual(["read", "write", "edit", "multiedit", "apply_patch", "glob", "grep", "list", "bash", "job_output", "job_list", "job_stop", "job_send", "web_search", "web_fetch", "task", "plan_write", "plan_update", "ask_user", "memory", "session_search", "notes_search", "note_view", "note_manage", "skills_list", "skill_view", "skill_manage", "settings"]) + expect(fake!.requests[0].tools.map((t: any) => t.function.name).filter((n: string) => !n.startsWith("knowledge_"))).toEqual(["read", "write", "edit", "multiedit", "apply_patch", "glob", "grep", "list", "bash", "job_output", "job_list", "job_stop", "job_send", "web_search", "web_fetch", "task", "plan_write", "plan_update", "ask_user", "memory", "session_search", "notes_search", "note_view", "note_manage", "skills_list", "skill_view", "skill_manage", "settings", "recall"]) const ends = events.filter((e) => e.type === "tool_end") expect(ends.map((e) => e.type === "tool_end" && e.result.title)).toEqual(["hello.txt · 1–1 of 1", "hello.txt +1 −1"]) const sid = store.sessions(1)[0]!.id diff --git a/tests/harness.test.ts b/tests/harness.test.ts index 93715d4..456e282 100644 --- a/tests/harness.test.ts +++ b/tests/harness.test.ts @@ -69,8 +69,7 @@ test("every tool says how it is shown (block), from the spec's list", () => { }) test("every tool the spec gives the CLI is built in, and every built-in tool is in the spec", () => { - // recall comes with context hygiene (harness/context.md), in the commit that implements it. - expect(ours.map((t) => t.name).sort()).toEqual(forUs.map((t) => t.name).filter((n) => n !== "recall").sort()) + expect(ours.map((t) => t.name).sort()).toEqual(forUs.map((t) => t.name).sort()) }) for (const t of ours) { diff --git a/tests/jobs.test.ts b/tests/jobs.test.ts index 67bbb47..1ae76fa 100644 --- a/tests/jobs.test.ts +++ b/tests/jobs.test.ts @@ -204,7 +204,7 @@ describe("in an app", () => { a.newSession() expect(a.jobs()).toEqual([]) await a.turns.prompt("list") - expect(fake!.requests[3].messages.at(-1).content).toBe("No background jobs.") + expect(fake!.requests[3].messages.at(-1).content).toBe("[#0001]\nNo background jobs.") expect(a.registry.running(first)).toHaveLength(1) // Deleting a session stops its jobs. expect(a.registry.stopAll(first)).toBe(1) diff --git a/tests/mcp.test.ts b/tests/mcp.test.ts index 2c7dbba..852dc4d 100644 --- a/tests/mcp.test.ts +++ b/tests/mcp.test.ts @@ -191,7 +191,7 @@ describe("through the engine", () => { expect(asked.join()).toContain("mcp__fix__echo") expect(fake.requests[0].tools.map((t: any) => t.function.name)).toContain("mcp__fix__echo") expect(fake.requests[0].messages[0].content).toContain("") - expect(fake.requests[1].messages.at(-1)).toMatchObject({ role: "tool", content: "echo: via engine" }) + expect(fake.requests[1].messages.at(-1)).toMatchObject({ role: "tool", content: "[#0001]\necho: via engine" }) await app.close() writeFileSync(join(paths.config, "config.yaml"), "") }) diff --git a/tests/plan.test.ts b/tests/plan.test.ts index d0b2e9d..c7e7eff 100644 --- a/tests/plan.test.ts +++ b/tests/plan.test.ts @@ -44,7 +44,7 @@ describe("plan mode", () => { expect(readFileSync(join(a.project.root, seen[0]!.path), "utf8")).toBe(RENDERED + "\n") expect(a.engine.plan?.steps).toEqual(["calc.py: divide by len(values)"]) // The model read the plan back with its ids from plan_write. - expect(fake!.requests[1].messages.at(-1).content).toBe(RENDERED) + expect(fake!.requests[1].messages.at(-1).content).toBe(`[#0001]\n${RENDERED}`) expect(names(fake!.requests[0])).toContain("plan_submit") expect(names(fake!.requests[0])).toContain("plan_write") expect(fake!.requests[0].messages[0].content).toContain("Permission mode: plan") diff --git a/tests/results.test.ts b/tests/results.test.ts new file mode 100644 index 0000000..500c7fa --- /dev/null +++ b/tests/results.test.ts @@ -0,0 +1,97 @@ +// Context hygiene through the engine (harness spec 3.0.0, context.md): ids, stubs, the cap, recall, +// and the rule that a stub only ever points at a result still in the conversation. The texts are +// the conformance area `context`'s; here they are checked where the engine inserts them. +import { afterEach, expect, test } from "bun:test" +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { createApp } from "../src/app.ts" +import type { AskReply } from "../src/bus/index.ts" +import { paths } from "../src/config/paths.ts" +import { Store } from "../src/session/store.ts" +import { delta, fakeProvider, toolCall, type Fake } from "./fake-provider.ts" + +let fake: Fake | undefined +afterEach(() => fake?.stop()) + +const yes = { ask: async (): Promise => ({ kind: "once" }) } + +function setup(script: Parameters[0]) { + fake = fakeProvider(script) + mkdirSync(paths.config, { recursive: true }) + writeFileSync(join(paths.config, "connections.yaml"), `connections:\n fake:\n dialect: openai-chat\n base_url: ${fake.url}\n models:\n m: { context: 32768 }\n`, { mode: 0o600 }) + writeFileSync(join(paths.config, "config.yaml"), "model: fake/m\n") + const dir = mkdtempSync(join(tmpdir(), "lembas-results-")) + writeFileSync(join(dir, "a.txt"), "one\ntwo\nthree\n") + return dir +} + +/** The tool results the model was sent in request `i`, in order. */ +const sent = (i: number): string[] => fake!.requests[i].messages.filter((m: any) => m.role === "tool").map((m: any) => m.content as string) + +test("an unchanged re-read is a stub; after an edit the file is read in full; a repeated listing is a stub", async () => { + const dir = setup([ + { chunks: [toolCall(0, "c1", "read", '{"path":"a.txt"}')] }, + { chunks: [toolCall(0, "c2", "read", '{"path":"a.txt","offset":2,"limit":1}')] }, + { chunks: [toolCall(0, "c3", "glob", '{"pattern":"*.txt"}')] }, + { chunks: [toolCall(0, "c4", "glob", '{"pattern":"*.txt"}')] }, + { chunks: [toolCall(0, "c5", "edit", JSON.stringify({ path: "a.txt", old: "two", new: "2" }))] }, + { chunks: [toolCall(0, "c6", "read", '{"path":"a.txt"}')] }, + { chunks: [delta({ content: "done" })] }, + ]) + const app = createApp({ cwd: dir, mode: "edit", asker: yes, store: false }) + await app.engine.prompt("go") + const r = sent(6) + expect(r[0]).toMatch(/^\[#0001 a\.txt lines 1-3 of 3 sha ([0-9a-f]{8})\]\n1: one\n2: two\n3: three$/) + const sha = /sha ([0-9a-f]{8})/.exec(r[0]!)![1] + expect(r[1]).toBe(`[#0002 a.txt lines 2-2 of 3 sha ${sha}] Unchanged since #0001, which already shows lines 1-3.`) + expect(r[2]).toBe("[#0003]\na.txt") + expect(r[3]).toBe("[#0004] Same result as #0003.") + expect(r[4]).toMatch(/^\[#0005 a\.txt sha [0-9a-f]{8}\]\n@@ -1,3 \+1,3 @@\n one\n-two\n\+2\n three$/) + expect(r[5]).toMatch(/^\[#0006 a\.txt lines 1-3 of 3 sha [0-9a-f]{8}\]\n1: one\n2: 2\n3: three$/) + // The prompt says what the ids and stubs are. + expect(fake!.requests[0].messages[0].content).toContain("# Tool results\n- Every tool result starts with its id in brackets") +}) + +test("a long output keeps its head and tail; recall shows it whole, from the store, after a resume too", async () => { + const dir = setup([ + { chunks: [toolCall(0, "c1", "bash", JSON.stringify({ command: "seq 1 1000", description: "count" }))] }, + { chunks: [toolCall(0, "c2", "recall", '{"id":"#0001","offset":500,"limit":3}')] }, + { chunks: [delta({ content: "done" })] }, + // After a resume: a new id, never one given before; an unknown one said so. + { chunks: [toolCall(0, "c3", "recall", '{"id":"0001","offset":999}'), toolCall(1, "c4", "recall", '{"id":"#ffff"}')] }, + { chunks: [delta({ content: "again" })] }, + ]) + const store = new Store() + const app = createApp({ cwd: dir, mode: "auto", asker: yes, store, modelTitles: false }) + await app.engine.prompt("count") + const r = sent(2) + const lines = r[0]!.split("\n") + expect(lines[0]).toBe("[#0001]") + expect(lines.slice(1, 3)).toEqual(["1", "2"]) + expect(r[0]).toContain("\n200\n[") + expect(r[0]).toMatch(/\n\[\d+ lines elided, full output #0001\]\n/) + expect(r[0]).toContain("\n1000\n\n[exit 0]") + expect(r[1]).toBe("[#0002 recall #0001 lines 500-502 of 1002]\n500\n501\n502") + const id = app.engine.sessionId! + const again = createApp({ cwd: dir, mode: "auto", asker: yes, store, resuming: id, modelTitles: false }) + again.resume(id) + await again.engine.prompt("more") + const s = sent(4) + expect(s.slice(-2)).toEqual(["[#0003 recall #0001 lines 999-1002 of 1002]\n999\n1000\n\n[exit 0]", "[#0004] No stored result #ffff in this session."]) +}) + +test("a stub never points at a result that left the conversation: after /undo the file is read in full", async () => { + const dir = setup([ + { chunks: [toolCall(0, "c1", "read", '{"path":"a.txt"}')] }, + { chunks: [delta({ content: "read" })] }, + { chunks: [toolCall(0, "c2", "read", '{"path":"a.txt"}')] }, + { chunks: [delta({ content: "read again" })] }, + ]) + const app = createApp({ cwd: dir, mode: "edit", asker: yes, store: false }) + await app.turns.prompt("read it", [], "read it") + await app.turns.undo() + await app.turns.prompt("read it", [], "read it") + expect(sent(3)).toHaveLength(1) + expect(sent(3)[0]).toMatch(/^\[#0002 a\.txt lines 1-3 of 3 sha [0-9a-f]{8}\]\n1: one/) +}) diff --git a/tests/subagent.test.ts b/tests/subagent.test.ts index 3c1d8d4..f99de10 100644 --- a/tests/subagent.test.ts +++ b/tests/subagent.test.ts @@ -76,7 +76,7 @@ describe("subagents", () => { expect(child.messages[0].content).not.toContain("## Skills") // the parent gets the report, not the reading const result = fake!.requests[3].messages.at(-1).content as string - expect(result).toStartWith("calc.py:2 divides by len(v) - 1") + expect(result).toStartWith("[#0001]\ncalc.py:2 divides by len(v) - 1") expect(result).toContain("[explore subagent · 2 steps · 1 tool call]") expect(fake!.requests[3].messages.some((m: any) => m.role === "tool" && m.content.includes("return sum(v)"))).toBe(false) expect(events.find((e) => e.type === "sub_tool")).toMatchObject({ type: "sub_tool", callId: "t1", name: "read" }) @@ -198,7 +198,7 @@ describe("background subagents (spec 2.0.0)", () => { ) await a.turns.prompt("look into it in the background") const started = fake!.requests[1].messages.at(-1).content as string - expect(started).toStartWith("Started subagent explore as job1 (find the bug).") + expect(started).toStartWith("[#0001]\nStarted subagent explore as job1 (find the bug).") await settled(a, 3) const note = fake!.requests[2].messages.at(-1).content as string expect(note).toStartWith(MARKS.subagent) @@ -298,7 +298,7 @@ describe("background subagents (spec 2.0.0)", () => { ) await a.turns.prompt("go") const results = fake!.requests[1].messages.filter((m: any) => m.role === "tool").map((m: any) => m.content as string) - expect(results[0]).toStartWith("Started subagent explore as job1") + expect(results[0]).toStartWith("[#0001]\nStarted subagent explore as job1") expect(results[1]).toContain("holds one model at a time, and a subagent is already running on g/m there") expect(other!.requests.map((r: any) => r.model)).toEqual(["m"]) await settled(a, 3)