From 8e06590d57711b51fcfbc03ab51239f71446d98c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaroslav=20Bene=C5=A1?= Date: Thu, 5 Nov 2026 13:22:02 +0000 Subject: [PATCH] Harness spec 3.1.0: the prefix stays still, in the CLI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The system prompt is made once per session (start, compaction, model switch), kept in session_meta and resent byte for byte; it holds no date. What changes mid-session -- the date, a mode the person chose, a plan or memory or instruction file they edited, a personality, MCP instructions that arrived later -- is a on the next user message, stored with it. The tool list is fixed for the session: plan_submit is always offered and answers unavailable outside plan mode, the wrap-up keeps the tools and runs no call, a withdrawn tool stays listed and answers its refusal; the first request waits up to mcp_wait_ms for MCP servers. Compaction asks for its summary as the next turn of the same conversation. Cached input tokens are parsed (incl. llama.cpp timings), stored per request and shown in /context. Spec: context.md §7/§10, loop.json context.note (+ plan_ended, plan_own, personal, empty, mcp_wait_ms), modes.json plan.refused, compact.md without the transcript heading. Conformance area prefix. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 28 ++ harness/conformance/prefix.json | 535 +++++++++++++++++++++++++++++++ harness/context.md | 20 +- harness/loop.json | 13 +- harness/permission/modes.json | 3 +- harness/prompts/tasks/compact.md | 2 - src/acp/agent.ts | 5 +- src/app.ts | 82 +++-- src/harness.ts | 18 +- src/mcp/index.ts | 16 +- src/prompt/assemble.ts | 31 +- src/provider/openai-chat.ts | 22 +- src/session/context.ts | 11 +- src/session/engine.ts | 287 +++++++++++++---- src/session/prefix.ts | 155 +++++++++ src/session/store.ts | 32 +- src/session/turns.ts | 3 + src/tool/plan_submit.ts | 5 +- src/tool/session_search.ts | 4 +- src/tool/tool.ts | 8 +- src/tui/commands.ts | 3 +- src/tui/state.ts | 5 +- tests/acp-v2.test.ts | 19 +- tests/budget.test.ts | 8 +- tests/conformance.ts | 55 ++++ tests/context.test.ts | 12 +- tests/e2e.test.ts | 29 +- tests/harness.test.ts | 4 +- tests/images.test.ts | 4 +- tests/lembas-library.test.ts | 6 +- tests/library.test.ts | 5 +- tests/plan.test.ts | 25 +- tests/prefix.test.ts | 232 ++++++++++++++ tests/subagent.test.ts | 3 +- tests/trust-hardening.test.ts | 2 +- tests/tui/app.test.tsx | 10 +- 36 files changed, 1533 insertions(+), 169 deletions(-) create mode 100644 harness/conformance/prefix.json create mode 100644 src/session/prefix.ts create mode 100644 tests/prefix.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 68ec563..2a6713d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,34 @@ All notable changes to LLeMbas CLI. The format follows ## [Unreleased] +Harness spec 3.1.0 (draft): what the model has already read stays exactly as it was, so its server +can reuse it instead of reading the whole session again. + +### Changed + +- **The system prompt is made once per session** — at the start, after a compaction, on a model + switch — kept with the session and sent byte for byte on every request; a resumed session sends + the one it kept. It holds no date any more. +- **What changes during a session is said in a note** after your words on your next message: the + date (on the first message, and when it changes), a permission mode you chose, a plan you + changed, memory or an instruction file you edited, a personality you picked, an MCP server's + instructions that arrived later. The model's own changes (its plan, its edits, a plan approval) + are not said again. Notes are not shown as your words. +- **The tool list is the same for the whole session.** `plan_submit` is offered in every mode and + answers that it works only in plan mode; `show` is offered from the start where the session can + have a viewer; knowledge search is decided as the session starts. The tools stay listed when a + step limit or budget winds a prompt up (a call in that answer is not run), and a tool refused + for good stays listed and answers that refusal. The first request waits up to 3 seconds for the + configured MCP servers, so their tools are in the first list. +- **Compaction asks for its summary as the next turn of the same conversation** instead of a + separate request with a pasted transcript, so it reuses what the server has read and no longer + loses the middle of a long session. A tool call in the summary's reply is not run. + +### Added + +- **Cached input tokens are recorded for every request** (the usage's own count, or llama.cpp's + `timings` and progress), kept with the session, and shown by `/context` for the last request. + ## [2.0.0-beta.1] — 2026-11-03 LLeMbas CLI 2.0 works with LLeMbas 2.0 and nothing else: it uses the server it is logged in to, diff --git a/harness/conformance/prefix.json b/harness/conformance/prefix.json new file mode 100644 index 0000000..13764fb --- /dev/null +++ b/harness/conformance/prefix.json @@ -0,0 +1,535 @@ +{ + "about": "The prefix (context.md §10, spec 3.1.0): one session's requests, in order — which system prompt each sends, which tools it offers, and the note on a user message. `start` is the session as it begins: `date`, `mode`, and what the prompt is made from that may change (`plan` as rendered, `memory`, `personality`, `files` — instruction texts by path or heading — `mcp` — instructions by server; each \"\" or {} for none), `could` (the requirements the session could ever meet: `plan` for any session that may enter plan mode, `display`, `page`, `forge`) and `met` (those met now besides plan, which is met in plan mode; default: all it could). `tools`: the names a case looks at; a tool's `requires` is the spec's. Steps: `user` (a user message: the request that carries it), `tool` (a tool step's request: no user message), `set` (something changes — any of the start's fields, or `met`; `by: model` when the model's own tool did it, whose result told the model: `plan` from plan_write, `mode` from plan_submit's approval, `memory` and `files` it changed itself; otherwise the person or the clock did), `call` (a call of `tool` now), `compact` (the summary is asked for: `{{transcript}}` and `{{previous_summary}}` empty — an earlier summary is already the first message), `wrapup` (the request after a step limit or a budget: the same prompt and tools, the wrap-up message as its last user message, no note; a call in its reply is not run) and `model` (the session moves to another model). A request answers `{system, tools, note}`: `system` numbers the system prompts the session has made, from 1, the same number being the same bytes; `tools` the case's tools it offers; `note` the note after the person's words, or null. `call` answers the tool's `unavailable` text, or null where it runs; `compact` answers the request's prompt, its tools and `ask`, the user message that follows every message as it was; `wrapup` answers like a request; `set` and `model` answer null; a value that became empty reads `context.note.empty`. `plan_dir` fills modes/plan.md. LLeMbas: services/context.py; CLI: session/prefix.ts, session/engine.ts.", + "cases": [ + { + "about": "A date that changes during a session is a note on the next user message; the system prompt and the tools stay the same bytes.", + "start": { + "date": "2026-11-04", + "mode": "manual", + "could": [ + "plan" + ] + }, + "tools": [ + "read", + "plan_submit" + ], + "steps": [ + { + "op": "user" + }, + { + "op": "tool" + }, + { + "op": "user" + }, + { + "op": "set", + "date": "2026-11-05" + }, + { + "op": "tool" + }, + { + "op": "user" + } + ], + "expect": [ + { + "system": 1, + "tools": [ + "read", + "plan_submit" + ], + "note": "\nToday is 2026-11-04.\n" + }, + { + "system": 1, + "tools": [ + "read", + "plan_submit" + ], + "note": null + }, + { + "system": 1, + "tools": [ + "read", + "plan_submit" + ], + "note": null + }, + null, + { + "system": 1, + "tools": [ + "read", + "plan_submit" + ], + "note": null + }, + { + "system": 1, + "tools": [ + "read", + "plan_submit" + ], + "note": "\nToday is 2026-11-05.\n" + } + ] + }, + { + "about": "plan_submit is offered in every mode and answers its unavailable text outside plan mode; a mode the person chose is a note with the mode's text, one the approval set is not.", + "start": { + "date": "2026-11-05", + "mode": "manual", + "could": [ + "plan" + ] + }, + "tools": [ + "read", + "plan_write", + "plan_submit" + ], + "steps": [ + { + "op": "user" + }, + { + "op": "call", + "tool": "plan_submit" + }, + { + "op": "set", + "mode": "plan" + }, + { + "op": "user" + }, + { + "op": "call", + "tool": "plan_submit" + }, + { + "op": "set", + "mode": "edit", + "by": "model" + }, + { + "op": "tool" + }, + { + "op": "user" + } + ], + "expect": [ + { + "system": 1, + "tools": [ + "read", + "plan_write", + "plan_submit" + ], + "note": "\nToday is 2026-11-05.\n" + }, + "Not in plan mode: plan_submit is how a plan made in plan mode is put to the person for approval. Carry on with the work; keep the plan current with plan_update.", + null, + { + "system": 1, + "tools": [ + "read", + "plan_write", + "plan_submit" + ], + "note": "\nThe permission mode is now plan:\nPermission mode: plan. You are planning, not implementing. Read, search and investigate as much as you need, but change nothing: no file but a plan under .agent/plans, no command that is not already allowed, nothing saved to the library.\n\nWrite the plan with plan_write, so that someone else could follow it: findings with file:line references, what will be true when it is done (objectives), and the work as phases of tasks in order, with the risks and how to verify it as the last phase. Ask with ask_user if a decision is the user's. When the plan is written, call plan_submit — that is how the user approves it, and after approval you carry on and implement it, keeping the plan current with plan_update.\n" + }, + null, + null, + { + "system": 1, + "tools": [ + "read", + "plan_write", + "plan_submit" + ], + "note": null + }, + { + "system": 1, + "tools": [ + "read", + "plan_write", + "plan_submit" + ], + "note": null + } + ] + }, + { + "about": "The compaction request is the same prompt and tools, every message, then the compaction text with no transcript; after it the prompt is made again and the next user message says the date again.", + "start": { + "date": "2026-11-05", + "mode": "edit", + "could": [ + "plan" + ] + }, + "tools": [ + "read", + "plan_submit" + ], + "steps": [ + { + "op": "user" + }, + { + "op": "tool" + }, + { + "op": "compact" + }, + { + "op": "user" + }, + { + "op": "compact" + } + ], + "expect": [ + { + "system": 1, + "tools": [ + "read", + "plan_submit" + ], + "note": "\nToday is 2026-11-05.\n" + }, + { + "system": 1, + "tools": [ + "read", + "plan_submit" + ], + "note": null + }, + { + "system": 1, + "tools": [ + "read", + "plan_submit" + ], + "ask": "Summarise the conversation below so it can be carried forward after the earlier turns are dropped from your context. This is a working record, not a report for a reader.\n\nKeep, under these headings and in this order:\n\n## What we are doing\nThe goal, and where we have got to.\n\n## Decisions\nAnything settled, and why. A decision without its reason gets argued again.\n\n## Facts established\nNames, numbers, versions, file paths, commands, URLs and identifiers, copied exactly. Do not round them, paraphrase them or reconstruct one from memory — if it is not in the transcript, leave it out.\n\n## Files touched\nEach file read or changed that still matters, and what was done to it.\n\n## Open threads\nWhat is unfinished, and what was about to happen next.\n\nLeave out pleasantries, retracted ideas and anything already superseded. Do not answer the conversation: you are recording it. Write in the language of the conversation, and stay under 600 words." + }, + { + "system": 2, + "tools": [ + "read", + "plan_submit" + ], + "note": "\nToday is 2026-11-05.\n" + }, + { + "system": 2, + "tools": [ + "read", + "plan_submit" + ], + "ask": "Summarise the conversation below so it can be carried forward after the earlier turns are dropped from your context. This is a working record, not a report for a reader.\n\nKeep, under these headings and in this order:\n\n## What we are doing\nThe goal, and where we have got to.\n\n## Decisions\nAnything settled, and why. A decision without its reason gets argued again.\n\n## Facts established\nNames, numbers, versions, file paths, commands, URLs and identifiers, copied exactly. Do not round them, paraphrase them or reconstruct one from memory — if it is not in the transcript, leave it out.\n\n## Files touched\nEach file read or changed that still matters, and what was done to it.\n\n## Open threads\nWhat is unfinished, and what was about to happen next.\n\nLeave out pleasantries, retracted ideas and anything already superseded. Do not answer the conversation: you are recording it. Write in the language of the conversation, and stay under 600 words." + } + ] + }, + { + "about": "What the model changed itself is not said again; what the person changed is, once, in loop.json's order; an MCP server that connected later is said by its own instructions.", + "start": { + "date": "2026-11-05", + "mode": "manual", + "plan": "", + "memory": "Prefers tabs.", + "personality": "", + "files": { + "/p/AGENTS.md": "Build with make." + }, + "mcp": { + "docs": "Search the docs first." + }, + "could": [ + "plan" + ] + }, + "tools": [ + "read" + ], + "steps": [ + { + "op": "user" + }, + { + "op": "set", + "by": "model", + "plan": "# Fix\n## p1 P (pending)\n[t1] [ ] a", + "memory": "Prefers tabs.\n§\nUses fish.", + "files": { + "/p/AGENTS.md": "Build with make; test with make test." + } + }, + { + "op": "tool" + }, + { + "op": "user" + }, + { + "op": "set", + "plan": "", + "memory": "Uses fish.", + "personality": "Be brief.", + "files": { + "/p/AGENTS.md": "Build with make; test with make test.", + "/p/src/AGENTS.md": "Strict types." + }, + "mcp": { + "docs": "Search the docs first.", + "tickets": "Name the ticket in every commit." + } + }, + { + "op": "user" + }, + { + "op": "set", + "files": { + "/p/AGENTS.md": "Build with make; test with make test." + } + }, + { + "op": "user" + } + ], + "expect": [ + { + "system": 1, + "tools": [ + "read" + ], + "note": "\nToday is 2026-11-05.\n" + }, + null, + { + "system": 1, + "tools": [ + "read" + ], + "note": null + }, + { + "system": 1, + "tools": [ + "read" + ], + "note": null + }, + null, + { + "system": 1, + "tools": [ + "read" + ], + "note": "\nThe person changed the plan; it is now:\n(none)\n\nThe person changed what is remembered; it is now:\nUses fish.\n\n/p/src/AGENTS.md changed; it now reads:\nStrict types.\n\nThe person chose a different personality:\nBe brief.\n\nInstructions from an MCP server that connected since the session began:\n\nName the ticket in every commit.\n\n" + }, + null, + { + "system": 1, + "tools": [ + "read" + ], + "note": "\n/p/src/AGENTS.md changed; it now reads:\n(none)\n" + } + ] + }, + { + "about": "A move to another model makes the prompt again; the date said in the conversation still holds.", + "start": { + "date": "2026-11-05", + "mode": "auto", + "could": [ + "plan" + ] + }, + "tools": [ + "read" + ], + "steps": [ + { + "op": "user" + }, + { + "op": "model" + }, + { + "op": "user" + } + ], + "expect": [ + { + "system": 1, + "tools": [ + "read" + ], + "note": "\nToday is 2026-11-05.\n" + }, + null, + { + "system": 2, + "tools": [ + "read" + ], + "note": null + } + ] + }, + { + "about": "A display the session could have is offered from the start and answers unavailable while there is none; one it never could have leaves show out.", + "start": { + "date": "2026-11-05", + "mode": "manual", + "could": [ + "plan", + "display" + ], + "met": [] + }, + "tools": [ + "read", + "show" + ], + "steps": [ + { + "op": "user" + }, + { + "op": "call", + "tool": "show" + }, + { + "op": "set", + "met": [ + "display" + ] + }, + { + "op": "call", + "tool": "show" + } + ], + "expect": [ + { + "system": 1, + "tools": [ + "read", + "show" + ], + "note": "\nToday is 2026-11-05.\n" + }, + "Nothing can be shown now: this session has no screen to show a file on. Give the user the path instead.", + null, + null + ] + }, + { + "start": { + "date": "2026-11-05", + "mode": "manual", + "could": [ + "plan" + ] + }, + "tools": [ + "read", + "show" + ], + "steps": [ + { + "op": "user" + } + ], + "expect": [ + { + "system": 1, + "tools": [ + "read" + ], + "note": "\nToday is 2026-11-05.\n" + } + ] + }, + { + "about": "The wrap-up keeps the session's tool list and prompt, carries no note, and the next user message carries what changed meanwhile.", + "start": { + "date": "2026-11-05", + "mode": "edit", + "could": [ + "plan" + ] + }, + "tools": [ + "read", + "plan_submit" + ], + "steps": [ + { + "op": "user" + }, + { + "op": "tool" + }, + { + "op": "set", + "date": "2026-11-06" + }, + { + "op": "wrapup" + }, + { + "op": "user" + } + ], + "expect": [ + { + "system": 1, + "tools": [ + "read", + "plan_submit" + ], + "note": "\nToday is 2026-11-05.\n" + }, + { + "system": 1, + "tools": [ + "read", + "plan_submit" + ], + "note": null + }, + null, + { + "system": 1, + "tools": [ + "read", + "plan_submit" + ], + "note": null + }, + { + "system": 1, + "tools": [ + "read", + "plan_submit" + ], + "note": "\nToday is 2026-11-06.\n" + } + ] + } + ] +} diff --git a/harness/context.md b/harness/context.md index 9818009..7dbbe84 100644 --- a/harness/context.md +++ b/harness/context.md @@ -129,7 +129,8 @@ two results. Compaction is the one deliberate break of the cache, and it already exists. **From 3.1.0 the summary is asked for as the next turn of the same conversation**: the same system prompt, the same tools, every message as it was, and then one more user message — `prompts/tasks/compact.md`, whose -`{{transcript}}` is empty in this form, since the model has the conversation itself. So the request +`{{transcript}}` and `{{previous_summary}}` are empty in this form, since the model has the +conversation, an earlier summary included, itself. So the request that makes the summary costs only that message; a separate request with a pasted transcript was a full re-read of everything, and a transcript capped to fit it lost the session's middle. A tool call in the summary's reply is not run; its text before the call is the summary, and a reply with @@ -187,14 +188,25 @@ model): a date that changed at midnight sent a 16,912-token session from 99 % ca ``` - (`loop.json` `context.note`.) A note is part of that message from then on, like anything sent. - A request with no user message (a tool step) carries no note; it waits for the next one. + (`loop.json` `context.note`.) Each change is one block of the note, the blocks separated by a + blank line, in the order `context.note` lists them; a value that became empty reads + `context.note.empty`; the date is the person's own (their zone, else the machine's). A note is + part of that message from then on, like anything sent. A request with no user message (a tool + step), and words the person adds to a turn already running, carry no note; it waits for the next + message that starts a turn. A change the model made itself (its own `plan_write`, `memory`, + `settings`, a file it wrote) needs no note: its tool result already said it. 4. **The tool list is the same for the whole session.** A tool whose `requires` can come and go during a session — `page`, `display`, `forge`, and plan mode for `plan_submit` — is offered from the start whenever the session could ever meet it, and a call made while it is not met answers the tool's `unavailable` text. Only what is fixed for the session may leave a tool out: the session kind (`sessions`), the model (`vision`), a subagent's narrowing, what the project's - configuration or the person switched off before the session began. + configuration or the person switched off before the session began. The tools stay listed in + the wrap-up request too (`loop.json` `wrap_up`): its message says they are withdrawn, and a call + in that reply is not run. A tool withdrawn after its final refusal stays in the list and answers + that refusal. **The one exception is an MCP server**: a session waits up to `mcp_wait_ms` for + the servers it is configured with before its first request, so their tools are in the first + list; a server that connects later adds its tools when it connects — one re-read, once — and its + instructions come as a note. 5. **Nothing already sent is rewritten** — the one rule above, for results; old tool output is never pruned in place. 6. **Every request's cache use is recorded** where the provider reports it (cached input tokens), diff --git a/harness/loop.json b/harness/loop.json index 740ef96..d25f548 100644 --- a/harness/loop.json +++ b/harness/loop.json @@ -23,7 +23,7 @@ } }, "wrap_up": { - "about": "One more request with no tools and a user message saying which limit was reached: answer now — what is done, what is not, what would come next. The transcript says the prompt was stopped, so an answer given because the room ran out is never mistaken for one the model chose to give.", + "about": "One more request, with the session's tool list unchanged (context.md §10, rule 4) and a user message saying which limit was reached: answer now — what is done, what is not, what would come next. The transcript says the prompt was stopped, so an answer given because the room ran out is never mistaken for one the model chose to give. A tool call in that reply is not run.", "message": "The {{limit}} for this prompt has been reached and the tools have been withdrawn. Answer now from what you have: what is done, what is not, and what would come next." }, "nudges": { @@ -107,16 +107,21 @@ }, "reasoning": "Earlier turns' reasoning is left to the model's chat template: the harness sends the history as it does, neither stripping reasoning the template keeps nor adding what it drops.", "note": { - "about": "A note on the next user message (context.md §10, rule 3): what changed since the system prompt was made, after the person's words. Lines in this order, each only when it applies.", + "about": "A note on the next user message (context.md §10, rule 3): what changed since the system prompt was made, after the person's words. Lines in this order, each only when it applies. `plan_ended` is the mode block for a session that leaves plan mode for a mode with no text of its own (a chat). `plan_own` is the plan block for a session that does not resend earlier tool results (a chat): the model's own plan, as its tools left it. `personal` is the {{path}} of the instructions block for the person's own instructions.", "open": "", "close": "", "date": "Today is {{date}}.", "mode": "The permission mode is now {{mode}}:\n{{mode_text}}", + "plan_ended": "Plan mode has ended: make the changes the work needs; nothing is refused for planning any more.", "plan": "The person changed the plan; it is now:\n{{plan}}", + "plan_own": "Your plan, as your plan tools left it:\n{{plan}}", "memory": "The person changed what is remembered; it is now:\n{{memory}}", "instructions": "{{path}} changed; it now reads:\n{{text}}", + "personal": "the person's own instructions", "personality": "The person chose a different personality:\n{{personality}}", - "mcp": "Instructions from an MCP server that connected since the session began:\n{{instructions}}" - } + "mcp": "Instructions from an MCP server that connected since the session began:\n{{instructions}}", + "empty": "(none)" + }, + "mcp_wait_ms": 3000 } } diff --git a/harness/permission/modes.json b/harness/permission/modes.json index ce87483..0681deb 100644 --- a/harness/permission/modes.json +++ b/harness/permission/modes.json @@ -32,7 +32,8 @@ "read": "rules", "write": "allow only for files under the plans directory; anything else is refused", "execute": "only what the rules already allow; anything else is refused", - "interact": "rules" + "interact": "rules", + "refused": "Plan mode: nothing changes but the plan. Put what this would do into the plan with plan_write or plan_update; it can be done once the person approves the plan." } }, "unattended": "With nobody to answer, whatever a mode would ask is refused instead, and the model is told so up front (prompts/modes/unattended.md)." diff --git a/harness/prompts/tasks/compact.md b/harness/prompts/tasks/compact.md index c7d2f77..f896238 100644 --- a/harness/prompts/tasks/compact.md +++ b/harness/prompts/tasks/compact.md @@ -23,6 +23,4 @@ Leave out pleasantries, retracted ideas and anything already superseded. Do not {{previous_summary}} -## Transcript - {{transcript}} diff --git a/src/acp/agent.ts b/src/acp/agent.ts index 219c3bf..710f357 100644 --- a/src/acp/agent.ts +++ b/src/acp/agent.ts @@ -42,6 +42,7 @@ import type { Message } from "../provider/types.ts" import type { Hub } from "./hub.ts" import { INVALID_PARAMS, METHOD_NOT_FOUND, RpcError, type Peer } from "./rpc.ts" import { jobList, jobOutput, jobSend, jobStop, jobTranscript, storedJobs } from "./jobs.ts" +import { isNote } from "../session/prefix.ts" export const ACP_VERSION = 1 /** The link's version (LLeMbas `services/device_link.py:PROTOCOL`, harness/acp.md *Versions*). 3: @@ -369,7 +370,9 @@ export function planReplyFrom(answer: any, maxMode?: Mode): PlanReply { } /** Text the expander added to a user message (an @file, an image, a listing, a `!` command's output). */ -const ATTACHMENT = /^<(file|directory|shell|image) / +// What the model was given beside the words typed — attachments, and a session note +// (session/prefix.ts) — is not shown as typed. +const ATTACHMENT = { test: (t: string) => /^<(file|directory|shell|image) /.test(t) || isNote(t) } /** A stored session as turns the web UI can draw: each person's message, then what the * model said, thought and ran in answer, up to the next one. A message's attachments are chips, diff --git a/src/app.ts b/src/app.ts index 449d3c1..bfd0d61 100644 --- a/src/app.ts +++ b/src/app.ts @@ -17,7 +17,8 @@ import { DEFAULT_RULES, toRules } from "./permission/evaluate.ts" import { hardlineRules } from "./permission/hardline.ts" import { defaultEffort } from "./provider/effort.ts" import { clientFor, familyOf, resolveModel } from "./provider/index.ts" -import { assembleSystem, instructionFiles, knowledgeScope } from "./prompt/assemble.ts" +import { assembleSystem, instructionFiles, knowledgeScope, modeText } from "./prompt/assemble.ts" +import type { Changing } from "./session/prefix.ts" import { describeRepo, repoState } from "./git/repo.ts" import { findProject, persistProjectRule, trustOf, type Project } from "./project/root.ts" import type { Message } from "./provider/types.ts" @@ -41,7 +42,7 @@ import { Library, type Embedder } from "./library/store.ts" import { instances as lembasInstances, keyFile as lembasKeyFile } from "./lembas/login.ts" import { PERSONAL_KEYS, personalConfig, pushPersonalization } from "./lembas/personal.ts" import type { Plan } from "./plan/plan.ts" -import { LIBRARY_SERVER } from "./mcp/index.ts" +import { LIBRARY_SERVER, mcpInstructions } from "./mcp/index.ts" import type { McpServer } from "./config/schema.ts" import { embedderFor } from "./library/embed.ts" import { effortsFor } from "./provider/effort.ts" @@ -253,8 +254,9 @@ export function createApp(o: AppOptions): App { return s ? describeRepo(s) : undefined } - // Memory and skills go into the system prompt as they stood when the session started: a write - // during the session reaches the file, not the prompt (Hermes' frozen snapshot). + // Memory and skills go into the system prompt as they stood when it was made: a write during the + // session reaches the file, not the prompt (Hermes' frozen snapshot) — the person's own edit + // reaches the model as a note (session/prefix.ts). const memoryOn = loaded.config.memory?.enabled !== false const memory = new MemoryStore({ memory: loaded.config.memory?.memory_chars ?? DEFAULT_LIMITS.memory, user: loaded.config.memory?.user_chars ?? DEFAULT_LIMITS.user }) const projectDir = () => (trusted && existsSync(project.dir) ? project.dir : undefined) @@ -263,17 +265,18 @@ export function createApp(o: AppOptions): App { ? new MemoryStore({ memory: loaded.config.memory?.memory_chars ?? DEFAULT_LIMITS.memory, user: 0 }, join(project.dir, "local"), { memory: "PROJECT MEMORY (this project only)" }) : undefined const skillList = () => loadSkills({ projectDir: projectDir(), external: loaded.config.skills?.external_dirs, disabled: loaded.config.skills?.disabled }) - // Everything in the system prompt that could change during a session is taken once, here, for - // the same reason as the git block: the prompt must stay byte for byte the same, or the server - // reads the whole conversation again. + // What the system prompt holds that nothing during the session updates: taken as the session + // starts, for the same reason as the git block. The engine makes the prompt once and resends it + // byte for byte (harness context.md §10); what may change and is said in a note when it does — + // memory, instruction files, personality, MCP instructions — is read by `changing` below. const freeze = () => ({ - // The server's memory, when it is the server's: as last read for this session (readServerMemory). - memory: !memoryOn ? undefined : serverMemory !== undefined ? serverMemory.text : [memory.snapshot(), projectMemory?.block("memory") ?? ""].filter(Boolean).join("\n\n"), skills: skillLines(skillList()), git: gitSummary(), tasks: projectDir() && !unsafeProjectFile(join(project.dir, "tasks.md"), project.root) ? boardSummary(project.dir) : undefined, - files: instructionFiles(o.cwd, project.root, loaded.instructions), }) + /** The memory block as it is now: the server's when it is the server's (as last read for this + * session, readServerMemory), else this machine's files; undefined when memory is off. */ + const memoryNow = () => (!memoryOn ? undefined : serverMemory !== undefined ? serverMemory.text : [memory.snapshot(), projectMemory?.block("memory") ?? ""].filter(Boolean).join("\n\n")) // `library: lembas`: the server keeps the memory (acp.md *Memory*); its entries for the prompt are // read from it as a session starts. Until the read answers, none. const memoryOnServer = loaded.config.library === "lembas" && memoryOn @@ -299,7 +302,7 @@ export function createApp(o: AppOptions): App { const lembasLibrary = memoryOnServer ? lembasLibraryServer(loaded.warnings) : undefined if (lembasLibrary) loaded.mcp[LIBRARY_SERVER] = lembasLibrary // Without a way to the server's memory, this machine's is used for now. - if (memoryOnServer && !lembasLibrary) serverMemory = undefined, frozen = freeze() + if (memoryOnServer && !lembasLibrary) serverMemory = undefined const mcp = new McpManager(loaded.mcp, { root: project.root, version: VERSION, @@ -340,8 +343,29 @@ export function createApp(o: AppOptions): App { const prototypesBlock = () => (context.prototypes?.offered || (!context.prototypes && loaded.config.prototypes === true) ? { sources: context.prototypes?.sources ?? "" } : undefined) /** The three personal keys this session uses: the client's for it, else the config's. */ const personal = () => (context.personalization ? personalConfig({ enabled: true, available: true, ...context.personalization }) : loaded.config) - const system = (m: Mode, current: ResolvedModel, subagent?: { name: string; instructions: string; report: boolean; skills: boolean; recall: boolean }, at?: { cwd: string; root: string }, plan?: Plan) => - assembleSystem({ + /** What may change during the session, as it is now (session/prefix.ts `Changing`): what the + * prompt is made from, and what a note compares. A subagent's prompt holds no memory and no + * personality; a worktree subagent reads its own checkout's instruction files. */ + type Source = { files: { path: string; text: string; chain?: true }[]; memory?: string; personality?: string; userInstructions?: string; instance?: { name: string; instructions: string }; mcp: Record } + const changing = (subagent?: boolean, at?: { cwd: string; root: string }): Changing => { + const files = instructionFiles(at?.cwd ?? o.cwd, at?.root ?? project.root, loaded.instructions) + const p = subagent ? {} : personalityOf(personal()) + const src: Source = { + files, + memory: subagent ? undefined : memoryNow(), + ...p, + ...(!subagent && context.instructions ? { instance: { name: contextFrom, instructions: context.instructions } } : {}), + mcp: mcp.serverInstructions(), + } + const named: Record = Object.fromEntries(files.map((f) => [f.path, f.text])) + // What a note names the person's own instructions by (loop.json context.note.personal). + if (src.userInstructions) named[LOOP.context.note.personal] = src.userInstructions + if (src.instance) named[`Instructions from ${src.instance.name}`] = src.instance.instructions + return { memory: src.memory ?? "", personality: src.personality ?? "", files: named, mcp: src.mcp, source: src } + } + const system = (m: Mode, current: ResolvedModel, ch: Changing, subagent?: { name: string; instructions: string; report: boolean; skills: boolean; recall: boolean }, at?: { cwd: string; root: string }, plan?: Plan) => { + const src = ch.source as Source + return assembleSystem({ modelRef: current.ref, family: familyOf(current), cwd: at?.cwd ?? o.cwd, @@ -355,22 +379,26 @@ export function createApp(o: AppOptions): App { // A worktree subagent works elsewhere: its own snapshot, taken as it starts. git: at ? gitSummary() : frozen.git, tasks: frozen.tasks, - files: at ? undefined : frozen.files, + files: src.files, subagent, - memory: frozen.memory, + memory: src.memory, // The skills block tells of skill_view; a subagent without the tool gets no block. skills: subagent && !subagent.skills ? undefined : frozen.skills, manageSkills: true, - // Read on every prompt: /personality and /settings change it without a restart. - ...personalityOf(personal()), - mcp: mcp.instructions(), + // As the prompt is made; /personality and /settings later change it by a note. + personality: src.personality, + userInstructions: src.userInstructions, + mcp: mcpInstructions(src.mcp), roster: roster(current.ref), plan, - ...(context.instructions ? { instance: { name: contextFrom, instructions: context.instructions } } : {}), + ...(src.instance ? { instance: src.instance } : {}), library: libraryBlock(), prototypes: prototypesBlock(), context: subagent ? subagent.recall : true, }) + } + /** A mode's text for a note, as the prompt would have it. */ + const modeNote = (m: Mode) => modeText(m, planDir, o.unattended ? UNATTENDED[m] : undefined) // The other configured models, for the task tool's `model`, or a switch. const roster = (current: string) => loaded.refs @@ -507,10 +535,13 @@ export function createApp(o: AppOptions): App { jobs: { registry, owner: () => ({ sessionId: childSessionId, job }) }, }, steerMark: MARKS.to_subagent, - system: (mode, current, plan) => + changing: () => changing(true, at), + modeText: modeNote, + system: (mode, current, plan, ch) => system( mode, current, + ch, { name: spec.name, instructions: wt @@ -572,7 +603,11 @@ export function createApp(o: AppOptions): App { compaction, perm, toolCtx: { ...baseToolCtx(), jobs: { registry, owner: baseOwner } }, - system: (m, current, plan) => system(m, current, undefined, undefined, plan), + system: (m, current, plan, ch) => system(m, current, ch, undefined, undefined, plan), + changing: () => changing(), + modeText: modeNote, + // The configured MCP servers, up to loop.json's wait: their tools in the first list. + beforeFirst: () => Promise.race([mcpReady.catch(() => {}), Bun.sleep(LOOP.context.mcp_wait_ms)]), persistRule: trusted ? (rule) => persistProjectRule(project, rule) : undefined, spawn, }) @@ -597,14 +632,13 @@ export function createApp(o: AppOptions): App { }) const mcpReady = mcp.start() /** The server's memory entries for the prompt (`lembas://memory`), read as a session starts, in - * the session's scope: put into the prompt while nothing has been sent yet, kept for the next - * session otherwise — the prompt does not change under a session. */ + * the session's scope: in the prompt when it is made after this, else said in a note on the next + * user message — the prompt does not change under a session. */ const readServerMemory = async () => { if (!lembasLibrary || !serverMemory) return try { const text = await mcp.readResource(LIBRARY_SERVER, MEMORY_RESOURCE, engine.o.toolCtx.libraryScope) serverMemory = { text } - if (!engine.messages.length && memoryOn) frozen = { ...frozen, memory: text } } catch (e) { bus.emit({ type: "notice", level: "warn", message: `memory: the server's entries could not be read (${(e as Error).message}); none are in this session's prompt` }) } diff --git a/src/harness.ts b/src/harness.ts index bffc7b1..659c7f1 100644 --- a/src/harness.ts +++ b/src/harness.ts @@ -76,8 +76,10 @@ export interface ToolSpecDef { offered?: "subagent" argument_aliases?: Record /** Spec 3.0.0: what must hold for it to be offered — `display` (the session is shown where a file - * can be opened: over the link, to a client that offers `show`), `page`, `forge` (the chat's). */ - requires?: ("display" | "page" | "forge")[] + * can be opened: over the link, to a client that offers `show`), `page`, `forge` (the chat's); + * `plan` (3.1.0): plan mode. From 3.1.0 one that can come and go during a session does not + * leave the tool out: it is offered and answers `unavailable` while unmet (context.md §10). */ + requires?: ("display" | "page" | "forge" | "plan")[] /** What a call answers, word for word, when a requirement went away after the offer. */ unavailable?: string /** `always`: put to the person in every mode, auto included; never remembered. */ @@ -131,6 +133,18 @@ export const LOOP: { dedupe: { by_hash: string[]; by_result: string[] } compaction: { heading: string; line: string; rows: number } reasoning: string + /** A note on the next user message (spec 3.1.0, context.md §10): its tags and its lines, in + * the order they are said. */ + note: { open: string; close: string; date: string; mode: string; plan: string; memory: string; instructions: string; personality: string; mcp: string; empty: string + /** What a note names the person's own instructions by (the instructions block's `{{path}}`). */ + personal: string + /** For a chat, whose modes have no text of their own; the CLI says modes/.md. */ + plan_ended: string + /** For a chat, which does not resend earlier tool results; the CLI's model has them. */ + plan_own: string + } + /** How long a session waits for its configured MCP servers before its first request. */ + mcp_wait_ms: number } } = LOOP_JSON diff --git a/src/mcp/index.ts b/src/mcp/index.ts index 57e9609..f017b93 100644 --- a/src/mcp/index.ts +++ b/src/mcp/index.ts @@ -25,6 +25,7 @@ import type { ImagePart } from "../provider/types.ts" import { ToolError, truncate, type Tool, type ToolResult } from "../tool/tool.ts" import { match } from "../permission/wildcard.ts" import { codeFrom, forget, OAuthProvider, waitForCallback, CALLBACK_PORT } from "./oauth.ts" +import { mcpBlock } from "../session/prefix.ts" export type ServerConfig = McpServer & { source: "global" | "project" } export type Status = "disabled" | "connecting" | "connected" | "failed" | "needs_auth" @@ -480,8 +481,13 @@ export class McpManager { /** The system prompt's block of server instructions, or "". */ instructions(): string { - const parts = [...this.servers.values()].filter((s) => s.status === "connected" && s.instructions).map((s) => `\n${s.instructions}\n`) - return parts.length ? `Instructions from the MCP servers you have tools from (their tools are named mcp____…):\n\n${parts.join("\n")}\n` : "" + return mcpInstructions(this.serverInstructions()) + } + + /** Each connected server's own instructions, by server: what the prompt shows (mcpInstructions), + * and what a session note compares (session/prefix.ts). */ + serverInstructions(): Record { + return Object.fromEntries([...this.servers.values()].filter((s) => s.status === "connected" && s.instructions).map((s) => [s.name, s.instructions!])) } /** Sign in to a remote server: show the page, catch the redirect (or take a pasted one). */ @@ -554,3 +560,9 @@ function withTimeout(p: Promise, ms: number, message: string): Promise let t: ReturnType return Promise.race([p, new Promise((_, reject) => (t = setTimeout(() => reject(new Error(message)), ms)))]).finally(() => clearTimeout(t)) } + +/** MCP servers' instructions as the system prompt carries them, "" for none. */ +export function mcpInstructions(by: Record): string { + const parts = Object.entries(by).map(([name, text]) => mcpBlock(name, text)) + return parts.length ? `Instructions from the MCP servers you have tools from (their tools are named mcp____…):\n\n${parts.join("\n")}\n` : "" +} diff --git a/src/prompt/assemble.ts b/src/prompt/assemble.ts index b4b850d..347845f 100644 --- a/src/prompt/assemble.ts +++ b/src/prompt/assemble.ts @@ -231,8 +231,8 @@ export interface PromptContext { /** Set for a subagent: who it is and what it is for; replaces the mode's own instructions. * `report`: it has the report tool. */ subagent?: { name: string; instructions: string; report?: boolean } - /** The memory snapshot taken when the session started ("" when both files are empty); - * undefined when memory is off. */ + /** The memory as it was when the prompt was made ("" when both files are empty); undefined + * when memory is off. A later change by the person goes as a note. */ memory?: string /** The skills list (skillLines), taken when the session started. */ skills?: string @@ -246,7 +246,8 @@ export interface PromptContext { mcp?: string /** The other configured models, one line each (with their notes). */ roster?: string - /** The session's plan (plan_write / plan_update), put into the prompt while it exists. */ + /** The session's plan (plan_write / plan_update) as it stood when the prompt was made; a change + * after that is the model's own tool result, or a note when the person made it. */ plan?: Plan /** An instance's own instructions for this session (the chat's authored prompt, over the link), * appended after the shared text under `## Instructions from `. */ @@ -263,6 +264,13 @@ export interface PromptContext { const MAX_INSTRUCTION_CHARS = 40_000 +/** The system prompt (harness spec; its stability is context.md §10, from 3.1.0). It is made once + * per session — at the start, after a compaction, on a model switch — and the engine resends + * those bytes on every request, so nothing in it may be a clock or anything else that moves on + * its own: no date (a note on the first user message says it, session/prefix.ts), the permission + * mode's text as it was at the start, the plan as it stood then, memory, instruction files, + * personality and MCP instructions as they were read for it. What changes later reaches the model + * as a `` on the next user message, never by making this again. */ export function assembleSystem(ctx: PromptContext): string { const vars = { agent_name: ctx.agentName ?? "LLeMbas", @@ -297,7 +305,7 @@ export function assembleSystem(ctx: PromptContext): string { `Project root: ${ctx.root}`, `Git repository: ${ctx.isGit ? "yes" : "no"}`, `Platform: ${platform()} ${release()}`, - `Today: ${new Date().toISOString().slice(0, 10)}`, + // No date: it would change at midnight and the whole session be read again (§10 rule 2). `Model: ${ctx.modelRef}`, "", ].join("\n"), @@ -320,8 +328,8 @@ export function assembleSystem(ctx: PromptContext): string { if (ctx.mcp) sections.push(ctx.mcp) if (!ctx.subagent) sections.push(agentsBlock(ctx, files)) - // Memory and skills: snapshots taken when the session started, so the prompt does not change - // under a session (Hermes' frozen snapshot). + // Memory and skills: as they were when the prompt was made, so the prompt does not change under + // a session (Hermes' frozen snapshot); the person's later edit to memory goes as a note. if (ctx.memory !== undefined && !ctx.subagent) { sections.push(promptText("blocks/memory.md")) if (ctx.memory) sections.push(ctx.memory) @@ -353,8 +361,7 @@ export function assembleSystem(ctx: PromptContext): string { read_only: ctx.mode === "plan" ? SUBAGENT_READ_ONLY : "", }), ) - else sections.push(fillPrompt(promptText(`modes/${ctx.mode}.md`), vars)) - if (ctx.unattended) sections.push(fillPrompt(promptText("modes/unattended.md"), { ...vars, refused: ctx.unattended })) + else sections.push(modeText(ctx.mode, ctx.planDir, ctx.unattended)) // What the person said about themselves goes last, under the web UI's headings and in its order. // Not for a subagent: it answers the main agent, not the person. if (ctx.userInstructions && !ctx.subagent) sections.push(`## How the person you are talking to wants to be helped\n\n${ctx.userInstructions}`) @@ -362,6 +369,14 @@ export function assembleSystem(ctx: PromptContext): string { return sections.join("\n\n") } +/** A permission mode's text (modes/.md), with the unattended block when nobody can approve + * (`refused`: what will be refused, in words): the prompt's own as it was made, and the body of a + * note when the mode changes later (session/prefix.ts). */ +export function modeText(mode: Mode, planDir: string, refused?: string): string { + const own = fillPrompt(promptText(`modes/${mode}.md`), { plan_dir: planDir }) + return refused ? `${own}\n\n${fillPrompt(promptText("modes/unattended.md"), { plan_dir: planDir, refused })}` : own +} + const GIT_QUESTION = "do you commit on your own once a change is finished and checked, only when they ask, or never; and which branch the work goes on (the current one, or a new branch for each task)." const GIT_QUESTION_NO_REPO = "this folder is not a git repository yet — should it be one, and if so, do you commit on your own once a change is finished and checked, or only when they ask." diff --git a/src/provider/openai-chat.ts b/src/provider/openai-chat.ts index c51dfc5..91287bc 100644 --- a/src/provider/openai-chat.ts +++ b/src/provider/openai-chat.ts @@ -207,6 +207,10 @@ export class OpenAIChatClient implements Client { const addText = (kind: "text" | "reasoning", text: string) => out.add(kind, text) let finish = "stop" let usage: StreamEvent | undefined + // llama.cpp's own count, in its last chunk: `timings` with `cache_n` (read from its cache) and + // `prompt_n` (read now). The cached part of a request when `usage` does not say it. + let timings: { cache: number; read: number; out?: number } | undefined + let progressCache: number | undefined for await (const { data } of sseJson(stream)) { const chunk = data as any @@ -217,6 +221,9 @@ export class OpenAIChatClient implements Client { // string code; statusOf() still reads it as the server error it is. throw new ProviderError(`${m.ref}: ${String(msg).trim()}`, statusOf(err), undefined, !out.output) } + const tm = chunk?.timings + if (tm && typeof tm === "object" && Number.isFinite(tm.cache_n)) + timings = { cache: Number(tm.cache_n), read: Number(tm.prompt_n) || 0, ...(Number.isFinite(tm.predicted_n) ? { out: Number(tm.predicted_n) } : {}) } if (chunk?.usage && typeof chunk.usage === "object") { const u = chunk.usage usage = { @@ -232,8 +239,10 @@ export class OpenAIChatClient implements Client { // llama.cpp, while it reads the prompt: { total, cache, processed, time_ms }. Anything not // shaped like that is ignored, never an error. const pp = chunk?.prompt_progress - if (pp && typeof pp === "object" && Number.isFinite(pp.total) && pp.total > 0 && !out.output) - yield { type: "progress", total: Number(pp.total), cache: Number(pp.cache) || 0, processed: Number(pp.processed) || 0, ms: Number(pp.time_ms) || 0 } + if (pp && typeof pp === "object" && Number.isFinite(pp.total) && pp.total > 0 && !out.output) { + progressCache = Number(pp.cache) || 0 + yield { type: "progress", total: Number(pp.total), cache: progressCache, processed: Number(pp.processed) || 0, ms: Number(pp.time_ms) || 0 } + } const choice = chunk?.choices?.[0] if (!choice) continue const delta = choice.delta ?? choice.message ?? {} @@ -272,9 +281,16 @@ export class OpenAIChatClient implements Client { if (calls.length && finish !== "length") finish = "tool_calls" const message = out.message() const parts = message.parts + // The cached input tokens (harness context.md §10 rule 6): the usage's own field + // (`prompt_tokens_details.cached_tokens`, or `cache_read_input_tokens` where an Anthropic-shaped + // count comes through), else llama.cpp's timings, else its last progress report. + const cached = timings?.cache ?? progressCache + if (usage?.type === "usage" && usage.usage.cached === undefined && cached !== undefined) usage.usage.cached = cached + if (!usage && timings?.out !== undefined) + usage = { type: "usage", usage: { input: timings.cache + timings.read, output: timings.out, cached: timings.cache } } if (!usage) { const out = parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.name + p.args : p.text), 0) - usage = { type: "usage", usage: { input: estimateValue(body.messages), output: out, estimated: true } } + usage = { type: "usage", usage: { input: estimateValue(body.messages), output: out, estimated: true, ...(cached !== undefined ? { cached } : {}) } } } yield usage yield { type: "finish", reason: finish, message } diff --git a/src/session/context.ts b/src/session/context.ts index 93923ac..8e0dd6c 100644 --- a/src/session/context.ts +++ b/src/session/context.ts @@ -41,7 +41,7 @@ export function contextBreakdown(engine: Engine, system: string, instructions: { if (imageCount) rows.push({ label: `images (${imageCount}, rough)`, tokens: by.images }) const estimated = rows.filter((r) => !r.indent).reduce((n, r) => n + r.tokens, 0) const reported = engine.usage ? engine.contextUsed() : undefined - return { rows, estimated, reported, window: contextWindow(engine.model), model: engine.model.ref } + return { rows, estimated, reported, window: contextWindow(engine.model), model: engine.model.ref, last: engine.lastRequest } } /** Plain text, for the transcript. */ @@ -58,5 +58,14 @@ export function renderBreakdown(b: ReturnType): string const used = b.reported ?? b.estimated lines.push(` ${"in use".padEnd(33)} ${fmt(used).padStart(7)} ${bar(used)}${b.window ? ` ${Math.round((used / b.window) * 100)}%` : ""}`) lines.push(b.reported !== undefined ? " (the server's own count from the last reply, plus an estimate since)" : " (estimated: ≈4 characters a token — no reply yet to count from)") + // What the server reused of the last request (harness context.md §10 rule 6): a slow reply with + // little of it cached was a re-read. + const last = b.last?.usage + if (last && !last.estimated) + lines.push( + last.cached !== undefined + ? ` last request${b.last!.kind === "compaction" ? " (compaction)" : ""}: ${fmt(last.input)} in, ${fmt(last.cached)} of it cached${last.input ? ` (${Math.round((last.cached / last.input) * 100)}%)` : ""}` + : ` last request: ${fmt(last.input)} in — the server did not say how much was cached`, + ) return lines.join("\n") } diff --git a/src/session/engine.ts b/src/session/engine.ts index 7cf6bba..ef57391 100644 --- a/src/session/engine.ts +++ b/src/session/engine.ts @@ -4,6 +4,7 @@ // backstop, not a budget (LLeMbas generation._run / _wrap_up). // The text is the harness spec's, so LLeMbas's Continue sends the same words. import continueText from "../../harness/prompts/tasks/continue.md" with { type: "text" } +import { isAbsolute } from "node:path" import type { Asker, AttachmentInfo, Bus } from "../bus/index.ts" import type { Effort, Mode } from "../config/schema.ts" import { evaluate, shadowsDeny, type Context as PermContext, type Rule } from "../permission/evaluate.ts" @@ -12,13 +13,15 @@ import type { Plan } from "../plan/plan.ts" import { LOOP } from "../harness.ts" import { fillPrompt } from "../prompt/fill.ts" import { splitPurpose, toSpec, ToolError, type Tool, type ToolContext, type ToolResult } from "../tool/tool.ts" -import { fill, promptText } from "../prompt/assemble.ts" +import { promptText } from "../prompt/assemble.ts" import { resolveCall } from "../tool/names.ts" import { contextWindow } from "../provider/index.ts" import { summaryTurns, type Store } from "./store.ts" import { estimateTokens, estimateValue } from "../provider/tokens.ts" import { duration } from "../duration.ts" import { memoryResults, presentIds, ResultContext } from "./results.ts" +import { render } from "../plan/plan.ts" +import { compactAsk, isNote, NO_CHANGING, noteFor, offeredFor, PREFIX_META, summaryOf, today, toldAt, unavailable, type Changing, type Kept, type Now, type Requirement, type Told } from "./prefix.ts" export interface EngineOptions { bus: Bus @@ -28,8 +31,20 @@ export interface EngineOptions { tools: Tool[] store?: Store sessionId?: string - /** The system prompt for a step; `plan`: the session's plan, for `{{plan_context}}`. */ - system: (mode: Mode, model: ResolvedModel, plan?: Plan) => string + /** The session's system prompt, made from `changing` (what may change during the session, as it + * is now); `plan`: the session's plan, for `{{plan_context}}`. Called once per session, after a + * compaction and on a model switch — never per request (harness context.md §10). */ + system: (mode: Mode, model: ResolvedModel, plan: Plan | undefined, changing: Changing) => string + /** What may change during the session, as it is now: compared at every user message, and a + * difference said in a note. Absent: nothing (a subagent's prompt holds none of it). */ + changing?: () => Changing + /** The text a note gives for a permission mode (modes/.md as the prompt fills it). */ + modeText?: (mode: Mode) => string + /** Awaited before a prompt is made at a turn's start: the app waits up to loop.json + * `context.mcp_wait_ms` for its MCP servers. */ + beforeFirst?: () => Promise + /** Today's date, as the note says it (tests set it). */ + today?: () => string mode: Mode effort: Effort | null perm: Omit @@ -126,8 +141,9 @@ export class Engine { plan: Plan | undefined private sessionRules: Rule[] = [] private recentCalls: string[] = [] - /** Tools refused with no one able to approve them: no longer offered. */ - private withdrawn = new Set() + /** Tools refused with no one able to approve them, and the refusal: still listed (the tool list + * is the session's, harness context.md §10), and every further call answers that refusal. */ + private withdrawn = new Map() /** This prompt's spending, against `budget`: time waiting for the user, tool output, tokens written. */ private turnWaitedMs = 0 private turnOutputBytes = 0 @@ -140,6 +156,16 @@ export class Engine { * (session/results.ts, harness context.md). */ results: ResultContext + /** The prefix (harness context.md §10): the system prompt this session sends, the tools left out + * of its list, and what the model has been told — made at the first request, kept with the + * session, made again only after a compaction or on a model switch. */ + private kept: Kept | undefined + /** What was told when the prompt was made: where a rewind (/undo) goes back to, since the notes + * after it may be gone from the conversation. */ + private keptBase: Told | undefined + /** The last request's usage, cached input tokens included, and what it was for. */ + lastRequest: { usage: Usage; kind: "step" | "compaction" } | undefined + constructor(readonly o: EngineOptions) { this.mode = o.mode this._effort = o.effort @@ -148,6 +174,121 @@ export class Engine { this.sessionId = o.sessionId this.plan = this.storedPlan() this.results = this.resultsFor(o.sessionId) + this.kept = this.storedPrefix() + } + + /** The prefix the store holds for this session, if any: a resumed session sends what it kept. */ + private storedPrefix(): Kept | undefined { + if (!this.o.store || !this.sessionId) return undefined + const k = this.o.store.meta(this.sessionId, PREFIX_META) + if (!k || typeof k !== "object" || typeof k.system !== "string" || !Array.isArray(k.left) || !k.told) return undefined + this.keptBase = { ...structuredClone(k.told), date: undefined } + return k + } + + private saveKept() { + if (this.o.store && this.sessionId) this.o.store.setMeta(this.sessionId, PREFIX_META, this.kept ?? null) + } + + /** What could ever be met in this session (§10 rule 4), decided as the prompt is made: plan mode + * always can be entered; vision, a project directory, knowledge with documents and a display + * are as they are now, for the whole session. */ + private could(): Set { + const could = new Set(["plan"]) + if (this.model.spec.vision === true) could.add("vision") + if (this.o.toolCtx.projectDir !== undefined) could.add("project") + if (this.o.toolCtx.show !== undefined) could.add("display") + const L = this.o.toolCtx.library + const have = L?.lib.bases().filter((b) => b.documents > 0).map((b) => b.name) ?? [] + if (have.some((b) => !L!.bases || L!.bases.includes(b))) could.add("knowledge") + return could + } + + /** What is met now, for a call: the mode for plan, the display as it is; the rest are fixed. */ + private met(): Set { + const met = this.could() + if (this.mode !== "plan") met.delete("plan") + if (this.o.toolCtx.show === undefined) met.delete("display") + return met + } + + /** Everything the note compares, as it is now. */ + private now(changing = this.o.changing?.() ?? NO_CHANGING): Now { + const { source: _source, ...rest } = changing + return { ...structuredClone(rest), date: (this.o.today ?? today)(), mode: this.mode, plan: this.plan ? render(this.plan) : "" } + } + + /** The session's prefix, made now if there is none yet (or it was made for another model). */ + prefix(): Kept { + if (this.kept && this.kept.model === this.model.ref) return this.kept + const changing = this.o.changing?.() ?? NO_CHANGING + const now = this.now(changing) + const could = this.could() + // Left out for the whole session: what it can never meet, and task where nothing can spawn. + // Everything else — MCP tools arriving as their servers connect among them — is offered. + const left = this.o.tools.filter((t) => (t.name === "task" && !this.o.spawn) || !offeredFor(t.requires ? [t.requires] : undefined, could)).map((t) => t.name) + const told = toldAt(now) + // A model switch keeps the conversation, and with it the date already said. + if (this.kept?.told.date) told.date = this.kept.told.date + this.keptBase = { ...structuredClone(told), date: undefined } + this.kept = { system: this.o.system(this.mode, this.model, this.plan, changing), left, model: this.model.ref, told } + this.saveKept() + return this.kept + } + + /** A system prompt as it would be made now, kept nowhere (what /context and tests look at + * before a session has made its own). */ + compose(mode: Mode = this.mode, plan: Plan | undefined = this.plan): string { + return this.o.system(mode, this.model, plan, this.o.changing?.() ?? NO_CHANGING) + } + + /** The system prompt this session sends (made now if it has none yet). */ + systemPrompt(): string { + return this.prefix().system + } + + /** The tools a request offers: the session's list, the same on every request — the wrap-up's + * included, and a tool withdrawn after a final refusal too (it answers that refusal). */ + private toolSpecs() { + const left = new Set(this.prefix().left) + return this.o.tools.filter((t) => !left.has(t.name)).map(toSpec) + } + + /** The note for the next user message (§10 rule 3), "" when nothing changed; what it tells is + * the model's from then on. */ + private takeNote(): string { + const kept = this.prefix() + const { note, told } = noteFor(kept.told, this.now(), (m) => this.o.modeText?.(m) ?? fillPrompt(promptText(`modes/${m}.md`), { plan_dir: this.o.toolCtx.planDir ?? "" })) + if (note) { + kept.told = told + this.saveKept() + } + return note + } + + /** Something the model learnt from its own tool's result (a plan it wrote, the mode its + * plan_submit set, a memory or file it changed): told, so no note says it again. */ + private learnt(fields: Partial) { + if (!this.kept) return + Object.assign(this.kept.told, structuredClone(fields)) + this.saveKept() + } + + /** The conversation was cut back (/undo, a rewind): the notes after the cut may be gone, so what + * the model knows is the prompt again, and the next user message says the rest. */ + rewound() { + if (!this.kept || !this.keptBase) return + this.kept.told = structuredClone(this.keptBase) + this.saveKept() + } + + /** A request's usage, kept with the session (cached input tokens included, §10 rule 6). */ + private recordUsage(usage: Usage, kind: "step" | "compaction") { + this.lastRequest = { usage, kind } + if (this.o.store && this.sessionId) + try { + this.o.store.request(this.sessionId, { model: this.model.ref, kind, ...usage }) + } catch {} } /** A session's results: kept with it in the store, or in memory without one. A stub may point @@ -164,10 +305,12 @@ export class Engine { } /** Replace the plan (null clears it): kept with the session, and said on the bus so every - * front end's card follows. */ - setPlan(plan: Plan | null) { + * front end's card follows. `by`: the model's own tool, whose result shows it the plan; a + * change by the person reaches the model as a note on the next user message. */ + setPlan(plan: Plan | null, by: "person" | "model" = "person") { this.plan = plan ?? undefined if (this.o.store && this.sessionId) this.o.store.setMeta(this.sessionId, PLAN_META, plan) + if (by === "model") this.learnt({ plan: this.plan ? render(this.plan) : "" }) this.o.bus.emit({ type: "plan", plan: this.plan }) } @@ -182,7 +325,8 @@ export class Engine { this.o.bus.emit({ type: "model", ref: this.model.ref, effort: v ?? "off", connection: this.model.connectionName }) } - /** Switch model mid-session. The conversation carries over; the effort resets to the new model's. */ + /** Switch model mid-session. The conversation carries over; the effort resets to the new model's. + * The prompt is made again for it at the next request (prefix(): it was made for another). */ setModel(model: ResolvedModel, client: Client, effort: Effort | null) { const changed = model.ref !== this.model.ref || effort !== this._effort this.model = model @@ -201,8 +345,12 @@ export class Engine { this.withdrawn.clear() this.results = this.resultsFor(sessionId) // A new session's plan is whatever the store holds for it: none for a fresh one, the old one - // on a resume. + // on a resume; its prefix likewise. this.plan = this.storedPlan() + this.kept = undefined + this.keptBase = undefined + this.lastRequest = undefined + this.kept = this.storedPrefix() this.o.bus.emit({ type: "plan", plan: this.plan }) this.o.toolCtx.readFiles.clear() this.o.toolCtx.fileStamps.clear() @@ -227,17 +375,17 @@ export class Engine { } } + /** The summary, asked for as the next turn of this same conversation (harness context.md §7): + * the same system prompt and tools, every message as it is, then tasks/compact.md with an empty + * transcript and previous summary. Nothing of it is shown or stored as a turn; a tool call in the reply is not run. */ private async summarise(signal: AbortSignal): Promise { - const previous = this.messages[0]?.role === "user" && this.messages[0].parts[0]?.type === "text" && this.messages[0].parts[0].text.startsWith("The earlier part of this conversation was compacted") - ? this.messages[0].parts[0].text - : "" - const prompt = fill(promptText("tasks/compact.md"), { - previous_summary: previous ? `## Summary of even earlier turns\n\n${previous}` : "", - transcript: transcript(this.messages, 60_000), - }) + // No previous summary in the ask: an earlier one is already the conversation's first message. + const ask: Message = { role: "user", parts: [{ type: "text", text: compactAsk() }] } let summary = "" - for await (const ev of this.client.stream({ system: "", messages: [{ role: "user", parts: [{ type: "text", text: prompt }] }], tools: [], effort: null, signal })) - if (ev.type === "finish") summary = ev.message.parts.filter((p) => p.type === "text").map((p) => (p as { text: string }).text).join("").trim() + for await (const ev of this.client.stream({ system: this.systemPrompt(), messages: mergeUsers([...this.messages, ask]), tools: this.toolSpecs(), effort: this.effort, signal })) { + if (ev.type === "finish") summary = summaryOf(ev.message.parts) + else if (ev.type === "usage") this.recordUsage(ev.usage, "compaction") + } if (!summary) throw new Error("the model returned an empty summary") // The harness's own table of the reads still in the conversation (context.md §7), after the // summary; the conversation is then the summary alone, so no stub can point at a result before it. @@ -247,6 +395,11 @@ export class Engine { this.messages = summaryTurns(summary) this.usage = undefined this.usageAt = 0 + // Everything is read again now anyway: the prompt is made again at the next request, and the + // next user message says the date again (what said it is in the summary, if anywhere). + this.kept = undefined + this.keptBase = undefined + this.saveKept() // Said on the bus: the web UI marks the point in a shared session, as the TUI does. this.o.bus.emit({ type: "compacted", summary }) return summary @@ -293,14 +446,18 @@ export class Engine { await this.summarise(signal) // The task is never left to the summary: the prompt that started this turn comes back word // for word (found on a real run, where a small model's summary lost the task entirely). - const original = this.turnPrompt?.role === "user" ? this.turnPrompt.parts : [] + // Its note is not: what it said may be out of date, and the new prompt's first user message + // gets a note of its own. + const original = this.turnPrompt?.role === "user" ? this.turnPrompt.parts.filter((p) => p.type !== "text" || !isNote(p.text)) : [] const steers = this.turnSteers.map((s) => ({ type: "text" as const, text: `(Sent while you were working:) ${s}` })) + const note = this.takeNote() this.push({ role: "user", parts: [ ...original, ...steers, { type: "text", text: "(The conversation was compacted to fit the context window; the summary above records what was done so far. This was the request — continue it from where the summary leaves off.)" }, + ...(note ? [{ type: "text" as const, text: note }] : []), ], }) this.o.bus.emit({ type: "notice", message: "compacted — carrying on from the summary, with your request kept word for word" }) @@ -332,23 +489,6 @@ export class Engine { return undefined } - /** Tools a model can use at all: view_image only with vision, plan_submit only in plan mode - * (`requires: "plan"`; plan_write and plan_update are offered in every mode). */ - private offered(name: string): boolean { - const t = this.o.tools.find((x) => x.name === name) - if (name === "task" && !this.o.spawn) return false - if (!t?.requires) return true - if (t.requires === "vision") return this.model.spec.vision === true - if (t.requires === "project") return this.o.toolCtx.projectDir !== undefined - if (t.requires === "display") return this.o.toolCtx.show !== undefined - if (t.requires === "knowledge") { - const L = this.o.toolCtx.library - const have = L?.lib.bases().filter((b) => b.documents > 0).map((b) => b.name) ?? [] - return have.some((b) => !L!.bases || L!.bases.includes(b)) - } - return this.mode === "plan" - } - cancel() { this.abort?.abort() } @@ -449,12 +589,21 @@ export class Engine { const { bus } = this.o this.abort = new AbortController() const signal = this.abort.signal - this.push({ role: "user", parts: [{ type: "text", text }, ...extra.map((t) => (typeof t === "string" ? { type: "text" as const, text: t } : t))], ...(turnId ? { turnId } : {}), ...(machine ? { machine } : {}) }) + // Before a session's first request: the configured MCP servers get a moment to connect, so + // their tools are in the first list (§10 rule 4). + if (!this.kept || this.kept.model !== this.model.ref) await this.o.beforeFirst?.() + // What changed since the prompt was made goes after the person's words, as a note (§10 rule 3). + const note = this.takeNote() + this.push({ + role: "user", + parts: [{ type: "text", text }, ...extra.map((t) => (typeof t === "string" ? { type: "text" as const, text: t } : t)), ...(note ? [{ type: "text" as const, text: note }] : [])], + ...(turnId ? { turnId } : {}), + ...(machine ? { machine } : {}), + }) this.turnPrompt = this.messages[this.messages.length - 1] this.turnSteers = [] this.autoCompactions = 0 const toolMap = new Map(this.o.tools.map((t) => [t.name, t])) - const allSpecs = this.o.tools.map(toSpec) const maxSteps = this.o.maxSteps ?? LOOP.steps.default const started = performance.now() this.turnWaitedMs = 0 @@ -486,10 +635,13 @@ export class Engine { // What was said before a stop: kept, so /continue picks up from there. let partial = "" try { + // The same system prompt and tool list on every request of the session (§10). const stream = this.client.stream({ - system: this.o.system(this.mode, this.model, this.plan), + system: this.systemPrompt(), messages: mergeUsers(wrapUp ? [...this.messages, { role: "user", parts: [{ type: "text", text: wrapUpText(ranOut ? ranOut.limit : "step limit") }] }] : this.messages), - tools: wrapUp ? [] : allSpecs.filter((s) => !this.withdrawn.has(s.name) && this.offered(s.name)), + // The wrap-up too (loop.json wrap_up): its message says the tools are withdrawn, and a + // call in its reply is not run. + tools: this.toolSpecs(), effort: this.effort, signal, }) @@ -533,8 +685,16 @@ export class Engine { bus.emit({ type: "done", reason: "error" }) return "error" } + // The wrap-up's answer is its text before any call: a call there is not run, and none is kept + // without a result to go with it. + if (wrapUp && assistant.parts.some((p) => p.type === "tool_call")) { + const first = assistant.parts.findIndex((p) => p.type === "tool_call") + assistant = { ...assistant, parts: assistant.parts.slice(0, first) } + bus.emit({ type: "notice", message: "the wrap-up reply called a tool; it was not run" }) + } this.push(assistant) if (usage) { + this.recordUsage(usage, "step") this.turnCompletion += usage.output // Counted once the reply is in the history, so the reply itself is not estimated again. this.usage = usage @@ -608,11 +768,13 @@ export class Engine { question: this.o.asker.question ? (req, id) => this.waiting(this.o.asker.question!(req, id)) : undefined, plan: { get: () => this.plan, - set: (p) => this.setPlan(p), + set: (p) => this.setPlan(p, "model"), ask: this.o.asker.plan ? (req, id) => this.waiting(this.o.asker.plan!(req, id)) : undefined, }, setMode: (m) => { this.mode = m + // Its own plan_submit's result tells the model the mode. + this.learnt({ mode: m }) bus.emit({ type: "mode", mode: m }) }, spawn: this.o.spawn, @@ -647,6 +809,16 @@ export class Engine { ) } this.bgNow = undefined + // A memory or instruction file changed by this step's tools is the model's own doing, which + // their results showed it: no note says it again. Changed between turns, it is the person's. + if (runnable.some((r) => r.writes) && this.o.changing && this.kept) { + const now = this.now() + // Files by their path only: the person's own instructions and an instance's (named by a + // heading) are not something a tool of the model writes. + const told = this.kept.told.files + const files = Object.fromEntries([...Object.entries(told).filter(([k]) => !isAbsolute(k)), ...Object.entries(now.files).filter(([k]) => isAbsolute(k))]) + this.learnt({ memory: now.memory, files, ...(runnable.some((r) => r.tool.name === "settings") ? { personality: now.personality } : {}) }) + } const images: ImagePart[] = [] for (const call of calls) { const r = results.get(call.id) ?? { output: "Not run: the reply was cancelled.", isError: true } @@ -702,6 +874,12 @@ export class Engine { const resolved = resolveCall(call.name, raw) tool = toolMap.get(resolved.name) ?? tool raw = resolved.raw + // Offered for the whole session; while what it needs is not met, the spec's sentence (§10 rule 4). + const no = unavailable(tool.name, tool.requires ? [tool.requires] : undefined, this.met()) + if (no) return { result: { output: no, isError: true } } + // Withdrawn after a final refusal: still listed, and the refusal is the answer. + const refused = this.withdrawn.get(tool.name) + if (refused !== undefined) return { result: { output: refused, isError: true }, denied: true } // What the call is for goes to the user when they are asked; the tool never sees an added `purpose`. const { args: own, purpose } = splitPurpose(tool, raw) const parsed = tool.schema.safeParse(own) @@ -746,15 +924,10 @@ export class Engine { if (reply.kind === "deny" && withdraw) { // Every tool behind the same permission goes: edit and write share "edit". const siblings = this.o.tools.filter((t) => t.access === tool.access) - for (const t of siblings) this.withdrawn.add(t.name) const gone = siblings.map((t) => t.name).join(", ") - return { - result: { - output: `Refused: ${reply.feedback ?? "nobody can approve this"} It will be refused every time in this session, so ${gone} ${gone.includes(",") ? "are" : "is"} no longer offered. Do not look for a way around it. Finish with what you can do without it, and say exactly what you would have done — the change, the command — so the user can do it.`, - isError: true, - }, - denied: true, - } + const output = `Refused: ${reply.feedback ?? "nobody can approve this"} It will be refused every time in this session: every further call to ${gone} answers this same refusal. Do not look for a way around it. Finish with what you can do without it, and say exactly what you would have done — the change, the command — so the user can do it.` + for (const t of siblings) this.withdrawn.set(t.name, output) + return { result: { output, isError: true }, denied: true } } if (reply.kind === "deny") { const output = reply.feedback @@ -790,19 +963,3 @@ export class Engine { return { tool, args, paths: request.command === undefined ? (request.paths ?? []) : [], writes: request.class !== "read", edited } } } - - -/** The conversation as plain text for the summariser, newest kept when it is too long. */ -export function transcript(messages: Message[], maxChars: number): string { - const lines: string[] = [] - for (const m of messages) { - if (m.role === "tool") lines.push(`[${m.name} result${m.isError ? ", error" : ""}]\n${m.content.slice(0, 2000)}`) - else - for (const p of m.parts) { - if (p.type === "text") lines.push(`${m.role === "user" ? "User" : "Assistant"}: ${p.text}`) - else if (p.type === "tool_call") lines.push(`[assistant called ${p.name} ${p.args.slice(0, 400)}]`) - } - } - const all = lines.join("\n\n") - return all.length <= maxChars ? all : "[…earlier turns cut…]\n\n" + all.slice(-maxChars) -} diff --git a/src/session/prefix.ts b/src/session/prefix.ts new file mode 100644 index 0000000..4579ba8 --- /dev/null +++ b/src/session/prefix.ts @@ -0,0 +1,155 @@ +// The prefix (harness spec 3.1.0, context.md §10): what the model has already read stays exactly as +// it was, so its server can reuse it. The system prompt and the tool list are made once per session +// and resent byte for byte; what changes during the session goes as a note on the next user +// message. This file holds the rules and the texts, without the engine: what the model was last +// told, what a note says about the difference, which tools are offered, what a call answers when +// its requirement is not met, and what compaction asks for. The engine keeps the state +// (session/engine.ts); the conformance area `prefix` drives these functions directly. +import type { Mode } from "../config/schema.ts" +import { LOOP, TOOL_SPECS } from "../harness.ts" +import { fillPrompt } from "../prompt/fill.ts" +import { fill, promptText } from "../prompt/assemble.ts" + +/** The note's texts (loop.json `context.note`). */ +export const NOTE = LOOP.context.note + +/** What a note says in place of something that is now empty: no plan, no memory, a file removed. */ +export const NOTHING = NOTE.empty + +/** The session_meta key the prefix is kept under (the prompt, the tools left out, what was told). */ +export const PREFIX_META = "prefix" + +/** What the system prompt holds that can change during a session: the app reads it as it is now, + * builds the prompt from it, and the engine compares it at every user message. Each is "" (or + * empty) where there is none. */ +export interface Changing { + /** The memory block as the prompt shows it. */ + memory: string + /** The personality as the prompt shows it. */ + personality: string + /** Instruction texts by what names them: an instruction file by its path; the person's own + * instructions and an instance's by their heading. */ + files: Record + /** MCP servers' instructions, by server. */ + mcp: Record + /** What the app needs to make the prompt from these same values; never compared or kept. */ + source?: unknown +} + +export const NO_CHANGING: Changing = { memory: "", personality: "", files: {}, mcp: {} } + +/** What the model was last told of each: by the system prompt when it was made, by a note since, or + * by its own tool's result (a plan it wrote, a mode its plan_submit set). `date` is absent until a + * note has said it. */ +export interface Told extends Changing { + date?: string + mode: Mode + /** The plan as rendered (plan.ts `render`), or "". */ + plan: string +} + +/** What is true now, against which a Told is compared. */ +export interface Now extends Changing { + date: string + mode: Mode + plan: string +} + +/** The prefix a session keeps (session_meta `prefix`): the prompt it sends, the tools left out of + * its list for the whole session (every other tool is offered, MCP tools as their servers + * connect), the model it was made for, and what the model has been told. */ +export interface Kept { + system: string + left: string[] + model: string + told: Told +} + +/** Told as of a prompt made now: everything as it is, except the date, which no prompt holds. */ +export function toldAt(now: Now): Told { + const { date: _date, source: _source, ...rest } = now + return structuredClone(rest) +} + +/** The note for a user message: a line for each thing that differs from what the model was told, + * in loop.json's order, inside the note's tags — "" when nothing does. `modeText` is the mode's + * text (modes/.md, filled as the prompt fills it). Returns what is told after it. */ +export function noteFor(told: Told, now: Now, modeText: (mode: Mode) => string): { note: string; told: Told } { + const lines: string[] = [] + const say = (text: string, vars: Record) => lines.push(fillPrompt(text, vars)) + const or = (s: string) => (s.trim() ? s : NOTHING) + if (told.date !== now.date) say(NOTE.date, { date: now.date }) + if (told.mode !== now.mode) say(NOTE.mode, { mode: now.mode, mode_text: modeText(now.mode) }) + if (told.plan !== now.plan) say(NOTE.plan, { plan: or(now.plan) }) + if (told.memory !== now.memory) say(NOTE.memory, { memory: or(now.memory) }) + for (const path of union(told.files, now.files)) + if (told.files[path] !== now.files[path]) say(NOTE.instructions, { path, text: or(now.files[path] ?? "") }) + if (told.personality !== now.personality) say(NOTE.personality, { personality: or(now.personality) }) + // Only a server that has connected or changed since: one that went away takes its tools with it. + const arrived = Object.keys(now.mcp).filter((name) => told.mcp[name] !== now.mcp[name]) + if (arrived.length) say(NOTE.mcp, { instructions: arrived.map((name) => mcpBlock(name, now.mcp[name]!)).join("\n") }) + return { note: lines.length ? noteBlock(lines) : "", told: { ...toldAt(now), date: now.date } } +} + +/** The block a note is: its lines between the tags, a blank line between two. */ +export function noteBlock(lines: string[]): string { + return `${NOTE.open}\n${lines.join("\n\n")}\n${NOTE.close}` +} + +/** Whether a text part of a user message is a note (not shown as the person's words). */ +export function isNote(text: string): boolean { + return text.startsWith(NOTE.open) && text.trimEnd().endsWith(NOTE.close) +} + +/** One MCP server's instructions as the prompt and a note show them. */ +export function mcpBlock(name: string, text: string): string { + return `\n${text}\n` +} + +function union(a: Record, b: Record): string[] { + return [...new Set([...Object.keys(a), ...Object.keys(b)])] +} + +/** Today's date where this runs, as the note says it (YYYY-MM-DD). */ +export function today(at = new Date()): string { + const p = (n: number) => String(n).padStart(2, "0") + return `${at.getFullYear()}-${p(at.getMonth() + 1)}-${p(at.getDate())}` +} + +/** A requirement a tool can have (the spec's `requires`, and the CLI's own: vision, project, + * knowledge). */ +export type Requirement = "vision" | "plan" | "project" | "knowledge" | "display" | "page" | "forge" + +/** Rule 4: the tool list is fixed for the session. A tool is offered when every requirement it has + * is one the session could meet at some point (`could`), decided once as the prompt is made; a + * call while one is not met now answers `unavailable`. */ +export function offeredFor(requires: readonly Requirement[] | undefined, could: ReadonlySet): boolean { + return (requires ?? []).every((r) => could.has(r)) +} + +/** What a call answers when a requirement of its tool is not met now: the spec's text, word for + * word, or undefined when it is met (or the tool has no such text). */ +export function unavailable(tool: string, requires: readonly Requirement[] | undefined, met: ReadonlySet): string | undefined { + if ((requires ?? []).every((r) => met.has(r))) return undefined + return TOOL_SPECS[tool]?.unavailable +} + +/** §7: what the summary is asked with, as one more user message after the conversation — + * tasks/compact.md with `{{transcript}}` and `{{previous_summary}}` empty, since the model has the + * conversation, an earlier summary included, itself. */ +export function compactAsk(): string { + return fill(promptText("tasks/compact.md"), { previous_summary: "", transcript: "" }) + .replace(/\n{3,}/g, "\n\n") + .trim() +} + +/** The summary in a reply to the compaction request: its text before any tool call (a call in it + * is not run), "" when there is none. */ +export function summaryOf(parts: { type: string; text?: string }[]): string { + const first = parts.findIndex((p) => p.type === "tool_call") + return (first < 0 ? parts : parts.slice(0, first)) + .filter((p) => p.type === "text") + .map((p) => p.text ?? "") + .join("") + .trim() +} diff --git a/src/session/store.ts b/src/session/store.ts index 2bf92f9..7e25626 100644 --- a/src/session/store.ts +++ b/src/session/store.ts @@ -2,7 +2,8 @@ import { Database } from "bun:sqlite" import { mkdirSync } from "node:fs" import { join } from "node:path" import { paths } from "../config/paths.ts" -import type { Message } from "../provider/types.ts" +import type { Message, Usage } from "../provider/types.ts" +import { isNote } from "./prefix.ts" // Sessions and their messages, in ~/.local/share/lembas/sessions.db. Message text is also // indexed with FTS5 so past sessions can be searched (session_search). @@ -53,6 +54,11 @@ CREATE TABLE IF NOT EXISTS results ( json TEXT NOT NULL, text TEXT NOT NULL, PRIMARY KEY (session_id, id) ); +CREATE TABLE IF NOT EXISTS requests ( + id INTEGER PRIMARY KEY AUTOINCREMENT, session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + created INTEGER NOT NULL, model TEXT NOT NULL, kind TEXT NOT NULL, + input INTEGER NOT NULL, output INTEGER NOT NULL, reasoning INTEGER, cached INTEGER, estimated INTEGER NOT NULL DEFAULT 0 +); CREATE TABLE IF NOT EXISTS session_meta ( session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, key TEXT NOT NULL, value TEXT NOT NULL, PRIMARY KEY (session_id, key) @@ -70,7 +76,16 @@ export const CWD_META = "cwd" export function plainText(m: Message): string { if (m.role === "tool") return m.content - return m.parts.map((p) => (p.type === "text" || p.type === "reasoning" ? p.text : p.type === "tool_call" ? `${p.name} ${p.args}` : "")).join("\n") + // A session note is the harness's, not the person's words: not searched. + return m.parts.map((p) => (p.type === "text" ? (m.role === "user" && isNote(p.text) ? "" : p.text) : p.type === "reasoning" ? p.text : p.type === "tool_call" ? `${p.name} ${p.args}` : "")).join("\n") +} + +/** One request's usage as the store keeps it (harness context.md §10 rule 6): what it was for — a + * step of a turn, or a compaction's summary — and the cached input tokens where the server said. */ +export interface RequestRow extends Usage { + model: string + kind: "step" | "compaction" + created?: number } export interface Row { @@ -225,6 +240,19 @@ export class Store { } } + /** Record one request's usage. */ + request(sessionId: string, r: RequestRow) { + this.db + .query("INSERT INTO requests (session_id, created, model, kind, input, output, reasoning, cached, estimated) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)") + .run(sessionId, Date.now(), r.model, r.kind, r.input, r.output, r.reasoning ?? null, r.cached ?? null, r.estimated ? 1 : 0) + } + + /** A session's requests, oldest first. */ + requests(sessionId: string): RequestRow[] { + const rows = this.db.query("SELECT created, model, kind, input, output, reasoning, cached, estimated FROM requests WHERE session_id = ? ORDER BY id").all(sessionId) as (Omit & { estimated: number; reasoning: number | null; cached: number | null })[] + return rows.map(({ estimated, reasoning, cached, ...r }) => ({ ...r, ...(reasoning !== null ? { reasoning } : {}), ...(cached !== null ? { cached } : {}), ...(estimated ? { estimated: true } : {}) })) + } + /** Something a session keeps besides its messages, as JSON. */ meta(sessionId: string, key: string): T | undefined { const r = this.db.query("SELECT value FROM session_meta WHERE session_id = ? AND key = ?").get(sessionId, key) as { value: string } | null diff --git a/src/session/turns.ts b/src/session/turns.ts index 0ed2916..ba532e2 100644 --- a/src/session/turns.ts +++ b/src/session/turns.ts @@ -96,6 +96,7 @@ export class Turns { this.snaps.restore(turn.before, files) } const messages = this.engine.messages.splice(turn.at) + this.engine.rewound() if (this.store && this.engine.sessionId && turn.lastRow >= turn.firstRow) this.store.setKind(this.engine.sessionId, turn.firstRow, turn.lastRow, "reverted") this.undone.push({ turn, current, messages }) return { prompt: turn.prompt, files } @@ -129,6 +130,7 @@ export class Turns { const first = dropped[0]! const last = dropped[dropped.length - 1]! this.engine.messages.splice(first.at) + this.engine.rewound() if (this.store && sessionId && last.lastRow >= first.firstRow) this.store.setKind(sessionId, first.firstRow, last.lastRow, "reverted") this.undone = [] return dropped.map((t) => t.turnId) @@ -142,6 +144,7 @@ export class Turns { this.store.setKind(sessionId, row, Number.MAX_SAFE_INTEGER, "reverted") const at = this.engine.messages.findIndex((m) => m.role === "user" && m.turnId === turnId) if (at >= 0) this.engine.messages.splice(at) + this.engine.rewound() this.done = [] this.undone = [] return ids diff --git a/src/tool/plan_submit.ts b/src/tool/plan_submit.ts index 986e87b..2292512 100644 --- a/src/tool/plan_submit.ts +++ b/src/tool/plan_submit.ts @@ -1,5 +1,6 @@ -// plan_submit: the plan written with plan_write, presented for approval — plan mode only, where -// it is the approval. The plan itself is the session's (src/plan/plan.ts); a copy goes under the +// plan_submit: the plan written with plan_write, presented for approval — in plan mode, where it is +// the approval. Offered in every mode (spec 3.1.0, `requires: plan`): outside plan mode the engine +// answers the spec's `unavailable` text and nothing runs. The plan itself is the session's (src/plan/plan.ts); a copy goes under the // project's plans directory for the record, since that is the one place plan mode may write. import { mkdirSync, writeFileSync } from "node:fs" import { join } from "node:path" diff --git a/src/tool/session_search.ts b/src/tool/session_search.ts index 38f8849..6193132 100644 --- a/src/tool/session_search.ts +++ b/src/tool/session_search.ts @@ -7,6 +7,7 @@ import type { Message } from "../provider/types.ts" import type { Row, SessionRow, Store } from "../session/store.ts" import { defineTool, ToolError, type ToolContext } from "./tool.ts" import { described } from "../harness.ts" +import { isNote } from "../session/prefix.ts" /** A user's words made safe for FTS5 MATCH: quoted phrases kept, operators' punctuation dropped, * terms with dots or hyphens quoted (FTS5 would read them as syntax). */ @@ -35,7 +36,8 @@ export function readable(json: string): { role: string; text: string } { const m = JSON.parse(json) as Message if (m.role === "tool") return { role: "tool", text: `[${m.name} result, ${m.content.length} chars]` } const text = m.parts - .map((p) => (p.type === "text" ? p.text : p.type === "tool_call" ? `[called ${p.name}]` : p.type === "image" ? "[image]" : "")) + // A session note is the harness's, not what was said. + .map((p) => (p.type === "text" ? (isNote(p.text) ? "" : p.text) : p.type === "tool_call" ? `[called ${p.name}]` : p.type === "image" ? "[image]" : "")) .filter(Boolean) .join("\n") return { role: m.role, text } diff --git a/src/tool/tool.ts b/src/tool/tool.ts index f1a3a6d..7ab7d6c 100644 --- a/src/tool/tool.ts +++ b/src/tool/tool.ts @@ -118,9 +118,11 @@ export interface Tool { access: string /** Runs alone, never alongside other calls of the same step (it waits on the user). */ exclusive?: boolean - /** Offered only when this holds: a model with vision, plan mode, a project directory, knowledge - * bases with documents, or a display (a client over the link that shows files: the spec's - * `requires: display`). */ + /** What the tool needs: a model with vision, plan mode, a project directory, knowledge bases with + * documents, or a display (a client over the link that shows files: the spec's `requires: + * display`). The tool list is the session's (harness context.md §10): one the session could + * never meet leaves the tool out; one that comes and goes (plan mode, a display) keeps it in, and + * a call while it is unmet answers the spec's `unavailable` text. */ requires?: "vision" | "plan" | "project" | "knowledge" | "display" description: string schema: S diff --git a/src/tui/commands.ts b/src/tui/commands.ts index 0d6aa19..883a510 100644 --- a/src/tui/commands.ts +++ b/src/tui/commands.ts @@ -627,7 +627,8 @@ export const COMMANDS: Command[] = [ description: "where the context window goes", run(c) { const e = c.app.engine - const system = e.o.system(e.mode, e.model) + // The prompt the session sends (made now if no request has gone yet). + const system = e.systemPrompt() const git = /\n([\s\S]*?)\n<\/git>/.exec(system)?.[1] c.view.info(renderBreakdown(contextBreakdown(e, system, instructionFiles(e.o.toolCtx.cwd, c.app.project.root, c.app.loaded.instructions), git, e.o.tools))) }, diff --git a/src/tui/state.ts b/src/tui/state.ts index a90eba3..7096c9d 100644 --- a/src/tui/state.ts +++ b/src/tui/state.ts @@ -16,6 +16,7 @@ import type { JobInfo } from "../jobs/registry.ts" import { splitPrototypes, titleFor, type PrototypeKind } from "../prototype/fence.ts" import { savePrototype } from "../prototype/store.ts" import { CHARS_PER_TOKEN } from "../provider/tokens.ts" +import { isNote } from "../session/prefix.ts" export type Item = /** `mid`: sent while the agent worked, and given to it at a step boundary. `machine`: the harness @@ -121,7 +122,9 @@ export interface ViewState { jobs: TrackedJob[] } -const ATTACHMENT = /^<(file|directory|shell|image) / +// What the model was given beside the words typed — attachments, and a session note +// (session/prefix.ts) — is not shown as typed. +const ATTACHMENT = { test: (t: string) => /^<(file|directory|shell|image) /.test(t) || isNote(t) } /** Running first, then by start — the registry's order. */ const byRunning = (a: JobInfo, b: JobInfo) => Number(b.state === "running") - Number(a.state === "running") || a.started - b.started let seq = 0 diff --git a/tests/acp-v2.test.ts b/tests/acp-v2.test.ts index bdb6890..956e9e7 100644 --- a/tests/acp-v2.test.ts +++ b/tests/acp-v2.test.ts @@ -360,14 +360,21 @@ test("the context reaches the prompt: the instance's instructions, its personali // names nothing to the model — nothing unfilled reaches it either. expect(system()).not.toContain("Manuals") expect(system()).not.toContain("{{") - // A later prompt replaces the context: personalization back to the device's, nothing from the instance. + // A later prompt replaces the context: personalization back to the device's, nothing from the + // instance. The system prompt stays as it was made (harness context.md §10); the change goes as a + // note on that prompt's message. + const first = system() await c.request("session/prompt", { sessionId: s.sessionId, prompt: [{ type: "text", text: "again" }], _meta: { lembas: { turnId: "T2", system_prompt: "", personalization: null, prototypes: { offered: false, sources: "" } } } }) - expect(system()).not.toContain("Instructions from example") - expect(system()).toContain("local words") - expect(system()).not.toContain("prototype-chat") - // A prompt that says only its turn id changes nothing. + expect(system()).toBe(first) + const said = () => fake!.requests.at(-1)!.messages.at(-1).content as string + expect(said()).toStartWith("again\n\n") + expect(said()).toContain("the person's own instructions changed; it now reads:\nlocal words") + expect(said()).toContain("Instructions from example changed; it now reads:\n(none)") + expect(said()).toContain("The person chose a different personality:\nBe brief.") + // A prompt that says only its turn id changes nothing, and so says nothing. await c.request("session/prompt", { sessionId: s.sessionId, prompt: [{ type: "text", text: "more" }], _meta: { lembas: { turnId: "T3" } } }) - expect(system()).toContain("local words") + expect(system()).toBe(first) + expect(said()).toBe("more") }) test("the plan is sent after a session is opened from the store, and cleared with plan: null", async () => { diff --git a/tests/budget.test.ts b/tests/budget.test.ts index 589af10..559f13b 100644 --- a/tests/budget.test.ts +++ b/tests/budget.test.ts @@ -25,7 +25,7 @@ function app(limits: string, script: Parameters[0], ask: () const tools = (r: any) => (r.tools ?? []).map((t: any) => t.function.name) -test("past the output budget the tools are withdrawn, the model is told, and it answers", async () => { +test("past the output budget the model is told the tools are withdrawn, and it answers; the list stays the session's", async () => { const a = app("output_bytes: 1000", [ { chunks: [toolCall(0, "r1", "read", '{"path":"big.txt"}')] }, { chunks: [delta({ content: "It is a file of x." }, "stop")] }, @@ -34,7 +34,7 @@ test("past the output budget the tools are withdrawn, the model is told, and it a.bus.on((e: Event) => e.type === "notice" && notices.push(e.message)) expect(await a.engine.prompt("what is in big.txt?")).toBe("budget") expect(tools(fake!.requests[0])).toContain("read") - expect(tools(fake!.requests[1])).toEqual([]) + expect(tools(fake!.requests[1])).toEqual(tools(fake!.requests[0])) expect(JSON.stringify(fake!.requests[1].messages.at(-1))).toContain("The tool output budget for this prompt has been reached") expect(notices.join("\n")).toContain("with too much tool output to read") }) @@ -44,8 +44,8 @@ test("the token budget counts what the model wrote this prompt", async () => { { chunks: [toolCall(0, "r1", "read", '{"path":"big.txt"}'), usage(10, 80)] }, { chunks: [delta({ content: "done" }, "stop")] }, ]) - await a.engine.prompt("go") - expect(tools(fake!.requests[1])).toEqual([]) + expect(await a.engine.prompt("go")).toBe("budget") + expect(JSON.stringify(fake!.requests[1].messages.at(-1))).toContain("The output budget for this prompt has been reached") }) test("time spent waiting for an approval is not spent from the wall-clock budget", async () => { diff --git a/tests/conformance.ts b/tests/conformance.ts index 1fc09e7..1692a98 100644 --- a/tests/conformance.ts +++ b/tests/conformance.ts @@ -23,6 +23,9 @@ import { CONTEXT, hunks, lineCount, memoryResults, presentIds, ResultContext, sh import { readBody } from "../src/tool/read.ts" import { applyOps, type Op, type Target } from "../src/memory/store.ts" import { jobNote, type NoteCase } from "../src/jobs/note.ts" +import { compactAsk, noteFor, offeredFor, toldAt, unavailable, type Now, type Requirement, type Told } from "../src/session/prefix.ts" +import { promptText } from "../src/prompt/assemble.ts" +import { TOOL_SPECS } from "../src/harness.ts" type Case = Record @@ -150,6 +153,58 @@ export const AREAS: Record unknown> = { return text }) }, + // The prefix (context.md §10): one session's requests, step by step — which prompt each sends (a + // new number each time one is made), which tools it offers, and the note on a user message. + prefix: (c) => { + const start = c.start as Partial & { could?: Requirement[]; met?: Requirement[] } + const could = new Set(start.could ?? []) + let met = new Set(start.met ?? could) + const now: Now = { date: start.date!, mode: start.mode ?? "manual", plan: start.plan ?? "", memory: start.memory ?? "", personality: start.personality ?? "", files: start.files ?? {}, mcp: start.mcp ?? {} } + const modeText = (m: Mode) => fillPrompt(promptText(`modes/${m}.md`), { plan_dir: (c.plan_dir as string | undefined) ?? ".agent/plans" }) + const requires = (t: string) => TOOL_SPECS[t]?.requires as Requirement[] | undefined + let made: { system: number; tools: string[]; told: Told } | undefined + let prompts = 0 + let carried: string | undefined + const make = () => { + if (made) return made + made = { system: ++prompts, tools: (c.tools as string[]).filter((t) => offeredFor(requires(t), could)), told: { ...toldAt(now), ...(carried ? { date: carried } : {}) } } + carried = undefined + return made + } + const fields = ["date", "mode", "plan", "memory", "personality", "files", "mcp"] as const + return (c.steps as Record[]).map((st) => { + if (st.op === "set") { + for (const f of fields) if (f in st) (now as unknown as Record)[f] = structuredClone(st[f]) + if (st.met) met = new Set(st.met) + // The model's own tool changed it: its result told the model. + if (st.by === "model" && made) for (const f of fields) if (f in st && f !== "date") (made.told as unknown as Record)[f] = structuredClone(st[f]) + return null + } + if (st.op === "call") { + const here = new Set(met) + if (now.mode === "plan") here.add("plan") + else here.delete("plan") + return unavailable(st.tool, requires(st.tool), here) ?? null + } + if (st.op === "model") { + carried = made?.told.date + made = undefined + return null + } + const k = make() + if (st.op === "user") { + const { note, told } = noteFor(k.told, now, modeText) + if (note) k.told = told + return { system: k.system, tools: k.tools, note: note || null } + } + if (st.op === "tool" || st.op === "wrapup") return { system: k.system, tools: k.tools, note: null } + if (st.op === "compact") { + made = undefined + return { system: k.system, tools: k.tools, ask: compactAsk() } + } + throw new Error(`prefix: unknown op ${String(st.op)}`) + }) + }, library_scope: (c) => resolveBase(c.bases as { id: string; name: string }[], c.base as string | null), toolcall_index: (c) => { // The deltas as the OpenAI stream carries them: `function.name` and `function.arguments`. diff --git a/tests/context.test.ts b/tests/context.test.ts index a32f1d4..f3a23c2 100644 --- a/tests/context.test.ts +++ b/tests/context.test.ts @@ -66,7 +66,15 @@ describe("context", () => { a.bus.on((e: Event) => e.type === "notice" && notices.push(e.message)) await a.engine.prompt("go") expect(notices.some((n) => n.includes("compacting the conversation"))).toBe(true) - expect(fake!.requests[1].messages[0].content).toContain("## Transcript") + // The summary is the next turn of the same conversation (context.md §7, spec 3.1.0): the same + // system prompt and tools, every message as it was, then the compaction text — no transcript. + const [first, compaction] = [fake!.requests[0], fake!.requests[1]] + expect(compaction.messages[0]).toEqual(first.messages[0]) + expect(compaction.tools).toEqual(first.tools) + expect(compaction.messages.slice(0, first.messages.length)).toEqual(first.messages) + expect(compaction.messages.at(-2).role).toBe("tool") + expect(compaction.messages.at(-1).content).toStartWith("Summarise the conversation") + expect(compaction.messages.at(-1).content).not.toContain("yyyy") const last = fake!.requests[2].messages expect(JSON.stringify(last[1])).toContain("A long task.") // the original request comes back verbatim, with the note @@ -77,7 +85,7 @@ describe("context", () => { test("/context: parts of the system prompt, tools, the conversation, against the window", async () => { const a = app(`connections:\n f:\n type: webui\n url: URL\n models: { f/m: { context: 32768 } }\n`, [{ chunks: [toolCall(0, "c1", "read", '{"path":"big.txt"}'), usage(1500, 20)] }, { chunks: [delta({ content: "ok" }), usage(3100, 5)] }]) await a.engine.prompt("read it") - const system = a.engine.o.system(a.engine.mode, a.engine.model) + const system = a.engine.compose(a.engine.mode) const text = renderBreakdown(contextBreakdown(a.engine, system, [], undefined, a.engine.o.tools)) expect(text).toContain("window 32.8k") expect(text).toContain("system prompt") diff --git a/tests/e2e.test.ts b/tests/e2e.test.ts index c66ea81..3a683e1 100644 --- a/tests/e2e.test.ts +++ b/tests/e2e.test.ts @@ -53,7 +53,7 @@ describe("end to end", () => { expect(fake!.requests[0].messages[0].content).toContain("Permission mode: edit") // knowledge_search/get are offered once any knowledge base has documents, and the suite shares // one LEMBAS_HOME: another file's base must not decide this list (found in the full run). - expect(fake!.requests[0].tools.map((t: any) => t.function.name).filter((n: string) => !n.startsWith("knowledge_"))).toEqual(["read", "write", "edit", "multiedit", "apply_patch", "glob", "grep", "list", "bash", "job_output", "job_list", "job_stop", "job_send", "web_search", "web_fetch", "task", "plan_write", "plan_update", "ask_user", "memory", "session_search", "notes_search", "note_view", "note_manage", "skills_list", "skill_view", "skill_manage", "settings", "recall"]) + expect(fake!.requests[0].tools.map((t: any) => t.function.name).filter((n: string) => !n.startsWith("knowledge_"))).toEqual(["read", "write", "edit", "multiedit", "apply_patch", "glob", "grep", "list", "bash", "job_output", "job_list", "job_stop", "job_send", "web_search", "web_fetch", "task", "plan_write", "plan_update", "ask_user", "plan_submit", "memory", "session_search", "notes_search", "note_view", "note_manage", "skills_list", "skill_view", "skill_manage", "settings", "recall"]) const ends = events.filter((e) => e.type === "tool_end") expect(ends.map((e) => e.type === "tool_end" && e.result.title)).toEqual(["hello.txt · 1–1 of 1", "hello.txt +1 −1"]) const sid = store.sessions(1)[0]!.id @@ -109,16 +109,20 @@ describe("end to end", () => { expect(Bun.YAML.parse(saved)).toEqual({ mode: "manual", permission: { bash: { "echo *": "allow" } } }) }) - test("step ceiling withdraws tools and asks for an answer", async () => { + test("step ceiling: the same tools are listed, the model is asked for an answer, and a call in it is not run", async () => { const dir = setup([ { chunks: [toolCall(0, "c1", "list", "{}")] }, - { chunks: [delta({ content: "summary" })] }, + { chunks: [delta({ content: "summary" }), toolCall(0, "c2", "list", "{}")] }, ]) writeFileSync(join(paths.config, "config.yaml"), "model: fake/m\nlimits: { steps: 1 }\n") const app = createApp({ cwd: dir, mode: "edit", asker: noAsk, store: false }) expect(await app.engine.prompt("go")).toBe("steps") - expect(fake!.requests[1].tools).toBeUndefined() + // The tool list is the session's, the wrap-up's request included (harness context.md §10). + expect(fake!.requests[1].tools).toEqual(fake!.requests[0].tools) expect(fake!.requests[1].messages.at(-1).content).toContain("step limit") + expect(fake!.requests).toHaveLength(2) + const last = app.engine.messages.at(-1)! + expect(last).toEqual({ role: "assistant", parts: [{ type: "text", text: "summary" }] }) }) test("the CLI: lembas run", async () => { @@ -137,9 +141,11 @@ describe("end to end", () => { }) describe("nobody to approve", () => { - test("a final denial withdraws every tool under that permission, and the prompt says so up front", async () => { + test("a final denial withdraws every tool under that permission: still listed, each further call answers the refusal", async () => { + const asked: string[] = [] const dir = setup([ { chunks: [toolCall(0, "c1", "read", '{"path":"hello.txt"}'), toolCall(1, "c2", "edit", JSON.stringify({ path: "hello.txt", old: "world", new: "x" }))] }, + { chunks: [toolCall(0, "c3", "write", JSON.stringify({ path: "hello.txt", content: "x" }))] }, { chunks: [delta({ content: "I would change world to x." })] }, ]) const app = createApp({ @@ -147,16 +153,17 @@ describe("nobody to approve", () => { mode: "manual", store: false, unattended: true, - asker: { ask: async ({ tool }) => ({ kind: "deny", final: true, feedback: `nobody is present to approve ${tool}.` }) }, + asker: { ask: async ({ tool }) => (asked.push(tool), { kind: "deny", final: true, feedback: `nobody is present to approve ${tool}.` }) }, }) await app.engine.prompt("go") expect(fake!.requests[0].messages[0].content).toContain("nobody can approve anything") expect(fake!.requests[0].messages[0].content).toContain("any file change") - const names = fake!.requests[1].tools.map((t: any) => t.function.name) - expect(names).not.toContain("edit") - expect(names).not.toContain("write") - expect(names).toContain("read") - expect(fake!.requests[1].messages.at(-1).content).toContain("no longer offered") + expect(fake!.requests[1].tools).toEqual(fake!.requests[0].tools) + const refusal = fake!.requests[1].messages.at(-1).content as string + expect(refusal).toContain("every further call to write, edit, multiedit, apply_patch answers this same refusal") + // The write is not put to anybody: its answer is that same refusal. + expect(asked).toEqual(["edit"]) + expect(fake!.requests[2].messages.at(-1).content).toBe(refusal.replace(/^\[#0002\]/, "[#0003]")) }) test("a 5xx before any output is retried once, then given up", async () => { diff --git a/tests/harness.test.ts b/tests/harness.test.ts index f1d1809..28aaef9 100644 --- a/tests/harness.test.ts +++ b/tests/harness.test.ts @@ -206,7 +206,7 @@ test("every {{variable}} in a description is declared, and none reaches a model" const plan = normalise({ title: "T", phases: [{ title: "P", tasks: ["a"] }] }) as Plan for (const mode of MODES) { for (const p of [undefined, plan]) { - const system = app.engine.o.system(mode, app.engine.model, p) + const system = app.engine.compose(mode, p) expect(system).not.toContain("{{") expect(system).toContain("Use the instructions below and the tools you have to help with software engineering and with running the project.") expect(system).toContain("- Your output is shown in a terminal and rendered as GitHub-flavoured markdown in a monospace font. Be concise") @@ -221,7 +221,7 @@ test("every {{variable}} in a description is declared, and none reaches a model" } } app.setContext({ instructions: "Be brief.", prototypes: { offered: true, sources: "" } }, "example") - const withContext = app.engine.o.system("manual", app.engine.model, plan) + const withContext = app.engine.compose("manual", plan) expect(withContext).not.toContain("{{") expect(withContext).toContain("## Instructions from example\n\nBe brief.") expect(withContext).toContain("prototype-page") diff --git a/tests/images.test.ts b/tests/images.test.ts index 15d5b17..a01a4de 100644 --- a/tests/images.test.ts +++ b/tests/images.test.ts @@ -59,6 +59,8 @@ describe("images", () => { const atts = att("see @shot.png", app.engine.o.toolCtx, true) await app.engine.prompt("see @shot.png", atts.flatMap((a) => (a.image ? [a.text, a.image] : [a.text]))) const content = fake!.requests[0].messages.at(-1).content - expect(content.map((p: any) => p.type)).toEqual(["text", "text", "image_url"]) + // The session's first user message also carries the date, as a note after everything else. + expect(content.map((p: any) => p.type)).toEqual(["text", "text", "image_url", "text"]) + expect(content.at(-1).text).toStartWith("\nToday is ") }) }) diff --git a/tests/lembas-library.test.ts b/tests/lembas-library.test.ts index 4a477ab..5de5e87 100644 --- a/tests/lembas-library.test.ts +++ b/tests/lembas-library.test.ts @@ -82,8 +82,8 @@ test("memory is the server's; notes, knowledge and skills stay this machine's", // The entries for the prompt: read from the resource as the session started. for (let i = 0; i < 50 && !lembas.calls.some((c) => c.uri); i++) await Bun.sleep(10) expect(lembas.calls.find((c) => c.uri)).toEqual({ auth: "Bearer lmb_lib", uri: "lembas://memory" }) - for (let i = 0; i < 50 && !app.engine.o.system("manual", app.engine.model).includes("Prefers tabs."); i++) await Bun.sleep(10) - expect(app.engine.o.system("manual", app.engine.model)).toContain("MEMORY (the account's)\nPrefers tabs.") + for (let i = 0; i < 50 && !app.engine.compose("manual").includes("Prefers tabs."); i++) await Bun.sleep(10) + expect(app.engine.compose("manual")).toContain("MEMORY (the account's)\nPrefers tabs.") await app.close() }) @@ -106,7 +106,7 @@ test("a session the client opened sends its id on every memory call and on the r for (let i = 0; i < 50 && !lembas.calls.some((c) => c.uri && c.meta); i++) await Bun.sleep(10) expect(lembas.calls.find((c) => c.uri && c.meta)).toEqual({ auth: "Bearer lmb_lib", uri: "lembas://memory", meta: { scope: app.engine.sessionId } }) // Knowledge is this machine's: nothing the server holds is named to the model. - expect(app.engine.o.system("manual", app.engine.model)).not.toContain("Knowledge searches here cover only") + expect(app.engine.compose("manual")).not.toContain("Knowledge searches here cover only") }) test("without a login it says so and keeps this machine's memory", () => { diff --git a/tests/library.test.ts b/tests/library.test.ts index 9a3141f..d98e56c 100644 --- a/tests/library.test.ts +++ b/tests/library.test.ts @@ -155,10 +155,11 @@ test("the tools: knowledge_search is offered only with something to search; a pr lib.putDocument("private", { title: "Diary", source: "d", text: "The pump primes in my dreams." }) await app.engine.prompt("anything") expect(offered()).not.toContain("knowledge_search") - // A document in the project's base: offered, and only that base is searched. + // A document in the project's base: offered from the next session — the tool list is the + // session's, decided as it starts (harness context.md §10) — and only that base is searched. const pump = lib.putDocument("manuals", { title: "Pump", source: "p", text: "The pump primes in thirty seconds." }) fake.requests.length = 0 - app.engine.messages = [] + app.newSession() await app.engine.prompt("how long does the pump take to prime?") expect(offered()).toContain("knowledge_search") const results = app.engine.messages.filter((m) => m.role === "tool").map((m) => (m.role === "tool" ? m.content : "")) diff --git a/tests/plan.test.ts b/tests/plan.test.ts index 08b9aba..72f36c1 100644 --- a/tests/plan.test.ts +++ b/tests/plan.test.ts @@ -7,6 +7,7 @@ import type { Asker, AskReply } from "../src/bus/index.ts" import { paths } from "../src/config/paths.ts" import type { PlanReply, PlanRequest } from "../src/tool/plan_submit.ts" import { delta, fakeProvider, toolCall, type Fake } from "./fake-provider.ts" +import { TOOL_SPECS } from "../src/harness.ts" let fake: Fake | undefined afterEach(() => fake?.stop()) @@ -29,7 +30,7 @@ const exitPlan = () => toolCall(0, "x1", "plan_submit", JSON.stringify({ path: " const names = (r: any) => r.tools.map((t: any) => t.function.name) describe("plan mode", () => { - test("plan_write, plan_submit, approve → edit mode in the same turn, and plan_submit is no longer offered", async () => { + test("plan_write, plan_submit, approve → edit mode in the same turn; the prompt and tools stay as they were", async () => { const seen: PlanRequest[] = [] const a = app([{ chunks: [writePlan()] }, { chunks: [exitPlan()] }, { chunks: [delta({ content: "Implementing." })] }], async (r) => { seen.push(r) @@ -48,8 +49,11 @@ describe("plan mode", () => { expect(names(fake!.requests[0])).toContain("plan_submit") expect(names(fake!.requests[0])).toContain("plan_write") expect(fake!.requests[0].messages[0].content).toContain("Permission mode: plan") - expect(fake!.requests[2].messages[0].content).toContain("Permission mode: edit") - expect(names(fake!.requests[2])).not.toContain("plan_submit") + // Spec 3.1.0 (context.md §10): the system prompt and the tool list are the session's, byte for + // byte; the approval's own result tells the model the mode, so no note says it again. + expect(fake!.requests[2].messages[0]).toEqual(fake!.requests[0].messages[0]) + expect(fake!.requests[2].tools).toEqual(fake!.requests[0].tools) + expect(names(fake!.requests[2])).toContain("plan_submit") // plan_write and plan_update are offered in every mode: the plan is kept current while implementing. expect(names(fake!.requests[2])).toContain("plan_update") expect(fake!.requests[2].messages.at(-1).content).toContain("The user approved the plan") @@ -69,13 +73,20 @@ describe("plan mode", () => { expect(b.engine.mode).toBe("plan") }) - test("plan_submit with no plan written is an error naming plan_write; edit mode does not offer it", async () => { - const a = app([{ chunks: [exitPlan()] }, { chunks: [delta({ content: "ok" })] }], async () => ({ kind: "approve", mode: "edit" })) + test("plan_submit with no plan written is an error naming plan_write; outside plan mode it is offered and answers unavailable", async () => { + const a = app([{ chunks: [exitPlan()] }, { chunks: [delta({ content: "ok" })] }, { chunks: [exitPlan()] }, { chunks: [delta({ content: "carrying on" })] }], async () => ({ kind: "approve", mode: "edit" })) await a.engine.prompt("go") expect(fake!.requests[1].messages.at(-1).content).toContain("Write it with plan_write") expect(a.engine.mode).toBe("plan") a.engine.mode = "edit" - await a.engine.prompt("again").catch(() => {}) - expect(names(fake!.requests.at(-1))).not.toContain("plan_submit") + await a.engine.prompt("again") + const [again, after] = fake!.requests.slice(-2) + expect(names(again)).toContain("plan_submit") + expect(again.tools).toEqual(fake!.requests[0].tools) + expect(again.messages[0]).toEqual(fake!.requests[0].messages[0]) + // The person's mode change comes as a note on their next message, the mode's own text with it. + expect(again.messages.at(-1).content).toStartWith("again\n\nThe permission mode is now edit:\nPermission mode: edit.") + expect(after.messages.at(-1).content).toBe(`[#0002]\n${TOOL_SPECS.plan_submit!.unavailable}`) + expect(a.engine.mode).toBe("edit") }) }) diff --git a/tests/prefix.test.ts b/tests/prefix.test.ts new file mode 100644 index 0000000..75cc34f --- /dev/null +++ b/tests/prefix.test.ts @@ -0,0 +1,232 @@ +// The prefix (harness spec 3.1.0, context.md §10): the system prompt and the tool list are made once +// per session and resent byte for byte; what changes during it goes as a note on the next user +// message; compaction asks for its summary as the next turn of the same conversation; every +// request's cached input tokens are kept. These drive the real engine against a scripted server. +import { afterEach, expect, test } from "bun:test" +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { createApp } from "../src/app.ts" +import type { AskReply } from "../src/bus/index.ts" +import { paths } from "../src/config/paths.ts" +import { TOOL_SPECS } from "../src/harness.ts" +import { Store } from "../src/session/store.ts" +import { isNote, NOTE, noteFor, PREFIX_META, summaryOf, toldAt, type Kept } from "../src/session/prefix.ts" +import { delta, fakeProvider, toolCall, usage, type Fake } from "./fake-provider.ts" + +let fake: Fake | undefined +afterEach(() => fake?.stop()) + +const once = { ask: async (): Promise => ({ kind: "once" }) } + +function setup(script: Parameters[0], extra = "") { + fake = fakeProvider(script) + mkdirSync(paths.config, { recursive: true }) + writeFileSync(join(paths.config, "connections.yaml"), `connections:\n fake:\n type: webui\n url: ${fake.url}\n models:\n fake/m: { context: 32768 }\n fake/n: { context: 32768 }\n`, { mode: 0o600 }) + writeFileSync(join(paths.config, "config.yaml"), `model: fake/m\ntitles: prompt\n${extra}`) + const dir = mkdtempSync(join(tmpdir(), "lembas-prefix-")) + Bun.spawnSync(["git", "init", "-q", dir]) + writeFileSync(join(dir, "AGENTS.md"), "# Demo\nBuild with make.\n") + return dir +} + +const reply = (text: string) => ({ chunks: [delta({ content: text }, "stop")] }) +const system = (i: number) => fake!.requests.at(i)!.messages[0].content as string +const lastUser = (i: number) => { + const m = fake!.requests.at(i)!.messages + return [...m].reverse().find((x: { role: string }) => x.role === "user").content as string +} +const names = (i: number) => (fake!.requests.at(i)!.tools as { function: { name: string } }[]).map((t) => t.function.name) + +test("no clock in the prompt: two requests across a date change send the same system bytes, and the date goes as a note", async () => { + const dir = setup([reply("one"), reply("two"), reply("three")]) + const app = createApp({ cwd: dir, asker: once, store: false, snapshots: false }) + let day = "2026-11-04" + app.engine.o.today = () => day + await app.engine.prompt("first") + await app.engine.prompt("second") + day = "2026-11-05" + await app.engine.prompt("third") + expect(system(0)).not.toContain("Today") + expect(system(0)).not.toContain("2026-11-0") + expect(system(1)).toBe(system(0)) + expect(system(2)).toBe(system(0)) + expect(fake!.requests[2].tools).toEqual(fake!.requests[0].tools) + expect(lastUser(0)).toBe(`first\n${NOTE.open}\nToday is 2026-11-04.\n${NOTE.close}`) + expect(lastUser(1)).toBe("second") + expect(lastUser(2)).toBe(`third\n${NOTE.open}\nToday is 2026-11-05.\n${NOTE.close}`) + // A note is part of its message from then on: the next request sends the first one as it was. + expect(fake!.requests[2].messages[1].content).toBe(lastUser(0)) +}) + +test("a tool step carries no note; the note waits for the next user message", async () => { + const dir = setup([{ chunks: [toolCall(0, "c1", "read", '{"path":"AGENTS.md"}')] }, reply("read it"), reply("ok")]) + const app = createApp({ cwd: dir, asker: once, store: false, snapshots: false }) + let day = "2026-11-04" + app.engine.o.today = () => day + app.bus.on((e) => { + // The clock passes midnight while the tool runs. + if (e.type === "tool_start") day = "2026-11-05" + }) + await app.engine.prompt("look") + expect(fake!.requests[1].messages.at(-1).role).toBe("tool") + expect(system(1)).toBe(system(0)) + await app.engine.prompt("and now") + expect(lastUser(2)).toBe(`and now\n${NOTE.open}\nToday is 2026-11-05.\n${NOTE.close}`) +}) + +test("the person's changes come as one note in the spec's order; the model's own do not", async () => { + const dir = setup([ + { chunks: [toolCall(0, "w1", "plan_write", JSON.stringify({ title: "Fix", phases: [{ title: "P", tasks: ["a"] }] }))] }, + { chunks: [toolCall(0, "e1", "edit", JSON.stringify({ path: "AGENTS.md", old: "Build with make.", new: "Build with make; test with make test." }))] }, + reply("done"), + reply("ok"), + reply("fine"), + ]) + const app = createApp({ cwd: dir, asker: once, mode: "edit", store: false, snapshots: false }) + app.engine.o.today = () => "2026-11-05" + // The model reads AGENTS.md before it may edit it: read-before-edit is the tool's rule. + app.engine.o.toolCtx.readFiles.add(join(dir, "AGENTS.md")) + await app.engine.prompt("plan and fix") + // Its own plan and its own edit: their results told it. + await app.engine.prompt("thanks") + expect(lastUser(3)).toBe("thanks") + // Between turns the person changes the mode, the plan, an instruction file, and the personality. + app.engine.mode = "manual" + app.engine.setPlan(null) + writeFileSync(join(dir, "AGENTS.md"), "# Demo\nBuild with ninja.\n") + app.settings.set("personality", "concise", "session") + await app.engine.prompt("again") + const note = lastUser(4).slice("again\n".length) + expect(isNote(note)).toBe(true) + const order = ["The permission mode is now manual:\nPermission mode: manual.", "The person changed the plan; it is now:\n(none)", `${join(dir, "AGENTS.md")} changed; it now reads:\n# Demo\nBuild with ninja.`, "The person chose a different personality:\nBe brief."] + for (const line of order) expect(note).toContain(line) + expect(order.map((l) => note.indexOf(l))).toEqual([...order.map((l) => note.indexOf(l))].sort((a, b) => a - b)) + expect(system(4)).toBe(system(0)) +}) + +test("plan_submit is in the list in every mode, and answers its unavailable text outside plan mode", async () => { + const dir = setup([{ chunks: [toolCall(0, "x1", "plan_submit", "{}")] }, reply("carrying on")]) + const app = createApp({ cwd: dir, asker: once, mode: "manual", store: false, snapshots: false }) + await app.engine.prompt("submit it") + expect(names(0)).toContain("plan_submit") + expect(fake!.requests[1].messages.at(-1).content).toBe(`[#0001]\n${TOOL_SPECS.plan_submit!.unavailable}`) + // A terminal session can never show a file: show is not in its list at all. + expect(names(0)).not.toContain("show") +}) + +test("compaction: the same system and tools, every message, then the compaction text; a call in the reply is not run, and the request is no turn", async () => { + const dir = setup([ + reply("The answer is 42."), + { chunks: [delta({ content: "## What we are doing\nFinding the answer." }), toolCall(0, "c1", "bash", '{"command":"rm -rf x","description":"x"}'), usage(900, 40)] }, + reply("ok"), + ]) + const store = new Store(":memory:") + const app = createApp({ cwd: dir, asker: { ask: async () => { throw new Error("nothing may ask") } }, store, snapshots: false }) + await app.engine.prompt("what is the answer") + const sid = app.engine.sessionId! + const summary = await app.engine.compact() + const [first, compaction] = [fake!.requests[0], fake!.requests[1]] + expect(compaction.messages[0]).toEqual(first.messages[0]) + expect(compaction.tools).toEqual(first.tools) + expect(compaction.messages.slice(1, -1).map((m: { content: string }) => m.content)).toEqual([first.messages[1].content, "The answer is 42."]) + expect(compaction.messages.at(-1).content).toStartWith("Summarise the conversation") + expect(compaction.messages.at(-1).content).not.toContain("User: what is the answer") + // The text before the call is the summary; the call was not run. + expect(summary).toStartWith("## What we are doing\nFinding the answer.") + expect(store.messages(sid).map((m) => (m.role === "tool" ? "tool" : m.parts.map((p) => (p.type === "text" ? p.text : p.type)).join("|")))).not.toContainEqual(expect.stringContaining("Summarise")) + expect(store.messages(sid).some((m) => m.role === "tool")).toBe(false) + // Its usage is kept, as the compaction's. + expect(store.requests(sid).map((r) => r.kind)).toEqual(["step", "compaction"]) + // After it the prompt is made again, and the next user message says the date again. + await app.engine.prompt("go on") + expect(lastUser(2)).toMatch(/^go on\n\nToday is \d{4}-\d{2}-\d{2}\.\n<\/session-note>$/) +}) + +test("a resumed session sends the prompt it kept, byte for byte; what changed meanwhile is a note", async () => { + const dir = setup([reply("one"), reply("two")]) + const store = new Store(":memory:") + const a = createApp({ cwd: dir, asker: once, store, snapshots: false }) + await a.engine.prompt("first") + const sid = a.engine.sessionId! + const kept = store.meta(sid, PREFIX_META)! + expect(kept.system).toBe(system(0)) + await a.close() + writeFileSync(join(dir, "AGENTS.md"), "# Demo\nBuild with ninja.\n") + const b = createApp({ cwd: dir, asker: once, store, snapshots: false, resuming: sid }) + b.resume(sid) + await b.engine.prompt("second") + expect(system(1)).toBe(system(0)) + expect(system(1)).toContain("Build with make.") + expect(lastUser(1)).toContain(`${join(dir, "AGENTS.md")} changed; it now reads:\n# Demo\nBuild with ninja.`) + // Shown as typed: the note is not the person's words. + expect(b.engine.messages.filter((m) => m.role === "user").length).toBe(2) + await b.close() +}) + +test("a model switch makes the prompt again for the new model; the date already said holds", async () => { + const dir = setup([reply("one"), reply("two")]) + const app = createApp({ cwd: dir, asker: once, store: false, snapshots: false }) + await app.engine.prompt("first") + app.switchModel("fake/n") + await app.engine.prompt("second") + expect(system(0)).toContain("Model: fake/m") + expect(system(1)).toContain("Model: fake/n") + expect(lastUser(1)).toBe("second") +}) + +test("cached input tokens are kept with every request's usage and shown by /context", async () => { + const dir = setup([ + { chunks: [delta({ content: "one" }), { choices: [], usage: { prompt_tokens: 1000, completion_tokens: 5, prompt_tokens_details: { cached_tokens: 0 } } }] }, + // llama.cpp says it in its timings when the usage does not. + { chunks: [delta({ content: "two" }), usage(1100, 5), { choices: [], timings: { cache_n: 1000, prompt_n: 100, predicted_n: 5 } }] }, + ]) + const store = new Store(":memory:") + const app = createApp({ cwd: dir, asker: once, store, snapshots: false }) + await app.engine.prompt("first") + await app.engine.prompt("second") + expect(store.requests(app.engine.sessionId!).map((r) => [r.kind, r.input, r.cached])).toEqual([ + ["step", 1000, 0], + ["step", 1100, 1000], + ]) + const { contextBreakdown, renderBreakdown } = await import("../src/session/context.ts") + expect(renderBreakdown(contextBreakdown(app.engine, app.engine.systemPrompt(), [], undefined, app.engine.o.tools))).toContain("last request: 1.1k in, 1.0k of it cached (91%)") + await app.close() +}) + +test("noteFor: nothing changed says nothing; an MCP server that connected later is said by its own instructions", () => { + const now = { date: "2026-11-05", mode: "manual" as const, plan: "", memory: "", personality: "", files: {}, mcp: { docs: "Search the docs." } } + const told = { ...toldAt(now), date: "2026-11-05" } + expect(noteFor(told, now, () => "").note).toBe("") + const later = { ...now, mcp: { ...now.mcp, tickets: "Name the ticket." } } + expect(noteFor(told, later, () => "").note).toBe(`${NOTE.open}\nInstructions from an MCP server that connected since the session began:\n\nName the ticket.\n\n${NOTE.close}`) + expect(summaryOf([{ type: "reasoning", text: "x" }, { type: "text", text: " S " }, { type: "tool_call" }, { type: "text", text: "after" }])).toBe("S") +}) + +test("the first request waits for the configured MCP servers (up to loop.json's mcp_wait_ms): their tools and instructions are in the first list and prompt", async () => { + const BUN = process.execPath + const SERVER = join(import.meta.dir, "fixtures", "mcp", "server.ts") + const dir = setup([reply("one")], `mcp:\n fix:\n command: ["${BUN}", "${SERVER}"]\n`) + const app = createApp({ cwd: dir, asker: once, store: false, snapshots: false }) + // Not waiting for app.mcpReady here: the engine does, before it makes the prompt. + await app.engine.prompt("first") + expect(names(0)).toContain("mcp__fix__echo") + expect(system(0)).toContain("") + await app.close() + writeFileSync(join(paths.config, "config.yaml"), "") +}) + +test("an empty value in a note reads loop.json's context.note.empty", () => { + const now = { date: "2026-11-05", mode: "manual" as const, plan: "", memory: "", personality: "", files: {}, mcp: {} } + const told = { ...toldAt({ ...now, memory: "Prefers tabs." }), date: "2026-11-05" } + expect(noteFor(told, now, () => "").note).toBe(`${NOTE.open}\nThe person changed what is remembered; it is now:\n${NOTE.empty}\n${NOTE.close}`) +}) + +test("compaction's ask has no previous summary and no transcript heading", async () => { + const { compactAsk } = await import("../src/session/prefix.ts") + const ask = compactAsk() + expect(ask).not.toContain("## Transcript") + expect(ask).not.toContain("Summary of even earlier turns") + expect(ask).not.toMatch(/\n{3,}/) + expect(ask).toEndWith("stay under 600 words.") +}) diff --git a/tests/subagent.test.ts b/tests/subagent.test.ts index 7ecc5b8..54273cc 100644 --- a/tests/subagent.test.ts +++ b/tests/subagent.test.ts @@ -63,7 +63,8 @@ describe("subagents", () => { const child = fake!.requests[1] expect(child.messages[0].content).toContain('You are the "explore" subagent') expect(child.messages[0].content).not.toContain("Permission mode: plan") - expect(child.messages.at(-1).content).toBe("Find why average is wrong in calc.py") + // Its prompt has no date either: its first message says it (harness context.md §10). + expect(child.messages.at(-1).content).toMatch(/^Find why average is wrong in calc\.py\n\nToday is \d{4}-\d{2}-\d{2}\.\n<\/session-note>$/) // its own job tools for commands it starts, and report for a word to the main agent expect(names(child)).toEqual(["read", "glob", "grep", "list", "bash", "job_output", "job_list", "job_stop", "web_search", "web_fetch", "report"]) // the preamble is the spec's text; a subagent is told nothing of plan_write or ask_user diff --git a/tests/trust-hardening.test.ts b/tests/trust-hardening.test.ts index 1c28479..d0cf259 100644 --- a/tests/trust-hardening.test.ts +++ b/tests/trust-hardening.test.ts @@ -47,5 +47,5 @@ test("an untrusted project's task board is neither in the prompt nor writable", setTrust(root, "readonly") const app = createApp({ cwd: root, store: false, snapshots: false, asker: { ask: async () => ({ kind: "once" }) } }) expect(app.engine.o.toolCtx.projectDir).toBeUndefined() - expect(app.engine.o.system("plan", app.engine.model)).not.toContain("IGNORE PREVIOUS INSTRUCTIONS") + expect(app.engine.compose("plan")).not.toContain("IGNORE PREVIOUS INSTRUCTIONS") }) diff --git a/tests/tui/app.test.tsx b/tests/tui/app.test.tsx index 1cb04ee..0542220 100644 --- a/tests/tui/app.test.tsx +++ b/tests/tui/app.test.tsx @@ -104,7 +104,11 @@ test("/compact replaces the conversation with a summary", async () => { await t.input.typeText("/compact") t.input.pressEnter() expect(await t.until("compacted")).toBe(true) - expect(t.fake.requests[1].messages[0].content).toContain("## Transcript") + // Asked as the next turn of the same conversation: the same system prompt, the messages, then the + // compaction text (harness context.md §7). + expect(t.fake.requests[1].messages[0]).toEqual(t.fake.requests[0].messages[0]) + expect(t.fake.requests[1].messages.at(-2).content).toBe("The answer is 42.") + expect(t.fake.requests[1].messages.at(-1).content).toStartWith("Summarise the conversation") expect(t.app.engine.messages[0]).toMatchObject({ role: "user" }) expect(JSON.stringify(t.app.engine.messages[0])).toContain("It is 42.") }) @@ -429,8 +433,10 @@ test("/review sends the session's diff with the review brief in plan mode, then t.input.pressEnter() expect(await t.until("No findings.")).toBe(true) const req = t.fake.requests.at(-1) - expect(req.messages[0].content).toContain("Permission mode: plan") + // The session's prompt as it was made; plan mode comes as a note on the review's message. + expect(req.messages[0]).toEqual(t.fake.requests[0].messages[0]) const user = req.messages.at(-1).content as string + expect(user).toContain("\nThe permission mode is now plan:\nPermission mode: plan.") expect(user).toContain("Findings first, ordered by severity") expect(user).toContain("+hello there") expect(t.frame()).toContain("┃ /review")