diff --git a/harness/conformance/paths.json b/harness/conformance/paths.json new file mode 100644 index 0000000..a3b0dc1 --- /dev/null +++ b/harness/conformance/paths.json @@ -0,0 +1,89 @@ +{ + "about": "Where an agent session works, as the model is told it and as its tools read a path (context.md §11, spec 3.2.0). `op: env` — `root` is the project root and `started` the directory the session was started in (both absolute); the answer is the environment block's lines about them, in order, from loop.json context.env — none naming an absolute path. `op: resolve` — `path` as a tool call gives it; the answer is the absolute path it means: relative paths against the project root, whatever `started` is, `.` and `..` by their text (filesystem.md, Paths); an absolute path as it is. LLeMbas: not run — a chat's paths are the server's tree's; CLI: prompt/assemble.ts, tool/tool.ts.", + "cases": [ + { + "about": "Started at the root: the project's name and nothing else.", + "op": "env", + "root": "/home/ada/src/app", + "started": "/home/ada/src/app", + "expect": [ + "Project: app (paths are relative to its root)" + ] + }, + { + "about": "Started below the root: that directory, relative to the root.", + "op": "env", + "root": "/home/ada/src/app", + "started": "/home/ada/src/app/packages/web", + "expect": [ + "Project: app (paths are relative to its root)", + "Started in: packages/web (relative to the project root)" + ] + }, + { + "about": "A trailing slash changes nothing.", + "op": "env", + "root": "/srv/work/site/", + "started": "/srv/work/site", + "expect": [ + "Project: site (paths are relative to its root)" + ] + }, + { + "about": "A relative path is resolved against the project root.", + "op": "resolve", + "root": "/home/ada/src/app", + "started": "/home/ada/src/app", + "path": "src/main.ts", + "expect": "/home/ada/src/app/src/main.ts" + }, + { + "about": "…also when the session started below the root: one base for every relative path.", + "op": "resolve", + "root": "/home/ada/src/app", + "started": "/home/ada/src/app/packages/web", + "path": "src/main.ts", + "expect": "/home/ada/src/app/src/main.ts" + }, + { + "about": "The directory the session started in is reached by its path from the root.", + "op": "resolve", + "root": "/home/ada/src/app", + "started": "/home/ada/src/app/packages/web", + "path": "packages/web/index.ts", + "expect": "/home/ada/src/app/packages/web/index.ts" + }, + { + "about": "An absolute path is still accepted, as it is.", + "op": "resolve", + "root": "/home/ada/src/app", + "started": "/home/ada/src/app/packages/web", + "path": "/etc/hosts", + "expect": "/etc/hosts" + }, + { + "about": ". is the root.", + "op": "resolve", + "root": "/home/ada/src/app", + "started": "/home/ada/src/app/packages/web", + "path": ".", + "expect": "/home/ada/src/app" + }, + { + "about": ".. is resolved by its text (whether it may be reached is the permission rules' business).", + "op": "resolve", + "root": "/home/ada/src/app", + "started": "/home/ada/src/app", + "path": "../lib/x.ts", + "expect": "/home/ada/src/lib/x.ts" + }, + { + "about": "./ and repeated slashes are resolved by their text.", + "op": "resolve", + "root": "/home/ada/src/app", + "started": "/home/ada/src/app", + "path": "./docs//guide.md", + "expect": "/home/ada/src/app/docs/guide.md" + } + ] +} diff --git a/harness/conformance/reasoning.json b/harness/conformance/reasoning.json new file mode 100644 index 0000000..fca25f3 --- /dev/null +++ b/harness/conformance/reasoning.json @@ -0,0 +1,323 @@ +{ + "about": "What a request sends back of the model's reasoning (context.md §8, spec 3.2.0). `messages` is the history a request is built from, in order: `role` user, assistant or tool; `content`; an assistant message's stored `reasoning` (absent: it came with none); a user message's `turn` — false for a delivery at a step boundary (a job's note, a team message, words added to a running turn), which does not start a turn; absent or true for one that does (the person's, or a machine message starting a turn of its own). The answer is the messages as the OpenAI chat dialect sends them — `role`, `content`, and `reasoning_content` on an assistant message after the last user message that started a turn, when its reasoning is not empty (tool calls and ids are left out of the cases: they are sent as always). The Anthropic dialect sends the same messages' thinking blocks. LLeMbas: services/llm/openai_client.py, services/generation.py; CLI: provider/openai-chat.ts, session/engine.ts.", + "cases": [ + { + "about": "One turn, three steps: every reply of the turn carries its reasoning back.", + "messages": [ + { + "role": "user", + "content": "Fix the failing test." + }, + { + "role": "assistant", + "content": "", + "reasoning": "The test name points at the parser." + }, + { + "role": "tool", + "content": "[#0001 src/parse.ts lines 1-40 of 40 sha 1c9e04ab]\n…" + }, + { + "role": "assistant", + "content": "", + "reasoning": "Line 12 drops the last token." + }, + { + "role": "tool", + "content": "[#0002 src/parse.ts sha 7d0a91c3]\n@@ -12 +12 @@" + } + ], + "expect": [ + { + "role": "user", + "content": "Fix the failing test." + }, + { + "role": "assistant", + "content": "", + "reasoning_content": "The test name points at the parser." + }, + { + "role": "tool", + "content": "[#0001 src/parse.ts lines 1-40 of 40 sha 1c9e04ab]\n…" + }, + { + "role": "assistant", + "content": "", + "reasoning_content": "Line 12 drops the last token." + }, + { + "role": "tool", + "content": "[#0002 src/parse.ts sha 7d0a91c3]\n@@ -12 +12 @@" + } + ] + }, + { + "about": "A second turn: the first turn's replies go without their reasoning, the current one's with it.", + "messages": [ + { + "role": "user", + "content": "Where is the config loaded?" + }, + { + "role": "assistant", + "content": "", + "reasoning": "Search for load." + }, + { + "role": "tool", + "content": "[#0001] src/config/load.ts:40" + }, + { + "role": "assistant", + "content": "In src/config/load.ts.", + "reasoning": "Found it at line 40." + }, + { + "role": "user", + "content": "And who calls it?" + }, + { + "role": "assistant", + "content": "", + "reasoning": "Grep for loadConfig." + }, + { + "role": "tool", + "content": "[#0002] src/app.ts:120" + } + ], + "expect": [ + { + "role": "user", + "content": "Where is the config loaded?" + }, + { + "role": "assistant", + "content": "" + }, + { + "role": "tool", + "content": "[#0001] src/config/load.ts:40" + }, + { + "role": "assistant", + "content": "In src/config/load.ts." + }, + { + "role": "user", + "content": "And who calls it?" + }, + { + "role": "assistant", + "content": "", + "reasoning_content": "Grep for loadConfig." + }, + { + "role": "tool", + "content": "[#0002] src/app.ts:120" + } + ] + }, + { + "about": "A job's note delivered at a step boundary does not start a turn: the turn's earlier replies keep their reasoning.", + "messages": [ + { + "role": "user", + "content": "Run the tests in the background and fix the lint." + }, + { + "role": "assistant", + "content": "", + "reasoning": "Start the tests, then lint." + }, + { + "role": "tool", + "content": "[#0001] job1 started" + }, + { + "role": "user", + "content": "[A background job you started has finished:]\njob1 (npm test) exited with 0 after 5s.", + "turn": false + }, + { + "role": "assistant", + "content": "", + "reasoning": "Tests pass; now the lint." + }, + { + "role": "tool", + "content": "[#0002] 0 problems" + } + ], + "expect": [ + { + "role": "user", + "content": "Run the tests in the background and fix the lint." + }, + { + "role": "assistant", + "content": "", + "reasoning_content": "Start the tests, then lint." + }, + { + "role": "tool", + "content": "[#0001] job1 started" + }, + { + "role": "user", + "content": "[A background job you started has finished:]\njob1 (npm test) exited with 0 after 5s." + }, + { + "role": "assistant", + "content": "", + "reasoning_content": "Tests pass; now the lint." + }, + { + "role": "tool", + "content": "[#0002] 0 problems" + } + ] + }, + { + "about": "A machine message that starts a turn of its own (a team report to an idle lead) does: the replies before it go without.", + "messages": [ + { + "role": "user", + "content": "Split the work." + }, + { + "role": "assistant", + "content": "Started alice.", + "reasoning": "Add one member." + }, + { + "role": "user", + "content": "[Team report from alice to lead:]\nDone.", + "turn": true + }, + { + "role": "assistant", + "content": "", + "reasoning": "Review alice's change." + }, + { + "role": "tool", + "content": "[#0001 src/a.ts lines 1-10 of 10 sha 52b6e3a0]\n…" + } + ], + "expect": [ + { + "role": "user", + "content": "Split the work." + }, + { + "role": "assistant", + "content": "Started alice." + }, + { + "role": "user", + "content": "[Team report from alice to lead:]\nDone." + }, + { + "role": "assistant", + "content": "", + "reasoning_content": "Review alice's change." + }, + { + "role": "tool", + "content": "[#0001 src/a.ts lines 1-10 of 10 sha 52b6e3a0]\n…" + } + ] + }, + { + "about": "A reply that came with no reasoning, or an empty one, carries no field.", + "messages": [ + { + "role": "user", + "content": "Hello" + }, + { + "role": "assistant", + "content": "", + "reasoning": "Greet." + }, + { + "role": "tool", + "content": "[#0001] ok" + }, + { + "role": "assistant", + "content": "Hi." + }, + { + "role": "user", + "content": "Again" + }, + { + "role": "assistant", + "content": "Hi again.", + "reasoning": "" + } + ], + "expect": [ + { + "role": "user", + "content": "Hello" + }, + { + "role": "assistant", + "content": "" + }, + { + "role": "tool", + "content": "[#0001] ok" + }, + { + "role": "assistant", + "content": "Hi." + }, + { + "role": "user", + "content": "Again" + }, + { + "role": "assistant", + "content": "Hi again." + } + ] + }, + { + "about": "The last message the person wrote starts the turn even when nothing has answered it yet: every earlier reply goes without.", + "messages": [ + { + "role": "user", + "content": "One" + }, + { + "role": "assistant", + "content": "Done.", + "reasoning": "Easy." + }, + { + "role": "user", + "content": "Two" + } + ], + "expect": [ + { + "role": "user", + "content": "One" + }, + { + "role": "assistant", + "content": "Done." + }, + { + "role": "user", + "content": "Two" + } + ] + } + ] +} diff --git a/harness/context.md b/harness/context.md index 7dbbe84..825e0c3 100644 --- a/harness/context.md +++ b/harness/context.md @@ -1,6 +1,6 @@ # Context hygiene -Spec 3.1.0. What a session does to keep tool results from filling the model's context, and (§10, from +Spec 3.2.0. What a session does to keep tool results from filling the model's context, and (§10, from 3.1.0) what it keeps still so the model's server can reuse what it has already read — the same in a chat and in an agent session, in both projects. The texts and numbers are `loop.json` `context`; the cases are `conformance/context.json`; what the model is told is `prompts/blocks/context.md`. @@ -12,7 +12,8 @@ rewritten.** Changing earlier context invalidates the prompt cache from that poi server processes everything after it again, and a cloud API charges full input price for it again — on a long context that costs more than the measure saves. So no earlier result is ever shortened, replaced by a stub or removed afterwards. The one deliberate break is compaction, which already -replaces the history with a summary. +replaces the history with a summary. (From 3.2.0 a turn's reasoning is sent back during the turn and +not after it, §8: the common chat templates render the history the same either way.) ## 1. Every result has an id @@ -215,16 +216,67 @@ model): a date that changed at midnight sent a 16,912-token session from 99 % ca What a server does with a matching prefix is its own business; a harness that keeps these rules has done what it can. -## 8. Earlier reasoning +## 8. Reasoning goes back to the model during its turn -Left to the model's own chat template: the harness sends the history as it does, and neither strips -reasoning a template keeps nor adds what it drops. +From 3.2.0 (it was left to the chat template before). A model that thinks before each tool call +works on from what it thought: sent its own reasoning back, it carries on; sent its calls without it, +it reasons its way to them again, step after step. Measured on a local model: a step re-read about a +fifth fewer tokens with its reasoning sent back. + +- **The current turn** is every message after the last one that started a turn: a message the + person wrote, or a machine message that starts a turn of its own (a job's or a team's note to an + idle session, `loop.json` `jobs`). What is delivered at a step boundary while a turn runs — a + job's note, a team message, words the person adds — does not start a turn, and a tool result + never does. +- **Every assistant message of the current turn carries the reasoning it came with**, as the + model gave it, in full: in the OpenAI chat dialect as `reasoning_content` on the assistant message + (beside its `content` and `tool_calls`), and in the Anthropic dialect as its thinking blocks with + their signatures, unchanged and first (which that API already requires for tool use). Reasoning + read from inline tags (``; `quirks.json` `reasoning-fields`) goes back the same way, as + `reasoning_content`, and never as text in `content`. A message that came with no reasoning + carries none — no empty field. +- **Assistant messages of earlier turns carry none.** The common templates (Qwen3.x, Nemotron's + `truncate_history_thinking`, gpt-oss) drop it there themselves, so what the server reads of the + history is the same bytes either way and nothing is re-read; with a template that would keep it, + the turn boundary re-reads at most the previous turn. A compaction's summary starts the history + afresh: nothing before it carries reasoning. +- **What is stored is unchanged**: a session keeps each message's reasoning as before; this rule + is only about what a request carries (`conformance/reasoning.json`). +- A subagent, a team member and the main agent each have their own turns: a member's turn starts + with its brief or with a message that woke it. + +## 11. No absolute paths in the prompt + +From 3.2.0. An absolute path costs tokens on every request, says where a machine keeps its files, +and is copied by the model into calls where a relative one would do. So in an agent session (a +device's disk): + +- **The system prompt, every note and the harness's own task messages (`tasks/init.md`) name no + absolute path of the machine.** The environment + block names the project by its root's last path component and says paths are relative to its + root (`loop.json` `context.env`): `Project: app (paths are relative to its root)`. Where the + session was started in a directory below the root, one more line names that directory relative + to the root: `Started in: packages/web (relative to the project root)`. There is no + `Working directory:` or `Project root:` line with an absolute path any more. +- **A file in the project is named relative to its root** (an instruction file, `docs/AGENTS.md`); a + file under the person's home directory outside the project as `~/…` (the global instructions, + `~/.config/lembas/AGENTS.md`). Only a file outside both is named by its absolute path. +- **Relative paths are resolved against the project root** — by every file tool (`read`, `write`, + `edit`, `multiedit`, `apply_patch`, `glob`, `grep`, `list`, `view_image`, `show`) and as the + directory `bash` starts in (its `cwd` argument, relative, likewise). Where the person started + below the root is a fact told to the model, not a second base; it still ends the instruction + chain (`filesystem.md`). **Absolute paths are still accepted** everywhere, as before, and answers + still name paths as `filesystem.md` says. +- A chat on the server is not a machine's disk: its paths are the server's tree's, short and the + same for every instance (`/home/`, `blocks/workspace.md`). Its prompt names nothing longer. + +`conformance/paths.json`: the environment lines, and how a path is resolved. ## 9. What the model is told `prompts/blocks/context.md`, in every session that has `recall`. -## Not in 3.1.0 +## Not in 3.2.0 Eliding old content in batches behind a threshold, and measuring quality and tokens with and without these measures, are left for a later spec. Every measure above ships with its unit and conformance diff --git a/harness/filesystem.md b/harness/filesystem.md index 78f8c63..384a2c3 100644 --- a/harness/filesystem.md +++ b/harness/filesystem.md @@ -18,7 +18,8 @@ one access level for the whole working directory, decided by the CLI's permissio - **Absolute** paths begin with `/`, the root of the tree: the organisation's root on the server, the file system's root on a device. -- **Relative** paths resolve against the session's **start directory** (its working directory). +- **Relative** paths resolve against the session's **start directory** (its working directory) — + on a device, from spec 3.2.0, against the project root (*Node kinds*, below). - `.` and `..` are resolved by their text, before anything is looked up; `..` above `/` stays at `/`. Repeated `/` count as one, and a trailing `/` changes nothing. On the server there are no links: a path is what it says. On a device, a link is followed, and where the CLI's rules look at @@ -42,7 +43,11 @@ one access level for the whole working directory, decided by the CLI's permissio A session has a **boundary** (what it may see at all: the root, a directory, one repository) and a **start directory** inside it. On a device the boundary is the project root (the directory the -CLI's trust is given to) and the start directory is where the session started. +CLI's trust is given to) and the start directory is where the session started. From spec 3.2.0 a +device resolves relative paths against the **project root**, not the start directory (`context.md` +§11): the model is told the project and, when it started below the root, that directory relative to +the root — one base for every relative path, whatever the directory. The start directory still +ends the instruction chain (below). ## Access levels diff --git a/harness/loop.json b/harness/loop.json index c56e678..3df9140 100644 --- a/harness/loop.json +++ b/harness/loop.json @@ -105,7 +105,12 @@ "line": "Read a file again for its content; recall shows a result by its id.", "rows": 50 }, - "reasoning": "Earlier turns' reasoning is left to the model's chat template: the harness sends the history as it does, neither stripping reasoning the template keeps nor adding what it drops.", + "reasoning": "Spec 3.2.0 (context.md §8): every assistant message of the current turn — the messages after the last one that started a turn (the person's, or a machine message that starts a turn of its own; a delivery at a step boundary does not) — carries the reasoning it came with (OpenAI chat: `reasoning_content`; Anthropic: its thinking blocks, unchanged); assistant messages of earlier turns carry none. What is stored keeps it. conformance/reasoning.json.", + "env": { + "about": "Spec 3.2.0 (context.md §11): the environment block's lines about where the session works, in an agent session. `project` always, `name` being the project root's last path component; `started` only when the session started in a directory below the root, `path` being that directory relative to the root. No absolute path. conformance/paths.json.", + "project": "Project: {{name}} (paths are relative to its root)", + "started": "Started in: {{path}} (relative to the project root)" + }, "note": { "about": "A note on the next user message (context.md §10, rule 3): what changed since the system prompt was made, after the person's words. Lines in this order, each only when it applies. `plan_ended` is the mode block for a session that leaves plan mode for a mode with no text of its own (a chat). `plan_own` is the plan block for a session that does not resend earlier tool results (a chat): the model's own plan, as its tools left it. `personal` is the {{path}} of the instructions block for the person's own instructions.", "open": "", diff --git a/harness/prompts/blocks/agents-missing.md b/harness/prompts/blocks/agents-missing.md index 4318310..b2826bc 100644 --- a/harness/prompts/blocks/agents-missing.md +++ b/harness/prompts/blocks/agents-missing.md @@ -2,7 +2,7 @@ kept by the agent, and how git is used there is the user's answer, asked once and written down. --> ## AGENTS.md This project has no AGENTS.md yet. It is the file every later session here starts from, so making it is part of the work: -- Make it at the first natural pause in real work, once you know what the project is — not before you have done anything, and not in plan mode. It goes at the project root: {{root}}/AGENTS.md. +- Make it at the first natural pause in real work, once you know what the project is — not before you have done anything, and not in plan mode. It goes at the project root: `AGENTS.md`. - Before you write it, ask the user how git is to be used here, with ask_user, in one call: {{git_question}} Mark the options you would recommend. - Write their answer under a "## Git" heading. Above it, say briefly what the project is, how to build, run and test it, and the conventions that matter. - If the user does not want the file, do not bring it up again this session. diff --git a/harness/prompts/tasks/init.md b/harness/prompts/tasks/init.md index bbd0abd..f010ac6 100644 --- a/harness/prompts/tasks/init.md +++ b/harness/prompts/tasks/init.md @@ -1,6 +1,6 @@ -Create or update `AGENTS.md` at the project root, {{root}}/AGENTS.md: the file every later session here starts from. Every line should answer "would an agent likely get this wrong without being told?" — if not, leave it out. +Create or update `AGENTS.md` at the project root: the file every later session here starts from. Every line should answer "would an agent likely get this wrong without being told?" — if not, leave it out. {{focus}}