Custom HTTP tools an administrator defines
A row in custom_tools becomes a ToolDef like any built-in, offered beside the thirteen. The registry had to stop being an import-time constant for that: `resolve_tools` now returns the schemas *and* the runners together, carried to the loop on the ToolContext. That closes a hole on the way. `run_tool` looked names up in the global REGISTRY with no reference to what had been offered, so a model naming a tool its chat was gated out of -- a family switched off, a permission the reader lacks -- had it run anyway. The resolved set is now authoritative. Arguments come from a model, so an argument may fill a hole but never move the target: the scheme and host of a URL template are literal, values are escaped for where they land, and the origin is pinned afterwards. Every redirect hop is checked the way services/fetch.py checks one, and the secret is dropped if a hop leaves the origin it was issued for. Also fixes the tool-activity block claiming every library tool had "searched the web", which it has done since the second family landed. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jaroslav Beneš
@@ -0,0 +1,303 @@
|
||||
"""The custom-tools admin screen."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import select
|
||||
|
||||
from lembas.db.models import CustomTool, Group, User
|
||||
from lembas.security.passwords import hash_password
|
||||
from lembas.services.crypto import UNCHANGED_SENTINEL, decrypt, encrypt
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def plain_user(client: TestClient, db, registered):
|
||||
"""A second account, which is never an administrator."""
|
||||
client.post("/auth/logout")
|
||||
client.post(
|
||||
"/auth/register",
|
||||
data={"name": "Sam", "email": "sam@shire.test", "password": "correct horse battery"},
|
||||
follow_redirects=False,
|
||||
)
|
||||
user = db.scalar(select(User).where(User.email == "sam@shire.test"))
|
||||
user.role = "user"
|
||||
user.active = True
|
||||
db.commit()
|
||||
client.post(
|
||||
"/auth/login",
|
||||
data={"email": "sam@shire.test", "password": "correct horse battery"},
|
||||
follow_redirects=False,
|
||||
)
|
||||
return user
|
||||
|
||||
|
||||
def _form(**overrides) -> dict:
|
||||
base = {
|
||||
"name": "Weather",
|
||||
"slug": "weather",
|
||||
"description": "Look up the weather for a city.",
|
||||
"parameters": json.dumps(
|
||||
{"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}
|
||||
),
|
||||
"method": "GET",
|
||||
"url_template": "https://api.test/v1/{{city}}",
|
||||
"headers": "Accept: application/json",
|
||||
"secret_placement": "bearer",
|
||||
"secret_name": "Authorization",
|
||||
"response_mode": "json",
|
||||
"max_chars": "8000",
|
||||
"timeout": "20",
|
||||
"position": "0",
|
||||
"enabled": "true",
|
||||
"public": "true",
|
||||
}
|
||||
base.update(overrides)
|
||||
return {key: value for key, value in base.items() if value is not None}
|
||||
|
||||
|
||||
def _create(client: TestClient, **overrides):
|
||||
return client.post("/admin/tools", data=_form(**overrides), follow_redirects=False)
|
||||
|
||||
|
||||
# --- Guards ------------------------------------------------------------------
|
||||
def test_the_pages_are_refused_to_a_plain_user(client: TestClient, plain_user):
|
||||
assert client.get("/admin/tools").status_code == 403
|
||||
assert client.get("/admin/tools/new").status_code == 403
|
||||
assert client.post("/admin/tools", data=_form()).status_code == 403
|
||||
|
||||
|
||||
def test_a_tool_that_does_not_exist_is_a_404(client: TestClient, registered):
|
||||
assert client.get("/admin/tools/nope/edit").status_code == 404
|
||||
|
||||
|
||||
def test_new_is_not_parsed_as_a_tool_id(client: TestClient, registered):
|
||||
"""FastAPI matches in registration order; /admin/models has been bitten by
|
||||
exactly this."""
|
||||
response = client.get("/admin/tools/new")
|
||||
assert response.status_code == 200
|
||||
assert "New tool" in response.text
|
||||
|
||||
|
||||
# --- Creating and editing ----------------------------------------------------
|
||||
def test_creating_a_tool_then_editing_it(client: TestClient, db, registered):
|
||||
_create(client, secret="s3cret")
|
||||
|
||||
tool = db.scalar(select(CustomTool))
|
||||
assert tool.slug == "weather"
|
||||
assert tool.headers_json == {"Accept": "application/json"}
|
||||
assert tool.parameters_json["properties"]["city"] == {"type": "string"}
|
||||
assert decrypt(tool.secret_encrypted) == "s3cret"
|
||||
assert tool.enabled is True
|
||||
|
||||
client.post(
|
||||
f"/admin/tools/{tool.id}",
|
||||
data=_form(name="Forecast", enabled=None, secret=UNCHANGED_SENTINEL),
|
||||
follow_redirects=False,
|
||||
)
|
||||
db.refresh(tool)
|
||||
assert tool.name == "Forecast"
|
||||
# An unticked checkbox is simply absent from the post, which is the signal.
|
||||
assert tool.enabled is False
|
||||
assert decrypt(tool.secret_encrypted) == "s3cret", "the dots must keep the secret"
|
||||
|
||||
|
||||
def test_clearing_the_field_removes_the_secret(client: TestClient, db, registered):
|
||||
_create(client, secret="s3cret")
|
||||
tool = db.scalar(select(CustomTool))
|
||||
|
||||
client.post(f"/admin/tools/{tool.id}", data=_form(secret=""), follow_redirects=False)
|
||||
db.refresh(tool)
|
||||
assert tool.secret_encrypted == ""
|
||||
|
||||
|
||||
def test_a_secret_is_never_rendered_in_full(client: TestClient, db, registered):
|
||||
_create(client, secret="super-secret-value")
|
||||
tool = db.scalar(select(CustomTool))
|
||||
|
||||
page = client.get(f"/admin/tools/{tool.id}/edit").text
|
||||
assert "super-secret-value" not in page
|
||||
assert UNCHANGED_SENTINEL in page
|
||||
|
||||
|
||||
# --- Validation reports back into the form -----------------------------------
|
||||
def test_bad_parameter_json_is_reported_not_a_422(client: TestClient, db, registered):
|
||||
response = _create(client, parameters="{not json")
|
||||
assert response.status_code == 200
|
||||
assert "not valid JSON" in response.text
|
||||
assert db.scalar(select(CustomTool)) is None
|
||||
|
||||
|
||||
def test_parameters_that_are_not_an_object_are_refused(client: TestClient, db, registered):
|
||||
response = _create(client, parameters='{"type": "string"}')
|
||||
assert "must be a JSON object" in response.text
|
||||
assert db.scalar(select(CustomTool)) is None
|
||||
|
||||
|
||||
def test_a_slug_colliding_with_a_builtin_is_refused(client: TestClient, db, registered):
|
||||
response = _create(client, slug="web_search")
|
||||
assert "built-in tool" in response.text
|
||||
assert db.scalar(select(CustomTool)) is None
|
||||
|
||||
|
||||
def test_a_duplicate_slug_is_refused(client: TestClient, db, registered):
|
||||
_create(client)
|
||||
response = _create(client, name="Other")
|
||||
assert "already a tool" in response.text
|
||||
assert len(list(db.scalars(select(CustomTool)))) == 1
|
||||
|
||||
|
||||
def test_a_url_whose_host_is_a_variable_is_refused(client: TestClient, db, registered):
|
||||
response = _create(client, url_template="https://{{city}}.test/x")
|
||||
assert "literal" in response.text
|
||||
assert db.scalar(select(CustomTool)) is None
|
||||
|
||||
|
||||
def test_a_rejected_save_keeps_what_was_typed(client: TestClient, db, registered):
|
||||
_create(client)
|
||||
tool = db.scalar(select(CustomTool))
|
||||
|
||||
response = client.post(
|
||||
f"/admin/tools/{tool.id}",
|
||||
data=_form(name="Renamed", parameters="{oops"),
|
||||
follow_redirects=False,
|
||||
)
|
||||
assert "Renamed" in response.text, "the form still holds the submitted name"
|
||||
db.refresh(tool)
|
||||
assert tool.name == "Weather", "and the stored row is untouched"
|
||||
|
||||
|
||||
# --- Access ------------------------------------------------------------------
|
||||
def test_making_a_tool_public_clears_its_groups(client: TestClient, db, registered):
|
||||
group = Group(name="Council")
|
||||
db.add(group)
|
||||
db.commit()
|
||||
|
||||
_create(client, public=None, group_ids=group.id)
|
||||
tool = db.scalar(select(CustomTool))
|
||||
assert [g.name for g in tool.groups] == ["Council"]
|
||||
|
||||
client.post(
|
||||
f"/admin/tools/{tool.id}",
|
||||
data={**_form(public="true"), "group_ids": group.id},
|
||||
follow_redirects=False,
|
||||
)
|
||||
db.refresh(tool)
|
||||
assert tool.groups == []
|
||||
|
||||
|
||||
def test_a_restricted_tool_is_hidden_from_a_user_outside_its_groups(db, registered):
|
||||
from lembas.services import tool_access
|
||||
|
||||
group = Group(name="Council")
|
||||
outsider = User(name="Sam", email="s@shire.test", password_hash=hash_password("x"))
|
||||
db.add_all([group, outsider])
|
||||
db.add(
|
||||
CustomTool(
|
||||
slug="weather", name="Weather", url_template="https://api.test/", public=False
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
tool = db.scalar(select(CustomTool))
|
||||
tool.groups = [group]
|
||||
db.commit()
|
||||
|
||||
assert tool_access.visible_custom_tools(db, outsider) == []
|
||||
outsider.groups = [group]
|
||||
db.commit()
|
||||
assert len(tool_access.visible_custom_tools(db, outsider)) == 1
|
||||
|
||||
|
||||
# --- Running one by hand -----------------------------------------------------
|
||||
def test_the_test_button_shows_what_the_model_would_read(
|
||||
client: TestClient, db, registered, mock_http, monkeypatch
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
"socket.getaddrinfo", lambda *a, **k: [(2, 1, 6, "", ("93.184.216.34", 80))]
|
||||
)
|
||||
mock_http(lambda _r: httpx.Response(200, json={"summary": "Sunny"}))
|
||||
_create(client, response_mode="json", response_path="summary")
|
||||
tool = db.scalar(select(CustomTool))
|
||||
|
||||
response = client.post(
|
||||
f"/admin/tools/{tool.id}/test", data={"arguments": '{"city": "Minas Tirith"}'}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert "Sunny" in response.text
|
||||
|
||||
|
||||
def test_a_failing_test_is_recorded_on_the_row(
|
||||
client: TestClient, db, registered, mock_http, monkeypatch
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
"socket.getaddrinfo", lambda *a, **k: [(2, 1, 6, "", ("93.184.216.34", 80))]
|
||||
)
|
||||
mock_http(lambda _r: httpx.Response(503, text="down"))
|
||||
_create(client)
|
||||
tool = db.scalar(select(CustomTool))
|
||||
|
||||
client.post(f"/admin/tools/{tool.id}/test", data={"arguments": "{}"})
|
||||
db.refresh(tool)
|
||||
assert "503" in tool.last_error
|
||||
assert tool.last_checked_at is not None
|
||||
|
||||
|
||||
def test_bad_test_arguments_are_reported(client: TestClient, db, registered):
|
||||
_create(client)
|
||||
tool = db.scalar(select(CustomTool))
|
||||
response = client.post(f"/admin/tools/{tool.id}/test", data={"arguments": "[1, 2]"})
|
||||
assert "JSON object" in response.text
|
||||
|
||||
|
||||
# --- The list ----------------------------------------------------------------
|
||||
def test_the_list_searches_and_filters(client: TestClient, db, registered):
|
||||
_create(client)
|
||||
_create(client, name="Tickets", slug="tickets", url_template="https://jira.test/{{city}}")
|
||||
tool = db.scalar(select(CustomTool).where(CustomTool.slug == "tickets"))
|
||||
tool.enabled = False
|
||||
db.commit()
|
||||
|
||||
page = client.get("/admin/tools").text
|
||||
assert "Weather" in page and "Tickets" in page
|
||||
|
||||
assert "Tickets" not in client.get("/admin/tools?filter=enabled").text
|
||||
assert "Weather" not in client.get("/admin/tools?q=tick").text
|
||||
|
||||
|
||||
def test_deleting_a_tool_leaves_its_prompt_override_alone(client: TestClient, db, registered):
|
||||
"""The override outlives the row, which is what lets a tool be recreated
|
||||
under the same slug without losing the wording somebody chose."""
|
||||
from lembas.services import prompts as prompts_service
|
||||
|
||||
_create(client, guidance="- Default wording.")
|
||||
tool = db.scalar(select(CustomTool))
|
||||
prompts_service.save(db, {"tool.custom_weather": "- Edited wording."})
|
||||
|
||||
client.post(f"/admin/tools/{tool.id}/delete", follow_redirects=False)
|
||||
assert db.scalar(select(CustomTool)) is None
|
||||
assert prompts_service.stored(db)["tool.custom_weather"] == "- Edited wording."
|
||||
|
||||
|
||||
def test_the_secret_survives_a_round_trip_through_the_form(client: TestClient, db, registered):
|
||||
"""A regression guard on the sentinel: the field is rendered with dots, and
|
||||
submitting the page unchanged must not overwrite the key with them."""
|
||||
db.add(
|
||||
CustomTool(
|
||||
slug="weather",
|
||||
name="Weather",
|
||||
url_template="https://api.test/",
|
||||
secret_encrypted=encrypt("s3cret"),
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
tool = db.scalar(select(CustomTool))
|
||||
|
||||
client.post(
|
||||
f"/admin/tools/{tool.id}", data=_form(secret=UNCHANGED_SENTINEL), follow_redirects=False
|
||||
)
|
||||
db.refresh(tool)
|
||||
assert decrypt(tool.secret_encrypted) == "s3cret"
|
||||
@@ -0,0 +1,382 @@
|
||||
"""Custom HTTP tools: filling the template, and refusing to be pointed elsewhere.
|
||||
|
||||
Most of this file is about the second. The arguments come from a model, which
|
||||
can be talked into things by whatever it just read, so an argument filling a
|
||||
hole in a URL is the same kind of input as a URL typed by a stranger.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from lembas.db.models import (
|
||||
RESPONSE_JSON,
|
||||
RESPONSE_RAW,
|
||||
RESPONSE_TEXT,
|
||||
SECRET_BEARER,
|
||||
SECRET_HEADER,
|
||||
SECRET_NONE,
|
||||
CustomTool,
|
||||
)
|
||||
from lembas.services import custom_tools
|
||||
from lembas.services.crypto import encrypt
|
||||
from lembas.services.fetch import FetchError
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def dns(monkeypatch):
|
||||
"""Resolve invented hostnames to a public address.
|
||||
|
||||
`api.test` does not exist and `check_url` resolves for real, which is the
|
||||
whole point of it. A literal IP is handed back as itself, so the tests that
|
||||
check a private address are still checking one.
|
||||
"""
|
||||
import ipaddress
|
||||
import socket
|
||||
|
||||
def resolve(host, *_args, **_kwargs):
|
||||
try:
|
||||
ipaddress.ip_address(host)
|
||||
except ValueError:
|
||||
return [(2, 1, 6, "", ("93.184.216.34", 80))]
|
||||
return [(2, 1, 6, "", (host, 80))]
|
||||
|
||||
monkeypatch.setattr(socket, "getaddrinfo", resolve)
|
||||
|
||||
|
||||
def _spec(**overrides) -> custom_tools.HttpSpec:
|
||||
base = {
|
||||
"slug": "weather",
|
||||
"label": "Weather",
|
||||
"method": "GET",
|
||||
"url_template": "https://api.test/v1/city/{{city}}",
|
||||
"secret_placement": SECRET_NONE,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}, "days": {"type": "integer"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
}
|
||||
return custom_tools.HttpSpec(**{**base, **overrides})
|
||||
|
||||
|
||||
def _ok(body: str = "sunny", *, status: int = 200, content_type: str = "text/plain"):
|
||||
seen = {}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
seen["request"] = request
|
||||
seen.setdefault("urls", []).append(str(request.url))
|
||||
return httpx.Response(status, text=body, headers={"content-type": content_type})
|
||||
|
||||
return handler, seen
|
||||
|
||||
|
||||
# --- Filling the URL ---------------------------------------------------------
|
||||
def test_an_argument_is_percent_encoded_into_the_url():
|
||||
url = custom_tools.fill_url(_spec(), {"city": "Minas Tirith"})
|
||||
assert url == "https://api.test/v1/city/Minas%20Tirith"
|
||||
|
||||
|
||||
def test_an_argument_cannot_add_a_path_segment_or_a_query():
|
||||
"""safe="" is the whole point. Everything structural encodes."""
|
||||
url = custom_tools.fill_url(_spec(), {"city": "../../admin?token=x#y"})
|
||||
assert "/admin" not in url
|
||||
assert "?" not in url and "#" not in url
|
||||
assert url.startswith("https://api.test/v1/city/")
|
||||
|
||||
|
||||
def test_an_argument_cannot_reach_another_host():
|
||||
url = custom_tools.fill_url(_spec(), {"city": "evil.test/steal"})
|
||||
assert url.startswith("https://api.test/")
|
||||
|
||||
|
||||
def test_an_undeclared_name_is_removed_rather_than_passed_through():
|
||||
"""Unlike the prompt fragments, where an unrecognised {{x}} is left alone.
|
||||
A literal {{x}} in a URL is not a feature."""
|
||||
spec = _spec(url_template="https://api.test/{{city}}/{{unknown}}")
|
||||
assert custom_tools.fill_url(spec, {"city": "a", "unknown": "b"}) == "https://api.test/a/"
|
||||
|
||||
|
||||
def test_an_argument_the_model_did_not_send_becomes_nothing():
|
||||
assert custom_tools.fill_url(_spec(), {}) == "https://api.test/v1/city/"
|
||||
|
||||
|
||||
def test_a_template_whose_host_is_a_variable_is_refused():
|
||||
with pytest.raises(FetchError, match="literal"):
|
||||
custom_tools.fill_url(_spec(url_template="https://{{city}}.test/x"), {"city": "a"})
|
||||
|
||||
|
||||
def test_a_template_that_is_not_http_is_refused():
|
||||
with pytest.raises(FetchError):
|
||||
custom_tools.fill_url(_spec(url_template="file:///etc/passwd"), {})
|
||||
|
||||
|
||||
# --- Reaching the network ----------------------------------------------------
|
||||
async def test_a_private_address_is_refused_unless_the_row_allows_it(mock_http):
|
||||
handler, _seen = _ok()
|
||||
mock_http(handler)
|
||||
|
||||
spec = _spec(url_template="http://127.0.0.1:11434/api/tags", parameters={})
|
||||
refused = await custom_tools.call(spec, {})
|
||||
assert refused.event["status"] == "error"
|
||||
assert "private or local" in refused.event["error"]
|
||||
|
||||
allowed = await custom_tools.call(
|
||||
_spec(
|
||||
url_template="http://127.0.0.1:11434/api/tags", parameters={}, allow_private=True
|
||||
),
|
||||
{},
|
||||
)
|
||||
assert allowed.event["status"] == "ok"
|
||||
|
||||
|
||||
async def test_a_hostname_resolving_to_loopback_is_refused(mock_http, monkeypatch):
|
||||
"""A name pointing at 127.0.0.1 walks past any check that only reads the
|
||||
text of the URL."""
|
||||
handler, _seen = _ok()
|
||||
mock_http(handler)
|
||||
monkeypatch.setattr(
|
||||
"socket.getaddrinfo",
|
||||
lambda *a, **k: [(2, 1, 6, "", ("127.0.0.1", 80))],
|
||||
)
|
||||
|
||||
outcome = await custom_tools.call(_spec(parameters={}), {})
|
||||
assert outcome.event["status"] == "error"
|
||||
assert "private or local" in outcome.event["error"]
|
||||
|
||||
|
||||
async def test_every_redirect_hop_is_checked(mock_http, monkeypatch):
|
||||
"""httpx's own following would validate the first address and then happily
|
||||
land on localhost."""
|
||||
hops = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
hops.append(str(request.url))
|
||||
if request.url.host == "api.test":
|
||||
return httpx.Response(302, headers={"location": "http://inside.test/secrets"})
|
||||
return httpx.Response(200, text="secrets")
|
||||
|
||||
mock_http(handler)
|
||||
|
||||
def resolve(host, *_args, **_kwargs):
|
||||
if host == "inside.test":
|
||||
return [(2, 1, 6, "", ("10.0.0.5", 80))]
|
||||
return [(2, 1, 6, "", ("93.184.216.34", 80))]
|
||||
|
||||
monkeypatch.setattr("socket.getaddrinfo", resolve)
|
||||
|
||||
outcome = await custom_tools.call(_spec(parameters={}), {})
|
||||
assert outcome.event["status"] == "error"
|
||||
assert len(hops) == 1, "the second hop must never be requested"
|
||||
|
||||
|
||||
async def test_the_secret_is_dropped_on_a_cross_host_redirect(mock_http):
|
||||
seen = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
seen.append((request.url.host, request.headers.get("authorization")))
|
||||
if request.url.host == "api.test":
|
||||
return httpx.Response(302, headers={"location": "https://elsewhere.test/x"})
|
||||
return httpx.Response(200, text="ok")
|
||||
|
||||
mock_http(handler)
|
||||
|
||||
outcome = await custom_tools.call(
|
||||
_spec(parameters={}, secret="s3cret", secret_placement=SECRET_BEARER), {}
|
||||
)
|
||||
assert outcome.event["status"] == "ok"
|
||||
assert seen[0] == ("api.test", "Bearer s3cret")
|
||||
assert seen[1] == ("elsewhere.test", None)
|
||||
|
||||
|
||||
async def test_a_bearer_secret_reaches_the_request_and_never_the_event(mock_http):
|
||||
handler, seen = _ok()
|
||||
mock_http(handler)
|
||||
|
||||
outcome = await custom_tools.call(
|
||||
_spec(parameters={}, secret="s3cret", secret_placement=SECRET_BEARER), {}
|
||||
)
|
||||
assert seen["request"].headers["authorization"] == "Bearer s3cret"
|
||||
assert "s3cret" not in json.dumps(outcome.event)
|
||||
|
||||
|
||||
async def test_a_header_secret_uses_the_name_it_was_given(mock_http):
|
||||
handler, seen = _ok()
|
||||
mock_http(handler)
|
||||
|
||||
await custom_tools.call(
|
||||
_spec(
|
||||
parameters={},
|
||||
secret="k",
|
||||
secret_placement=SECRET_HEADER,
|
||||
secret_name="X-Api-Key",
|
||||
),
|
||||
{},
|
||||
)
|
||||
assert seen["request"].headers["x-api-key"] == "k"
|
||||
|
||||
|
||||
async def test_a_header_value_cannot_carry_a_newline(mock_http):
|
||||
handler, seen = _ok()
|
||||
mock_http(handler)
|
||||
|
||||
await custom_tools.call(
|
||||
_spec(headers={"X-Trace": "{{city}}"}), {"city": "a\r\nX-Admin: yes"}
|
||||
)
|
||||
assert "\n" not in seen["request"].headers["x-trace"]
|
||||
|
||||
|
||||
async def test_a_body_argument_cannot_end_the_json_string(mock_http):
|
||||
handler, seen = _ok()
|
||||
mock_http(handler)
|
||||
|
||||
await custom_tools.call(
|
||||
_spec(
|
||||
method="POST",
|
||||
url_template="https://api.test/v1/ask",
|
||||
body_template='{"city": "{{city}}"}',
|
||||
),
|
||||
{"city": 'x", "admin": "yes'},
|
||||
)
|
||||
body = json.loads(seen["request"].content)
|
||||
assert set(body) == {"city"}
|
||||
|
||||
|
||||
# --- Reading the response ----------------------------------------------------
|
||||
async def test_a_json_response_is_narrowed_by_the_path(mock_http):
|
||||
def handler(_request):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"data": {"items": [{"title": "Mallorn"}, {"title": "Elanor"}]}},
|
||||
)
|
||||
|
||||
mock_http(handler)
|
||||
outcome = await custom_tools.call(
|
||||
_spec(parameters={}, response_mode=RESPONSE_JSON, response_path="data.items.0.title"), {}
|
||||
)
|
||||
assert outcome.content == "Mallorn"
|
||||
|
||||
|
||||
async def test_a_path_that_leads_nowhere_yields_the_whole_document(mock_http):
|
||||
mock_http(lambda _r: httpx.Response(200, json={"a": 1}))
|
||||
outcome = await custom_tools.call(
|
||||
_spec(parameters={}, response_mode=RESPONSE_JSON, response_path="nope.nope"), {}
|
||||
)
|
||||
assert json.loads(outcome.content) == {"a": 1}
|
||||
|
||||
|
||||
async def test_an_html_response_becomes_text(mock_http):
|
||||
handler, _seen = _ok(
|
||||
"<html><head><title>T</title></head><body><p>Hello</p></body></html>",
|
||||
content_type="text/html",
|
||||
)
|
||||
mock_http(handler)
|
||||
outcome = await custom_tools.call(_spec(parameters={}, response_mode=RESPONSE_TEXT), {})
|
||||
assert outcome.content == "Hello"
|
||||
|
||||
|
||||
async def test_a_raw_response_is_left_alone(mock_http):
|
||||
handler, _seen = _ok("<p>kept</p>", content_type="text/html")
|
||||
mock_http(handler)
|
||||
outcome = await custom_tools.call(_spec(parameters={}, response_mode=RESPONSE_RAW), {})
|
||||
assert outcome.content == "<p>kept</p>"
|
||||
|
||||
|
||||
async def test_the_response_is_capped_and_says_so(mock_http):
|
||||
handler, _seen = _ok("x" * 5000)
|
||||
mock_http(handler)
|
||||
outcome = await custom_tools.call(_spec(parameters={}, max_chars=500), {})
|
||||
assert len(outcome.content) < 600
|
||||
assert outcome.content.endswith("(truncated)")
|
||||
|
||||
|
||||
async def test_the_event_preview_is_capped_independently(mock_http):
|
||||
"""`max_chars` is spent once; the event is stored on the message row."""
|
||||
handler, _seen = _ok("x" * 30_000)
|
||||
mock_http(handler)
|
||||
outcome = await custom_tools.call(_spec(parameters={}, max_chars=20_000), {})
|
||||
assert len(outcome.event["text"]) <= custom_tools.MAX_EVENT_CHARS
|
||||
assert len(outcome.content) > custom_tools.MAX_EVENT_CHARS
|
||||
|
||||
|
||||
async def test_an_http_error_becomes_an_explanation_not_an_exception(mock_http):
|
||||
handler, _seen = _ok("no such city", status=404)
|
||||
mock_http(handler)
|
||||
outcome = await custom_tools.call(_spec(parameters={}), {})
|
||||
assert outcome.event["status"] == "error"
|
||||
assert "404" in outcome.content
|
||||
assert "no such city" in outcome.content
|
||||
|
||||
|
||||
async def test_a_transport_failure_is_reported_to_the_model(mock_http):
|
||||
def handler(request):
|
||||
raise httpx.ConnectTimeout("too slow", request=request)
|
||||
|
||||
mock_http(handler)
|
||||
outcome = await custom_tools.call(_spec(parameters={}), {})
|
||||
assert outcome.event["status"] == "error"
|
||||
assert "Could not reach" in outcome.content
|
||||
|
||||
|
||||
async def test_the_event_names_the_host_and_not_the_filled_url(mock_http):
|
||||
"""A path segment carries an argument, and the event is rendered and kept."""
|
||||
handler, _seen = _ok()
|
||||
mock_http(handler)
|
||||
outcome = await custom_tools.call(_spec(), {"city": "Minas Tirith"})
|
||||
assert outcome.event["detail"] == "GET api.test"
|
||||
assert "Minas" not in outcome.event["detail"]
|
||||
assert "Minas Tirith" in outcome.event["query"]
|
||||
|
||||
|
||||
# --- Turning rows into tools -------------------------------------------------
|
||||
def _row(db, **overrides) -> CustomTool:
|
||||
row = CustomTool(
|
||||
**{
|
||||
"slug": "weather",
|
||||
"name": "Weather",
|
||||
"description": "Look up the weather.",
|
||||
"url_template": "https://api.test/{{city}}",
|
||||
"parameters_json": {"type": "object", "properties": {"city": {"type": "string"}}},
|
||||
**overrides,
|
||||
}
|
||||
)
|
||||
db.add(row)
|
||||
db.commit()
|
||||
return row
|
||||
|
||||
|
||||
def test_a_row_becomes_a_tool_definition(db, user_id):
|
||||
from lembas.db.models import User
|
||||
|
||||
_row(db)
|
||||
defs = custom_tools.tool_defs(db, db.get(User, user_id))
|
||||
assert [tool.name for tool in defs] == ["weather"]
|
||||
assert defs[0].family == "custom:weather"
|
||||
assert defs[0].schema["function"]["description"] == "Look up the weather."
|
||||
|
||||
|
||||
def test_a_disabled_row_is_not_offered(db, user_id):
|
||||
from lembas.db.models import User
|
||||
|
||||
_row(db, enabled=False)
|
||||
assert custom_tools.tool_defs(db, db.get(User, user_id)) == []
|
||||
|
||||
|
||||
def test_a_schema_that_is_not_an_object_is_replaced(db, user_id):
|
||||
"""An endpoint rejects the whole request over a malformed tools array, so
|
||||
one bad row must not take the reply with it."""
|
||||
from lembas.db.models import User
|
||||
|
||||
_row(db, parameters_json={"type": "string"})
|
||||
defs = custom_tools.tool_defs(db, db.get(User, user_id))
|
||||
assert defs[0].parameters == {"type": "object", "properties": {}}
|
||||
|
||||
|
||||
def test_the_secret_is_decrypted_into_the_snapshot_and_nowhere_else(db):
|
||||
row = _row(db, secret_encrypted=encrypt("s3cret"))
|
||||
spec = custom_tools.spec_from(row)
|
||||
assert spec.secret == "s3cret"
|
||||
assert "s3cret" not in row.secret_encrypted
|
||||
@@ -54,6 +54,27 @@ def test_only_the_guidance_for_offered_tools_appears(db, owner):
|
||||
assert "Skills are procedures" not in text
|
||||
|
||||
|
||||
def test_a_custom_tools_guidance_appears_only_when_it_is_offered(db, owner):
|
||||
"""The row supplies the default, and the fragment is gated on the tool's own
|
||||
family -- which is why the registry had to stop being a module constant."""
|
||||
from lembas.db.models import CustomTool
|
||||
|
||||
db.add(
|
||||
CustomTool(
|
||||
slug="weather",
|
||||
name="Weather",
|
||||
description="Look up the weather.",
|
||||
url_template="https://api.test/{{city}}",
|
||||
guidance="- Check the weather rather than guessing at it.",
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
offered = tools_service.registry(db)["weather"].schema
|
||||
assert "Check the weather" in harness.compose(db, owner, [offered])
|
||||
assert "Check the weather" not in harness.compose(db, owner, _tools("web_search"))
|
||||
|
||||
|
||||
def test_the_memory_block_is_included_when_memory_is_offered(db, owner):
|
||||
memories_service.add(db, owner=owner, content="Prefers metric units.")
|
||||
text = harness.compose(db, owner, _tools("memory_add"))
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Rendering what a tool did.
|
||||
|
||||
The block is written from four places and read from stored rows written by
|
||||
earlier versions, so it has to render anything shaped roughly like an event --
|
||||
and everything in it is third-party text.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from lembas.web.templating import templates
|
||||
|
||||
|
||||
def _render(*events, live: bool = False) -> str:
|
||||
return templates.get_template("chat/_tool_activity.html").render(
|
||||
{"tool_events": list(events), "live": live}
|
||||
)
|
||||
|
||||
|
||||
def test_a_search_still_says_it_searched_the_web():
|
||||
html = _render(
|
||||
{
|
||||
"name": "web_search",
|
||||
"kind": "search",
|
||||
"query": "mallorn",
|
||||
"status": "ok",
|
||||
"results": [
|
||||
{
|
||||
"title": "Mallorn",
|
||||
"url": "https://a.test/m",
|
||||
"host": "a.test",
|
||||
"snippet": "A tree.",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
assert "Searched the web for “mallorn”" in html
|
||||
assert '<a class="tool-result__title" href="https://a.test/m"' in html
|
||||
assert "1 result" in html
|
||||
|
||||
|
||||
def test_a_library_tool_no_longer_claims_to_have_searched_the_web():
|
||||
"""Stored rows predate `kind`, and every one of them used to render a globe
|
||||
and "Searched the web for <the note title>"."""
|
||||
html = _render({"name": "notes_search", "query": "shopping", "status": "ok", "results": []})
|
||||
assert "Searched the web" not in html
|
||||
assert "notes_search" in html
|
||||
|
||||
|
||||
def test_a_custom_tool_is_named_and_its_host_shown():
|
||||
html = _render(
|
||||
{
|
||||
"name": "weather",
|
||||
"kind": "custom",
|
||||
"label": "Weather",
|
||||
"query": "city='Minas Tirith'",
|
||||
"detail": "GET api.test",
|
||||
"status": "ok",
|
||||
"results": [],
|
||||
"text": "Sunny.",
|
||||
}
|
||||
)
|
||||
assert "Weather" in html
|
||||
assert "GET api.test" in html
|
||||
assert "Sunny." in html
|
||||
|
||||
|
||||
def test_a_tools_own_text_is_escaped_and_never_rendered_as_markdown():
|
||||
"""Hard rule 6. A tool's reply is exactly as untrusted as a search result,
|
||||
and markdown is the one path allowed to emit HTML."""
|
||||
html = _render(
|
||||
{
|
||||
"name": "weather",
|
||||
"kind": "custom",
|
||||
"label": "Weather",
|
||||
"status": "ok",
|
||||
"results": [],
|
||||
"text": "<img src=x onerror=alert(1)> [click](javascript:alert(1))",
|
||||
}
|
||||
)
|
||||
assert "<img" not in html
|
||||
assert "<img" in html
|
||||
# The markdown link is shown as the text it is, not turned into an anchor.
|
||||
assert "<a " not in html
|
||||
assert "[click](javascript:alert(1))" in html
|
||||
|
||||
|
||||
def test_a_result_url_that_is_not_http_never_becomes_a_link():
|
||||
html = _render(
|
||||
{
|
||||
"name": "web_search",
|
||||
"kind": "search",
|
||||
"status": "ok",
|
||||
"results": [{"title": "Bad", "url": "javascript:alert(1)", "host": "", "snippet": ""}],
|
||||
}
|
||||
)
|
||||
assert "<a " not in html
|
||||
assert '<span class="tool-result__title">Bad</span>' in html
|
||||
|
||||
|
||||
def test_a_result_with_no_url_at_all_does_not_explode():
|
||||
html = _render(
|
||||
{
|
||||
"name": "notes_search",
|
||||
"status": "ok",
|
||||
"results": [{"title": "A note", "id": "abc"}],
|
||||
}
|
||||
)
|
||||
assert "A note" in html
|
||||
|
||||
|
||||
def test_a_failure_shows_its_reason():
|
||||
html = _render(
|
||||
{
|
||||
"name": "weather",
|
||||
"kind": "custom",
|
||||
"label": "Weather",
|
||||
"status": "error",
|
||||
"error": "HTTP 503",
|
||||
"results": [],
|
||||
}
|
||||
)
|
||||
assert "tool-activity--error" in html
|
||||
assert "Weather failed" in html
|
||||
assert "HTTP 503" in html
|
||||
@@ -166,6 +166,57 @@ def _context(**kwargs):
|
||||
return tools_service.ToolContext(owner_id="someone", **kwargs)
|
||||
|
||||
|
||||
# --- The offer and the runner travel together --------------------------------
|
||||
def test_the_resolved_set_carries_the_runners_with_the_schemas(db, user_id):
|
||||
"""A tool that is a database row is not reachable through the import-time
|
||||
registry, so the resolution has to travel with the offer."""
|
||||
settings_store.update(db, {"enabled": True}, key=settings_store.SEARCH)
|
||||
chat = _chat_with(db, user_id, capabilities={"tools": True})
|
||||
resolved = tools_service.resolve_tools(db, chat, _user(db, user_id))
|
||||
|
||||
assert _names(resolved.schemas) == set(resolved.by_name)
|
||||
assert all(callable(tool.run) for tool in resolved.defs)
|
||||
# The old accessor is the same set, so nothing that only wants schemas moved.
|
||||
assert resolved.schemas == tools_service.enabled_tools(db, chat, _user(db, user_id))
|
||||
|
||||
|
||||
async def test_a_tool_that_was_not_offered_is_refused(db, user_id):
|
||||
"""The lookup is against what was offered, not against everything that
|
||||
exists. A model naming a tool its chat was gated out of used to have it run,
|
||||
because only the offer was ever filtered."""
|
||||
from lembas.db.models import User
|
||||
from lembas.services.library import notes as notes_service
|
||||
|
||||
owner = db.get(User, user_id)
|
||||
note = notes_service.create(db, owner=owner, title="Keep me", body="...")
|
||||
|
||||
chat = _chat_with(db, user_id, capabilities={"tools": True, "tool_notes": False})
|
||||
resolved = tools_service.resolve_tools(db, chat, owner)
|
||||
context = tools_service.context_for(db, owner, chat, tools=resolved)
|
||||
|
||||
outcome = await tools_service.run_tool(
|
||||
context, "notes_delete", json.dumps({"id": note.id})
|
||||
)
|
||||
assert outcome.event["status"] == "error"
|
||||
assert notes_service.get(db, note.id, owner) is not None
|
||||
|
||||
|
||||
async def test_a_context_with_no_toolset_still_finds_the_builtins(monkeypatch):
|
||||
"""None means nobody resolved a set. An empty dict does not -- it means
|
||||
nothing was offered, and is authoritative."""
|
||||
|
||||
async def fake_run(_config, query, *, limit=None):
|
||||
return [SearchResult("A title", "https://a.test", "a snippet")]
|
||||
|
||||
monkeypatch.setattr("lembas.services.search.run", fake_run)
|
||||
|
||||
unresolved = await tools_service.run_tool(_context(), "web_search", '{"query": "x"}')
|
||||
assert unresolved.event["status"] == "ok"
|
||||
|
||||
empty = await tools_service.run_tool(_context(tools={}), "web_search", '{"query": "x"}')
|
||||
assert empty.event["status"] == "error"
|
||||
|
||||
|
||||
# --- Knowledge is scoped to the chat's bases ---------------------------------
|
||||
async def test_knowledge_search_is_limited_to_the_attached_bases(db, user_id):
|
||||
""""Answer from the contracts folder" is a different question from "answer
|
||||
|
||||
Reference in New Issue
Block a user