"""Listing a project directory: the ladder, the cache, and the budget. The ladder is exercised against a fake executor rather than a real host, because what is being tested is *which rung is chosen* and what happens when one falls through -- and a real box either has git or does not, which would make the interesting cases unreachable. `scan_dir` itself is tested against a real SFTP server in test_agent_browse.py. """ from __future__ import annotations import time import pytest from lembas.services.agent import index as index_service from lembas.services.agent.base import ExecError, ExecResult, RemoteEntry class _Fake: """An executor that answers whatever the test says, and records the asking.""" def __init__(self, *, answers: dict[str, ExecResult] | None = None, tree=None): self.answers = answers or {} self.tree = tree or {} self.commands: list[str] = [] self.scans: list[str] = [] async def run(self, request) -> ExecResult: self.commands.append(request.command) for fragment, result in self.answers.items(): if fragment in request.command: return result return ExecResult(exit_status=1, output="") async def scan_dir(self, path: str) -> list[RemoteEntry]: self.scans.append(path) if path not in self.tree: raise ExecError(f"There is no directory at {path}.") return self.tree[path] def _ok(output: str) -> ExecResult: return ExecResult(exit_status=0, output=output) @pytest.fixture(autouse=True) def _clean_cache(): index_service.clear() yield index_service.clear() # --- The ladder -------------------------------------------------------------- async def test_git_is_tried_first_and_wins_where_it_works(): """`--exclude-standard` is the whole reason. A repository already carries somebody's considered list of what is not part of the project, and reproducing it by hand is how an index ends up mostly build output.""" executor = _Fake(answers={"git ls-files": _ok("README.md\nsrc/main.py\n")}) found = await index_service.build(executor, "/work") assert found.source == "git" assert found.paths == ("README.md", "src/main.py") assert executor.commands == [executor.commands[0]] assert "--exclude-standard" in executor.commands[0] async def test_find_takes_over_when_the_directory_is_not_a_repository(): executor = _Fake(answers={"find .": _ok("./README.md\n./src/main.py\n")}) found = await index_service.build(executor, "/work") assert found.source == "find" assert found.paths == ("README.md", "src/main.py") assert len(executor.commands) == 2 # git was tried, and fell through async def test_find_prunes_the_usual_noise(): """Not applied to the git rung, which has already applied the repository's own rules -- a checked-in `vendor/` is checked in on purpose.""" executor = _Fake(answers={"find .": _ok("./a\n")}) await index_service.build(executor, "/work") assert "node_modules" in executor.commands[1] assert "'.git'" in executor.commands[1] async def test_sftp_is_the_last_resort_and_always_works(): executor = _Fake( tree={ "/work": [RemoteEntry("src", True), RemoteEntry("README.md", False)], "src": [RemoteEntry("main.py", False)], } ) found = await index_service.build(executor, "/work") assert found.source == "sftp" assert "README.md" in found.paths assert "src/main.py" in found.paths async def test_a_directory_that_cannot_be_read_at_all_is_empty_not_an_error(): """A reply must not fail because a listing did. The fragment carrying it disappears instead, which is what `requires` is for.""" executor = _Fake(tree={}) found = await index_service.build(executor, "/work") assert found.paths == () assert not found.ok async def test_control_characters_are_stripped_from_a_filename(): """These end up inside a system prompt. An escape sequence in a filename could otherwise repaint the transcript it is quoted in.""" executor = _Fake(answers={"git ls-files": _ok("ok.py\nevil\x1b[31m.py\n")}) found = await index_service.build(executor, "/work") assert "evil[31m.py" in found.paths assert "\x1b" not in "".join(found.paths) async def test_the_number_of_paths_is_capped_and_says_so(): """"There is nothing else here" and "I stopped looking" are different answers, and it must not give the first for the second.""" many = "\n".join(f"f{i}.py" for i in range(index_service.MAX_ENTRIES + 50)) executor = _Fake(answers={"git ls-files": _ok(many)}) found = await index_service.build(executor, "/work") assert len(found.paths) == index_service.MAX_ENTRIES assert found.truncated # --- The cache --------------------------------------------------------------- async def test_a_second_ask_does_not_walk_again(): executor = _Fake(answers={"git ls-files": _ok("a.py\n")}) await index_service.ensure(executor, "profile-1", "/work") await index_service.ensure(executor, "profile-1", "/work") assert len(executor.commands) == 1 async def test_refreshing_walks_again(): executor = _Fake(answers={"git ls-files": _ok("a.py\n")}) await index_service.ensure(executor, "profile-1", "/work") await index_service.ensure(executor, "profile-1", "/work", refresh=True) assert len(executor.commands) == 2 async def test_a_stale_index_is_not_returned(): executor = _Fake(answers={"git ls-files": _ok("a.py\n")}) await index_service.ensure(executor, "profile-1", "/work") index_service._CACHE[("profile-1", "/work")] = index_service.ProjectIndex( paths=("a.py",), total=1, source="git", built_at=time.monotonic() - index_service.TTL - 1 ) assert index_service.cached("profile-1", "/work") is None async def test_forgetting_a_connection_drops_its_listings(): """A connection somebody has just revoked must not leave a listing of the machine behind it. Called from the same three places that close its terminals.""" executor = _Fake(answers={"git ls-files": _ok("a.py\n")}) await index_service.ensure(executor, "profile-1", "/work") await index_service.ensure(executor, "profile-2", "/work") assert index_service.forget("profile-1") == 1 assert index_service.cached("profile-1", "/work") is None assert index_service.cached("profile-2", "/work") is not None async def test_a_host_that_refuses_to_run_commands_still_gets_a_listing(): """SFTP is the rung for exactly this, and the ladder used to skip it. An `ExecError` from git or find -- an SFTP-only account, a forced command, a shell that is `/bin/false` -- escaped the loop and was caught outside it, which returned an empty index without ever trying the one method that would have worked. """ class _NoExec(_Fake): async def run(self, request): raise ExecError("This account may not run commands.") executor = _NoExec(tree={"/work": [RemoteEntry("README.md", False, 5)]}) found = await index_service.build(executor, "/work") assert found.source == "sftp" assert "README.md" in found.paths async def test_forgetting_one_tree_leaves_the_others(): """What a write invalidates is the directory it wrote into, not the machine. Two chats on one box in different trees share nothing but the connection, and dropping both would make every write cost somebody else a walk. """ executor = _Fake(answers={"git ls-files": _ok("a.py\n")}) await index_service.ensure(executor, "profile-1", "/work") await index_service.ensure(executor, "profile-1", "/other") index_service.forget_dir("profile-1", "/work") assert index_service.cached("profile-1", "/work") is None assert index_service.cached("profile-1", "/other") is not None def test_reading_the_cache_never_does_work(): """`harness` calls this synchronously while assembling the system message, so it must never be the thing that opens a connection.""" assert index_service.cached("nobody", "/nowhere") is None # --- The budget -------------------------------------------------------------- def _index(paths) -> index_service.ProjectIndex: return index_service.ProjectIndex( paths=tuple(sorted(paths)), total=len(paths), source="git", built_at=time.monotonic() ) def test_a_small_tree_is_shown_whole(): text = index_service.render(_index(["README.md", "src/main.py"]), 2000) assert "README.md" in text assert "main.py" in text assert "files)" not in text def test_a_big_directory_becomes_a_count(): paths = ["README.md"] + [f"vendor/f{i}.js" for i in range(400)] text = index_service.render(_index(paths), 300) assert "vendor/ (400 files)" in text assert "README.md" in text assert "file_list" in text def test_the_deepest_big_directory_is_collapsed_first(): """Collapsing by size alone takes `src/` before `src/.../vendor/` -- it is bigger because it *contains* it -- and loses every name worth having in order to fold away one directory of third-party files.""" paths = [f"src/api/{n}.py" for n in "abcde"] + [ f"src/web/vendor/f{i}.js" for i in range(300) ] text = index_service.render(_index(paths), 400) assert "vendor/ (300 files)" in text assert "a.py" in text # src/api survived def test_a_flat_directory_of_thousands_is_still_bounded(): """The one shape collapsing cannot help with: no directory to fold them into, so the running budget has to bite instead.""" text = index_service.render(_index([f"dump{i:05d}.log" for i in range(5000)]), 400) assert len(text) < 1200 assert "more files" in text def test_a_budget_of_zero_renders_nothing(): """Which is how "index it for the picker, but say nothing to the model" is expressed. The fragment vanishes rather than appearing empty.""" assert index_service.render(_index(["a.py"]), 0) == "" def test_an_empty_index_renders_nothing(): assert index_service.render(index_service.ProjectIndex(), 2000) == "" def test_a_truncated_index_says_it_is_a_sample(): sample = index_service.ProjectIndex( paths=("a.py", "b.py"), total=9000, truncated=True, source="find", built_at=time.monotonic() ) assert "sample" in index_service.render(sample, 2000)