feat(harness): probe instruction files in one batch

Stat every candidate of every ancestor concurrently instead of one round trip at a time, and log a stat failure that is not absence. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
sanchitrk · Oct 10, 2026 · Oct 10, 2026 · Oct 10, 2026 · Oct 10, 2026 · Oct 10, 2026
commit 041a12b2d519c3f9ec7d10e98190a38e72098e84
diff --git a/packages/harness/src/plugin/builtin/prompt/instruction.ts b/packages/harness/src/plugin/builtin/prompt/instruction.ts
@@ -1,11 +1,13 @@
 /*
- * Files are read only through the sandbox, never from the host (`hostDir`, `hostCwd`, `<home>`):
- * a host-side global file would be read twice whenever a local sandbox's walk passes through it.
+ * Discovery happens only in the sandbox namespace, local or remote: never through the host
+ * filesystem (`hostDir`, `hostCwd`) and never from a global `<home>` file. The walk ends at the
+ * sandbox root, which on a local sandbox is `/`, so a `~/AGENTS.md` above the cwd loads like any
+ * other ancestor's file.
  */
 
 import * as Section from "@codeworksh/plugin/plugin/section";
 import { Effect, Option } from "effect";
-import type { SandboxFileSystem } from "../../../sandbox/fs/filesystem.ts";
+import { SandboxFileSystem } from "../../../sandbox/fs/filesystem.ts";
 import { SandboxIO } from "../../../sandbox/io.ts";
 import { posix } from "../../../util/posix.ts";
 import type { SharedPluginContext } from "../../context.ts";
@@ -21,7 +23,7 @@ const names: ReadonlyArray<string> = [
 	"CLAUDE.MD",
 ];
 
-interface Found {
+export interface Found {
 	readonly name: string;
 	readonly path: string;
 	/** Undefined for a file with nothing to say, which still claims its directory. */
@@ -33,16 +35,45 @@ const content = (raw: string): string | undefined => {
 	return text.trim().length === 0 ? undefined : text;
 };
 
-const first = Effect.fnUntraced(function* (fs: SandboxFileSystem.Interface, directory: string) {
+/** `directory` and every ancestor up to the sandbox root, closest first. */
+const ancestors = (directory: string): string[] => {
+	const chain: string[] = [];
+	for (let current = directory; ; current = posix.dirname(current)) {
+		chain.push(current);
+		if (posix.dirname(current) === current) return chain;
+	}
+};
+
+/**
+ * Which of `paths` are files. Absence is the expected answer and stays silent; any other failure
+ * (permissions, a remote blip) is logged so a dropped file is not a mystery, then counts as absent.
+ */
+const present = (fs: SandboxFileSystem.Interface, paths: ReadonlyArray<string>) =>
+	Effect.forEach(
+		paths,
+		(path) =>
+			fs.stat(path).pipe(
+				Effect.map((stat) => stat.isFile),
+				Effect.catch((error) =>
+					SandboxFileSystem.isNotFoundError(error.cause)
+						? Effect.succeed(false)
+						: Effect.logWarning(`could not stat instruction file ${path}`, error.cause).pipe(Effect.as(false)),
+				),
+			),
+		{ concurrency: "unbounded" },
+	).pipe(Effect.map((hits) => new Set(paths.filter((_, index) => hits[index]))));
+
+/** A file that stats but does not read falls through to the next candidate. */
+const first = Effect.fnUntraced(function* (
+	fs: SandboxFileSystem.Interface,
+	directory: string,
+	files: ReadonlySet<string>,
+) {
 	for (const name of names) {
 		const path = posix.join(directory, name);
-		const isFile = yield* fs.stat(path).pipe(
-			Effect.map((stat) => stat.isFile),
-			Effect.orElseSucceed(() => false),
-		);
-		if (!isFile) continue;
+		if (!files.has(path)) continue;
 		const raw = yield* fs.readFile(path).pipe(
-			Effect.tapError((cause) => Effect.logWarning(`could not read instruction file ${path}`, cause)),
+			Effect.tapError((error) => Effect.logWarning(`could not read instruction file ${path}`, error.cause)),
 			Effect.option,
 		);
 		if (Option.isSome(raw)) return { name, path, content: content(raw.value) } satisfies Found;
@@ -66,34 +97,39 @@ const shadowing = (location: SharedPluginContext["location"]): string | undefine
 	return location.space.location.startsWith(`${main}/`) ? main : undefined;
 };
 
+/** One file per directory from the sandbox root down to the cwd, closest last so it has the final word. */
+export const discover = Effect.fnUntraced(function* (
+	fs: SandboxFileSystem.Interface,
+	location: SharedPluginContext["location"],
+) {
+	const chain = ancestors(location.directory);
+	// Every candidate of every directory is probed in one batch: on a remote sandbox each stat is a
+	// network round trip, so a serial walk cost 6 x depth of them before every model call.
+	const files = yield* present(
+		fs,
+		chain.flatMap((directory) => names.map((name) => posix.join(directory, name))),
+	);
+	const found = yield* Effect.forEach(chain, (directory) => first(fs, directory, files), {
+		concurrency: "unbounded",
+	});
+
+	// The worktree root is always on the chain, since the cwd is at or beneath it.
+	const main = shadowing(location);
+	const worktree = main === undefined ? undefined : found[chain.indexOf(location.space.location)];
+	const walked: Found[] = [];
+	for (const [index, here] of found.entries()) {
+		const shadowed = chain[index] === main && worktree !== undefined && here?.name === worktree.name;
+		if (here !== undefined && !shadowed) walked.push(here);
+	}
+	return walked.reverse();
+});
+
 export const instructionPlugin = define({
 	id: "codework.prompt.instruction",
 	kind: "prompt",
 	setup: Effect.fn("InstructionPlugin.setup")(function* (ctx) {
 		const fs = yield* SandboxIO.FileSystem;
-
-		// The worktree root is probed before the walk and again during it.
-		const probed = new Map<string, Found | undefined>();
-		const at = Effect.fnUntraced(function* (directory: string) {
-			if (!probed.has(directory)) probed.set(directory, yield* first(fs, directory));
-			return probed.get(directory);
-		});
-
-		const walked: Found[] = [];
-		const main = shadowing(ctx.location);
-		const worktree = main === undefined ? undefined : yield* at(ctx.location.space.location);
-		let current: string = ctx.location.directory;
-		while (true) {
-			const here = yield* at(current);
-			const shadowed = current === main && worktree !== undefined && here?.name === worktree.name;
-			if (here !== undefined && !shadowed) walked.push(here);
-			const parent = posix.dirname(current);
-			if (parent === current) break;
-			current = parent;
-		}
-
-		// Closest file last, so it has the final word.
-		const files = walked.reverse().filter((file) => file.content !== undefined);
+		const files = (yield* discover(fs, ctx.location)).filter((file) => file.content !== undefined);
 		if (files.length === 0) return;
 
 		const prompt = ctx.plugin.prompt;

diff --git a/packages/harness/test/__artifacts__/prompt.instruction.unreadable.txt b/packages/harness/test/__artifacts__/prompt.instruction.unreadable.txt
@@ -0,0 +1,15 @@
+<project_context>
+Project-Specific Instructions & Guidelines:
+
+<project_instructions path="<root>/repo/AGENTS.md">
+Repo.
+
+</project_instructions>
+
+<project_instructions path="<root>/repo/src/AGENTS.md">
+Src: still loaded.
+
+</project_instructions>
+</project_context>
+
+could not stat instruction file <root>/repo/src/AGENTS.override.md Error: ELOOP: too many symbolic links encountered, stat '<root>/repo/src/AGENTS.override.md'
diff --git a/packages/harness/test/plugin.instruction.test.ts b/packages/harness/test/plugin.instruction.test.ts
@@ -1,12 +1,20 @@
 import "./utils/env.ts";
-import { Effect } from "effect";
+import { Effect, Logger } from "effect";
 import { execFile } from "node:child_process";
-import { chmod, mkdir, realpath, writeFile } from "node:fs/promises";
+import { chmod, mkdir, realpath, symlink, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { promisify } from "node:util";
 import { describe, expect, it } from "vite-plus/test";
 import { Harness } from "../src/effect/harness.ts";
 import { Session } from "../src/effect/session.ts";
+import { Location } from "../src/location/location.ts";
+import { discover } from "../src/plugin/builtin/prompt/instruction.ts";
+import { ProjectSchema } from "../src/project/schema.ts";
+import { SandboxFileSystem } from "../src/sandbox/fs/filesystem.ts";
+import { SandboxInstance } from "../src/sandbox/instance.ts";
+import { AbsolutePath } from "../src/schema.ts";
+import { SpaceSchema } from "../src/space/schema.ts";
+import { Space } from "../src/space/space.ts";
 import { scenario } from "./fixtures/instruction.spec.ts";
 import { immediateOpen } from "./fixtures/llm.ts";
 import { withSettings } from "./fixtures/settings.ts";
@@ -20,9 +28,13 @@ const write = async (path: string, text: string) => {
 	await writeFile(path, text);
 };
 
-const prompt = async (input: { root: string; directory: string; hostCwd: string }) => {
+/** Runs one exchange; `warnings` collects every warning the harness logged while it ran. */
+const prompt = async (input: { root: string; directory: string; hostCwd: string; warnings?: string[] }) => {
 	const observed: string[] = [];
 	const open = immediateOpen();
+	const collector = Logger.make(({ logLevel, message }) => {
+		if (logLevel === "Warn") input.warnings?.push((Array.isArray(message) ? message : [message]).join(" "));
+	});
 	await Effect.runPromise(
 		Effect.gen(function* () {
 			const session = yield* Session.create({ directory: input.directory });
@@ -40,12 +52,65 @@ const prompt = async (input: { root: string; directory: string; hostCwd: string
 				}),
 			),
 			Effect.scoped,
+			Effect.provide(Logger.layer([collector])),
 		),
 	);
 	const real = await realpath(input.root);
 	return (observed[0] ?? "").replaceAll(real, "<root>").replaceAll(input.root, "<root>");
 };
 
+/**
+ * A filesystem over `files`. Paths in `slow` answer `stat` and `readFile` late, so a probe that took
+ * completion order for candidate order would be caught; `failing` paths fail `stat` with EIO.
+ */
+const instrumented = (
+	files: Readonly<Record<string, string>>,
+	slow: ReadonlySet<string>,
+	failing: ReadonlySet<string>,
+) => {
+	const stats = { calls: 0, inFlight: 0, peak: 0 };
+	const unused = () => Promise.reject(new Error("unused"));
+	const settle = (path: string) => new Promise((resolve) => setTimeout(resolve, slow.has(path) ? 30 : 1));
+	const fs = SandboxFileSystem.fromProvider({
+		readFile: async (path) => {
+			await settle(path);
+			return path in files ? (files[path] ?? "") : unused();
+		},
+		readFileBuffer: unused,
+		writeFile: unused,
+		stat: async (path) => {
+			stats.calls += 1;
+			stats.peak = Math.max(stats.peak, ++stats.inFlight);
+			await settle(path);
+			stats.inFlight -= 1;
+			if (failing.has(path)) throw Object.assign(new Error(`EIO: i/o error, stat '${path}'`), { code: "EIO" });
+			if (!(path in files)) throw Object.assign(new Error(`ENOENT: stat '${path}'`), { code: "ENOENT" });
+			return { isFile: true, isDirectory: false };
+		},
+		readdir: unused,
+		exists: unused,
+		mkdir: unused,
+		rm: unused,
+		realpath: unused,
+		scanLines: unused,
+	});
+	return { fs, stats };
+};
+
+const plain = (directory: string) =>
+	new Location.Info({
+		directory: AbsolutePath.make(directory),
+		space: new SpaceSchema.Info({
+			id: Space.id(SandboxInstance.ID.local, directory),
+			projectId: ProjectSchema.ID.make("local"),
+			location: AbsolutePath.make(directory),
+			kind: "plain",
+			env: SandboxInstance.ID.local,
+			status: "active",
+		}),
+		project: new ProjectSchema.Info({ id: ProjectSchema.ID.make("local"), name: "local", status: "active" }),
+	});
+
 const context = (text: string) => /<project_context>\n[\s\S]*?\n<\/project_context>/.exec(text)?.[0];
 
 describe("codework.prompt.instruction", () => {
@@ -97,6 +162,59 @@ describe("codework.prompt.instruction", () => {
 			expect(main).toContain("Main checkout copy.");
 		}));
 
+	it("logs a stat failure that is not absence and still uses the next candidate", () =>
+		withSettings(async ({ root }) => {
+			const src = join(root, "repo", "src");
+			await write(join(root, "repo", "AGENTS.md"), "Repo.\n");
+			await write(join(src, "AGENTS.md"), "Src: still loaded.\n");
+			// A symlink to itself: `stat` fails with ELOOP, which is not "no such file".
+			await symlink("AGENTS.override.md", join(src, "AGENTS.override.md"));
+
+			const warnings: string[] = [];
+			const text = await prompt({ root, directory: src, hostCwd: root, warnings });
+			const real = await realpath(root);
+			const normalised = warnings.map((line) => line.replaceAll(real, "<root>").replaceAll(root, "<root>"));
+			expect(
+				normalised.some((line) =>
+					line.includes("could not stat instruction file <root>/repo/src/AGENTS.override.md"),
+				),
+			).toBe(true);
+			await expect(
+				[context(text), ...normalised.map((line) => line.split("\n")[0])].join("\n\n"),
+			).toMatchFileSnapshot("./__artifacts__/prompt.instruction.unreadable.txt");
+		}));
+
+	it("probes every candidate of every ancestor in one batch and keeps candidate order", async () => {
+		// The winners answer last: the root's file after the cwd's, `AGENTS.MD` after `CLAUDE.md`.
+		const { fs, stats } = instrumented(
+			{
+				"/AGENTS.md": "root",
+				"/a/b/CLAUDE.md": "b: lower priority",
+				"/a/b/AGENTS.MD": "b: wins",
+				"/a/b/c/d/e/f/AGENTS.md": "cwd",
+			},
+			new Set(["/AGENTS.md", "/a/b/AGENTS.MD"]),
+			new Set(["/a/b/c/CLAUDE.override.md"]),
+		);
+		const warnings: string[] = [];
+		const collector = Logger.make(({ logLevel, message }) => {
+			if (logLevel === "Warn") warnings.push(String(message));
+		});
+		const found = await Effect.runPromise(
+			discover(fs, plain("/a/b/c/d/e/f")).pipe(Effect.provide(Logger.layer([collector]))),
+		);
+		// Seven directories, six names each: all in flight at once instead of 42 serial round trips.
+		expect(stats.calls).toBe(42);
+		expect(stats.peak).toBe(42);
+		expect(found.map((file) => `${file.path}=${file.content}`)).toEqual([
+			"/AGENTS.md=root",
+			"/a/b/AGENTS.MD=b: wins",
+			"/a/b/c/d/e/f/AGENTS.md=cwd",
+		]);
+		expect(warnings).toHaveLength(1);
+		expect(warnings[0]).toContain("could not stat instruction file /a/b/c/CLAUDE.override.md");
+	});
+
 	it("adds nothing when it is disabled in settings", () =>
 		withSettings(async ({ root }) => {
 			await write(join(root, "repo", "AGENTS.md"), "Repo.\n");