Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changeset/fix-pi-smoke-completion.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions .github/workflows/smoke-pi-sub-agents.lock.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

36 changes: 20 additions & 16 deletions .github/workflows/smoke-pi-sub-agents.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,8 @@ This is the Pi variant of `smoke-copilot-sub-agents.md`. Keep output short.
1. Use the managed `subagent` tool exactly once per declared agent, sequentially:
- `{"agent":"haiku-whoami","task":"who am i?"}`
- `{"agent":"mini-whoami","task":"who am i?"}`
- `{"agent":"nano-whoami","task":"who am i?"}`
2. Check the exact responses: `claude-haiku-4.5`, `gpt-5-mini`, and `gpt-5-nano`.
- `{"agent":"compact-whoami","task":"who am i?"}`
2. Check the exact responses: `claude-haiku-4.5`, `gpt-5-mini`, and `gpt-4o-mini`.
3. Mark a missing tool, failed delegation, or unexpected response as FAIL. Do not
answer on behalf of a sub-agent or substitute a different agent.
4. Do not make other tool calls except the staged `create_issue` report below.
Expand All @@ -55,36 +55,40 @@ Create a staged issue titled **"Smoke Test: Pi Sub Agents - ${{ github.run_id }}

These fixed responses test delegation, not a model's ability to identify itself.
The dispatch records and inference model usage provide model-selection evidence.
Provider-qualified model IDs pin the exact models rather than resolving aliases.

## agent: `haiku-whoami`
---
description: Returns the Haiku model identity for smoke testing
model: claude-haiku-4.5
model: copilot/claude-haiku-4.5
---
When asked `who am i?`, reply with exactly:
When asked `who am i?`, reply with this single plain-text line, without Markdown
or backticks:

`claude-haiku-4.5`
claude-haiku-4.5

No extra words, punctuation, or formatting.
Do not call tools. No extra words, punctuation, or formatting.

## agent: `mini-whoami`
---
description: Returns the GPT-5 mini model identity for smoke testing
model: gpt-5-mini
model: copilot/gpt-5-mini
---
When asked `who am i?`, reply with exactly:
When asked `who am i?`, reply with this single plain-text line, without Markdown
or backticks:

`gpt-5-mini`
gpt-5-mini

No extra words, punctuation, or formatting.
Do not call tools. No extra words, punctuation, or formatting.

## agent: `nano-whoami`
## agent: `compact-whoami`
---
description: Returns the GPT-5 nano model identity for smoke testing
model: gpt-5-nano
description: Returns the GPT-4o mini model identity for smoke testing
model: copilot/gpt-4o-mini
---
When asked `who am i?`, reply with exactly:
When asked `who am i?`, reply with this single plain-text line, without Markdown
or backticks:

`gpt-5-nano`
gpt-4o-mini

No extra words, punctuation, or formatting.
Do not call tools. No extra words, punctuation, or formatting.
4 changes: 2 additions & 2 deletions actions/setup/js/pi_steering_extension.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@ function piSteeringExtension(pi) {
process.stderr.write(`[gh-aw/steering] Session started. timeout=${config.timeoutMinutes}min, warn<${config.timeWarningMinutes}min, critical<${config.timeCriticalMinutes}min\n`);
});

pi.on("turn_end", async (/** @type {any} */ _event, /** @type {any} */ ctx) => {
if (startTime === undefined) {
pi.on("turn_end", async (/** @type {any} */ event, /** @type {any} */ ctx) => {
if (startTime === undefined || event.message?.stopReason !== "toolUse") {
return;
}

Expand Down
27 changes: 22 additions & 5 deletions actions/setup/js/pi_steering_extension.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -49,13 +49,13 @@ describe("Pi timeout steering", () => {
const { handlers, sendUserMessage } = harness();
await handlers.agent_start();
vi.advanceTimersByTime(6 * 60_000);
await handlers.turn_end({}, {});
await handlers.turn_end({}, {});
await handlers.turn_end({ message: { role: "assistant", stopReason: "toolUse" } }, {});
await handlers.turn_end({ message: { role: "assistant", stopReason: "toolUse" } }, {});
expect(sendUserMessage).toHaveBeenCalledTimes(1);
expect(sendUserMessage).toHaveBeenNthCalledWith(1, expect.stringContaining("wrap up"), { deliverAs: "steer" });
vi.advanceTimersByTime(3 * 60_000);
await handlers.turn_end({}, {});
await handlers.turn_end({}, {});
await handlers.turn_end({ message: { role: "assistant", stopReason: "toolUse" } }, {});
await handlers.turn_end({ message: { role: "assistant", stopReason: "toolUse" } }, {});
expect(sendUserMessage).toHaveBeenCalledTimes(2);
expect(sendUserMessage).toHaveBeenNthCalledWith(2, expect.stringContaining("CRITICAL"), { deliverAs: "steer" });
});
Expand All @@ -65,7 +65,24 @@ describe("Pi timeout steering", () => {
await handlers.agent_start();
vi.advanceTimersByTime(9 * 60_000);
await handlers.agent_start();
await handlers.turn_end({}, {});
await handlers.turn_end({ message: { role: "assistant", stopReason: "toolUse" } }, {});
expect(sendUserMessage).toHaveBeenCalledOnce();
expect(sendUserMessage).toHaveBeenCalledWith(expect.stringContaining("CRITICAL"), { deliverAs: "steer" });
});

it.each(["stop", "error", "aborted", "unknown", undefined])("does not steer terminal assistant turns with stopReason %s", async stopReason => {
const { handlers, sendUserMessage } = harness();
await handlers.agent_start();
vi.advanceTimersByTime(9 * 60_000);
await handlers.turn_end({ message: { role: "assistant", stopReason } }, {});
expect(sendUserMessage).not.toHaveBeenCalled();
});

it("steers only ongoing tool turns", async () => {
const { handlers, sendUserMessage } = harness();
await handlers.agent_start();
vi.advanceTimersByTime(9 * 60_000);
await handlers.turn_end({ message: { role: "assistant", stopReason: "toolUse" } }, {});
expect(sendUserMessage).toHaveBeenCalledOnce();
expect(sendUserMessage).toHaveBeenCalledWith(expect.stringContaining("CRITICAL"), { deliverAs: "steer" });
});
Expand Down
6 changes: 5 additions & 1 deletion actions/setup/js/pi_subagent_extension.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,11 @@ async function runPiSubagent(agent, task, ctx, signal, launch = spawn) {
const promptPath = path.join(dir, "system.txt");
const systemPath = process.env.GH_AW_PI_SYSTEM_PROMPT;
const system = systemPath ? fs.readFileSync(systemPath, "utf8") + "\n\n" : "";
fs.writeFileSync(promptPath, system + agent.prompt, { mode: 0o600 });
const delegationScope =
"You are a delegated sub-agent, not the workflow's main agent. Complete only the delegated task and return your answer to the parent, following the declared agent's output format exactly. " +
"The parent is responsible for finalizing the workflow and its required safe-output reporting. Do not emit noop safe outputs or workflow-completion reports for this child session. " +

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[/diagnosing-bugs] This fix relies entirely on prompt text ("Do not emit noop safe outputs or workflow-completion reports") rather than structural enforcement. The codebase already has a working pattern for this exact scenario one line above: GH_AW_PI_SUBAGENT_CHILD env var gates piSubagentExtension registration structurally. A model can ignore prompt instructions.

💡 Why this matters and a possible structural fix

The root cause of the original bug ("staged report FAIL despite green Actions... aborted compact child") was that children could independently trigger workflow-completion/noop safe outputs. Pinning this down with a text instruction treats the symptom, not the mechanism. Since GH_AW_PI_SUBAGENT_CHILD=1 is already set in the child's env (line 52), consider filtering/disabling the noop/completion-reporting tool(s) from the child's MCP tool list when that env var is set, the same way piSubagentExtension itself is disabled for children. This would make the constraint unbypassable regardless of what the model decides to do with the extra prompt text.

If a structural fix is infeasible for this PR (e.g. tool registration happens in a separate process/extension not easily reached here), it would help to note in the PR description or a code comment why prompt-only mitigation was chosen, and whether gh aw audit/tests can detect a regression if a child does emit a completion report despite the instruction.

@copilot please address this.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The child Pi tool policy now blocks noop and incomplete-reporting tools while leaving task safe outputs available. The existing prompt assertions remain, with added policy regression coverage. Pushed as 1994aa9.

"Safe-output actions required to perform the delegated task remain permitted.";
fs.writeFileSync(promptPath, system + agent.prompt + "\n\n" + delegationScope, { mode: 0o600 });
process.stdout.write(JSON.stringify({ type: "gh_aw_subagent_dispatch", agent: agent.name, requested_model: agent.declaredModel || agent.modelId, resolved_model: agent.modelId }) + "\n");
return await new Promise((resolve, reject) => {
const child = launch(process.env.GH_AW_PI_COMMAND || "pi", subagentArgs(agent, promptPath), {
Expand Down
22 changes: 19 additions & 3 deletions actions/setup/js/pi_subagent_extension.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,21 @@ describe("Pi managed delegation", () => {
await expect(runPiSubagent(agent, "Read.", { cwd: dir }, undefined, launcher({ stopReason: "error", errorMessage: "Model unavailable" }))).rejects.toThrow("Model unavailable");
});

it("inherits workflow controls without making the child finalize the workflow", async () => {
const systemPath = path.join(dir, "workflow-system.txt");
fs.writeFileSync(systemPath, "Workflow instructions and tool restrictions.");
vi.stubEnv("GH_AW_PI_SYSTEM_PROMPT", systemPath);
const launch = launcher();
await runPiSubagent(agent, "Read.", { cwd: dir }, undefined, (...args) => {
const prompt = fs.readFileSync(args[1].at(-1), "utf8");
expect(prompt).toContain("Workflow instructions and tool restrictions.\n\nRead only.");
expect(prompt).toContain("The parent is responsible for finalizing the workflow");
expect(prompt).toContain("Do not emit noop safe outputs");
expect(prompt).toContain("Safe-output actions required to perform the delegated task remain permitted");
return launch(...args);
});
});

it("propagates nonzero child exits", async () => {
await expect(runPiSubagent(agent, "Read.", { cwd: dir }, undefined, launcher({}, 1))).rejects.toThrow();
});
Expand Down Expand Up @@ -171,13 +186,14 @@ process.stdin.on("end", () => {
GH_AW_SUB_AGENT_EXT: ".md",
GH_AW_PI_SUBAGENT_ARGS: originalEnv.GH_AW_PI_SUBAGENT_ARGS,
GH_AW_PI_TOOL_POLICY: originalEnv.GH_AW_PI_TOOL_POLICY,
GH_AW_PI_MODEL_ALIASES: JSON.stringify({ "gpt-5-mini": ["copilot/gpt-5*mini*"], "gpt-5-nano": ["copilot/gpt-5*nano*"] }),
};
vi.stubGlobal("core", { info: vi.fn() });
const source = fs.readFileSync(new URL("../../../.github/workflows/smoke-pi-sub-agents.md", import.meta.url), "utf8");
const parentPrompt = writeInlineSubAgents(source, dir, dir, "pi");
expect(parentPrompt).not.toContain("## agent:");
stagePiArtifacts(process.env.PI_CODING_AGENT_DIR);
const expected = ["claude-haiku-4.5", "gpt-5-mini", "gpt-5-nano"];
const expected = ["claude-haiku-4.5", "gpt-5-mini", "gpt-4o-mini"];
const sdk = {
parseFrontmatter: content => {
const [, header, body] = content.split("---");
Expand All @@ -197,7 +213,7 @@ process.stdin.on("end", () => {
agentDir: process.env.PI_CODING_AGENT_DIR,
sdk,
provider: "github-copilot",
catalog: expected.map(model => `github-copilot/${model}`),
catalog: [...expected, "gpt-5.4-mini"].map(model => `github-copilot/${model}`),
gateway: true,
parentModel: "github-copilot/gpt-5.3-codex",
});
Expand All @@ -221,7 +237,7 @@ process.stdin.on("end", () => {
process.env.GH_AW_PI_COMMAND = executable;
for (const agent of agents) {
const result = await runPiSubagent(agent, "who am i?", { cwd: dir });
expect(result.content[0].text).toBe(agent.declaredModel);
expect(result.content[0].text).toBe(agent.modelId);
expect(result.details.model).toBe(agent.modelId);
}
} finally {
Expand Down
1 change: 1 addition & 0 deletions actions/setup/js/pi_tool_policy.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ function piToolPolicy(pi) {
callCount = reservePiBudget("calls", callCount, limit);
let reason;
if (limit !== undefined && callCount > limit) reason = `Workflow tool-call budget exhausted (${limit} calls)`;
else if (process.env.GH_AW_PI_SUBAGENT_CHILD === "1" && /(?:^|[^a-z0-9])(?:noop|report_incomplete|create_report_incomplete_issue)$/i.test(event.toolName || "")) reason = "Workflow-completion reporting is owned by the parent agent";
else if (event.toolName === "bash" && !isPiBashAllowed(event.input?.command, policy.bash)) reason = "Command is outside the workflow tools.bash allowlist";
else if (policy.edit === false && ["edit", "write"].includes(event.toolName)) reason = "File editing is disabled by the workflow";
if (!reason) return;
Expand Down
15 changes: 15 additions & 0 deletions actions/setup/js/pi_tool_policy.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,21 @@ describe("Pi tool policy", () => {
for (const toolName of ["edit", "write"]) expect(handlers.tool_call({ toolName }, {})).toMatchObject({ block: true });
});

it("blocks child workflow-completion reporting without blocking task safe outputs", () => {
vi.stubEnv("GH_AW_PI_SUBAGENT_CHILD", "1");
const child = {};
policyExtension({ on: (event, handler) => (child[event] = handler) });
for (const toolName of ["noop", "safeoutputs__report_incomplete", "safeoutputs__create_report_incomplete_issue"]) {
expect(child.tool_call({ toolName }, {})).toMatchObject({ block: true, reason: expect.stringContaining("parent agent") });
}
expect(child.tool_call({ toolName: "safeoutputs__create_issue" }, {})).toBeUndefined();

vi.stubEnv("GH_AW_PI_SUBAGENT_CHILD", "");
const parent = {};
policyExtension({ on: (event, handler) => (parent[event] = handler) });
expect(parent.tool_call({ toolName: "safeoutputs__noop" }, {})).toBeUndefined();
});

it("aborts inference at the configured repeated-denial threshold", () => {
const originalExitCode = process.exitCode;
vi.stubEnv("GH_AW_PI_TOOL_POLICY", '{"bash":false}');
Expand Down
2 changes: 2 additions & 0 deletions docs/src/content/docs/engines/pi.md
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,8 @@ An omitted model inherits the parent's configured model. Bare model IDs use the

Children retain the workflow's inference gateway, MCP configuration, steering, bash allowlist, and edit restrictions. Tool-call and denial budgets are shared with the parent. Children use compiler-managed execution arguments rather than inheriting `engine.args` overrides. Optional automatically discovered extensions are disabled in children; only the managed infrastructure extensions are loaded. Custom drivers do not support this delegation path.

Children return task results to the parent instead of emitting workflow-completion reports or `noop` safe outputs. Task-required safe-output actions remain available. Time-pressure steering applies to ongoing tool turns, not completed answers, so it does not reopen a finished child session.

Dispatch logs record both the requested alias and resolved model. `gh aw audit` reports `subagent_model_not_observed` when usage evidence exists but no agent inference matches a declared model. This means the agent may not have run or its model was not honored; model presence alone does not prove delegation. Detection requests are not treated as agent-model evidence.

## Additional providers
Expand Down
19 changes: 19 additions & 0 deletions pkg/cli/token_usage_declared_subagents.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,16 @@ import (
"maps"
"os"
"path"
"regexp"
"slices"
"strings"
)

var (
effectiveModelDateSuffix = regexp.MustCompile(`-[0-9]{4}(?:-[0-9]{2}-[0-9]{2}|[0-9]{4})$`)
claudeHyphenatedModelVersion = regexp.MustCompile(`^(claude-.+)-([0-9]+)-([0-9]+)$`)
)

func augmentDeclaredSubagentModels(runDir string, summary *TokenUsageSummary) {
infoPath := findAwInfoPath(runDir)
if infoPath == "" {
Expand Down Expand Up @@ -98,9 +104,22 @@ func matchesDeclaredModel(pattern, observed, provider string) bool {
pattern = model
}
matched, err := path.Match(pattern, observed)
if err == nil && matched {
return true
}
normalized := normalizeEffectiveModelName(observed)
if normalized == observed {
return false
}
matched, err = path.Match(pattern, normalized)
return err == nil && matched
}

func normalizeEffectiveModelName(model string) string {
model = effectiveModelDateSuffix.ReplaceAllString(model, "")
return claudeHyphenatedModelVersion.ReplaceAllString(model, "$1-$2.$3")
}

func generateSubagentModelFindings(summary *TokenUsageSummary) []AuditFinding {
if summary == nil {
return nil
Expand Down
Loading
Loading