Repository navigation
Make safeoutputs CLI transport fail loudly instead of fail open #61427
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 5 commits
6b5ad0d
cc0d873
e786783
f83f180
4a8b4a9
13bdf8c
6042238
67f059e
a444378
556c34c
f0d8b29
cd61925
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -3160,6 +3160,90 @@ function buildEngineFailureContext(options = {}) { | |
| } | ||
| } | ||
|
|
||
| /** Maximum number of safeoutputs CLI command excerpts rendered in the missing-safe-outputs report */ | ||
| const SAFEOUTPUTS_CLI_EXCERPT_LIMIT = 10; | ||
| /** Maximum rendered length of a single safeoutputs CLI command excerpt */ | ||
| const SAFEOUTPUTS_CLI_EXCERPT_MAX_LENGTH = 300; | ||
|
|
||
| /** | ||
| * Detect a safeoutputs CLI invocation where the binary never runs because the | ||
| * pipe into it was dropped, e.g. `printf '{...}' safeoutputs noop .` instead of | ||
| * `printf '{...}' | safeoutputs noop .`. In the dropped-pipe form `printf`/`echo` | ||
| * consume `safeoutputs`, the tool name and `.` as trailing arguments, print the | ||
| * payload and exit 0, so the failure is indistinguishable from success. | ||
| * | ||
| * @param {string} line - Single transcript line containing `safeoutputs` | ||
| * @returns {boolean} | ||
| */ | ||
| function isDroppedPipeSafeOutputsCommand(line) { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [/diagnosing-bugs] The dropped-pipe heuristic can false-positive on prose. 💡 ReproductionisDroppedPipeSafeOutputsCommand(`echo "See docs: safeoutputs add_comment example"`) // => true (false positive)Consider requiring @copilot please address this. |
||
| const match = /(printf|echo)\b([^\n]*?)safeoutputs\b/.exec(line); | ||
| if (!match) return false; | ||
| // A pipe (or command separator) between the writer and `safeoutputs` means the | ||
| // binary does run; only the separator-free form is silently skipped. | ||
| return !/[|;&><]/.test(match[2]); | ||
|
|
||
| } | ||
|
|
||
| /** | ||
| * Extract `safeoutputs` CLI commands from the agent transcript so that a run | ||
| * ending with zero safe outputs is self-diagnosing: it distinguishes "the agent | ||
| * never tried to emit anything" from "the agent tried and the shell command | ||
| * silently skipped the CLI". | ||
| * | ||
| * @param {string} [stdioLogPathOverride] - Optional explicit transcript path (testing) | ||
| * @returns {string} Markdown context block, or an empty string when nothing was found | ||
| */ | ||
| function buildSafeOutputsCliInvocationContext(stdioLogPathOverride) { | ||
| const agentOutputFile = process.env.GH_AW_AGENT_OUTPUT; | ||
| const stdioLogPath = stdioLogPathOverride || (agentOutputFile ? path.join(path.dirname(agentOutputFile), "agent-stdio.log") : "/tmp/gh-aw/agent-stdio.log"); | ||
|
|
||
| let logContent; | ||
| try { | ||
| logContent = fs.readFileSync(stdioLogPath, "utf8"); | ||
| } catch { | ||
| core.info(`agent-stdio.log not readable at ${stdioLogPath}; skipping safeoutputs CLI invocation context`); | ||
| return ""; | ||
| } | ||
|
|
||
| const maskedValues = collectAddMaskedValues(logContent); | ||
| /** @type {string[]} */ | ||
| const commands = []; | ||
| const seen = new Set(); | ||
| let hasDroppedPipe = false; | ||
| for (const rawLine of logContent.split("\n")) { | ||
| const line = rawLine.trim(); | ||
| if (!line || !line.includes("safeoutputs")) continue; | ||
| if (isAddMaskCommandLine(line)) continue; | ||
| // Only keep lines that look like a shell invocation of the CLI. | ||
| if (!/(^|[|;&(\s"'`])safeoutputs\s+[a-z_][a-z0-9_]*/i.test(line)) continue; | ||
| const redacted = applyAddMaskRedaction(line, maskedValues); | ||
| const truncated = redacted.length > SAFEOUTPUTS_CLI_EXCERPT_MAX_LENGTH ? `${redacted.slice(0, SAFEOUTPUTS_CLI_EXCERPT_MAX_LENGTH)}…` : redacted; | ||
| if (isDroppedPipeSafeOutputsCommand(line)) { | ||
| hasDroppedPipe = true; | ||
| } | ||
| if (seen.has(truncated)) continue; | ||
| seen.add(truncated); | ||
| if (commands.length < SAFEOUTPUTS_CLI_EXCERPT_LIMIT) { | ||
| commands.push(truncated); | ||
| } | ||
| } | ||
|
|
||
| if (commands.length === 0) { | ||
| core.info("No safeoutputs CLI commands found in agent-stdio.log"); | ||
| return ""; | ||
| } | ||
|
|
||
| core.info(`Found ${commands.length} safeoutputs CLI command(s) in agent-stdio.log${hasDroppedPipe ? " (dropped-pipe pattern detected)" : ""}`); | ||
|
|
||
| let context = "**`safeoutputs` commands found in the agent transcript:**\n\n```bash\n"; | ||
| context += commands.join("\n"); | ||
| context += "\n```\n\n"; | ||
| if (hasDroppedPipe) { | ||
| context += "A command writes a JSON payload with `printf`/`echo` but has no `|` before `safeoutputs`, so the payload was printed and the CLI never ran (exit code 0, no safe output). "; | ||
| context += 'Pass the payload inline instead: `safeoutputs <tool> \'{"key":"value"}\'`.\n\n'; | ||
| } | ||
| return context; | ||
| } | ||
|
|
||
| /** Cascade detection constants */ | ||
| const CASCADE_WINDOW_MINUTES = 60; | ||
| const CASCADE_WINDOW_MS = CASCADE_WINDOW_MINUTES * 60 * 1000; | ||
|
|
@@ -4077,7 +4161,9 @@ async function main() { | |
| missingSafeOutputsContext += "\n\nThis typically indicates:\n"; | ||
| missingSafeOutputsContext += "- The safe output server failed to run\n"; | ||
| missingSafeOutputsContext += "- The prompt failed to generate any meaningful result\n"; | ||
| missingSafeOutputsContext += "- The agent should have called `noop` to explicitly indicate no action was taken\n\n"; | ||
| missingSafeOutputsContext += "- The agent should have called `noop` to explicitly indicate no action was taken\n"; | ||
| missingSafeOutputsContext += "- A `safeoutputs` CLI command was malformed and never invoked the CLI\n\n"; | ||
| missingSafeOutputsContext += buildSafeOutputsCliInvocationContext(); | ||
| } | ||
|
|
||
| // Build fork context hint | ||
|
|
@@ -4311,7 +4397,9 @@ async function main() { | |
| missingSafeOutputsContext += "\n\nThis typically indicates:\n"; | ||
| missingSafeOutputsContext += "- The safe output server failed to run\n"; | ||
| missingSafeOutputsContext += "- The prompt failed to generate any meaningful result\n"; | ||
| missingSafeOutputsContext += "- The agent should have called `noop` to explicitly indicate no action was taken\n\n"; | ||
| missingSafeOutputsContext += "- The agent should have called `noop` to explicitly indicate no action was taken\n"; | ||
| missingSafeOutputsContext += "- A `safeoutputs` CLI command was malformed and never invoked the CLI\n\n"; | ||
| missingSafeOutputsContext += buildSafeOutputsCliInvocationContext(); | ||
| } | ||
|
|
||
| // Build fork context hint | ||
|
|
@@ -4522,6 +4610,8 @@ module.exports = { | |
| setFailureIssueOutputs, | ||
| buildAssignCopilotFailureContext, | ||
| buildEngineFailureContext, | ||
| buildSafeOutputsCliInvocationContext, | ||
| isDroppedPipeSafeOutputsCommand, | ||
| detectAWFFirewallStartupFailureFromLog, | ||
| buildReportIncompleteContext, | ||
| buildMCPPolicyErrorContext, | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2537,6 +2537,71 @@ describe("handle_agent_failure", () => { | |
| // buildEngineFailureContext | ||
| // ────────────────────────────────────────────────────── | ||
|
|
||
| describe("buildSafeOutputsCliInvocationContext", () => { | ||
| let buildSafeOutputsCliInvocationContext; | ||
| let isDroppedPipeSafeOutputsCommand; | ||
| const fs = require("fs"); | ||
| const path = require("path"); | ||
| const os = require("os"); | ||
|
|
||
| /** @type {string} */ | ||
| let tmpDir; | ||
| /** @type {string} */ | ||
| let stdioLogPath; | ||
|
|
||
| beforeEach(() => { | ||
| vi.resetModules(); | ||
| tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "aw-test-")); | ||
| stdioLogPath = path.join(tmpDir, "agent-stdio.log"); | ||
| process.env.GH_AW_AGENT_OUTPUT = path.join(tmpDir, "agent_output.json"); | ||
| ({ buildSafeOutputsCliInvocationContext, isDroppedPipeSafeOutputsCommand } = require("./handle_agent_failure.cjs")); | ||
| }); | ||
|
|
||
| afterEach(() => { | ||
| delete process.env.GH_AW_AGENT_OUTPUT; | ||
| if (fs.existsSync(tmpDir)) { | ||
| fs.rmSync(tmpDir, { recursive: true, force: true }); | ||
| } | ||
| }); | ||
|
|
||
| it("returns empty string when the transcript is missing", () => { | ||
| expect(buildSafeOutputsCliInvocationContext()).toBe(""); | ||
| }); | ||
|
|
||
| it("returns empty string when the transcript has no safeoutputs commands", () => { | ||
| fs.writeFileSync(stdioLogPath, "analysis complete\nnothing to report\n"); | ||
| expect(buildSafeOutputsCliInvocationContext()).toBe(""); | ||
| }); | ||
|
|
||
| it("lists safeoutputs commands found in the transcript", () => { | ||
| fs.writeFileSync(stdioLogPath, `printf '{"message":"no action needed"}' | safeoutputs noop .\n`); | ||
| const result = buildSafeOutputsCliInvocationContext(); | ||
| expect(result).toContain("safeoutputs` commands found in the agent transcript"); | ||
| expect(result).toContain("safeoutputs noop ."); | ||
| expect(result).not.toContain("never ran"); | ||
| }); | ||
|
|
||
| it("diagnoses a dropped pipe before safeoutputs", () => { | ||
| fs.writeFileSync(stdioLogPath, `printf '{"message":"no action needed"}' safeoutputs noop .\n`); | ||
| const result = buildSafeOutputsCliInvocationContext(); | ||
| expect(result).toContain("the CLI never ran"); | ||
| expect(result).toContain('safeoutputs <tool> \'{"key":"value"}\''); | ||
| }); | ||
|
|
||
| it("deduplicates repeated commands", () => { | ||
| fs.writeFileSync(stdioLogPath, `safeoutputs noop --message "done"\nsafeoutputs noop --message "done"\n`); | ||
| const result = buildSafeOutputsCliInvocationContext(); | ||
| const occurrences = result.split(`safeoutputs noop --message "done"`).length - 1; | ||
| expect(occurrences).toBe(1); | ||
| }); | ||
|
|
||
| it("classifies piped and unpiped invocations", () => { | ||
| expect(isDroppedPipeSafeOutputsCommand(`printf '{"message":"x"}' safeoutputs noop .`)).toBe(true); | ||
| expect(isDroppedPipeSafeOutputsCommand(`printf '{"message":"x"}' | safeoutputs noop .`)).toBe(false); | ||
| expect(isDroppedPipeSafeOutputsCommand(`safeoutputs noop --message "x"`)).toBe(false); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [/tdd] No regression test covers the false-positive case above (prose mentioning 💡 Suggested testit("does not misclassify prose containing 'safeoutputs' as a dropped pipe", () => {
expect(isDroppedPipeSafeOutputsCommand(`echo "See docs: safeoutputs add_comment example"`)).toBe(false);
});@copilot please address this. |
||
| }); | ||
| }); | ||
|
|
||
| describe("buildEngineFailureContext", () => { | ||
| let buildEngineFailureContext; | ||
| const fs = require("fs"); | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
[/codebase-design] This
.github/aw/actions-lock.jsonchange downgradesdocker/build-push-actionv7.4.0→v7.3.0 anddocker/setup-buildx-actionv4.4.0→v4.3.0, unrelated to this PR's stated purpose (safeoutputs CLI transport). Every generated.lock.ymlin the repo (e.g.detection-analysis-report.lock.yml,daily-cache-strategy-analyzer.lock.yml) still references the newer v7.4.0/v4.4.0 SHAs, so this diff introduces a lock-file/workflow-output inconsistency rather than fixing one.💡 Suggestion
Revert this hunk unless it was an intentional, separately-justified downgrade (e.g. reverting a bad upstream release). If unintentional, it's likely a stale rebase artifact — regenerate the lock file from the current
mainbefore merging.@copilot please address this.