Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/safeoutputs-cli-inline-json-payload.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 6 additions & 6 deletions .github/aw/actions-lock.json
Original file line number Diff line number Diff line change
Expand Up @@ -118,10 +118,10 @@
"version": "v2.0.5",
"sha": "22d081ff2d3a40755e97629de92e3bcbfa7cf2ed"
},
"docker/build-push-action@v7.4.0": {
"docker/build-push-action@v7.3.0": {
"repo": "docker/build-push-action",
"version": "v7.4.0",
"sha": "c3c9e263c25d99ce0380d002d59b67737d91b0dc"
"version": "v7.3.0",
"sha": "53b7df96c91f9c12dcc8a07bcb9ccacbed38856a"
},

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[/codebase-design] This .github/aw/actions-lock.json change downgrades docker/build-push-action v7.4.0→v7.3.0 and docker/setup-buildx-action v4.4.0→v4.3.0, unrelated to this PR's stated purpose (safeoutputs CLI transport). Every generated .lock.yml in the repo (e.g. detection-analysis-report.lock.yml, daily-cache-strategy-analyzer.lock.yml) still references the newer v7.4.0/v4.4.0 SHAs, so this diff introduces a lock-file/workflow-output inconsistency rather than fixing one.

💡 Suggestion

Revert this hunk unless it was an intentional, separately-justified downgrade (e.g. reverting a bad upstream release). If unintentional, it's likely a stale rebase artifact — regenerate the lock file from the current main before merging.

@copilot please address this.

"docker/login-action@v4.6.0": {
"repo": "docker/login-action",
Expand All @@ -133,10 +133,10 @@
"version": "v6.2.0",
"sha": "dc802804100637a589fabce1cb79ff13a1411302"
},
"docker/setup-buildx-action@v4.4.0": {
"docker/setup-buildx-action@v4.3.0": {
"repo": "docker/setup-buildx-action",
"version": "v4.4.0",
"sha": "594f3bf4285d9ea8dc53c9a0c9c4092420091003"
"version": "v4.3.0",
"sha": "37fe631027851001ddb9b187196cc803df7f5f0e"
},
"erlef/setup-beam@v1.24.1": {
"repo": "erlef/setup-beam",
Expand Down
94 changes: 92 additions & 2 deletions actions/setup/js/handle_agent_failure.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -3160,6 +3160,90 @@ function buildEngineFailureContext(options = {}) {
}
}

/** Maximum number of safeoutputs CLI command excerpts rendered in the missing-safe-outputs report */
const SAFEOUTPUTS_CLI_EXCERPT_LIMIT = 10;
/** Maximum rendered length of a single safeoutputs CLI command excerpt */
const SAFEOUTPUTS_CLI_EXCERPT_MAX_LENGTH = 300;

/**
* Detect a safeoutputs CLI invocation where the binary never runs because the
* pipe into it was dropped, e.g. `printf '{...}' safeoutputs noop .` instead of
* `printf '{...}' | safeoutputs noop .`. In the dropped-pipe form `printf`/`echo`
* consume `safeoutputs`, the tool name and `.` as trailing arguments, print the
* payload and exit 0, so the failure is indistinguishable from success.
*
* @param {string} line - Single transcript line containing `safeoutputs`
* @returns {boolean}
*/
function isDroppedPipeSafeOutputsCommand(line) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[/diagnosing-bugs] The dropped-pipe heuristic can false-positive on prose. isDroppedPipeSafeOutputsCommand only checks for printf/echo followed by safeoutputs with no |/;/&/>/< in between — it doesn't require safeoutputs to actually be the argument of printf/echo. A transcript line like echo "See docs: safeoutputs add_comment example" gets misclassified as a dropped pipe, and the failure report tells the author "the CLI never ran" when it may never have been invoked at all here (it's just prose in an echo statement).

💡 Reproduction
isDroppedPipeSafeOutputsCommand(`echo "See docs: safeoutputs add_comment example"`) // => true (false positive)

Consider requiring safeoutputs to appear as a quoted-string boundary token (e.g. right after '/") rather than anywhere in the echo/printf argument, or tightening the regex to the actual invocation shape (safeoutputs <tool_name>) followed by no closing quote before it.

@copilot please address this.

const match = /(printf|echo)\b([^\n]*?)safeoutputs\b/.exec(line);
if (!match) return false;
// A pipe (or command separator) between the writer and `safeoutputs` means the
// binary does run; only the separator-free form is silently skipped.
return !/[|;&><]/.test(match[2]);
}

/**
* Extract `safeoutputs` CLI commands from the agent transcript so that a run
* ending with zero safe outputs is self-diagnosing: it distinguishes "the agent
* never tried to emit anything" from "the agent tried and the shell command
* silently skipped the CLI".
*
* @param {string} [stdioLogPathOverride] - Optional explicit transcript path (testing)
* @returns {string} Markdown context block, or an empty string when nothing was found
*/
function buildSafeOutputsCliInvocationContext(stdioLogPathOverride) {
const agentOutputFile = process.env.GH_AW_AGENT_OUTPUT;
const stdioLogPath = stdioLogPathOverride || (agentOutputFile ? path.join(path.dirname(agentOutputFile), "agent-stdio.log") : "/tmp/gh-aw/agent-stdio.log");

let logContent;
try {
logContent = fs.readFileSync(stdioLogPath, "utf8");
} catch {
core.info(`agent-stdio.log not readable at ${stdioLogPath}; skipping safeoutputs CLI invocation context`);
return "";
}

const maskedValues = collectAddMaskedValues(logContent);
/** @type {string[]} */
const commands = [];
const seen = new Set();
let hasDroppedPipe = false;
for (const rawLine of logContent.split("\n")) {
const line = rawLine.trim();
if (!line || !line.includes("safeoutputs")) continue;
if (isAddMaskCommandLine(line)) continue;
// Only keep lines that look like a shell invocation of the CLI.
if (!/(^|[|;&(\s"'`])safeoutputs\s+[a-z_][a-z0-9_]*/i.test(line)) continue;
const redacted = applyAddMaskRedaction(line, maskedValues);
const truncated = redacted.length > SAFEOUTPUTS_CLI_EXCERPT_MAX_LENGTH ? `${redacted.slice(0, SAFEOUTPUTS_CLI_EXCERPT_MAX_LENGTH)}…` : redacted;
if (isDroppedPipeSafeOutputsCommand(line)) {
hasDroppedPipe = true;
}
if (seen.has(truncated)) continue;
seen.add(truncated);
if (commands.length < SAFEOUTPUTS_CLI_EXCERPT_LIMIT) {
commands.push(truncated);
}
}

if (commands.length === 0) {
core.info("No safeoutputs CLI commands found in agent-stdio.log");
return "";
}

core.info(`Found ${commands.length} safeoutputs CLI command(s) in agent-stdio.log${hasDroppedPipe ? " (dropped-pipe pattern detected)" : ""}`);

let context = "**`safeoutputs` commands found in the agent transcript:**\n\n```bash\n";
context += commands.join("\n");
context += "\n```\n\n";
if (hasDroppedPipe) {
context += "A command writes a JSON payload with `printf`/`echo` but has no `|` before `safeoutputs`, so the payload was printed and the CLI never ran (exit code 0, no safe output). ";
context += 'Pass the payload inline instead: `safeoutputs <tool> \'{"key":"value"}\'`.\n\n';
}
return context;
}

/** Cascade detection constants */
const CASCADE_WINDOW_MINUTES = 60;
const CASCADE_WINDOW_MS = CASCADE_WINDOW_MINUTES * 60 * 1000;
Expand Down Expand Up @@ -4077,7 +4161,9 @@ async function main() {
missingSafeOutputsContext += "\n\nThis typically indicates:\n";
missingSafeOutputsContext += "- The safe output server failed to run\n";
missingSafeOutputsContext += "- The prompt failed to generate any meaningful result\n";
missingSafeOutputsContext += "- The agent should have called `noop` to explicitly indicate no action was taken\n\n";
missingSafeOutputsContext += "- The agent should have called `noop` to explicitly indicate no action was taken\n";
missingSafeOutputsContext += "- A `safeoutputs` CLI command was malformed and never invoked the CLI\n\n";
missingSafeOutputsContext += buildSafeOutputsCliInvocationContext();
}

// Build fork context hint
Expand Down Expand Up @@ -4311,7 +4397,9 @@ async function main() {
missingSafeOutputsContext += "\n\nThis typically indicates:\n";
missingSafeOutputsContext += "- The safe output server failed to run\n";
missingSafeOutputsContext += "- The prompt failed to generate any meaningful result\n";
missingSafeOutputsContext += "- The agent should have called `noop` to explicitly indicate no action was taken\n\n";
missingSafeOutputsContext += "- The agent should have called `noop` to explicitly indicate no action was taken\n";
missingSafeOutputsContext += "- A `safeoutputs` CLI command was malformed and never invoked the CLI\n\n";
missingSafeOutputsContext += buildSafeOutputsCliInvocationContext();
}

// Build fork context hint
Expand Down Expand Up @@ -4522,6 +4610,8 @@ module.exports = {
setFailureIssueOutputs,
buildAssignCopilotFailureContext,
buildEngineFailureContext,
buildSafeOutputsCliInvocationContext,
isDroppedPipeSafeOutputsCommand,
detectAWFFirewallStartupFailureFromLog,
buildReportIncompleteContext,
buildMCPPolicyErrorContext,
Expand Down
65 changes: 65 additions & 0 deletions actions/setup/js/handle_agent_failure.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -2537,6 +2537,71 @@ describe("handle_agent_failure", () => {
// buildEngineFailureContext
// ──────────────────────────────────────────────────────

describe("buildSafeOutputsCliInvocationContext", () => {
let buildSafeOutputsCliInvocationContext;
let isDroppedPipeSafeOutputsCommand;
const fs = require("fs");
const path = require("path");
const os = require("os");

/** @type {string} */
let tmpDir;
/** @type {string} */
let stdioLogPath;

beforeEach(() => {
vi.resetModules();
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "aw-test-"));
stdioLogPath = path.join(tmpDir, "agent-stdio.log");
process.env.GH_AW_AGENT_OUTPUT = path.join(tmpDir, "agent_output.json");
({ buildSafeOutputsCliInvocationContext, isDroppedPipeSafeOutputsCommand } = require("./handle_agent_failure.cjs"));
});

afterEach(() => {
delete process.env.GH_AW_AGENT_OUTPUT;
if (fs.existsSync(tmpDir)) {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
});

it("returns empty string when the transcript is missing", () => {
expect(buildSafeOutputsCliInvocationContext()).toBe("");
});

it("returns empty string when the transcript has no safeoutputs commands", () => {
fs.writeFileSync(stdioLogPath, "analysis complete\nnothing to report\n");
expect(buildSafeOutputsCliInvocationContext()).toBe("");
});

it("lists safeoutputs commands found in the transcript", () => {
fs.writeFileSync(stdioLogPath, `printf '{"message":"no action needed"}' | safeoutputs noop .\n`);
const result = buildSafeOutputsCliInvocationContext();
expect(result).toContain("safeoutputs` commands found in the agent transcript");
expect(result).toContain("safeoutputs noop .");
expect(result).not.toContain("never ran");
});

it("diagnoses a dropped pipe before safeoutputs", () => {
fs.writeFileSync(stdioLogPath, `printf '{"message":"no action needed"}' safeoutputs noop .\n`);
const result = buildSafeOutputsCliInvocationContext();
expect(result).toContain("the CLI never ran");
expect(result).toContain('safeoutputs <tool> \'{"key":"value"}\'');
});

it("deduplicates repeated commands", () => {
fs.writeFileSync(stdioLogPath, `safeoutputs noop --message "done"\nsafeoutputs noop --message "done"\n`);
const result = buildSafeOutputsCliInvocationContext();
const occurrences = result.split(`safeoutputs noop --message "done"`).length - 1;
expect(occurrences).toBe(1);
});

it("classifies piped and unpiped invocations", () => {
expect(isDroppedPipeSafeOutputsCommand(`printf '{"message":"x"}' safeoutputs noop .`)).toBe(true);
expect(isDroppedPipeSafeOutputsCommand(`printf '{"message":"x"}' | safeoutputs noop .`)).toBe(false);
expect(isDroppedPipeSafeOutputsCommand(`safeoutputs noop --message "x"`)).toBe(false);

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[/tdd] No regression test covers the false-positive case above (prose mentioning safeoutputs inside an echo/printf string being misclassified as a dropped pipe). The existing tests only cover clean-piped and cleanly-dropped-pipe cases, not the ambiguous "safeoutputs appears in the string content, not as the invoked command" case.

💡 Suggested test
it("does not misclassify prose containing 'safeoutputs' as a dropped pipe", () => {
  expect(isDroppedPipeSafeOutputsCommand(`echo "See docs: safeoutputs add_comment example"`)).toBe(false);
});

@copilot please address this.

});
});

describe("buildEngineFailureContext", () => {
let buildEngineFailureContext;
const fs = require("fs");
Expand Down
57 changes: 56 additions & 1 deletion actions/setup/js/mcp_cli_bridge.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -739,6 +739,24 @@ function tryExtractJsonFieldFromStdin(trimmedStdin, canonicalKey, schemaProperti
}
}

/**
* Detect inline JSON payload mode: a single positional argument that is a JSON
* object, e.g. `safeoutputs noop '{"message":"done"}'`.
*
* Returns the trimmed JSON string when the sole argument looks like a JSON
* object (starts with `{`), otherwise null. Detection is intentionally
* permissive about validity so malformed JSON produces a loud parse error
* instead of being silently skipped as a non-flag argument.
*
* @param {string[]} args - User arguments after the tool name
* @returns {string | null}
*/
function findInlineJsonPayloadArg(args) {
if (args.length !== 1) return null;
const trimmed = args[0].trim();
return trimmed.startsWith("{") ? trimmed : null;
}
Comment on lines +755 to +767

/**
* Parse user-provided --key value pairs into a tool arguments object.
* Supports both --key value and --key=value styles.
Expand Down Expand Up @@ -773,6 +791,29 @@ function parseToolArgs(args, schemaProperties = {}, stdinContent = null) {
let jsonOutput = false;
const hasSchemaProperties = Object.keys(schemaProperties).length > 0;
const { normalizedSchemaKeyMap, ambiguousNormalizedSchemaKeys } = buildNormalizedSchemaKeyMap(schemaProperties);

// Inline JSON payload mode: the whole payload is passed as a single quoted
// argument, e.g. `safeoutputs noop '{"message":"done"}'`. This keeps the
// binary first on the command line so a malformed invocation still runs the
// CLI and reports an error, unlike the piped `.` sentinel form where a
// dropped pipe silently skips the binary entirely.
const inlineJsonPayload = findInlineJsonPayloadArg(args);
if (inlineJsonPayload !== null) {
let parsed;
try {
parsed = JSON.parse(inlineJsonPayload);
} catch (err) {
throw new Error(`inline JSON argument is not valid JSON: ${getErrorMessage(err)}. Pass a single quoted JSON object, use --key value flags, or pipe JSON on stdin with '.'.`, { cause: err });
}
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
throw new Error("inline JSON argument must be a JSON object. Pass a single quoted JSON object, use --key value flags, or pipe JSON on stdin with '.'.");
}
for (const [key, value] of Object.entries(parsed)) {
const canonicalKey = resolveSchemaPropertyKey(key, schemaProperties, normalizedSchemaKeyMap, ambiguousNormalizedSchemaKeys);
result[canonicalKey] = value;
}
return { args: result, json: false };
}
// Trimmed stdin content used in both JSON payload mode and per-field stdin mode.
const trimmedStdin = stdinContent !== null ? stdinContent.trim() : null;
const isJsonPayloadMode = trimmedStdin !== null && (args.length === 0 || (args.length === 1 && args[0] === "."));
Expand Down Expand Up @@ -1246,7 +1287,7 @@ function showToolHelp(serverName, toolName, tools) {
...renderToolSignature(serverName, tool).split("\n"),
"Recommended:",
...renderToolRecommendedExample(serverName, tool).split("\n"),
`JSON mode: printf '{"param":"value",...}' | ${serverName} ${toolName} .`,
`JSON mode: ${serverName} ${toolName} '{"param":"value",...}' (or: printf '{"param":"value",...}' | ${serverName} ${toolName} .)`,
];

const props = tool.inputSchema?.properties;
Expand Down Expand Up @@ -1613,6 +1654,19 @@ async function main() {
}
const { args: toolArgs, json: jsonOutput } = parsedArgs;

// Fail loudly when arguments were supplied but none of them were recognized
// (e.g. `safeoutputs noop 'no action needed'`). Silently dropping them would
// leave the agent believing a safe output was emitted when none was. Structured
// payload modes are exempt: an explicit `{}` payload legitimately yields no arguments.
const usedStructuredPayload = shouldReadStdin || findInlineJsonPayloadArg(toolUserArgs) !== null;
if (serverName === SAFEOUTPUTS_SERVER_NAME && Object.keys(toolArgs).length === 0 && toolUserArgs.length > 0 && !jsonOutput && !usedStructuredPayload) {
const message = `no arguments were recognized for '${toolName}' from: ${toolUserArgs.join(" ")}. Pass a JSON object inline (${serverName} ${toolName} '{"key":"value"}'), use --key value flags, or pipe JSON on stdin with '.'.`;
auditLog(serverName, { event: "unrecognized_args", tool: toolName });
process.stderr.write(`Error: ${message}\n`);
core.setFailed(`[${serverName}] ${message}`);
return;
}

if (shouldShowToolHelpForEmptyArgs(serverName, toolArgs, matchedTool)) {
core.warning(`[${serverName}] No arguments provided for '${toolName}'; showing command help instead of calling the tool`);
auditLog(serverName, { event: "show_tool_help_empty_args", tool: toolName });
Expand Down Expand Up @@ -1697,6 +1751,7 @@ module.exports = {
showToolHelp,
shouldShowToolHelpForEmptyArgs,
hasStdinJsonPayload,
findInlineJsonPayloadArg,
readStdinSync,
ensureSafeOutputsTools,
refreshDeferredToolsIfNeeded,
Expand Down
Loading
Loading