Fix Codex MCP watchdog edge cases

Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com>
pelikhan · Oct 7, 2026 · Oct 6, 2026 · Oct 6, 2026 · Oct 6, 2026 · Oct 7, 2026
commit 41f97a2b7e214d4a494136e042624cc17f9e7913
diff --git a/actions/setup/js/codex_harness.cjs b/actions/setup/js/codex_harness.cjs
@@ -34,6 +34,7 @@
 
 const { getErrorMessage } = require("./error_helpers.cjs");
 const fs = require("fs");
+const { loadCompiledConfig, mergeConfig } = require("./codex_config.cjs");
 const { runProcess, formatDuration, sleep, MIN_POST_RESULT_WATCHDOG_TIMEOUT_MS, DEFAULT_POST_RESULT_WATCHDOG_IDLE_TIMEOUT_MS, MAX_POST_RESULT_WATCHDOG_TIMEOUT_MS, resolvePostResultWatchdogIdleTimeoutMs } = require("./process_runner.cjs");
 const { runHarnessRetryLoop, shouldSkipForNoopSafeOutputs, shouldStopForNoopSafeOutputs } = require("./harness_retry_runner.cjs");
 const {
@@ -92,9 +93,37 @@ const SERVER_ERROR_PATTERN = /InternalServerError|ServiceUnavailableError|500 In
 // an identical rejection: retrying only re-bills the turns that succeeded before the failure point.
 const INVALID_REQUEST_ERROR_PATTERN = /invalid_request_error/i;
 const DEFAULT_MCP_CALL_WATCHDOG_MS = 120_000;
+const MCP_CALL_TRANSPORT_GRACE_MS = 60_000;
+
+function resolveMCPServerToolTimeouts(config, runtimeToolTimeoutSeconds) {
+  const configuredServers = config.defaults?.mcp_servers && typeof config.defaults.mcp_servers === "object" ? config.defaults.mcp_servers : {};
+  const defaults = {
+    ...config.defaults,
+    mcp_servers: Object.fromEntries(
+      Object.entries(configuredServers).map(([name, value]) => [
+        name,
+        typeof value === "object" && value !== null && !Array.isArray(value)
+          ? { ...value, ...(Number.isSafeInteger(runtimeToolTimeoutSeconds) && runtimeToolTimeoutSeconds > 0 ? { tool_timeout_sec: runtimeToolTimeoutSeconds } : {}) }
+          : value,
+      ])
+    ),
+  };
+  const effectiveServers = mergeConfig(defaults, config.overrides || {}).mcp_servers || {};
+  return Object.fromEntries(
+    Object.entries(effectiveServers).flatMap(([name, value]) => (typeof value?.tool_timeout_sec === "number" && Number.isSafeInteger(value.tool_timeout_sec) && value.tool_timeout_sec > 0 ? [[name, value.tool_timeout_sec]] : []))
+  );
+}
 
 function createMCPCallWatchdog(timeoutMs, now = Date.now) {
   const pending = new Map();
+  function track(eventType, item) {
+    if (item?.type !== "mcp_tool_call" || typeof item.id !== "string") return;
+    if (eventType === "item.started") {
+      const callTimeoutMs = typeof timeoutMs === "function" ? timeoutMs(item) : timeoutMs;
+      pending.set(item.id, { startedAt: now(), timeoutMs: Number.isSafeInteger(callTimeoutMs) && callTimeoutMs > 0 ? callTimeoutMs : DEFAULT_MCP_CALL_WATCHDOG_MS });
+    }
+    if (eventType === "item.completed" || eventType === "item.failed") pending.delete(item.id);
+  }
   return {
     observe(line) {
       let event;
@@ -103,13 +132,39 @@ function createMCPCallWatchdog(timeoutMs, now = Date.now) {
       } catch {
         return;
       }
-      const item = event?.item;
-      if (item?.type !== "mcp_tool_call" || typeof item.id !== "string") return;
-      if (event.type === "item.started") pending.set(item.id, now());
-      if (event.type === "item.completed" || event.type === "item.failed") pending.delete(item.id);
+      track(event?.type, event?.item);
+    },
+    observePrefix(prefix) {
+      const eventType = /^\s*\{\s*"type"\s*:\s*"(item\.started|item\.completed|item\.failed)"/.exec(prefix)?.[1];
+      if (!eventType) return;
+      const itemStart = /,\s*"item"\s*:\s*\{/.exec(prefix);
+      if (!itemStart) return;
+      const itemPrefix = prefix.slice(itemStart.index + itemStart[0].length);
+      if (!/(?:^|,)\s*"type"\s*:\s*"mcp_tool_call"/.test(itemPrefix)) return;
+      const idMatch = /(?:^|,)\s*"id"\s*:\s*("(?:\\.|[^"\\])*")/.exec(itemPrefix);
+      if (!idMatch) return;
+      const serverMatch = /(?:^|,)\s*"server(?:_name)?"\s*:\s*("(?:\\.|[^"\\])*")/.exec(itemPrefix);
+      let item;
+      try {
+        item = {
+          type: "mcp_tool_call",
+          id: JSON.parse(idMatch[1]),
+          ...(serverMatch ? { server: JSON.parse(serverMatch[1]) } : {}),
+        };
+      } catch {
+        return;
+      }
+      track(eventType, item);
+    },
+    expiredTimeoutMs() {
+      const current = now();
+      for (const call of pending.values()) {
+        if (current - call.startedAt >= call.timeoutMs) return call.timeoutMs;
+      }
+      return null;
     },
     expired() {
-      return [...pending.values()].some(start => now() - start >= timeoutMs);
+      return this.expiredTimeoutMs() !== null;
     },
   };
 }
@@ -831,8 +886,12 @@ async function main() {
     runAttempt: async attempt => {
       const terminalErrors = [];
       const configuredToolTimeout = Number(codexEnv.GH_AW_TOOL_TIMEOUT);
-      const mcpWatchdogTimeoutMs = Number.isSafeInteger(configuredToolTimeout) && configuredToolTimeout > 0 ? configuredToolTimeout * 1000 + 60_000 : DEFAULT_MCP_CALL_WATCHDOG_MS;
-      const mcpWatchdog = createMCPCallWatchdog(mcpWatchdogTimeoutMs);
+      const fallbackToolTimeoutMs = Number.isSafeInteger(configuredToolTimeout) && configuredToolTimeout > 0 ? configuredToolTimeout * 1000 + MCP_CALL_TRANSPORT_GRACE_MS : DEFAULT_MCP_CALL_WATCHDOG_MS;
+      const serverToolTimeouts = resolveMCPServerToolTimeouts(loadCompiledConfig(), configuredToolTimeout);
+      const mcpWatchdog = createMCPCallWatchdog(item => {
+        const server = item.server ?? item.server_name ?? item.serverName;
+        return serverToolTimeouts[server] ? serverToolTimeouts[server] * 1000 + MCP_CALL_TRANSPORT_GRACE_MS : fallbackToolTimeoutMs;
+      });
       let nextContextCheckAt = Date.now() + contextRebuildCircuitBreaker.pollIntervalMs;
       // Track the file size before this attempt so the watchdog only arms on output
       // written by this attempt, not by a previous retry.
@@ -858,12 +917,14 @@ async function main() {
             }
           } catch {}
         },
+        onStdoutLinePrefix: prefix => mcpWatchdog.observePrefix(prefix),
         runtimeGuard: {
           pollIntervalMs: Math.min(contextRebuildCircuitBreaker.pollIntervalMs, 1000),
           termGraceMs: contextRebuildCircuitBreaker.termGraceMs,
           shouldTerminate: async () => {
             if (softTimeoutGuard && Date.now() >= softTimeoutGuard.softDeadlineMs) return { terminate: true, reason: "Codex reached the soft execution deadline; stopping to preserve structured output before the step timeout." };
-            if (mcpWatchdog.expired()) return { terminate: true, reason: `transport_wedge: MCP tool call timed out after ${Math.round(mcpWatchdogTimeoutMs / 1000)}s` };
+            const expiredMCPCallTimeoutMs = mcpWatchdog.expiredTimeoutMs();
+            if (expiredMCPCallTimeoutMs !== null) return { terminate: true, reason: `transport_wedge: MCP tool call timed out after ${Math.round(expiredMCPCallTimeoutMs / 1000)}s` };
             if (!contextRebuildCircuitBreaker.enabled) return false;
             if (Date.now() < nextContextCheckAt) return false;
             nextContextCheckAt = Date.now() + contextRebuildCircuitBreaker.pollIntervalMs;
@@ -1133,6 +1194,7 @@ if (typeof module !== "undefined" && module.exports) {
     getCodexModelEnvVar,
     resolvePostResultWatchdogIdleTimeoutMs,
     createMCPCallWatchdog,
+    resolveMCPServerToolTimeouts,
     POST_RESULT_WATCHDOG_IDLE_TIMEOUT_MS,
     DEFAULT_POST_RESULT_WATCHDOG_IDLE_TIMEOUT_MS,
     MIN_POST_RESULT_WATCHDOG_TIMEOUT_MS,

diff --git a/actions/setup/js/codex_harness.test.cjs b/actions/setup/js/codex_harness.test.cjs
@@ -42,6 +42,7 @@ const {
   DEFAULT_CONTEXT_REBUILD_TERM_GRACE_MS,
   resolvePostResultWatchdogIdleTimeoutMs,
   createMCPCallWatchdog,
+  resolveMCPServerToolTimeouts,
   DEFAULT_POST_RESULT_WATCHDOG_IDLE_TIMEOUT_MS,
   MIN_POST_RESULT_WATCHDOG_TIMEOUT_MS,
   MAX_POST_RESULT_WATCHDOG_TIMEOUT_MS,
@@ -141,6 +142,37 @@ describe("codex_harness.cjs", () => {
       time = 200;
       expect(watchdog.expired()).toBe(false);
     });
+
+    it("uses per-server timeouts after the global timeout override", () => {
+      const timeouts = resolveMCPServerToolTimeouts(
+        {
+          defaults: { mcp_servers: { github: { tool_timeout_sec: 60 }, search: { tool_timeout_sec: 30 } } },
+          overrides: { mcp_servers: { github: { tool_timeout_sec: 180 } } },
+        },
+        90
+      );
+      expect(timeouts).toEqual({ github: 180, search: 90 });
+
+      let time = 0;
+      const watchdog = createMCPCallWatchdog(
+        item => timeouts[item.server] * 1000 + 60_000,
+        () => time
+      );
+      watchdog.observe(JSON.stringify({ type: "item.started", item: { type: "mcp_tool_call", id: "1", server: "github" } }));
+      time = 239_999;
+      expect(watchdog.expired()).toBe(false);
+      time = 240_000;
+      expect(watchdog.expiredTimeoutMs()).toBe(240_000);
+    });
+
+    it("clears an oversized completion from its bounded lifecycle prefix", () => {
+      let time = 0;
+      const watchdog = createMCPCallWatchdog(100, () => time);
+      watchdog.observe(JSON.stringify({ type: "item.started", item: { type: "mcp_tool_call", id: "large" } }));
+      time = 100;
+      watchdog.observePrefix('{"type":"item.completed","item":{"id":"large","type":"mcp_tool_call","server":"github","result":"');
+      expect(watchdog.expired()).toBe(false);
+    });
   });
   describe("native exec orchestration", () => {
     it("preserves a native argument-parse exit without retrying a deterministic startup error", () => {

diff --git a/actions/setup/js/handle_agent_failure.cjs b/actions/setup/js/handle_agent_failure.cjs
@@ -333,7 +333,7 @@ function buildFailureMatchCategories(options) {
 }
 
 function hasMCPTransportWedge(logContent) {
-  return /\[codex-harness\][^\n]*runtime guard requested termination \(transport_wedge: MCP tool call timed out after \d+s\)/.test(logContent);
+  return /^\[codex-harness\](?: \d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z)?)? attempt \d+: runtime guard requested termination \(transport_wedge: MCP tool call timed out after \d+s\) \(SIGTERM\)\r?$/m.test(logContent);
 }
 
 /**

diff --git a/actions/setup/js/handle_agent_failure.test.cjs b/actions/setup/js/handle_agent_failure.test.cjs
@@ -6415,8 +6415,10 @@ describe("handle_agent_failure", () => {
 
     it("recognizes only the harness MCP watchdog termination record", () => {
       const { hasMCPTransportWedge } = require("./handle_agent_failure.cjs");
-      expect(hasMCPTransportWedge("[codex-harness] 2026-01-01 attempt 1: runtime guard requested termination (transport_wedge: MCP tool call timed out after 120s) (SIGTERM)")).toBe(true);
+      expect(hasMCPTransportWedge("[codex-harness] 2026-01-01T00:00:00.000Z attempt 1: runtime guard requested termination (transport_wedge: MCP tool call timed out after 120s) (SIGTERM)")).toBe(true);
+      expect(hasMCPTransportWedge("[codex-harness] attempt 1: runtime guard requested termination (transport_wedge: MCP tool call timed out after 120s) (SIGTERM)")).toBe(true);
       expect(hasMCPTransportWedge("transport_wedge: MCP tool call timed out after 120s")).toBe(false);
+      expect(hasMCPTransportWedge('{"type":"item.failed","message":"[codex-harness] 2026-01-01T00:00:00.000Z attempt 1: runtime guard requested termination (transport_wedge: MCP tool call timed out after 120s) (SIGTERM)"}')).toBe(false);
     });
 
     it("returns missing_safe_outputs category", () => {

diff --git a/actions/setup/js/process_runner.cjs b/actions/setup/js/process_runner.cjs
@@ -79,6 +79,7 @@ function sleep(ms) {
  *   env?: NodeJS.ProcessEnv,
  *   stdin?: string | Buffer,
  *   onStdoutLine?: (line: string) => void,
+ *   onStdoutLinePrefix?: (prefix: string) => void,
  *   onStderrLine?: (line: string) => void,
  *   shutdownGraceMs?: number,
  *   drainTimeoutMs?: number,
@@ -118,6 +119,7 @@ function runProcess({
   env,
   stdin,
   onStdoutLine,
+  onStdoutLinePrefix,
   onStderrLine,
   shutdownGraceMs = 5000,
   drainTimeoutMs = 2000,
@@ -214,14 +216,15 @@ function runProcess({
       // Complete output still goes to the parent's log streams. Keep a bounded classifier tail.
       return Buffer.from(combined).subarray(-outputLimit).toString("utf8");
     };
-    const lineObserver = callback => {
+    const lineObserver = (callback, onOversizedLine) => {
       let pending = "";
       let dropping = false;
       return (text, final = false) => {
         if (!callback) return;
         for (const part of text.split(/(?<=\n)/)) {
           if (!dropping) pending += part;
           if (pending.length > 1024 * 1024) {
+            onOversizedLine?.(pending.slice(0, 64 * 1024));
             pending = "";
             dropping = true;
           }
@@ -234,7 +237,7 @@ function runProcess({
         if (final) pending = "";
       };
     };
-    const observeStdout = lineObserver(onStdoutLine);
+    const observeStdout = lineObserver(onStdoutLine, onStdoutLinePrefix);
     const observeStderr = lineObserver(onStderrLine);
     /** @param {NodeJS.Signals} signal */
     function signalTree(signal) {

diff --git a/actions/setup/js/process_runner.test.cjs b/actions/setup/js/process_runner.test.cjs
@@ -139,6 +139,32 @@ describe("process_runner.cjs", () => {
       }
     });
 
+    it("preserves a bounded stdout prefix when dropping an oversized line", async () => {
+      const observedLines = [];
+      const observedPrefixes = [];
+      const original = process.stdout.write;
+      process.stdout.write = () => true;
+      try {
+        const result = await runProcess({
+          command: process.execPath,
+          args: ["-e", `process.stdout.write(JSON.stringify({type:"item.completed",item:{id:"call-1",type:"mcp_tool_call",server:"github",result:"x".repeat(1048576)}})+"\\n")`],
+          attempt: 0,
+          log: () => {},
+          maxCollectedOutputBytes: 1024,
+          onStdoutLine: line => observedLines.push(line),
+          onStdoutLinePrefix: prefix => observedPrefixes.push(prefix),
+        });
+        expect(result.exitCode).toBe(0);
+        expect(observedLines).toEqual([]);
+        expect(observedPrefixes).toHaveLength(1);
+        expect(observedPrefixes[0]).toContain('"id":"call-1"');
+        expect(observedPrefixes[0]).toContain('"server":"github"');
+        expect(observedPrefixes[0]).toHaveLength(64 * 1024);
+      } finally {
+        process.stdout.write = original;
+      }
+    });
+
     it("kills descendants holding inherited stdio when the runtime guard fires", async () => {
       const result = await runProcess({
         command: process.execPath,
-Original file line number
+Diff line change
@@ Expand Up / @@ -333,7 +333,7 @@ function buildFailureMatchCategories(options) { @@
     }
     function hasMCPTransportWedge(logContent) {
-      return /\[codex-harness\][^\n]*runtime guard requested termination \(transport_wedge: MCP tool call timed out after \d+s\)/.test(logContent);
+      return /^\[codex-harness\](?: \d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z)?)? attempt \d+: runtime guard requested termination \(transport_wedge: MCP tool call timed out after \d+s\) \(SIGTERM\)\r?$/m.test(logContent);
     }
     /**
@@ Expand Down @@