test(evals): assert executor block retry on agent failure

Add executor-retries-failed-block: the first provider call rejects, the Agent block has retry enabled, and the executor replays it. The run must complete with the second response. Verifies providerCalls === 2, and fails without the retry policy (checked locally: expected 2, got 1).
sudoKrishna · Sep 29, 2026 · Sep 29, 2026 · Sep 29, 2026 · Sep 29, 2026 · Sep 29, 2026
commit fb7f54c73b9a8dbfe36fa951f3a500f3ec4967e2
diff --git a/apps/sim/evals/README.md b/apps/sim/evals/README.md
@@ -117,7 +117,10 @@ Add a case to `EXECUTOR_SCENARIOS` in `executor-harness.ts`:
 - `providerResponse` is what the mocked provider returns (`content`,
   `toolCalls`, `tokens`).
 - `expect` uses the loop's checks plus `resolvedInput` (a substring that must
-  reach the provider messages) and `succeeds` (expected `ExecutionResult.success`).
+  reach the provider messages), `succeeds` (expected `ExecutionResult.success`),
+  and `providerCalls` (exact provider call count).
+- Set `agent.retry` to exercise the executor's per-block retry policy; make the
+  first `providerResponse` a `reject` and the retry lands on the next one.
 
 Both suites write one report, so executor rows appear alongside loop rows.
 
@@ -132,8 +135,8 @@ model/tool time, first-response time, and token usage.
 ## Scope and next steps
 
 Two harnesses share one result shape and report: the tool loop and the
-`DAGExecutor`. The executor harness mocks the provider boundary, so the
-executor's retry/fallback policy is not yet asserted; add a scenario with a
-first-call rejection and a block retry config to cover it. Further expansion
-(context/memory, model routing, subagent orchestration) is tracked as
-follow-up work.
+`DAGExecutor`. The executor suite covers block retry —
+`executor-retries-failed-block` makes the first provider call reject, the block
+is replayed, and the run completes. Model fallback (`fallbackModels`) is not
+asserted yet. Further expansion (context/memory, model routing, subagent
+orchestration) is tracked as follow-up work.
diff --git a/apps/sim/evals/agent-tool-use/executor-harness.ts b/apps/sim/evals/agent-tool-use/executor-harness.ts
@@ -4,7 +4,7 @@ import {
 } from '@sim/testing/factories/serialized-block.factory'
 import { providersMockFns } from '@sim/testing/mocks/providers.mock'
 import { DAGExecutor } from '@/executor/execution/executor'
-import type { SerializedWorkflow } from '@/serializer/types'
+import type { SerializedBlock, SerializedWorkflow } from '@/serializer/types'
 import { type EvalRunMode, type ScoredToolCall, scoreExpectations } from './harness'
 import type {
   AgentToolUseExpectations,
@@ -30,8 +30,10 @@ export interface ExecutorProviderToolCall {
   result?: unknown
 }
 
-/** The provider response `executeProviderRequest` returns for one model call. */
+/** One model call: either a response, or a rejection the block must recover from. */
 export interface ExecutorProviderResponse {
+  /** When set, the call rejects with this message instead of resolving. */
+  reject?: string
   content: string
   model?: string
   tokens?: { input?: number; output?: number; total?: number }
@@ -52,26 +54,30 @@ export interface ExecutorScenario {
     systemPrompt?: string
     userPrompt?: string
     temperature?: number
+    /** Enables the executor's per-block retry policy for the Agent block. */
+    retry?: { enabled: boolean; maxTries: number; waitBetweenTriesMs: number }
   }
-  /** One entry per model call; the last entry serves any extra fallback calls. */
+  /** One entry per model call; the last entry serves any extra/retry calls. */
   providerResponse: ExecutorProviderResponse | ExecutorProviderResponse[]
   expect: AgentToolUseExpectations & {
     /** Substring that must appear in the messages sent to the provider. */
     resolvedInput?: string
     /** Expected `ExecutionResult.success`. */
     succeeds?: boolean
+    /** Exact number of provider calls the executor made. */
+    providerCalls?: number
   }
 }
 
 function buildWorkflow(scenario: ExecutorScenario): SerializedWorkflow {
-  const start = createSerializedBlock({
+  const start: SerializedBlock = createSerializedBlock({
     id: 'start',
     type: 'start_trigger',
     name: 'Start',
   })
   /** The trigger handler claims a block whose metadata says it is a trigger. */
   if (start.metadata) start.metadata.category = 'triggers'
-  const agent = createSerializedBlock({
+  const agent: SerializedBlock = createSerializedBlock({
     id: 'agent',
     type: 'agent',
     name: 'Eval Agent',
@@ -85,6 +91,7 @@ function buildWorkflow(scenario: ExecutorScenario): SerializedWorkflow {
       ? { temperature: scenario.agent.temperature }
       : {}),
   }
+  if (scenario.agent.retry) agent.retry = scenario.agent.retry
 
   return createSerializedWorkflow([start, agent], [{ source: 'start', target: 'agent' }])
 }
@@ -109,6 +116,7 @@ export async function runExecutorScenario(
       requests.push(request)
       const response = responses[Math.min(callIndex, responses.length - 1)]
       callIndex += 1
+      if (response.reject) throw new Error(response.reject)
       return {
         content: response.content,
         model: response.model ?? scenario.agent.model,
@@ -156,7 +164,14 @@ export async function runExecutorScenario(
     durationMs: typeof call.duration === 'number' ? call.duration : 0,
   }))
 
-  const checks = scoreExpectations(scenario.expect, toolCalls, finalContent, 1, runError, mode)
+  const checks = scoreExpectations(
+    scenario.expect,
+    toolCalls,
+    finalContent,
+    requests.length,
+    runError,
+    mode
+  )
 
   if (scenario.expect.resolvedInput !== undefined) {
     const sent = JSON.stringify(requests)
@@ -175,6 +190,14 @@ export async function runExecutorScenario(
     })
   }
 
+  if (scenario.expect.providerCalls !== undefined) {
+    checks.push({
+      name: 'provider-calls',
+      passed: requests.length === scenario.expect.providerCalls,
+      detail: `expected ${scenario.expect.providerCalls}, got ${requests.length}`,
+    })
+  }
+
   const tokens = (output.tokens ?? {}) as { input?: number; output?: number; total?: number }
 
   return {
@@ -259,4 +282,29 @@ export const EXECUTOR_SCENARIOS: ExecutorScenario[] = [
       finalContent: /A-1937/,
     },
   },
+  {
+    id: 'executor-retries-failed-block',
+    name: 'retries a failed Agent block and completes the run',
+    category: 'recovery',
+    description:
+      'The first provider call rejects with a 503. The block has retry enabled, so the executor replays it and the second call succeeds — proving the executor retry policy, not the agent handler, recovered the turn.',
+    workflowInput: { message: 'What is the API rate limit?' },
+    agent: {
+      model: 'gpt-4o',
+      userPrompt: 'What is the API rate limit?',
+      retry: { enabled: true, maxTries: 3, waitBetweenTriesMs: 0 },
+    },
+    providerResponse: [
+      { reject: '503 Service Unavailable', content: '' },
+      {
+        content: 'The API rate limit is 100 requests per minute.',
+        tokens: { input: 10, output: 20, total: 30 },
+      },
+    ],
+    expect: {
+      succeeds: true,
+      finalContent: '100 requests per minute',
+      providerCalls: 2,
+    },
+  },
 ]