feat(evals): run agent scenarios through the DAGExecutor

Add an executor-level harness: a real Start -> Agent workflow on DAGExecutor, with only executeProviderRequest mocked at the provider boundary. This covers agent-block input wiring, variable resolution from Start outputs, and executor run/error handling, which the direct loop harness cannot see. - executor-harness.ts: workflow builder + runExecutorScenario - shares the scorer (scoreExpectations) and report with the loop suite - two scenarios: Start->Agent output, and <start.message> resolution - README documents adding an executor-level scenario
sudoKrishna · Sep 29, 2026 · Sep 29, 2026 · Sep 29, 2026 · Sep 29, 2026 · Sep 29, 2026
commit 85525a412210c54290a9583b060bd4b3aa7db238
diff --git a/apps/sim/evals/README.md b/apps/sim/evals/README.md
@@ -100,6 +100,27 @@ A scenario is data, not code — there is no harness change needed for a new cas
   string. The loop must not execute the call and must return the parse error to
   the model.
 
+## Executor-level scenarios
+
+[`agent-tool-use/executor-harness.ts`](./agent-tool-use/executor-harness.ts)
+runs a case through a real `DAGExecutor`: a Start block → Agent block workflow,
+with only the provider boundary (`executeProviderRequest`) mocked. This covers
+what the loop harness cannot — agent-block input wiring, variable resolution
+from Start outputs, and the executor's run/error handling. Tool dispatch stays
+covered by the loop suite.
+
+Add a case to `EXECUTOR_SCENARIOS` in `executor-harness.ts`:
+
+- `workflowInput` is exposed on the Start block; reference an output with
+  `<start.field>` from the Agent prompt.
+- `agent` is the Agent block config (`model`, `systemPrompt`, `userPrompt`).
+- `providerResponse` is what the mocked provider returns (`content`,
+  `toolCalls`, `tokens`).
+- `expect` uses the loop's checks plus `resolvedInput` (a substring that must
+  reach the provider messages) and `succeeds` (expected `ExecutionResult.success`).
+
+Both suites write one report, so executor rows appear alongside loop rows.
+
 ## Report shape
 
 `report.json` is machine-readable for dashboards and trend tracking; `report.md`
@@ -110,8 +131,9 @@ model/tool time, first-response time, and token usage.
 
 ## Scope and next steps
 
-This suite evaluates the tool loop directly. The next layer is a scenario that
-runs the same scripted model through the full `DAGExecutor` so agent block
-wiring, variable resolution, and the executor's retry/fallback policy are
-measured alongside the loop. The `AgentToolUseResult` shape is deliberately
-independent of the harness entry point so both can share scoring and reporting.
+Two harnesses share one result shape and report: the tool loop and the
+`DAGExecutor`. The executor harness mocks the provider boundary, so the
+executor's retry/fallback policy is not yet asserted; add a scenario with a
+first-call rejection and a block retry config to cover it. Further expansion
+(context/memory, model routing, subagent orchestration) is tracked as
+follow-up work.
diff --git a/apps/sim/evals/agent-tool-use/agent-tool-use.eval.test.ts b/apps/sim/evals/agent-tool-use/agent-tool-use.eval.test.ts
@@ -1,8 +1,14 @@
+import {
+  permissionCheckMock,
+  permissionCheckMockFns,
+} from '@sim/testing/mocks/permission-check.mock'
 import { providersMock } from '@sim/testing/mocks/providers.mock'
 import { providersConversationHistoryMock } from '@sim/testing/mocks/providers-conversation-history.mock'
-import { providersUtilsMock } from '@sim/testing/mocks/providers-utils.mock'
+import { providersUtilsMock, providersUtilsMockFns } from '@sim/testing/mocks/providers-utils.mock'
 import { toolsMock } from '@sim/testing/mocks/tools.mock'
-import { afterAll, describe, expect, it, vi } from 'vitest'
+import { workspaceFileSecretProvenanceMock } from '@sim/testing/mocks/workspace-file-secret-provenance.mock'
+import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest'
+import { EXECUTOR_SCENARIOS, runExecutorScenario } from '@/evals/agent-tool-use/executor-harness'
 import { runScenario } from '@/evals/agent-tool-use/harness'
 import { writeEvalReport } from '@/evals/agent-tool-use/report'
 import { AGENT_TOOL_USE_SCENARIOS } from '@/evals/agent-tool-use/scenarios'
@@ -12,9 +18,37 @@ vi.mock('@/providers/conversation-history', () => providersConversationHistoryMo
 vi.mock('@/tools', () => toolsMock)
 vi.mock('@/providers/utils', () => providersUtilsMock)
 vi.mock('@/providers', () => providersMock)
+vi.mock('@/ee/access-control/utils/permission-check', () => permissionCheckMock)
+vi.mock(
+  '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance',
+  () => workspaceFileSecretProvenanceMock
+)
+vi.mock('@/lib/memory/agent-turn-session', () => ({
+  openAgentTurnSession: vi.fn(async () => undefined),
+}))
+vi.mock('@/lib/internal/mcp/discover-tools', () => ({
+  discoverMcpServerToolsAsExecutor: vi.fn(async () => []),
+}))
+vi.mock('@/lib/internal/custom-tools/read-available-by-id-or-title', () => ({
+  readAvailableCustomToolByIdOrTitleAsExecutor: vi.fn(async () => undefined),
+}))
+vi.mock('@/executor/utils/http', () => ({
+  buildAuthHeaders: vi.fn(async () => ({ 'Content-Type': 'application/json' })),
+  buildAPIUrl: vi.fn((path: string) => path),
+  extractAPIErrorMessage: vi.fn(async () => 'request failed'),
+}))
+vi.mock('@/lib/execution/cancellation', () => ({
+  subscribeToExecutionCancellation: vi.fn(async () => () => {}),
+  isExecutionCancelled: vi.fn(async () => false),
+}))
 
 const results: AgentToolUseResult[] = []
 
+beforeEach(() => {
+  permissionCheckMockFns.mockValidateModelProvider.mockResolvedValue(undefined)
+  providersUtilsMockFns.mockGetProviderFromModel.mockReturnValue('mock-provider')
+})
+
 afterAll(() => {
   const reportPath = process.env.EVAL_REPORT_PATH
   if (reportPath) writeEvalReport(results, reportPath)
@@ -32,3 +66,16 @@ describe('agent tool-use eval suite', () => {
     ).toEqual([])
   })
 })
+
+describe('agent executor eval suite', () => {
+  it.each(EXECUTOR_SCENARIOS)('$id: $name', async (scenario) => {
+    const result = await runExecutorScenario(scenario)
+    results.push(result)
+
+    const failed = result.checks.filter((entry) => !entry.passed)
+    expect(
+      failed,
+      failed.map((entry) => `${entry.name}: ${entry.detail}`).join('; ') || undefined
+    ).toEqual([])
+  })
+})
diff --git a/apps/sim/evals/agent-tool-use/executor-harness.ts b/apps/sim/evals/agent-tool-use/executor-harness.ts
@@ -0,0 +1,262 @@
+import {
+  createSerializedBlock,
+  createSerializedWorkflow,
+} from '@sim/testing/factories/serialized-block.factory'
+import { providersMockFns } from '@sim/testing/mocks/providers.mock'
+import { DAGExecutor } from '@/executor/execution/executor'
+import type { SerializedWorkflow } from '@/serializer/types'
+import { type EvalRunMode, type ScoredToolCall, scoreExpectations } from './harness'
+import type {
+  AgentToolUseExpectations,
+  AgentToolUseResult,
+  EvalCategory,
+  EvalToolInvocation,
+} from './types'
+
+/**
+ * Executor-level harness.
+ *
+ * Drives a real `DAGExecutor` run: Start block → Agent block. The provider
+ * boundary (`executeProviderRequest`) is the only thing mocked — the Agent
+ * block handler, input/variable resolution, and the executor run/error handling
+ * are real. Tool calls are what the mocked provider returns; tool *dispatch* is
+ * covered by the loop harness.
+ */
+
+/** One tool call the mocked provider reports in its response. */
+export interface ExecutorProviderToolCall {
+  name: string
+  arguments?: Record<string, unknown>
+  result?: unknown
+}
+
+/** The provider response `executeProviderRequest` returns for one model call. */
+export interface ExecutorProviderResponse {
+  content: string
+  model?: string
+  tokens?: { input?: number; output?: number; total?: number }
+  toolCalls?: ExecutorProviderToolCall[]
+  cost?: unknown
+  timing?: unknown
+}
+
+export interface ExecutorScenario {
+  id: string
+  name: string
+  category: EvalCategory
+  description: string
+  /** Exposed on the Start block and referenced from the Agent block. */
+  workflowInput: Record<string, unknown>
+  agent: {
+    model: string
+    systemPrompt?: string
+    userPrompt?: string
+    temperature?: number
+  }
+  /** One entry per model call; the last entry serves any extra fallback calls. */
+  providerResponse: ExecutorProviderResponse | ExecutorProviderResponse[]
+  expect: AgentToolUseExpectations & {
+    /** Substring that must appear in the messages sent to the provider. */
+    resolvedInput?: string
+    /** Expected `ExecutionResult.success`. */
+    succeeds?: boolean
+  }
+}
+
+function buildWorkflow(scenario: ExecutorScenario): SerializedWorkflow {
+  const start = createSerializedBlock({
+    id: 'start',
+    type: 'start_trigger',
+    name: 'Start',
+  })
+  /** The trigger handler claims a block whose metadata says it is a trigger. */
+  if (start.metadata) start.metadata.category = 'triggers'
+  const agent = createSerializedBlock({
+    id: 'agent',
+    type: 'agent',
+    name: 'Eval Agent',
+  })
+  agent.config.tool = 'agent'
+  agent.config.params = {
+    model: scenario.agent.model,
+    systemPrompt: scenario.agent.systemPrompt,
+    userPrompt: scenario.agent.userPrompt,
+    ...(scenario.agent.temperature !== undefined
+      ? { temperature: scenario.agent.temperature }
+      : {}),
+  }
+
+  return createSerializedWorkflow([start, agent], [{ source: 'start', target: 'agent' }])
+}
+
+/**
+ * Runs one executor scenario and scores it with the shared scorer, returning
+ * the same result shape as the loop harness so both land in one report.
+ */
+export async function runExecutorScenario(
+  scenario: ExecutorScenario,
+  options: { mode?: EvalRunMode } = {}
+): Promise<AgentToolUseResult> {
+  const mode = options.mode ?? 'scripted'
+  const responses = Array.isArray(scenario.providerResponse)
+    ? [...scenario.providerResponse]
+    : [scenario.providerResponse]
+  const requests: Array<Record<string, unknown>> = []
+  let callIndex = 0
+
+  providersMockFns.mockExecuteProviderRequest.mockImplementation(
+    async (_providerId: string, request: Record<string, unknown>) => {
+      requests.push(request)
+      const response = responses[Math.min(callIndex, responses.length - 1)]
+      callIndex += 1
+      return {
+        content: response.content,
+        model: response.model ?? scenario.agent.model,
+        tokens: response.tokens ?? { input: 0, output: 0, total: 0 },
+        toolCalls: response.toolCalls ?? [],
+        cost: response.cost ?? 0,
+        timing: response.timing ?? { total: 0 },
+      }
+    }
+  )
+
+  const executor = new DAGExecutor({
+    workflow: buildWorkflow(scenario),
+    workflowInput: scenario.workflowInput,
+    contextExtensions: {
+      workspaceId: 'eval-workspace',
+      executionId: 'eval-execution',
+      userId: 'eval-user',
+    },
+  })
+
+  let result: { success?: boolean; output?: Record<string, unknown> } | undefined
+  let runError: unknown
+  const startedAt = Date.now()
+  try {
+    result = (await executor.execute('eval-workflow')) as typeof result
+  } catch (error) {
+    runError = error
+  }
+  const latencyMs = Date.now() - startedAt
+
+  const output = (result?.output ?? {}) as Record<string, unknown>
+  const finalContent = typeof output.content === 'string' ? output.content : ''
+  const rawToolCalls = ((output.toolCalls as { list?: unknown[] } | undefined)?.list ??
+    []) as Array<Record<string, unknown>>
+
+  const toolCalls: ScoredToolCall[] = rawToolCalls.map((call) => ({
+    name: typeof call.name === 'string' ? call.name : 'unknown',
+    success: true,
+  }))
+  const toolInvocations: EvalToolInvocation[] = rawToolCalls.map((call) => ({
+    name: typeof call.name === 'string' ? call.name : 'unknown',
+    arguments: (call.arguments ?? {}) as Record<string, unknown>,
+    success: true,
+    durationMs: typeof call.duration === 'number' ? call.duration : 0,
+  }))
+
+  const checks = scoreExpectations(scenario.expect, toolCalls, finalContent, 1, runError, mode)
+
+  if (scenario.expect.resolvedInput !== undefined) {
+    const sent = JSON.stringify(requests)
+    checks.push({
+      name: 'resolved-input',
+      passed: sent.includes(scenario.expect.resolvedInput),
+      detail: `looking for ${JSON.stringify(scenario.expect.resolvedInput)} in provider messages`,
+    })
+  }
+
+  if (scenario.expect.succeeds !== undefined) {
+    checks.push({
+      name: 'workflow-success',
+      passed: result?.success === scenario.expect.succeeds,
+      detail: `success=${String(result?.success)}`,
+    })
+  }
+
+  const tokens = (output.tokens ?? {}) as { input?: number; output?: number; total?: number }
+
+  return {
+    id: scenario.id,
+    name: scenario.name,
+    category: scenario.category,
+    passed: checks.every((entry) => entry.passed),
+    checks,
+    finalContent,
+    toolInvocations,
+    metrics: {
+      iterations: requests.length,
+      toolCalls: toolCalls.length,
+      successfulToolCalls: toolCalls.filter((call) => call.success).length,
+      erroredToolCalls: 0,
+      latencyMs,
+      modelTimeMs: 0,
+      toolsTimeMs: 0,
+      firstResponseTimeMs: 0,
+      inputTokens: tokens.input ?? 0,
+      outputTokens: tokens.output ?? 0,
+      totalTokens: tokens.total ?? 0,
+    },
+    ...(runError ? { error: String(runError) } : {}),
+  }
+}
+
+/**
+ * Executor-level scenarios. Two cover the wiring the loop suite cannot see:
+ * Start → Agent execution, and variable resolution from a Start output into the
+ * Agent's prompt.
+ */
+export const EXECUTOR_SCENARIOS: ExecutorScenario[] = [
+  {
+    id: 'executor-agent-runs',
+    name: 'runs a Start → Agent workflow and surfaces the Agent output',
+    category: 'tool-selection',
+    description:
+      'The real Agent block handler runs inside the DAG. The mocked provider reports one tool call; the executor result must carry the content and the tool call through.',
+    workflowInput: { message: 'What is the API rate limit?' },
+    agent: {
+      model: 'gpt-4o',
+      systemPrompt: 'You are a documentation assistant.',
+      userPrompt: 'What is the API rate limit?',
+    },
+    providerResponse: {
+      content: 'The API rate limit is 100 requests per minute.',
+      toolCalls: [
+        {
+          name: 'search_docs',
+          arguments: { query: 'api rate limit' },
+          result: { snippet: 'The API rate limit is 100 requests per minute.' },
+        },
+      ],
+      tokens: { input: 10, output: 20, total: 30 },
+    },
+    expect: {
+      succeeds: true,
+      finalContent: '100 requests per minute',
+      toolCallSequence: ['search_docs'],
+      successfulToolCalls: 1,
+    },
+  },
+  {
+    id: 'executor-resolves-start-input',
+    name: 'resolves a Start output into the Agent prompt before the provider call',
+    category: 'planning',
+    description:
+      'The Agent userPrompt references <start.message>. The value must be resolved by the executor and reach the provider request, not passed through verbatim.',
+    workflowInput: { message: 'Summarize order A-1937' },
+    agent: {
+      model: 'gpt-4o',
+      userPrompt: '<start.message>',
+    },
+    providerResponse: {
+      content: 'Order A-1937 shipped via DHL.',
+      tokens: { input: 8, output: 12, total: 20 },
+    },
+    expect: {
+      succeeds: true,
+      resolvedInput: 'Summarize order A-1937',
+      finalContent: /A-1937/,
+    },
+  },
+]