sudoKrishna · Sep 29, 2026 · Sep 29, 2026 · Sep 29, 2026 · Sep 29, 2026 · Sep 29, 2026
diff --git a/apps/sim/evals/README.md b/apps/sim/evals/README.md
@@ -145,6 +145,25 @@ Add a case to `EXECUTOR_SCENARIOS` in `executor-harness.ts`:
 
 Both suites write one report, so executor rows appear alongside loop rows.
 
+## Context evals
+
+[`agent-context/`](./agent-context/) drives the Agent block through the executor
+with conversation memory on. The memory read is stubbed per conversation id, so
+the provider request shows exactly what the handler assembled: prior history,
+then the new user prompt, with the system prompt preserved, and the conversation
+id must match. Windowing inside the memory service (`sliding_window`, token
+budgets) is covered by its unit tests; this suite covers the assembly the model
+sees.
+
+```sh
+cd apps/sim
+bun run test:evals:context   # writes test-results/evals/agent-context.{json,md}
+```
+
+Scenarios live in [`agent-context/scenarios.ts`](./agent-context/scenarios.ts)
+and reuse the executor harness, so a case is the same shape as an executor case
+plus `agent.memory`.
+
 ## Report shape
 
 `report.json` is machine-readable for dashboards and trend tracking; `report.md`

diff --git a/apps/sim/evals/agent-context/agent-context.eval.test.ts b/apps/sim/evals/agent-context/agent-context.eval.test.ts
@@ -0,0 +1,67 @@
+import {
+  permissionCheckMock,
+  permissionCheckMockFns,
+} from '@sim/testing/mocks/permission-check.mock'
+import { providersMock } from '@sim/testing/mocks/providers.mock'
+import { providersConversationHistoryMock } from '@sim/testing/mocks/providers-conversation-history.mock'
+import { providersUtilsMock, providersUtilsMockFns } from '@sim/testing/mocks/providers-utils.mock'
+import { toolsMock } from '@sim/testing/mocks/tools.mock'
+import { workspaceFileSecretProvenanceMock } from '@sim/testing/mocks/workspace-file-secret-provenance.mock'
+import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest'
+import { AGENT_CONTEXT_SCENARIOS } from '@/evals/agent-context/scenarios'
+import { runExecutorScenario } from '@/evals/agent-tool-use/executor-harness'
+import { writeEvalReport } from '@/evals/agent-tool-use/report'
+import type { AgentToolUseResult } from '@/evals/agent-tool-use/types'
+
+vi.mock('@/providers/conversation-history', () => providersConversationHistoryMock)
+vi.mock('@/tools', () => toolsMock)
+vi.mock('@/providers/utils', () => providersUtilsMock)
+vi.mock('@/providers', () => providersMock)
+vi.mock('@/ee/access-control/utils/permission-check', () => permissionCheckMock)
+vi.mock(
+  '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance',
+  () => workspaceFileSecretProvenanceMock
+)
+vi.mock('@/lib/memory/agent-turn-session', () => ({
+  openAgentTurnSession: vi.fn(async () => undefined),
+}))
+vi.mock('@/lib/internal/mcp/discover-tools', () => ({
+  discoverMcpServerToolsAsExecutor: vi.fn(async () => []),
+}))
+vi.mock('@/lib/internal/custom-tools/read-available-by-id-or-title', () => ({
+  readAvailableCustomToolByIdOrTitleAsExecutor: vi.fn(async () => undefined),
+}))
+vi.mock('@/executor/utils/http', () => ({
+  buildAuthHeaders: vi.fn(async () => ({ 'Content-Type': 'application/json' })),
+  buildAPIUrl: vi.fn((path: string) => path),
+  extractAPIErrorMessage: vi.fn(async () => 'request failed'),
+}))
+vi.mock('@/lib/execution/cancellation', () => ({
+  subscribeToExecutionCancellation: vi.fn(async () => () => {}),
+  isExecutionCancelled: vi.fn(async () => false),
+}))
+
+const results: AgentToolUseResult[] = []
+
+beforeEach(() => {
+  permissionCheckMockFns.mockValidateModelProvider.mockResolvedValue(undefined)
+  providersUtilsMockFns.mockGetProviderFromModel.mockReturnValue('mock-provider')
+})
+
+afterAll(() => {
+  const reportPath = process.env.EVAL_CONTEXT_REPORT_PATH
+  if (reportPath) writeEvalReport(results, reportPath)
+})
+
+describe('agent context eval suite', () => {
+  it.each(AGENT_CONTEXT_SCENARIOS)('$id: $name', async (scenario) => {
+    const result = await runExecutorScenario(scenario)
+    results.push(result)
+
+    const failed = result.checks.filter((entry) => !entry.passed)
+    expect(
+      failed,
+      failed.map((entry) => `${entry.name}: ${entry.detail}`).join('; ') || undefined
+    ).toEqual([])
+  })
+})
diff --git a/apps/sim/evals/agent-context/scenarios.ts b/apps/sim/evals/agent-context/scenarios.ts
@@ -0,0 +1,71 @@
+import type { ExecutorScenario } from '@/evals/agent-tool-use/executor-harness'
+
+/**
+ * Agent context evals.
+ *
+ * These drive the real Agent block inside the `DAGExecutor` with conversation
+ * memory on. The memory read is stubbed per conversation id, so the provider
+ * request shows exactly what the handler assembled: prior history, then the new
+ * user prompt, with the system prompt preserved. A handler that passed the
+ * wrong conversation id, dropped history, or reordered the prompt fails.
+ *
+ * Windowing inside the memory service (`sliding_window`, token budgets) is
+ * covered by its own unit tests; this suite covers the assembly the model sees.
+ */
+export const AGENT_CONTEXT_SCENARIOS: ExecutorScenario[] = [
+  {
+    id: 'context-remembers-prior-turns',
+    name: 'includes prior conversation memory before the new user prompt',
+    category: 'retrieval',
+    description:
+      'Memory holds two prior turns. The provider request must contain both, in order, followed by the new user prompt, with the system prompt present.',
+    workflowInput: {},
+    agent: {
+      model: 'gpt-4o',
+      systemPrompt: 'You are a helpful assistant.',
+      userPrompt: 'What is my name?',
+      memory: {
+        conversationId: 'conv-ada',
+        history: [
+          { role: 'user', content: 'My name is Ada.' },
+          { role: 'assistant', content: 'Nice to meet you, Ada.' },
+        ],
+      },
+    },
+    providerResponse: {
+      content: 'Your name is Ada.',
+      tokens: { input: 30, output: 5, total: 35 },
+    },
+    expect: {
+      succeeds: true,
+      finalContent: 'Ada',
+    },
+  },
+  {
+    id: 'context-isolates-conversations',
+    name: 'reads the conversation named by the block, not another',
+    category: 'retrieval',
+    description:
+      'The memory stub only returns history for the block conversation id; any other id yields a placeholder. A handler that passed the wrong id would surface the placeholder and fail.',
+    workflowInput: {},
+    agent: {
+      model: 'gpt-4o',
+      userPrompt: 'What did we decide?',
+      memory: {
+        conversationId: 'conv-b',
+        history: [
+          { role: 'user', content: 'We decided to ship on Friday.' },
+          { role: 'assistant', content: 'Shipping Friday.' },
+        ],
+      },
+    },
+    providerResponse: {
+      content: 'You decided to ship on Friday.',
+      tokens: { input: 25, output: 6, total: 31 },
+    },
+    expect: {
+      succeeds: true,
+      finalContent: 'Friday',
+    },
+  },
+]
diff --git a/apps/sim/evals/agent-tool-use/executor-harness.ts b/apps/sim/evals/agent-tool-use/executor-harness.ts
@@ -3,7 +3,9 @@ import {
   createSerializedWorkflow,
 } from '@sim/testing/factories/serialized-block.factory'
 import { providersMockFns } from '@sim/testing/mocks/providers.mock'
+import { vi } from 'vitest'
 import { DAGExecutor } from '@/executor/execution/executor'
+import { memoryService } from '@/executor/handlers/agent/memory'
 import type { SerializedBlock, SerializedWorkflow } from '@/serializer/types'
 import { type EvalRunMode, type ScoredToolCall, scoreExpectations } from './harness'
 import type {
@@ -58,6 +60,14 @@ export interface ExecutorScenario {
     retry?: { enabled: boolean; maxTries: number; waitBetweenTriesMs: number }
     /** Ordered models the Agent handler tries after the primary fails. */
     fallbackModels?: Array<{ model: string }>
+    /**
+     * Turns on conversation memory. `history` is what the mocked memory read
+     * returns for `conversationId`, so a wrong id surfaces as a failed check.
+     */
+    memory?: {
+      conversationId: string
+      history: Array<{ role: 'user' | 'assistant' | 'system'; content: string }>
+    }
   }
   /** One entry per model call; the last entry serves any extra/retry calls. */
   providerResponse: ExecutorProviderResponse | ExecutorProviderResponse[]
@@ -73,6 +83,13 @@ export interface ExecutorScenario {
   }
 }
 
+function lastMatchingIndex(contents: string[], needle: string): number {
+  for (let index = contents.length - 1; index >= 0; index--) {
+    if (contents[index].includes(needle)) return index
+  }
+  return -1
+}
+
 function buildWorkflow(scenario: ExecutorScenario): SerializedWorkflow {
   const start: SerializedBlock = createSerializedBlock({
     id: 'start',
@@ -95,6 +112,9 @@ function buildWorkflow(scenario: ExecutorScenario): SerializedWorkflow {
       ? { temperature: scenario.agent.temperature }
       : {}),
     ...(scenario.agent.fallbackModels ? { fallbackModels: scenario.agent.fallbackModels } : {}),
+    ...(scenario.agent.memory
+      ? { memoryType: 'conversation', conversationId: scenario.agent.memory.conversationId }
+      : {}),
   }
   if (scenario.agent.retry) agent.retry = scenario.agent.retry
 
@@ -133,6 +153,17 @@ export async function runExecutorScenario(
     }
   )
 
+  const fetchedConversationIds: unknown[] = []
+  if (scenario.agent.memory) {
+    const memory = scenario.agent.memory
+    vi.spyOn(memoryService, 'fetchMemoryMessages').mockImplementation(async (_ctx, inputs) => {
+      fetchedConversationIds.push(inputs.conversationId)
+      return inputs.conversationId === memory.conversationId
+        ? memory.history.map((message) => ({ ...message }))
+        : [{ role: 'user', content: '__WRONG_CONVERSATION__' }]
+    })
+  }
+
   const executor = new DAGExecutor({
     workflow: buildWorkflow(scenario),
     workflowInput: scenario.workflowInput,
@@ -212,6 +243,59 @@ export async function runExecutorScenario(
     })
   }
 
+  if (scenario.agent.memory) {
+    const memory = scenario.agent.memory
+    const requestMessages = ((requests[0] as { messages?: unknown[] } | undefined)?.messages ??
+      []) as Array<{ role?: string; content?: unknown }>
+    const contents = requestMessages.map((message) =>
+      typeof message.content === 'string' ? message.content : ''
+    )
+
+    const missingHistory = memory.history.filter(
+      (message) => !contents.some((content) => content.includes(message.content))
+    )
+    checks.push({
+      name: 'memory-history-in-request',
+      passed: missingHistory.length === 0,
+      detail:
+        missingHistory.length === 0
+          ? `all ${memory.history.length} history messages reached the provider`
+          : `missing [${missingHistory.map((message) => message.content).join(', ')}]`,
+    })
+
+    const lastHistoryIndex =
+      memory.history.length === 0
+        ? -1
+        : Math.max(...memory.history.map((message) => lastMatchingIndex(contents, message.content)))
+    const promptIndex = scenario.agent.userPrompt
+      ? lastMatchingIndex(contents, scenario.agent.userPrompt)
+      : -1
+    checks.push({
+      name: 'memory-before-user-prompt',
+      passed: promptIndex >= 0 && promptIndex > lastHistoryIndex,
+      detail: `history ends at ${lastHistoryIndex}, user prompt at ${promptIndex}`,
+    })
+
+    if (scenario.agent.systemPrompt) {
+      const systemPrompt = scenario.agent.systemPrompt
+      checks.push({
+        name: 'system-prompt-in-request',
+        passed: requestMessages.some(
+          (message) => message.role === 'system' && String(message.content).includes(systemPrompt)
+        ),
+        detail: 'configured system prompt reached the provider',
+      })
+    }
+
+    checks.push({
+      name: 'conversation-id',
+      passed:
+        fetchedConversationIds.length > 0 &&
+        fetchedConversationIds.every((id) => id === memory.conversationId),
+      detail: `expected ${memory.conversationId}, got [${fetchedConversationIds.join(', ')}]`,
+    })
+  }
+
   const tokens = (output.tokens ?? {}) as { input?: number; output?: number; total?: number }
 
   return {

@@ -27,6 +27,7 @@
     "test:evals": "EVAL_REPORT_PATH=test-results/evals/agent-tool-use.json vitest run evals/agent-tool-use",
     "test:evals:live": "EVAL_LIVE=1 vitest run --mode live evals/agent-tool-use",
     "test:evals:record": "EVAL_LIVE=1 EVAL_RECORD=1 vitest run --mode live evals/agent-tool-use",
+    "test:evals:context": "EVAL_CONTEXT_REPORT_PATH=test-results/evals/agent-context.json vitest run evals/agent-context",
     "email:dev": "email dev --dir components/emails",
     "type-check": "tsc --noEmit",
     "lint": "biome check --write --unsafe .",