fix(harness): retry transient provider failures in the loop (#47)

sanchitrk · Oct 8, 2026 · Oct 6, 2026 · Oct 7, 2026 · Oct 7, 2026 · Oct 7, 2026
commit aa0eb2d34d8792527bd4e212ae58abc0d4933895
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -14,9 +14,16 @@ This file is the canonical source for unreleased changes and published release n
 ### Added
 
 - Added `gpt-6-luna` to the OpenAI Codex catalog. The Codex backend serves it to ChatGPT accounts, with every reasoning level from `low` to `max` and reasoning off.
+- Added automatic retry of transient provider failures in the harness loop. A turn whose request fails with a rate limit, overload, 5xx, timeout or dropped connection, including one that fails mid-stream, is retried with exponential backoff. Configure it with a top-level `retry` block in `settings.jsonc`: `{ "maxRetries": 3, "baseDelayMs": 2000, "maxDelayMs": 60000 }`, where `maxRetries: 0` turns it off. Quota errors are never retried, and a server that asks to wait longer than `maxDelayMs` fails the turn. Each failed attempt stays in the session as an aborted entry that the model never sees, and clients receive `session.retry.scheduled` and `session.retry.finished` events to show progress.
+
+### Changed
+
+- `model.options.maxRetries`, the retry inside the provider SDK, now defaults to `0` (was `3`). The SDK retried silently, only before the stream started, and also retried quota errors. Combined with the loop retry, one outage would have cost up to 16 requests. Set it per model if an endpoint needs it.
 
 ### Fixed
 
+- Fixed an interrupted turn never reporting `session.turn.aborted`. The cleanup that settles the turn's open draft and publishes the event was skipped on interruption, so the draft was only cleaned up on the session's next run.
+- Fixed `retry-after-ms` and an HTTP-date `retry-after` being ignored when normalizing a provider failure. Only a `retry-after` given in seconds was read, so `retryAfterMs` was missing and a retry could come before the server's requested wait.
 - Fixed OpenAI and Codex reasoning levels that the API rejects. `minimal` is no longer offered for gpt-5.4 through gpt-6 and is clamped to `low`, since these models reject it. `xhigh` and `max` are now offered on gpt-5.6 and gpt-6 models through the OpenAI API, and `max` stays off gpt-5.5 and earlier. OpenAI gpt-5.6 and gpt-6 models now turn reasoning off with `none`, as do Codex `gpt-5.5`, `gpt-5.6-luna` and `gpt-6-luna`.
 - Fixed `gpt-6-luna` ignoring a request to turn reasoning off. `@ai-sdk/openai` 4.0.66 dropped `reasoningEffort: "none"` for the model; aikit now requires 4.0.81.
 - Fixed every request to Claude Sonnet 5.5, Opus 5.5 and the Fable 5 models failing when no reasoning level was set. aikit sent `thinking: {type: "disabled"}`, which these models reject; they cannot turn thinking off, so aikit now leaves thinking to the model.

diff --git a/packages/aikit/src/llm/failure.ts b/packages/aikit/src/llm/failure.ts
@@ -66,11 +66,16 @@ const header = (headers: Record<string, string> | undefined, name: string): stri
 	}
 };
 
+/** The server's requested wait, in the three forms providers send: `retry-after-ms`, seconds, or an HTTP date. */
 const retryAfter = (headers: Record<string, string> | undefined): number | undefined => {
+	const millis = Number(header(headers, "retry-after-ms"));
+	if (Number.isFinite(millis) && millis >= 0) return millis;
 	const value = header(headers, "retry-after");
 	if (value === undefined) return undefined;
 	const seconds = Number(value);
 	if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1_000;
+	const date = Date.parse(value);
+	if (Number.isFinite(date)) return Math.max(0, date - Date.now());
 };
 
 const apiMetadata = (error: APICallError): Metadata => {

diff --git a/packages/aikit/test/failure.test.ts b/packages/aikit/test/failure.test.ts
@@ -37,6 +37,25 @@ describe("provider failure normalization", () => {
 		expect(JSON.stringify(failure)).not.toContain("sensitive");
 	});
 
+	it("reads retry-after-ms and an HTTP-date retry-after", () => {
+		const retryAfter = (responseHeaders: Record<string, string>) =>
+			Failure.normalize(
+				new APICallError({
+					message: "Overloaded",
+					url: "https://provider.invalid/v1/chat",
+					requestBodyValues: {},
+					statusCode: 529,
+					responseHeaders,
+				}),
+			).retryAfterMs;
+
+		expect(retryAfter({ "retry-after-ms": "1500", "retry-after": "9" })).toBe(1_500);
+		const at = retryAfter({ "retry-after": new Date(Date.now() + 45_000).toUTCString() });
+		expect(at).toBeGreaterThan(43_000);
+		expect(at).toBeLessThanOrEqual(45_000);
+		expect(retryAfter({ "retry-after": "Thu, 01 Jan 1970 00:00:00 GMT" })).toBe(0);
+	});
+
 	it("distinguishes quota exhaustion from ordinary rate limiting", () => {
 		const failure = Failure.normalize(
 			new APICallError({

diff --git a/packages/codework/tsconfig.json b/packages/codework/tsconfig.json
@@ -12,7 +12,7 @@
 			"@codeworksh/harness/sandboxes/daytona": ["../harness/src/sandboxes/daytona/index.ts"],
 			"@codeworksh/harness/sandboxes/vercel": ["../harness/src/sandboxes/vercel/index.ts"],
 			"@codeworksh/aikit": ["../aikit/dist/pack/index.d.mts"],
-			"@codeworksh/aikit/failure": ["../aikit/src/llm/failure.ts"],
+			"@codeworksh/aikit/failure": ["../aikit/dist/pack/llm/failure.d.mts"],
 			"@codeworksh/aikit/modelgen": ["../aikit/dist/pack/modelgen.d.mts"],
 			"@codeworksh/aikit/oauth/openai/codex": ["../aikit/dist/pack/oauth/openai/codex.d.mts"],
 			"@codeworksh/aikit/oauth/github/copilot": ["../aikit/dist/pack/oauth/github/copilot.d.mts"],

diff --git a/packages/harness/src/event/list.ts b/packages/harness/src/event/list.ts
@@ -81,6 +81,34 @@ export const TurnAborted = EventSchema.define({
 });
 export type TurnAborted = typeof TurnAborted.Type;
 
+/**
+ * A failed provider request will be repeated after `delayMs`. `attempt` counts retries from 1, so
+ * the request that failed was attempt `attempt - 1`; the failed draft is already settled.
+ */
+export const RetryScheduled = EventSchema.define({
+	type: "session.retry.scheduled",
+	schema: {
+		...baseOptions,
+		attempt: NonNegativeInt,
+		maxRetries: NonNegativeInt,
+		delayMs: NonNegativeInt,
+		message: Schema.String,
+	},
+});
+export type RetryScheduled = typeof RetryScheduled.Type;
+
+/** Once per turn that retried: whether a retry succeeded, after how many. */
+export const RetryFinished = EventSchema.define({
+	type: "session.retry.finished",
+	schema: {
+		...baseOptions,
+		attempt: NonNegativeInt,
+		success: Schema.Boolean,
+		message: optional(Schema.String),
+	},
+});
+export type RetryFinished = typeof RetryFinished.Type;
+
 /** Durable insertion of the request's draft assistant placeholder. */
 export const LLMStarted = EventSchema.define({
 	type: "session.llm.started",
@@ -269,6 +297,8 @@ export const Definitions = EventSchema.inventory(
 	TurnStarted,
 	TurnEnded,
 	TurnAborted,
+	RetryScheduled,
+	RetryFinished,
 	LLMStarted,
 	LLMTextStart,
 	LLMTextDelta,

diff --git a/packages/harness/src/runner/loop.ts b/packages/harness/src/runner/loop.ts
@@ -6,7 +6,7 @@
  */
 
 import { type Message } from "@codeworksh/aikit";
-import { Cause, DateTime, Effect, Exit, Layer, Option } from "effect";
+import { Cause, DateTime, Duration, Effect, Exit, Layer, Option } from "effect";
 import { ContextCodec } from "../context/codec.ts";
 import { Context } from "../context/context.ts";
 import { Event } from "../event/event.ts";
@@ -19,6 +19,7 @@ import { State } from "../state/state.ts";
 import { errorMessage } from "../util/error.ts";
 import { LLMEventPublisher } from "./event.ts";
 import { LLM } from "./llm.ts";
+import { Retry } from "./retry.ts";
 import { Runner } from "./run.ts";
 
 export type Promotion = "steer" | "followUp" | undefined;
@@ -254,10 +255,22 @@ export const layer = (options: Options = {}) =>
 					return { needsContinuation: false } satisfies TurnResult;
 				}
 
+				// The latest attempt's publisher; a retried turn has one per provider request.
 				let publisher: LLMEventPublisher.Publisher | undefined;
 				let committed = false;
-				const turnWindow = Effect.gen(function* () {
-					yield* events.publish(EventList.TurnStarted, { timestamp: yield* DateTime.now, sessionId });
+				let retries = 0;
+
+				/** Abort the latest attempt's draft, unless its own terminal event already did. */
+				const settleDraft = (cause: EventList.TurnAbortCause) =>
+					Effect.gen(function* () {
+						if (publisher?.startedMessageId === undefined) return;
+						const stored = yield* sessions.entry(publisher.startedMessageId);
+						if (Option.isSome(stored) && stored.value.entry.state === "draft") {
+							yield* publishFailure(stored.value, cause);
+						}
+					});
+
+				const attempt = Effect.gen(function* () {
 					publisher = yield* LLMEventPublisher.make({ sessionId }).pipe(
 						Effect.provideService(Event.Service, events),
 					);
@@ -284,6 +297,64 @@ export const layer = (options: Options = {}) =>
 							LLM.messageFailure(terminal.message),
 						);
 					}
+					return terminal;
+				});
+
+				/*
+				 * Nothing is committed before TurnEnded, so a failed request can always be repeated
+				 * against the same assembled context. Each failed attempt keeps its aborted draft in
+				 * the journal and the next one starts a new draft. An interrupt during the backoff
+				 * finds no open draft: the failed one was settled before the sleep.
+				 */
+				const request = (): Effect.Effect<
+					Effect.Success<typeof attempt>,
+					Effect.Error<typeof attempt>,
+					Effect.Services<typeof attempt>
+				> =>
+					attempt.pipe(
+						Effect.catch((error) => {
+							const delayMs = Retry.delay(snapshot.retry, retries + 1, error);
+							if (delayMs === undefined) return Effect.fail(error);
+							return Effect.gen(function* () {
+								retries += 1;
+								yield* Effect.uninterruptible(settleDraft({ _tag: "error", message: error.message }));
+								yield* events.publish(EventList.RetryScheduled, {
+									timestamp: yield* DateTime.now,
+									sessionId,
+									attempt: retries,
+									maxRetries: snapshot.retry.maxRetries,
+									delayMs,
+									message: error.message,
+								});
+								yield* Effect.sleep(Duration.millis(delayMs));
+								return yield* request();
+							});
+						}),
+					);
+
+				const turnWindow = Effect.gen(function* () {
+					yield* events.publish(EventList.TurnStarted, { timestamp: yield* DateTime.now, sessionId });
+					const terminal = yield* request().pipe(
+						Effect.onExit((exit) =>
+							retries === 0
+								? Effect.void
+								: Effect.gen(function* () {
+										yield* events.publish(EventList.RetryFinished, {
+											timestamp: yield* DateTime.now,
+											sessionId,
+											attempt: retries,
+											success: Exit.isSuccess(exit),
+											...(Exit.isSuccess(exit)
+												? {}
+												: {
+														message: Cause.hasInterrupts(exit.cause)
+															? "interrupted"
+															: errorMessage(exit.cause),
+													}),
+										});
+									}),
+						),
+					);
 
 					const interrupted = yield* settleTools(snapshot, terminal.message, terminal.reason, () => {
 						committed = true;
@@ -295,26 +366,22 @@ export const layer = (options: Options = {}) =>
 					} satisfies TurnResult;
 				});
 
+				// `onError`, not `catchCause`: a catch handler never runs on an interrupted fiber, and
+				// an interrupt is exactly when the open draft most needs settling.
 				return yield* turnWindow.pipe(
-					Effect.catchCause((cause) => {
-						if (committed) return Effect.failCause(cause);
+					Effect.onError((cause) => {
+						if (committed) return Effect.void;
 						const turnCause: EventList.TurnAbortCause = Cause.hasInterrupts(cause)
 							? { _tag: "interrupted" }
 							: { _tag: "error", message: errorMessage(cause) };
-						const record = Effect.gen(function* () {
-							if (publisher?.startedMessageId !== undefined) {
-								const stored = yield* sessions.entry(publisher.startedMessageId);
-								if (Option.isSome(stored) && stored.value.entry.state === "draft") {
-									yield* publishFailure(stored.value, turnCause);
-								}
-							}
+						return Effect.gen(function* () {
+							yield* settleDraft(turnCause);
 							yield* events.publish(EventList.TurnAborted, {
 								timestamp: yield* DateTime.now,
 								sessionId,
 								cause: turnCause,
 							});
 						});
-						return Effect.uninterruptible(record).pipe(Effect.andThen(Effect.failCause(cause)));
 					}),
 				);
 			});

diff --git a/packages/harness/src/runner/retry.ts b/packages/harness/src/runner/retry.ts
@@ -0,0 +1,31 @@
+/*
+ * @file Agent-level retry policy for a failed provider request.
+ *
+ * Pure: the loop owns the attempts, the sleep and the events; this decides whether a failure is
+ * worth another attempt and how long to wait first.
+ */
+
+import { Duration, Schema } from "effect";
+import type { RetryPolicy } from "../settings/schema.ts";
+import { Runner } from "./run.ts";
+
+/**
+ * Milliseconds to wait before retry `attempt` (1-based), or `undefined` to give up.
+ *
+ * Only a provider failure the provider itself calls transient is retried. A quota error is
+ * excluded even though it often arrives as a retryable 429: waiting does not refill a quota.
+ * Context overflow is a non-retryable 400 and is left to compaction.
+ *
+ * A server-sent `retry-after` raises the delay, never lowers it. One longer than `maxDelayMs`
+ * gives up rather than retrying early into the same rejection.
+ */
+export const delay = (policy: RetryPolicy, attempt: number, error: unknown): number | undefined => {
+	if (attempt > policy.maxRetries) return undefined;
+	if (!Schema.is(Runner.ProviderError)(error) || !error.isRetryable) return undefined;
+	if (error.reason._tag === "Runner.ProviderQuotaError") return undefined;
+	const backoff = Math.min(policy.baseDelayMs * 2 ** (attempt - 1), policy.maxDelayMs);
+	const requested = error.reason.retryAfter === undefined ? 0 : Duration.toMillis(error.reason.retryAfter);
+	return requested > policy.maxDelayMs ? undefined : Math.max(backoff, Math.ceil(requested));
+};
+
+export * as Retry from "./retry.ts";
diff --git a/packages/harness/src/state/state.ts b/packages/harness/src/state/state.ts
@@ -30,7 +30,7 @@ import type { ID as SessionId } from "../session/schema.ts";
 import { Session as SessionStore } from "../session/session.ts";
 import { merge } from "../settings/merge.ts";
 import { compose, resolveOptions } from "../settings/resolve.ts";
-import type { Block, Info } from "../settings/schema.ts";
+import type { Block, Info, RetryPolicy } from "../settings/schema.ts";
 import { Settings } from "../settings/settings.ts";
 import type { Resolved } from "../tool/registry.ts";
 
@@ -120,6 +120,8 @@ export interface Snapshot {
 	/** Matched file attributes used to construct provider requests. */
 	readonly settings: Block;
 	readonly toolExecution: ToolExecutionMode;
+	/** Agent-level retry of a transiently failed provider request. */
+	readonly retry: RetryPolicy;
 }
 
 export interface Configuration {
@@ -470,6 +472,7 @@ export const layer = (
 						request,
 						settings: configured.block,
 						toolExecution,
+						retry: loadedSettings.retry,
 					} satisfies Snapshot;
 				}),
 			});

diff --git a/packages/harness/test/__artifacts__/runner.retry.json b/packages/harness/test/__artifacts__/runner.retry.json
@@ -0,0 +1,87 @@
+{
+	"events": [
+		{
+			"type": "session.turn.started"
+		},
+		{
+			"messageId": "assistant_1",
+			"type": "session.llm.started"
+		},
+		{
+			"messageId": "assistant_1",
+			"reason": "error",
+			"type": "session.llm.failed"
+		},
+		{
+			"attempt": 1,
+			"delayMs": 5,
+			"maxRetries": 3,
+			"message": "openai/gpt-5.6-luna: Unavailable failure",
+			"type": "session.retry.scheduled"
+		},
+		{
+			"messageId": "assistant_2",
+			"type": "session.llm.started"
+		},
+		{
+			"messageId": "assistant_2",
+			"type": "session.llm.text.delta"
+		},
+		{
+			"messageId": "assistant_2",
+			"reason": "error",
+			"type": "session.llm.failed"
+		},
+		{
+			"attempt": 2,
+			"delayMs": 8,
+			"maxRetries": 3,
+			"message": "openai/gpt-5.6-luna: Transport failure",
+			"type": "session.retry.scheduled"
+		},
+		{
+			"messageId": "assistant_3",
+			"type": "session.llm.started"
+		},
+		{
+			"messageId": "assistant_3",
+			"reason": "stop",
+			"type": "session.llm.ended"
+		},
+		{
+			"attempt": 2,
+			"success": true,
+			"type": "session.retry.finished"
+		},
+		{
+			"messageId": "assistant_3",
+			"type": "session.turn.ended"
+		}
+	],
+	"path": [
+		{
+			"id": "msg_retry",
+			"parts": 1,
+			"state": "committed",
+			"type": "user"
+		},
+		{
+			"id": "assistant_1",
+			"parts": 0,
+			"state": "aborted",
+			"type": "assistant"
+		},
+		{
+			"id": "assistant_2",
+			"parts": 0,
+			"state": "aborted",
+			"type": "assistant"
+		},
+		{
+			"id": "assistant_3",
+			"parts": 1,
+			"state": "committed",
+			"type": "assistant"
+		}
+	]
+}