Skip to content

Commit aa0eb2d

Browse files
authored
fix(harness): retry transient provider failures in the loop (#47)
1 parent 892b5fc commit aa0eb2d

18 files changed

Lines changed: 848 additions & 20 deletions

‎CHANGELOG.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,16 @@ This file is the canonical source for unreleased changes and published release n
1414
### Added
1515

1616
- Added `gpt-6-luna` to the OpenAI Codex catalog. The Codex backend serves it to ChatGPT accounts, with every reasoning level from `low` to `max` and reasoning off.
17+
- Added automatic retry of transient provider failures in the harness loop. A turn whose request fails with a rate limit, overload, 5xx, timeout or dropped connection, including one that fails mid-stream, is retried with exponential backoff. Configure it with a top-level `retry` block in `settings.jsonc`: `{ "maxRetries": 3, "baseDelayMs": 2000, "maxDelayMs": 60000 }`, where `maxRetries: 0` turns it off. Quota errors are never retried, and a server that asks to wait longer than `maxDelayMs` fails the turn. Each failed attempt stays in the session as an aborted entry that the model never sees, and clients receive `session.retry.scheduled` and `session.retry.finished` events to show progress.
18+
19+
### Changed
20+
21+
- `model.options.maxRetries`, the retry inside the provider SDK, now defaults to `0` (was `3`). The SDK retried silently, only before the stream started, and also retried quota errors. Combined with the loop retry, one outage would have cost up to 16 requests. Set it per model if an endpoint needs it.
1722

1823
### Fixed
1924

25+
- Fixed an interrupted turn never reporting `session.turn.aborted`. The cleanup that settles the turn's open draft and publishes the event was skipped on interruption, so the draft was only cleaned up on the session's next run.
26+
- Fixed `retry-after-ms` and an HTTP-date `retry-after` being ignored when normalizing a provider failure. Only a `retry-after` given in seconds was read, so `retryAfterMs` was missing and a retry could come before the server's requested wait.
2027
- Fixed OpenAI and Codex reasoning levels that the API rejects. `minimal` is no longer offered for gpt-5.4 through gpt-6 and is clamped to `low`, since these models reject it. `xhigh` and `max` are now offered on gpt-5.6 and gpt-6 models through the OpenAI API, and `max` stays off gpt-5.5 and earlier. OpenAI gpt-5.6 and gpt-6 models now turn reasoning off with `none`, as do Codex `gpt-5.5`, `gpt-5.6-luna` and `gpt-6-luna`.
2128
- Fixed `gpt-6-luna` ignoring a request to turn reasoning off. `@ai-sdk/openai` 4.0.66 dropped `reasoningEffort: "none"` for the model; aikit now requires 4.0.81.
2229
- Fixed every request to Claude Sonnet 5.5, Opus 5.5 and the Fable 5 models failing when no reasoning level was set. aikit sent `thinking: {type: "disabled"}`, which these models reject; they cannot turn thinking off, so aikit now leaves thinking to the model.

‎packages/aikit/src/llm/failure.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,11 +66,16 @@ const header = (headers: Record<string, string> | undefined, name: string): stri
6666
}
6767
};
6868

69+
/** The server's requested wait, in the three forms providers send: `retry-after-ms`, seconds, or an HTTP date. */
6970
const retryAfter = (headers: Record<string, string> | undefined): number | undefined => {
71+
const millis = Number(header(headers, "retry-after-ms"));
72+
if (Number.isFinite(millis) && millis >= 0) return millis;
7073
const value = header(headers, "retry-after");
7174
if (value === undefined) return undefined;
7275
const seconds = Number(value);
7376
if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1_000;
77+
const date = Date.parse(value);
78+
if (Number.isFinite(date)) return Math.max(0, date - Date.now());
7479
};
7580

7681
const apiMetadata = (error: APICallError): Metadata => {

‎packages/aikit/test/failure.test.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,25 @@ describe("provider failure normalization", () => {
3737
expect(JSON.stringify(failure)).not.toContain("sensitive");
3838
});
3939

40+
it("reads retry-after-ms and an HTTP-date retry-after", () => {
41+
const retryAfter = (responseHeaders: Record<string, string>) =>
42+
Failure.normalize(
43+
new APICallError({
44+
message: "Overloaded",
45+
url: "https://provider.invalid/v1/chat",
46+
requestBodyValues: {},
47+
statusCode: 529,
48+
responseHeaders,
49+
}),
50+
).retryAfterMs;
51+
52+
expect(retryAfter({ "retry-after-ms": "1500", "retry-after": "9" })).toBe(1_500);
53+
const at = retryAfter({ "retry-after": new Date(Date.now() + 45_000).toUTCString() });
54+
expect(at).toBeGreaterThan(43_000);
55+
expect(at).toBeLessThanOrEqual(45_000);
56+
expect(retryAfter({ "retry-after": "Thu, 01 Jan 1970 00:00:00 GMT" })).toBe(0);
57+
});
58+
4059
it("distinguishes quota exhaustion from ordinary rate limiting", () => {
4160
const failure = Failure.normalize(
4261
new APICallError({

‎packages/codework/tsconfig.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
"@codeworksh/harness/sandboxes/daytona": ["../harness/src/sandboxes/daytona/index.ts"],
1313
"@codeworksh/harness/sandboxes/vercel": ["../harness/src/sandboxes/vercel/index.ts"],
1414
"@codeworksh/aikit": ["../aikit/dist/pack/index.d.mts"],
15-
"@codeworksh/aikit/failure": ["../aikit/src/llm/failure.ts"],
15+
"@codeworksh/aikit/failure": ["../aikit/dist/pack/llm/failure.d.mts"],
1616
"@codeworksh/aikit/modelgen": ["../aikit/dist/pack/modelgen.d.mts"],
1717
"@codeworksh/aikit/oauth/openai/codex": ["../aikit/dist/pack/oauth/openai/codex.d.mts"],
1818
"@codeworksh/aikit/oauth/github/copilot": ["../aikit/dist/pack/oauth/github/copilot.d.mts"],

‎packages/harness/src/event/list.ts‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,34 @@ export const TurnAborted = EventSchema.define({
8181
});
8282
export type TurnAborted = typeof TurnAborted.Type;
8383

84+
/**
85+
* A failed provider request will be repeated after `delayMs`. `attempt` counts retries from 1, so
86+
* the request that failed was attempt `attempt - 1`; the failed draft is already settled.
87+
*/
88+
export const RetryScheduled = EventSchema.define({
89+
type: "session.retry.scheduled",
90+
schema: {
91+
...baseOptions,
92+
attempt: NonNegativeInt,
93+
maxRetries: NonNegativeInt,
94+
delayMs: NonNegativeInt,
95+
message: Schema.String,
96+
},
97+
});
98+
export type RetryScheduled = typeof RetryScheduled.Type;
99+
100+
/** Once per turn that retried: whether a retry succeeded, after how many. */
101+
export const RetryFinished = EventSchema.define({
102+
type: "session.retry.finished",
103+
schema: {
104+
...baseOptions,
105+
attempt: NonNegativeInt,
106+
success: Schema.Boolean,
107+
message: optional(Schema.String),
108+
},
109+
});
110+
export type RetryFinished = typeof RetryFinished.Type;
111+
84112
/** Durable insertion of the request's draft assistant placeholder. */
85113
export const LLMStarted = EventSchema.define({
86114
type: "session.llm.started",
@@ -269,6 +297,8 @@ export const Definitions = EventSchema.inventory(
269297
TurnStarted,
270298
TurnEnded,
271299
TurnAborted,
300+
RetryScheduled,
301+
RetryFinished,
272302
LLMStarted,
273303
LLMTextStart,
274304
LLMTextDelta,

‎packages/harness/src/runner/loop.ts‎

Lines changed: 80 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
*/
77

88
import { type Message } from "@codeworksh/aikit";
9-
import { Cause, DateTime, Effect, Exit, Layer, Option } from "effect";
9+
import { Cause, DateTime, Duration, Effect, Exit, Layer, Option } from "effect";
1010
import { ContextCodec } from "../context/codec.ts";
1111
import { Context } from "../context/context.ts";
1212
import { Event } from "../event/event.ts";
@@ -19,6 +19,7 @@ import { State } from "../state/state.ts";
1919
import { errorMessage } from "../util/error.ts";
2020
import { LLMEventPublisher } from "./event.ts";
2121
import { LLM } from "./llm.ts";
22+
import { Retry } from "./retry.ts";
2223
import { Runner } from "./run.ts";
2324

2425
export type Promotion = "steer" | "followUp" | undefined;
@@ -254,10 +255,22 @@ export const layer = (options: Options = {}) =>
254255
return { needsContinuation: false } satisfies TurnResult;
255256
}
256257

258+
// The latest attempt's publisher; a retried turn has one per provider request.
257259
let publisher: LLMEventPublisher.Publisher | undefined;
258260
let committed = false;
259-
const turnWindow = Effect.gen(function* () {
260-
yield* events.publish(EventList.TurnStarted, { timestamp: yield* DateTime.now, sessionId });
261+
let retries = 0;
262+
263+
/** Abort the latest attempt's draft, unless its own terminal event already did. */
264+
const settleDraft = (cause: EventList.TurnAbortCause) =>
265+
Effect.gen(function* () {
266+
if (publisher?.startedMessageId === undefined) return;
267+
const stored = yield* sessions.entry(publisher.startedMessageId);
268+
if (Option.isSome(stored) && stored.value.entry.state === "draft") {
269+
yield* publishFailure(stored.value, cause);
270+
}
271+
});
272+
273+
const attempt = Effect.gen(function* () {
261274
publisher = yield* LLMEventPublisher.make({ sessionId }).pipe(
262275
Effect.provideService(Event.Service, events),
263276
);
@@ -284,6 +297,64 @@ export const layer = (options: Options = {}) =>
284297
LLM.messageFailure(terminal.message),
285298
);
286299
}
300+
return terminal;
301+
});
302+
303+
/*
304+
* Nothing is committed before TurnEnded, so a failed request can always be repeated
305+
* against the same assembled context. Each failed attempt keeps its aborted draft in
306+
* the journal and the next one starts a new draft. An interrupt during the backoff
307+
* finds no open draft: the failed one was settled before the sleep.
308+
*/
309+
const request = (): Effect.Effect<
310+
Effect.Success<typeof attempt>,
311+
Effect.Error<typeof attempt>,
312+
Effect.Services<typeof attempt>
313+
> =>
314+
attempt.pipe(
315+
Effect.catch((error) => {
316+
const delayMs = Retry.delay(snapshot.retry, retries + 1, error);
317+
if (delayMs === undefined) return Effect.fail(error);
318+
return Effect.gen(function* () {
319+
retries += 1;
320+
yield* Effect.uninterruptible(settleDraft({ _tag: "error", message: error.message }));
321+
yield* events.publish(EventList.RetryScheduled, {
322+
timestamp: yield* DateTime.now,
323+
sessionId,
324+
attempt: retries,
325+
maxRetries: snapshot.retry.maxRetries,
326+
delayMs,
327+
message: error.message,
328+
});
329+
yield* Effect.sleep(Duration.millis(delayMs));
330+
return yield* request();
331+
});
332+
}),
333+
);
334+
335+
const turnWindow = Effect.gen(function* () {
336+
yield* events.publish(EventList.TurnStarted, { timestamp: yield* DateTime.now, sessionId });
337+
const terminal = yield* request().pipe(
338+
Effect.onExit((exit) =>
339+
retries === 0
340+
? Effect.void
341+
: Effect.gen(function* () {
342+
yield* events.publish(EventList.RetryFinished, {
343+
timestamp: yield* DateTime.now,
344+
sessionId,
345+
attempt: retries,
346+
success: Exit.isSuccess(exit),
347+
...(Exit.isSuccess(exit)
348+
? {}
349+
: {
350+
message: Cause.hasInterrupts(exit.cause)
351+
? "interrupted"
352+
: errorMessage(exit.cause),
353+
}),
354+
});
355+
}),
356+
),
357+
);
287358

288359
const interrupted = yield* settleTools(snapshot, terminal.message, terminal.reason, () => {
289360
committed = true;
@@ -295,26 +366,22 @@ export const layer = (options: Options = {}) =>
295366
} satisfies TurnResult;
296367
});
297368

369+
// `onError`, not `catchCause`: a catch handler never runs on an interrupted fiber, and
370+
// an interrupt is exactly when the open draft most needs settling.
298371
return yield* turnWindow.pipe(
299-
Effect.catchCause((cause) => {
300-
if (committed) return Effect.failCause(cause);
372+
Effect.onError((cause) => {
373+
if (committed) return Effect.void;
301374
const turnCause: EventList.TurnAbortCause = Cause.hasInterrupts(cause)
302375
? { _tag: "interrupted" }
303376
: { _tag: "error", message: errorMessage(cause) };
304-
const record = Effect.gen(function* () {
305-
if (publisher?.startedMessageId !== undefined) {
306-
const stored = yield* sessions.entry(publisher.startedMessageId);
307-
if (Option.isSome(stored) && stored.value.entry.state === "draft") {
308-
yield* publishFailure(stored.value, turnCause);
309-
}
310-
}
377+
return Effect.gen(function* () {
378+
yield* settleDraft(turnCause);
311379
yield* events.publish(EventList.TurnAborted, {
312380
timestamp: yield* DateTime.now,
313381
sessionId,
314382
cause: turnCause,
315383
});
316384
});
317-
return Effect.uninterruptible(record).pipe(Effect.andThen(Effect.failCause(cause)));
318385
}),
319386
);
320387
});
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
/*
2+
* @file Agent-level retry policy for a failed provider request.
3+
*
4+
* Pure: the loop owns the attempts, the sleep and the events; this decides whether a failure is
5+
* worth another attempt and how long to wait first.
6+
*/
7+
8+
import { Duration, Schema } from "effect";
9+
import type { RetryPolicy } from "../settings/schema.ts";
10+
import { Runner } from "./run.ts";
11+
12+
/**
13+
* Milliseconds to wait before retry `attempt` (1-based), or `undefined` to give up.
14+
*
15+
* Only a provider failure the provider itself calls transient is retried. A quota error is
16+
* excluded even though it often arrives as a retryable 429: waiting does not refill a quota.
17+
* Context overflow is a non-retryable 400 and is left to compaction.
18+
*
19+
* A server-sent `retry-after` raises the delay, never lowers it. One longer than `maxDelayMs`
20+
* gives up rather than retrying early into the same rejection.
21+
*/
22+
export const delay = (policy: RetryPolicy, attempt: number, error: unknown): number | undefined => {
23+
if (attempt > policy.maxRetries) return undefined;
24+
if (!Schema.is(Runner.ProviderError)(error) || !error.isRetryable) return undefined;
25+
if (error.reason._tag === "Runner.ProviderQuotaError") return undefined;
26+
const backoff = Math.min(policy.baseDelayMs * 2 ** (attempt - 1), policy.maxDelayMs);
27+
const requested = error.reason.retryAfter === undefined ? 0 : Duration.toMillis(error.reason.retryAfter);
28+
return requested > policy.maxDelayMs ? undefined : Math.max(backoff, Math.ceil(requested));
29+
};
30+
31+
export * as Retry from "./retry.ts";

‎packages/harness/src/state/state.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ import type { ID as SessionId } from "../session/schema.ts";
3030
import { Session as SessionStore } from "../session/session.ts";
3131
import { merge } from "../settings/merge.ts";
3232
import { compose, resolveOptions } from "../settings/resolve.ts";
33-
import type { Block, Info } from "../settings/schema.ts";
33+
import type { Block, Info, RetryPolicy } from "../settings/schema.ts";
3434
import { Settings } from "../settings/settings.ts";
3535
import type { Resolved } from "../tool/registry.ts";
3636

@@ -120,6 +120,8 @@ export interface Snapshot {
120120
/** Matched file attributes used to construct provider requests. */
121121
readonly settings: Block;
122122
readonly toolExecution: ToolExecutionMode;
123+
/** Agent-level retry of a transiently failed provider request. */
124+
readonly retry: RetryPolicy;
123125
}
124126

125127
export interface Configuration {
@@ -470,6 +472,7 @@ export const layer = (
470472
request,
471473
settings: configured.block,
472474
toolExecution,
475+
retry: loadedSettings.retry,
473476
} satisfies Snapshot;
474477
}),
475478
});
Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
{
2+
"events": [
3+
{
4+
"type": "session.turn.started"
5+
},
6+
{
7+
"messageId": "assistant_1",
8+
"type": "session.llm.started"
9+
},
10+
{
11+
"messageId": "assistant_1",
12+
"reason": "error",
13+
"type": "session.llm.failed"
14+
},
15+
{
16+
"attempt": 1,
17+
"delayMs": 5,
18+
"maxRetries": 3,
19+
"message": "openai/gpt-5.6-luna: Unavailable failure",
20+
"type": "session.retry.scheduled"
21+
},
22+
{
23+
"messageId": "assistant_2",
24+
"type": "session.llm.started"
25+
},
26+
{
27+
"messageId": "assistant_2",
28+
"type": "session.llm.text.delta"
29+
},
30+
{
31+
"messageId": "assistant_2",
32+
"reason": "error",
33+
"type": "session.llm.failed"
34+
},
35+
{
36+
"attempt": 2,
37+
"delayMs": 8,
38+
"maxRetries": 3,
39+
"message": "openai/gpt-5.6-luna: Transport failure",
40+
"type": "session.retry.scheduled"
41+
},
42+
{
43+
"messageId": "assistant_3",
44+
"type": "session.llm.started"
45+
},
46+
{
47+
"messageId": "assistant_3",
48+
"reason": "stop",
49+
"type": "session.llm.ended"
50+
},
51+
{
52+
"attempt": 2,
53+
"success": true,
54+
"type": "session.retry.finished"
55+
},
56+
{
57+
"messageId": "assistant_3",
58+
"type": "session.turn.ended"
59+
}
60+
],
61+
"path": [
62+
{
63+
"id": "msg_retry",
64+
"parts": 1,
65+
"state": "committed",
66+
"type": "user"
67+
},
68+
{
69+
"id": "assistant_1",
70+
"parts": 0,
71+
"state": "aborted",
72+
"type": "assistant"
73+
},
74+
{
75+
"id": "assistant_2",
76+
"parts": 0,
77+
"state": "aborted",
78+
"type": "assistant"
79+
},
80+
{
81+
"id": "assistant_3",
82+
"parts": 1,
83+
"state": "committed",
84+
"type": "assistant"
85+
}
86+
]
87+
}

0 commit comments

Comments
 (0)