Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 42 additions & 6 deletions actions/setup/js/awf_model_routing.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,10 @@ function mapAWFRoutingEffort(engine, effort) {
* @param {any} reflectData
* @param {boolean} routingRequired
* @param {string[]|null} [allowedEndpoints]
* @returns {{selection: {provider: string, model: string, wire_model: string, effort: string|null, endpoint: string}|null, error: string|null}}
* @param {boolean} [allowEndpointOverride]
* @returns {{selection: {provider: string, model: string, wire_model: string, effort: string|null, endpoint: string, selected_endpoint: string}|null, error: string|null}}
*/
function resolveAWFModelRoutingSelection(reflectData, routingRequired = false, allowedEndpoints = null) {
function resolveAWFModelRoutingSelection(reflectData, routingRequired = false, allowedEndpoints = null, allowEndpointOverride = false) {
const routing = reflectData && typeof reflectData === "object" ? reflectData.routing : null;
if (routing == null) {
return routingRequired ? { selection: null, error: "AWF /reflect did not return the required model-routing selection" } : { selection: null, error: null };
Expand All @@ -59,17 +60,51 @@ function resolveAWFModelRoutingSelection(reflectData, routingRequired = false, a
if (!["copilot", "github-copilot", "github"].includes(provider) || !wireModel || !endpoint) {
return { selection: null, error: "AWF /reflect returned an incomplete or unsupported Copilot routing selection" };
}
if (allowedEndpoints && !allowedEndpoints.includes(endpoint)) {
return { selection: null, error: `AWF /reflect selected endpoint ${endpoint}, which is not supported by this engine` };
}
if (!isModelAvailableInReflectData(wireModel, reflectData, REFLECT_PROVIDER_ALIASES.github)) {
return { selection: null, error: `AWF /reflect selected unavailable Copilot wire model ${wireModel}` };
}
let effectiveEndpoint = endpoint;
if (allowedEndpoints && !allowedEndpoints.includes(endpoint)) {
if (!allowEndpointOverride) {
return { selection: null, error: `AWF /reflect selected endpoint ${endpoint}, which is not supported by this engine` };
}
const routingModel = getAWFRoutingModel(reflectData, wireModel);
const supportedEndpoints = routingModel?.supported_endpoints;
if (reflectData.candidate_metadata_complete !== true || !Array.isArray(supportedEndpoints) || !supportedEndpoints.every(value => typeof value === "string")) {
Comment thread
Copilot marked this conversation as resolved.
Resolved
return { selection: null, error: `AWF /reflect cannot verify endpoints for model ${wireModel}; candidate metadata is incomplete for engine API ${allowedEndpoints.join(", ")}` };
}
effectiveEndpoint = allowedEndpoints.find(candidate => supportedEndpoints.includes(candidate)) || "";
if (!effectiveEndpoint) {
return {
selection: null,
error: `AWF model ${wireModel} advertises endpoints [${supportedEndpoints.join(", ")}], none compatible with this engine API [${allowedEndpoints.join(", ")}]`,
};
}
}
const effort = typeof selection.effort === "string" && selection.effort.trim() ? selection.effort.trim().toLowerCase() : null;
if (effort && !isRoutingReasoningEffort(effort)) {
return { selection: null, error: `AWF /reflect returned unsupported reasoning effort ${effort}` };
}
return { selection: { provider, model: String(selection.model || ""), wire_model: wireModel, effort, endpoint }, error: null };
return {
selection: { provider, model: String(selection.model || ""), wire_model: wireModel, effort, endpoint: effectiveEndpoint, selected_endpoint: endpoint },
error: null,
};
}

/**
* @param {any} reflectData
* @param {string} wireModel
* @returns {any|null}
*/
function getAWFRoutingModel(reflectData, wireModel) {
const endpoints = Array.isArray(reflectData?.endpoints) ? reflectData.endpoints : [];
const normalizedModel = wireModel.toLowerCase();
for (const endpoint of endpoints) {
if (endpoint?.configured !== true || !REFLECT_PROVIDER_ALIASES.github.has(normalizeReflectProviderName(endpoint.provider))) continue;
const model = endpoint.routing_models?.find(candidate => typeof candidate?.model_id === "string" && candidate.model_id.toLowerCase() === normalizedModel);
Comment thread
SivaKesava1 marked this conversation as resolved.
Resolved
if (model) return model;
}
return null;
}

/**
Expand All @@ -93,5 +128,6 @@ module.exports = {
isRoutingReasoningEffort,
mapAWFRoutingEffort,
resolveAWFModelRoutingSelection,
getAWFRoutingModel,
isModelAvailableInReflectData,
};
35 changes: 35 additions & 0 deletions actions/setup/js/awf_model_routing.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,41 @@ describe("awf_model_routing.cjs", () => {
expect(resolveAWFModelRoutingSelection(reflectData, true, ["/responses"]).error).toContain("not supported by this engine");
});

it("uses an engine-compatible endpoint from complete routing metadata and preserves AWF's selection", () => {
const reflect = {
...reflectData,
candidate_metadata_complete: true,
endpoints: [
{
...reflectData.endpoints[0],
routing_models: [{ model_id: "claude-opus-5", supported_endpoints: ["/v1/messages", "/chat/completions"] }],
},
],
routing: {
...reflectData.routing,
selection: { ...reflectData.routing.selection, endpoint: "/chat/completions" },
},
};
expect(resolveAWFModelRoutingSelection(reflect, true, ["/v1/messages"], true).selection).toMatchObject({
endpoint: "/v1/messages",
selected_endpoint: "/chat/completions",
});
});

it.each([
[{ candidate_metadata_complete: false, routing_models: [{ model_id: "claude-opus-5", supported_endpoints: ["/v1/messages"] }] }, "metadata is incomplete"],
[{ candidate_metadata_complete: true, routing_models: [{ model_id: "claude-opus-5", supported_endpoints: ["/responses"] }] }, "advertises endpoints"],
[{ candidate_metadata_complete: true, routing_models: [] }, "metadata is incomplete"],
])("fails closed when the model endpoint metadata cannot verify a compatible endpoint: %s", ({ candidate_metadata_complete, routing_models }, error) => {
const reflect = {
...reflectData,
candidate_metadata_complete,
endpoints: [{ ...reflectData.endpoints[0], routing_models }],
routing: { ...reflectData.routing, selection: { ...reflectData.routing.selection, endpoint: "/chat/completions" } },
};
expect(resolveAWFModelRoutingSelection(reflect, true, ["/v1/messages"], true).error).toContain(error);
});

it("checks routed model availability only against configured GitHub provider aliases", () => {
const mixedProviders = {
...reflectData,
Expand Down
7 changes: 4 additions & 3 deletions actions/setup/js/claude_harness.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -377,7 +377,7 @@ async function buildClaudeChildEnv(reflectData, env = process.env, logger = log,
const childEnv = { ...env };
applyClaudeRuntimeTimeouts(childEnv);
if (env.GH_AW_MODEL_ROUTING === "1") {
const result = modelRoutingSelection ? { selection: modelRoutingSelection, error: null } : resolveAWFModelRoutingSelection(reflectData, true, CLAUDE_ROUTING_ENDPOINTS);
const result = modelRoutingSelection ? { selection: modelRoutingSelection, error: null } : resolveAWFModelRoutingSelection(reflectData, true, CLAUDE_ROUTING_ENDPOINTS, true);
if (result.error || !result.selection) {
throw new Error(`${result.error || "AWF model routing selection is missing"}; refusing to start Claude`);
}
Expand All @@ -387,7 +387,8 @@ async function buildClaudeChildEnv(reflectData, env = process.env, logger = log,
delete childEnv.GH_AW_MODEL_FALLBACK;
if (mappedEffort.effort) childEnv.CLAUDE_CODE_EFFORT_LEVEL = mappedEffort.effort;
else delete childEnv.CLAUDE_CODE_EFFORT_LEVEL;
logger(`inference routing: mode=awf-routed model=${result.selection.wire_model} effort=${result.selection.effort || "(unset)"}`);
const endpointOverride = result.selection.selected_endpoint && result.selection.selected_endpoint !== result.selection.endpoint ? ` selected_endpoint=${result.selection.selected_endpoint}` : "";
logger(`inference routing: mode=awf-routed model=${result.selection.wire_model} effort=${result.selection.effort || "(unset)"} endpoint=${result.selection.endpoint}${endpointOverride}`);
} else {
applyModelFallback(childEnv, "ANTHROPIC_MODEL", logger);
}
Expand Down Expand Up @@ -464,7 +465,7 @@ async function main() {
/** @type {any} */
let modelRoutingSelection = null;
if (process.env.GH_AW_MODEL_ROUTING === "1") {
const result = resolveAWFModelRoutingSelection(reflection.reflectData, true, CLAUDE_ROUTING_ENDPOINTS);
const result = resolveAWFModelRoutingSelection(reflection.reflectData, true, CLAUDE_ROUTING_ENDPOINTS, true);
if (result.error || !result.selection) {
throw new Error(`${result.error || "AWF model routing selection is missing"}; refusing to start Claude`);
}
Expand Down
24 changes: 21 additions & 3 deletions actions/setup/js/claude_harness.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,19 @@ describe("claude_harness.cjs", () => {
expect(removeClaudeRoutingOverrides(["--model", "fixed", "--effort=low", "--print"])).toEqual(["--print"]);
});

it("uses the Messages endpoint for a routed model selected on Chat Completions", async () => {
const reflect = {
...routedReflect,
candidate_metadata_complete: true,
endpoints: [{ ...routedReflect.endpoints[0], routing_models: [{ model_id: "claude-opus-5", supported_endpoints: ["/chat/completions", "/v1/messages"] }] }],
routing: { ...routedReflect.routing, selection: { ...routedReflect.routing.selection, effort: "max", endpoint: "/chat/completions" } },
};
const messages = [];
const child = await buildClaudeChildEnv(reflect, { GH_AW_MODEL_ROUTING: "1", GH_AW_LLM_PROVIDER: "github" }, message => messages.push(message));
expect(child).toMatchObject({ ANTHROPIC_MODEL: "claude-opus-5", CLAUDE_CODE_EFFORT_LEVEL: "max" });
expect(messages[0]).toContain("endpoint=/v1/messages selected_endpoint=/chat/completions");
});

it("starts the harness with the routed model and refuses an endpoint mismatch", () => {
const stubScript = `
const fs = require("fs");
Expand All @@ -222,17 +235,22 @@ describe("claude_harness.cjs", () => {

const mismatch = runHarnessWithStub({
stubScript,
reflectData: { ...routedReflect, routing: { ...routedReflect.routing, selection: { ...routedReflect.routing.selection, endpoint: "/responses" } } },
reflectData: {
...routedReflect,
candidate_metadata_complete: true,
endpoints: [{ ...routedReflect.endpoints[0], routing_models: [{ model_id: "claude-opus-5", supported_endpoints: ["/responses"] }] }],
routing: { ...routedReflect.routing, selection: { ...routedReflect.routing.selection, endpoint: "/responses" } },
},
extraEnv: { GH_AW_MODEL_ROUTING: "1", GH_AW_LLM_PROVIDER: "github", GH_AW_SAFE_OUTPUTS: "" },
});
expect(mismatch.result.status).not.toBe(0);
expect(mismatch.result.stderr).toContain("not supported by this engine");
expect(mismatch.result.stderr).toContain("advertises endpoints [/responses]");
expect(mismatch.calls).toHaveLength(0);
});

it.each([
[{ endpoints: routedReflect.endpoints, routing: { status: "pending" } }, "pending"],
[{ endpoints: routedReflect.endpoints, routing: { ...routedReflect.routing, selection: { ...routedReflect.routing.selection, endpoint: "/responses" } } }, "not supported by this engine"],
[{ endpoints: routedReflect.endpoints, routing: { ...routedReflect.routing, selection: { ...routedReflect.routing.selection, endpoint: "/responses" } } }, "candidate metadata is incomplete"],
[{ ...routedReflect, endpoints: [{ ...routedReflect.endpoints[0], models: ["gpt-5.6-sol"] }] }, "unavailable Copilot wire model"],
])("fails closed when routing is invalid: %s", async (reflect, error) => {
await expect(buildClaudeChildEnv(reflect, { GH_AW_MODEL_ROUTING: "1", GH_AW_LLM_PROVIDER: "github" }, () => {})).rejects.toThrow(error);
Expand Down
5 changes: 3 additions & 2 deletions actions/setup/js/codex_harness.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -514,7 +514,7 @@ function applyCodexRoutingEffort(args, effort) {
* @returns {{selection: any, model: string, args: string[], error: string|null}}
*/
function resolveCodexModelRouting(reflectData, args) {
const result = resolveAWFModelRoutingSelection(reflectData, true, CODEX_ROUTING_ENDPOINTS);
const result = resolveAWFModelRoutingSelection(reflectData, true, CODEX_ROUTING_ENDPOINTS, true);
if (result.error || !result.selection) return { selection: null, model: "", args, error: result.error || "AWF model routing selection is missing" };
const mappedEffort = mapAWFRoutingEffort("codex", result.selection.effort);
if (mappedEffort.error) return { selection: null, model: "", args, error: mappedEffort.error };
Expand Down Expand Up @@ -881,7 +881,8 @@ async function main() {
}
resolvedModel = result.model;
resolvedArgs = result.args;
log(`inference routing: mode=awf-routed model=${resolvedModel} effort=${result.selection.effort || "(unset)"}`);
const endpointOverride = result.selection.selected_endpoint && result.selection.selected_endpoint !== result.selection.endpoint ? ` selected_endpoint=${result.selection.selected_endpoint}` : "";
log(`inference routing: mode=awf-routed model=${resolvedModel} effort=${result.selection.effort || "(unset)"} endpoint=${result.selection.endpoint}${endpointOverride}`);
} else {
resolvedModel = normalizeCodexModel(codexModelEnvVar ? applyModelFallback(process.env, codexModelEnvVar, log) : "", process.env.GH_AW_LLM_PROVIDER || "openai", modelOptions);
resolvedArgs = normalizeCodexModelArgs(resolvedArgs, process.env.GH_AW_LLM_PROVIDER || "openai", modelOptions);
Expand Down
24 changes: 22 additions & 2 deletions actions/setup/js/codex_harness.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -151,12 +151,32 @@ describe("codex_harness.cjs", () => {
it.each([
[null, "required model-routing selection"],
[{ endpoints: reflectData.endpoints, routing: { status: "pending" } }, "pending"],
[{ ...reflectData, routing: { ...reflectData.routing, selection: { ...reflectData.routing.selection, endpoint: "/v1/messages" } } }, "not supported by this engine"],
[
{
...reflectData,
candidate_metadata_complete: true,
endpoints: [{ ...reflectData.endpoints[0], routing_models: [{ model_id: "gpt-5.6-sol", supported_endpoints: ["/v1/messages"] }] }],
routing: { ...reflectData.routing, selection: { ...reflectData.routing.selection, endpoint: "/v1/messages" } },
},
"advertises endpoints [/v1/messages]",
],
[{ ...reflectData, endpoints: [{ provider: "github", configured: true, models: [] }] }, "unavailable Copilot wire model"],
])("fails closed when routing data is invalid: %s", (reflect, error) => {
expect(resolveCodexModelRouting(reflect, ["exec", "--model", "fixed"]).error).toContain(error);
});

it("uses the Responses endpoint for a routed GPT model selected on Chat Completions", () => {
const reflect = {
...reflectData,
candidate_metadata_complete: true,
endpoints: [{ ...reflectData.endpoints[0], routing_models: [{ model_id: "gpt-5.6-sol", supported_endpoints: ["/chat/completions", "/responses"] }] }],
routing: { ...reflectData.routing, selection: { ...reflectData.routing.selection, endpoint: "/chat/completions" } },
};
const result = resolveCodexModelRouting(reflect, ["exec"]);
expect(result.error).toBeNull();
expect(result.selection).toMatchObject({ endpoint: "/responses", selected_endpoint: "/chat/completions" });
});

it("drops fixed model and reasoning-effort configuration", () => {
const args = removeCodexRoutingOverrides(["exec", "-m", "fixed", "-c", 'model_reasoning_effort="low"', "--json"]);
expect(args).toEqual(["exec", "--json"]);
Expand Down Expand Up @@ -187,7 +207,7 @@ describe("codex_harness.cjs", () => {
expect(calls[0].args).toContain("gpt-5.6-sol");
expect(calls[0].args).toContain('model_reasoning_effort="xhigh"');
expect(calls[0].args).not.toContain("fixed-model");
expect(result.stderr).toContain("inference routing: mode=awf-routed model=gpt-5.6-sol effort=xhigh");
expect(result.stderr).toContain("inference routing: mode=awf-routed model=gpt-5.6-sol effort=xhigh endpoint=/responses");
});
});

Expand Down
Loading
Loading