Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions actions/setup/js/pi_session.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -158,9 +158,11 @@ function transformPiV3Entries(records) {
events.push(event);
if (event.type === "tool.execution_start" && event.data.toolCallId !== undefined) calls.set(event.data.toolCallId, event);
} else if (raw.type === "gh_aw_subagent_dispatch") {
emit(raw, "pi.subagent_dispatch", { agent: raw.agent, requestedModel: raw.requested_model, resolvedModel: raw.resolved_model });
emit(raw, "pi.subagent_dispatch", { invocationId: raw.invocation_id, agent: raw.agent, requestedModel: raw.requested_model, resolvedModel: raw.resolved_model });
} else if (raw.type === "gh_aw_subagent_event") {
emit(raw, "pi.subagent_event", { agent: raw.agent, event: raw.event });
emit(raw, "pi.subagent_event", { invocationId: raw.invocation_id, agent: raw.agent, event: raw.event });
} else if (raw.type === "gh_aw_subagent_result") {
emit(raw, "pi.subagent_result", { invocationId: raw.invocation_id, agent: raw.agent, outcome: raw.outcome, error: raw.error });
} else if (raw.type === "session") emit(raw, "session.init", { sourceEngine: "pi", sessionId: raw.id, cwd: raw.cwd, model: raw.model });
else if (raw.type === "turn_start") {
assistantState = activeState = newMessageState();
Expand Down
12 changes: 10 additions & 2 deletions actions/setup/js/pi_session.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,21 @@ describe("Pi CI stream regressions", () => {
const parent = { type: "message_end", message: { role: "assistant", content: [{ type: "text", text: "Parent answer" }], usage: { input: 10, output: 3 } } };
const child = {
type: "gh_aw_subagent_event",
invocation_id: "invocation-1",
agent: "reader",
event: { type: "message_end", message: { role: "assistant", model: "claude-haiku-4.5", content: [{ type: "text", text: "Child answer" }], usage: { input: 5, output: 2 } } },
};
const records = [{ type: "gh_aw_subagent_dispatch", agent: "reader", requested_model: "small", resolved_model: "claude-haiku-4.5" }, child, parent];
const records = [
{ type: "gh_aw_subagent_dispatch", invocation_id: "invocation-1", agent: "reader", requested_model: "small", resolved_model: "claude-haiku-4.5" },
child,
{ type: "gh_aw_subagent_result", invocation_id: "invocation-1", agent: "reader", outcome: "completed" },
parent,
];
const events = transformPiV3Entries(records);
expect(byType(events, "pi.subagent_dispatch")[0].data).toMatchObject({ requestedModel: "small", resolvedModel: "claude-haiku-4.5" });
expect(byType(events, "pi.subagent_dispatch")[0].data).toMatchObject({ invocationId: "invocation-1", requestedModel: "small", resolvedModel: "claude-haiku-4.5" });
expect(byType(events, "pi.subagent_event")[0].data.event).toEqual(child.event);
expect(byType(events, "pi.subagent_event")[0].data.invocationId).toBe("invocation-1");
expect(byType(events, "pi.subagent_result")[0].data).toMatchObject({ invocationId: "invocation-1", outcome: "completed" });
expect(byType(events, "assistant.message").map(e => e.data.content)).toEqual(["Parent answer"]);
expect(computePiV3Stats(records).usage.input_tokens).toBe(10);
});
Expand Down
27 changes: 24 additions & 3 deletions actions/setup/js/pi_subagent_extension.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
const fs = require("node:fs");
const path = require("node:path");
const os = require("node:os");
const { randomUUID } = require("node:crypto");
const { spawn } = require("node:child_process");
const { getErrorMessage } = require("./error_helpers.cjs");
const { sanitizeProviderErrorMessage } = require("./pi_provider_error.cjs");
Expand Down Expand Up @@ -32,6 +33,21 @@ async function runPiSubagent(agent, task, ctx, signal, launch = spawn) {
if (signal?.aborted) throw new Error("Pi sub-agent was aborted before dispatch");
const minutes = Number(process.env.GH_AW_TIMEOUT_MINUTES || "30");
if (!Number.isFinite(minutes) || minutes <= 0) throw new Error("GH_AW_TIMEOUT_MINUTES must be positive");
const invocationId = randomUUID();
let resultEmitted = false;
const emitResult = (outcome, error = "") => {
if (resultEmitted) return;
resultEmitted = true;
process.stdout.write(
JSON.stringify({
type: "gh_aw_subagent_result",
invocation_id: invocationId,
agent: agent.name,
outcome,
...(error ? { error: sanitizeProviderErrorMessage(error).slice(0, 300) } : {}),
}) + "\n"
);
};
let dir;
try {
dir = fs.mkdtempSync(path.join(os.tmpdir(), "gh-aw-pi-subagent-"));
Expand All @@ -43,7 +59,7 @@ async function runPiSubagent(agent, task, ctx, signal, launch = spawn) {
"The parent is responsible for finalizing the workflow and its required safe-output reporting. Do not emit noop safe outputs or workflow-completion reports for this child session. " +
"Safe-output actions required to perform the delegated task remain permitted.";
fs.writeFileSync(promptPath, system + agent.prompt + "\n\n" + delegationScope, { mode: 0o600 });
process.stdout.write(JSON.stringify({ type: "gh_aw_subagent_dispatch", agent: agent.name, requested_model: agent.declaredModel || agent.modelId, resolved_model: agent.modelId }) + "\n");
process.stdout.write(JSON.stringify({ type: "gh_aw_subagent_dispatch", invocation_id: invocationId, agent: agent.name, requested_model: agent.declaredModel || agent.modelId, resolved_model: agent.modelId }) + "\n");
return await new Promise((resolve, reject) => {
const child = launch(process.env.GH_AW_PI_COMMAND || "pi", subagentArgs(agent, promptPath), {
cwd: ctx.cwd,
Expand Down Expand Up @@ -91,7 +107,7 @@ async function runPiSubagent(agent, task, ctx, signal, launch = spawn) {
if (["error", "aborted"].includes(event.message.stopReason))
failure = typeof event.message.errorMessage === "string" && event.message.errorMessage ? event.message.errorMessage : `Pi sub-agent ended with ${event.message.stopReason}`;
// Preserve child telemetry without mixing it with the parent's session events.
process.stdout.write(JSON.stringify({ type: "gh_aw_subagent_event", agent: agent.name, event }) + "\n");
process.stdout.write(JSON.stringify({ type: "gh_aw_subagent_event", invocation_id: invocationId, agent: agent.name, event }) + "\n");
Comment thread
Copilot marked this conversation as resolved.
Resolved
}
};
child.stdout.on("data", data => {
Expand Down Expand Up @@ -123,6 +139,7 @@ async function runPiSubagent(agent, task, ctx, signal, launch = spawn) {
clearTimeout(timeout);
clearTimeout(killTimer);
signal?.removeEventListener("abort", abort);
emitResult("failed", getErrorMessage(error));
reject(new Error(`Cannot launch Pi sub-agent "${agent.name}": ${getErrorMessage(error)}`, { cause: error }));
});
child.on("close", (code, terminationSignal) => {
Expand All @@ -131,8 +148,11 @@ async function runPiSubagent(agent, task, ctx, signal, launch = spawn) {
signal?.removeEventListener("abort", abort);
if (buffer.trim()) parseLine(buffer);
if (code !== 0 || terminationSignal || failure || !output.trim()) {
reject(new Error(sanitizeProviderErrorMessage(failure || stderr || `Pi sub-agent exited ${code ?? terminationSignal} without an answer`)));
const error = sanitizeProviderErrorMessage(failure || stderr || `Pi sub-agent exited ${code ?? terminationSignal} without an answer`);
emitResult("failed", error);
reject(new Error(error));
} else {
emitResult("completed");
const text = output.length > 64000 ? output.slice(0, 64000) + "\n[Sub-agent output truncated at 64000 characters]" : output;
resolve({ content: [{ type: "text", text }], details: { agent: agent.name, requestedModel: agent.declaredModel, model: agent.modelId } });
}
Expand All @@ -144,6 +164,7 @@ async function runPiSubagent(agent, task, ctx, signal, launch = spawn) {
if (signal?.aborted) abort();
});
} catch (error) {
emitResult("failed", getErrorMessage(error));
throw new Error(`Pi sub-agent "${agent.name}" failed: ${getErrorMessage(error)}`, { cause: error });
} finally {
if (dir) {
Expand Down
11 changes: 11 additions & 0 deletions actions/setup/js/pi_subagent_extension.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -85,12 +85,23 @@ describe("Pi managed delegation", () => {
expect(result).toMatchObject({ content: [{ type: "text", text: "Python 3.12" }], details: { requestedModel: "small", model: "claude-haiku-4.5" } });
expect(process.stdout.write).toHaveBeenCalledWith(expect.stringContaining('"type":"gh_aw_subagent_dispatch"'));
expect(process.stdout.write).toHaveBeenCalledWith(expect.stringContaining('"resolved_model":"claude-haiku-4.5"'));
const records = process.stdout.write.mock.calls.map(([line]) => JSON.parse(line));
const dispatch = records.find(record => record.type === "gh_aw_subagent_dispatch");
const resultEvent = records.find(record => record.type === "gh_aw_subagent_result");
expect(dispatch.invocation_id).toBeTruthy();
expect(resultEvent).toMatchObject({ invocation_id: dispatch.invocation_id, outcome: "completed", agent: "reader" });
expect(launch.mock.calls[0][2]).toMatchObject({ shell: false, cwd: dir });
expect(fs.existsSync(launch.mock.calls[0][1].at(-1))).toBe(false);
});

it("does not claim success when inference ends with an error and CLI exits zero", async () => {
await expect(runPiSubagent(agent, "Read.", { cwd: dir }, undefined, launcher({ stopReason: "error", errorMessage: "Model unavailable" }))).rejects.toThrow("Model unavailable");
const records = process.stdout.write.mock.calls.map(([line]) => JSON.parse(line));
expect(records.find(record => record.type === "gh_aw_subagent_result")).toMatchObject({
invocation_id: records.find(record => record.type === "gh_aw_subagent_dispatch").invocation_id,
outcome: "failed",
error: "Model unavailable",
});
});

it("inherits workflow controls without making the child finalize the workflow", async () => {
Expand Down
11 changes: 7 additions & 4 deletions pkg/cli/audit_comparison.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,12 @@
}

type AuditComparisonRoute struct {
Model string `json:"model,omitempty"`
Effort string `json:"effort,omitempty"`
Mode string `json:"mode,omitempty"`
RouterVersion string `json:"router_version,omitempty"`
Model string `json:"model,omitempty"`
Effort string `json:"effort,omitempty"`
Mode string `json:"mode,omitempty"`
RouterVersion string `json:"router_version,omitempty"`
MainAgentCost ModelRoutingCost `json:"main_agent_cost,omitempty"`

Check failure on line 45 in pkg/cli/audit_comparison.go

View workflow job for this annotation

GitHub Actions / lint-go-golangci (2)

omitzero: Omitempty has no effect on nested struct fields (modernize)
SubagentCosts []ModelRoutingAgentCost `json:"subagent_costs,omitempty"`
}

type AuditComparisonDelta struct {
Expand Down Expand Up @@ -127,6 +129,7 @@
return &AuditComparisonRoute{
Model: routing.SelectedModel, Effort: routing.SelectedEffort,
Mode: routing.Mode, RouterVersion: routing.RouterVersion,
MainAgentCost: routing.MainAgentCost, SubagentCosts: routing.SubagentCosts,
}
}

Expand Down
12 changes: 12 additions & 0 deletions pkg/cli/audit_cross_run_render.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,18 @@ func renderMarkdownModelRoutingToWriter(w io.Writer, routing *ModelRoutingLogsSu
writeModelRoutingReportLine(w, "## Model Routing\n\n")
writeModelRoutingReportLine(w, "Classifier AIC: %.3f; deviated requests: %d/%d (%.1f%%)\n\n",
routing.ClassifierAIC, routing.DeviatedRequests, routing.TotalRequests, routing.DeviatedTrafficShare*100)
if routing.MainAgentCost.Requests > 0 || routing.MainAgentCost.AIC > 0 || len(routing.SubagentCosts) > 0 {
writeModelRoutingReportLine(w, "Main-agent AIC: %.3f\n\n", routing.MainAgentCost.AIC)
if len(routing.SubagentCosts) > 0 {
writeModelRoutingReportLine(w, "| Sub-agent | Effort | Requests | Total AIC |\n|---|---|---:|---:|\n")
for _, agent := range routing.SubagentCosts {
writeModelRoutingReportLine(w, "| %s | %s | %d | %.3f |\n",
modelRoutingMarkdownCell(agent.AgentName), modelRoutingMarkdownCell(agent.Effort),
agent.Requests, agent.AIC)
}
writeModelRoutingReportLine(w, "\n")
}
}
if len(routing.Routes) == 0 {
return
}
Expand Down
1 change: 1 addition & 0 deletions pkg/cli/audit_finding_codes.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ const (
AuditFindingBlockedNetworkRequests AuditFindingCode = "blocked_network_requests"
AuditFindingWorkflowSucceeded AuditFindingCode = "workflow_succeeded"
AuditFindingSubagentModelNotObserved AuditFindingCode = "subagent_model_not_observed"
AuditFindingSubagentFailed AuditFindingCode = "subagent_failed"
AuditFindingDetectionJobFailed AuditFindingCode = "threat_detection_job_failed"
AuditFindingThreatDetected AuditFindingCode = "threat_detected"

Expand Down
21 changes: 21 additions & 0 deletions pkg/cli/audit_report_render.go
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,17 @@ func renderConsoleTokenUsage(tokenUsage *TokenUsageSummary) {
tokenUsage.TotalRequests,
console.FormatNumber(tokenUsage.TotalSteeringEvents),
)
if len(tokenUsage.AgentUsage) > 0 {
fmt.Fprintln(os.Stderr, " agent_usage:")
fmt.Fprintln(os.Stderr, " agent | instances | outcome (completed/failed/incomplete) | requests | tokens (in/out) | credits | effort")
for _, agent := range tokenUsage.AgentUsage {
fmt.Fprintf(os.Stderr, " %s %s | %d | %d/%d/%d | %d | %s/%s | %.3f | %s\n",
agent.AgentType, safeModelRoutingText(agent.AgentName), agent.InstanceCount,
agent.CompletedCount, agent.FailedCount, agent.IncompleteCount, agent.Requests,
console.FormatNumber(agent.InputTokens), console.FormatNumber(agent.OutputTokens),
agent.AIC, safeModelRoutingText(agent.Effort))
}
}
if len(tokenUsage.Warnings) > 0 {
fmt.Fprintln(os.Stderr, " token_usage_warnings:")
for _, warning := range tokenUsage.Warnings {
Expand All @@ -224,6 +235,16 @@ func renderConsoleModelRouting(routing *ModelRoutingSummary) {
}
fmt.Fprintf(os.Stderr, " cost_aic: classifier=%.3f selected_model=%.3f deviated=%.3f\n",
routing.ClassifierCost.AIC, routing.SelectedModelCost.AIC, routing.DeviatedTrafficCost.AIC)
if routing.MainAgentCost.Requests > 0 || routing.MainAgentCost.AIC > 0 {
fmt.Fprintf(os.Stderr, " main_agent_cost: requests=%d input=%d output=%d aic=%.3f\n",
routing.MainAgentCost.Requests, routing.MainAgentCost.InputTokens,
routing.MainAgentCost.OutputTokens, routing.MainAgentCost.AIC)
}
for _, agent := range routing.SubagentCosts {
fmt.Fprintf(os.Stderr, " subagent_cost: %s effort=%s requests=%d failed=%d aic=%.3f\n",
safeModelRoutingText(agent.AgentName), safeModelRoutingText(agent.Effort),
agent.Requests, agent.FailedCount, agent.AIC)
}
}

func renderConsoleModelRoutingSelection(routing *ModelRoutingSummary) {
Expand Down
2 changes: 2 additions & 0 deletions pkg/cli/logs_report.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@
type ModelRoutingLogsSummary struct {
Routes []ModelRoutingRouteSummary `json:"routes,omitempty" console:"title:Model Routing Mix,omitempty"`
ClassifierAIC float64 `json:"classifier_aic,omitempty" console:"header:Classifier AIC"`
MainAgentCost ModelRoutingCost `json:"main_agent_cost,omitempty" console:"-"`

Check failure on line 53 in pkg/cli/logs_report.go

View workflow job for this annotation

GitHub Actions / lint-go-golangci (2)

omitzero: Omitempty has no effect on nested struct fields (modernize)
SubagentCosts []ModelRoutingAgentCost `json:"subagent_costs,omitempty" console:"-"`
TotalRequests int `json:"total_requests,omitempty" console:"header:Routing Requests"`
DeviatedRequests int `json:"deviated_requests,omitempty" console:"header:Deviated Requests"`
DeviatedTrafficShare float64 `json:"deviated_traffic_share,omitempty" console:"header:Deviated Traffic Share"`
Expand Down
156 changes: 156 additions & 0 deletions pkg/cli/model_identity.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
package cli

import (
"encoding/json"
"os"
"path"
"regexp"
"strings"

"github.com/github/gh-aw/pkg/workflow"
)

var (
modelDateSuffix = regexp.MustCompile(`-[0-9]{4}(?:-[0-9]{2}-[0-9]{2}|[0-9]{4})$`)
claudeModelVersionSpelling = regexp.MustCompile(`^(claude-.+)-([0-9]+)-([0-9]+)$`)
)

type modelIdentityResolver struct {
aliases map[string][]string
}

func newModelIdentityResolver(runDir string) *modelIdentityResolver {
aliases := workflow.BuiltinModelAliases()
infoPath := findAwInfoPath(runDir)
if infoPath != "" {
content, err := os.ReadFile(infoPath)
if err == nil {
var info struct {
Models []struct {
Model string `json:"model"`
Patterns []string `json:"patterns"`
} `json:"sub_agent_models"`
}
if json.Unmarshal(content, &info) == nil {
for _, model := range info.Models {
if model.Model != "" && len(model.Patterns) > 0 {
aliases[model.Model] = append([]string(nil), model.Patterns...)
}
}
}
}
}
return &modelIdentityResolver{aliases: aliases}
}

func normalizeModelIdentity(model string) string {
model, _, _ = strings.Cut(strings.ToLower(strings.TrimSpace(model)), "?")
model = modelDateSuffix.ReplaceAllString(model, "")
return claudeModelVersionSpelling.ReplaceAllString(model, "$1-$2.$3")
}

func (resolver *modelIdentityResolver) resolve(model, provider string, observed []string) string {
normalized := normalizeModelIdentity(model)
patterns := resolver.patternsFor(normalized)
if len(patterns) == 0 {
return normalized
}
for _, candidate := range observed {
for _, pattern := range patterns {
if modelPatternMatches(pattern, candidate, provider) {
return normalizeModelIdentity(modelNameWithoutProvider(candidate))
}
}
}
return normalized
}

func (resolver *modelIdentityResolver) matches(pattern, observed, provider string) bool {
patterns := resolver.patternsFor(normalizeModelIdentity(pattern))
if len(patterns) == 0 {
patterns = []string{pattern}
}
for _, candidatePattern := range patterns {
if modelPatternMatches(candidatePattern, observed, provider) {
return true
}
}
return modelPatternMatches(pattern, observed, provider)
}

func (resolver *modelIdentityResolver) patternsFor(model string) []string {
var patterns []string
visited := make(map[string]bool)
var expand func(string)
expand = func(alias string) {
alias = normalizeModelIdentity(alias)
if visited[alias] {
return
}
visited[alias] = true
for _, pattern := range resolver.aliases[alias] {
nested := normalizeModelIdentity(pattern)
if !strings.ContainsAny(pattern, "/*?") && len(resolver.aliases[nested]) > 0 {
expand(nested)
continue
}
patterns = append(patterns, pattern)
}
}
expand(model)
return patterns
}

func modelPatternMatches(pattern, observed, provider string) bool {
pattern, _, _ = strings.Cut(strings.ToLower(pattern), "?")
observed, _, _ = strings.Cut(strings.ToLower(observed), "?")
originalObserved := observed
patternProvider, patternModel, patternQualified := strings.Cut(pattern, "/")
observedProvider, observedModel, observedQualified := strings.Cut(observed, "/")
if patternQualified {
if provider == "" && observedQualified {
provider = observedProvider
}
if provider != "" && normalizeModelProvider(patternProvider) != normalizeModelProvider(provider) {
return false
}
pattern = patternModel
}
if observedQualified {
if provider != "" && normalizeModelProvider(observedProvider) != normalizeModelProvider(provider) {
return modelPatternMatches(patternModel, originalObserved, "")
}
observed = observedModel
}
patterns := []string{pattern, normalizeModelIdentity(pattern)}
observedModels := []string{observed, normalizeModelIdentity(observed)}
for _, candidatePattern := range patterns {
for _, candidate := range observedModels {
if matched, err := path.Match(candidatePattern, candidate); err == nil && matched {
return true
}
}
}
return false
}

func normalizeModelProvider(provider string) string {
switch provider {
case "github", "copilot", "github-copilot":
return "github-copilot"
case "codex", "openai":
return "openai"
case "gemini", "google":
return "google"
default:
return provider
}
}

func modelNameWithoutProvider(model string) string {
_, name, qualified := strings.Cut(model, "/")
if qualified {
return name
}
return model
}
Loading
Loading