Skip to content

Commit 10a7fac

Browse files
CopilotCopilot
authored
Add initial deterministic trace grading framework (#54678)
1 parent 8539305 commit 10a7fac

29 files changed

Lines changed: 3748 additions & 15 deletions

‎actions/setup/js/experiment_helpers.cjs‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,25 @@ function readExperimentAssignments() {
3636
}
3737
return null;
3838
} catch {
39-
return null;
39+
// Fall through to environment-variable fallback.
40+
}
41+
42+
/** @type {Record<string, string>} */
43+
const fromEnv = {};
44+
for (const [key, value] of Object.entries(process.env)) {
45+
if (!key.startsWith("GH_AW_EXPERIMENTS_")) {
46+
continue;
47+
}
48+
const name = key.substring("GH_AW_EXPERIMENTS_".length).toLowerCase();
49+
if (!name || typeof value !== "string" || value.length === 0) {
50+
continue;
51+
}
52+
fromEnv[name] = value;
53+
}
54+
if (Object.keys(fromEnv).length > 0) {
55+
return fromEnv;
4056
}
57+
return null;
4158
}
4259

4360
module.exports = { readExperimentAssignments, EXPERIMENT_ASSIGNMENTS_PATH };

‎actions/setup/js/experiment_helpers.test.cjs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ describe("readExperimentAssignments", () => {
1616

1717
afterEach(() => {
1818
readFileSpy.mockRestore();
19+
delete process.env.GH_AW_EXPERIMENTS_PROMPT_STYLE;
20+
delete process.env.GH_AW_EXPERIMENTS_REASONING_DEPTH;
1921
if (savedStateDir !== undefined) {
2022
process.env.GH_AW_EXPERIMENT_STATE_DIR = savedStateDir;
2123
} else {
@@ -67,4 +69,10 @@ describe("readExperimentAssignments", () => {
6769
});
6870
expect(readExperimentAssignments()).toEqual({ mode: "fast" });
6971
});
72+
73+
it("falls back to GH_AW_EXPERIMENTS_* environment variables when assignments file is unavailable", () => {
74+
process.env.GH_AW_EXPERIMENTS_PROMPT_STYLE = "concise";
75+
process.env.GH_AW_EXPERIMENTS_REASONING_DEPTH = "deep";
76+
expect(readExperimentAssignments()).toEqual({ prompt_style: "concise", reasoning_depth: "deep" });
77+
});
7078
});

‎actions/setup/js/redact_secrets.cjs‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -343,4 +343,40 @@ async function main() {
343343
}
344344
}
345345

346-
module.exports = { main, redactSecrets, redactBuiltInPatterns, redactStepSummaryContent, extractMCPGatewayTokens, BUILT_IN_PATTERNS, MCP_GATEWAY_CONFIG_PATHS };
346+
/**
347+
* Targeted redaction pass for files in a specific directory. Used by the
348+
* post-graders redaction step to scan grader output files that were written
349+
* after the initial full-workspace redaction pass.
350+
*
351+
* @param {string} dir - Absolute directory path to scan
352+
*/
353+
async function redactFilesInDir(dir) {
354+
try {
355+
const secretNames = (process.env.GH_AW_SECRET_NAMES || "").split(",").filter(n => n.trim());
356+
/** @type {string[]} */
357+
const secretValues = [];
358+
for (const secretName of secretNames) {
359+
const value = process.env[`SECRET_${secretName.trim()}`];
360+
if (typeof value === "string" && value.trim() !== "") {
361+
secretValues.push(value.trim());
362+
}
363+
}
364+
secretValues.push(...extractMCPGatewayTokens(MCP_GATEWAY_CONFIG_PATHS));
365+
366+
const targetExtensions = [".json"];
367+
const files = findFiles(dir, targetExtensions);
368+
if (files.length === 0) return;
369+
370+
let totalRedactions = 0;
371+
for (const file of files) {
372+
totalRedactions += processFile(file, secretValues);
373+
}
374+
if (totalRedactions > 0) {
375+
core.info(`Grader output redaction: ${totalRedactions} redaction(s) in ${dir}`);
376+
}
377+
} catch (error) {
378+
core.warning(`Grader output redaction failed: ${getErrorMessage(error)}`);
379+
}
380+
}
381+
382+
module.exports = { main, redactFilesInDir, redactSecrets, redactBuiltInPatterns, redactStepSummaryContent, extractMCPGatewayTokens, BUILT_IN_PATTERNS, MCP_GATEWAY_CONFIG_PATHS };

‎actions/setup/js/run_evals.cjs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ const path = require("path");
3636
const { ERR_VALIDATION, ERR_SYSTEM } = require("./error_codes.cjs");
3737
const { getErrorMessage } = require("./error_helpers.cjs");
3838
const { EVALS_OUTPUT_PATH } = require("./evals_constants.cjs");
39+
const { readExperimentAssignments } = require("./experiment_helpers.cjs");
3940
const { resolveModelWithFallback } = require("./model_fallback.cjs");
4041

4142
const EVALS_DIR = "/tmp/gh-aw/evals";
@@ -127,6 +128,7 @@ async function parseMain() {
127128
const questionsRaw = process.env.GH_AW_EVALS_QUESTIONS;
128129
const model = resolveModelWithFallback(process.env, "GH_AW_EVALS_MODEL") || "";
129130
const runID = process.env.GITHUB_RUN_ID || "unknown";
131+
const experimentAssignments = readExperimentAssignments();
130132

131133
/** @type {Array<{id: string, question: string}>} */
132134
let questions = [];
@@ -185,6 +187,7 @@ async function parseMain() {
185187
model,
186188
timestamp,
187189
runid: runID,
190+
experiments: experimentAssignments || undefined,
188191
};
189192
results.push(record);
190193
core.info(`Q[${q.id}]: ${answer}`);

‎actions/setup/js/run_evals.test.cjs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { createRequire } from "module";
55
const EVALS_DIR = "/tmp/gh-aw/evals";
66
const EVALS_LOG_PATH = `${EVALS_DIR}/evals.log`;
77
const EVALS_OUTPUT_PATH = "/tmp/gh-aw/evals.jsonl";
8+
const EXPERIMENT_ASSIGNMENTS_PATH = "/tmp/gh-aw/experiments/assignments.json";
89
const require = createRequire(import.meta.url);
910
const { MODEL_FALLBACK_ENV_VAR } = require("./model_fallback.cjs");
1011
const { setupMain, parseMain, extractAssistantTextFromJsonlLog } = require("./run_evals.cjs");
@@ -32,6 +33,9 @@ describe("run_evals.cjs", () => {
3233
if (fs.existsSync(EVALS_OUTPUT_PATH)) {
3334
fs.unlinkSync(EVALS_OUTPUT_PATH);
3435
}
36+
if (fs.existsSync(EXPERIMENT_ASSIGNMENTS_PATH)) {
37+
fs.unlinkSync(EXPERIMENT_ASSIGNMENTS_PATH);
38+
}
3539
});
3640

3741
afterEach(() => {
@@ -42,6 +46,9 @@ describe("run_evals.cjs", () => {
4246
if (fs.existsSync(EVALS_OUTPUT_PATH)) {
4347
fs.unlinkSync(EVALS_OUTPUT_PATH);
4448
}
49+
if (fs.existsSync(EXPERIMENT_ASSIGNMENTS_PATH)) {
50+
fs.unlinkSync(EXPERIMENT_ASSIGNMENTS_PATH);
51+
}
4552
});
4653

4754
it("stores the workflow run id when writing eval records", async () => {
@@ -64,6 +71,20 @@ describe("run_evals.cjs", () => {
6471
});
6572
});
6673

74+
it("includes experiment assignments in eval records when available", async () => {
75+
vi.stubEnv("GH_AW_EVALS_QUESTIONS", JSON.stringify([{ id: "labels-applied", question: "Did labels get applied?" }]));
76+
vi.stubEnv("GH_AW_EVALS_MODEL", "small");
77+
vi.stubEnv("GITHUB_RUN_ID", "123456789");
78+
fs.mkdirSync("/tmp/gh-aw/experiments", { recursive: true });
79+
fs.writeFileSync("/tmp/gh-aw/experiments/assignments.json", JSON.stringify({ prompt_style: "concise" }) + "\n", "utf8");
80+
fs.writeFileSync(EVALS_LOG_PATH, "labels-applied: YES\n", "utf8");
81+
82+
await parseMain();
83+
84+
const [line] = fs.readFileSync(EVALS_OUTPUT_PATH, "utf8").trim().split("\n");
85+
expect(JSON.parse(line).experiments).toEqual({ prompt_style: "concise" });
86+
});
87+
6788
it('falls back to "unknown" when the workflow run id is absent', async () => {
6889
vi.stubEnv("GH_AW_EVALS_QUESTIONS", JSON.stringify([{ id: "labels-applied", question: "Did labels get applied?" }]));
6990
vi.stubEnv("GH_AW_EVALS_MODEL", "small");

0 commit comments

Comments
 (0)