Share grader path constants and try all grader result candidates

Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>
Copilot · Sep 13, 2026 · Sep 13, 2026 · Sep 13, 2026 · Sep 13, 2026 · Sep 13, 2026
commit 301b8b9823b19388475d7a1d499cad670ce963f5
diff --git a/actions/setup/js/constants.cjs b/actions/setup/js/constants.cjs
@@ -160,6 +160,27 @@ const RUN_CREATED_AT_INITIAL_DELAY_MS = 500;
  */
 const RUN_CREATED_AT_MAX_DELAY_MS = 2000;
 
+/**
+ * Directory name holding deterministic grader output files.
+ * Counterpart of GradersDirName in pkg/constants/job_constants.go.
+ * @type {string}
+ */
+const GRADERS_DIR_NAME = "graders";
+
+/**
+ * Filename of the deterministic grader results document.
+ * Counterpart of GraderResultsFilename in pkg/constants/job_constants.go.
+ * @type {string}
+ */
+const GRADER_RESULTS_FILENAME = "grader_results.json";
+
+/**
+ * Filename of the deterministic grader manifest.
+ * Counterpart of GraderManifestFilename in pkg/constants/job_constants.go.
+ * @type {string}
+ */
+const GRADER_MANIFEST_FILENAME = "grader_manifest.json";
+
 module.exports = {
   AGENT_OUTPUT_FILENAME,
   TMP_GH_AW_PATH,
@@ -180,4 +201,7 @@ module.exports = {
   RUN_CREATED_AT_MAX_RETRIES,
   RUN_CREATED_AT_INITIAL_DELAY_MS,
   RUN_CREATED_AT_MAX_DELAY_MS,
+  GRADERS_DIR_NAME,
+  GRADER_RESULTS_FILENAME,
+  GRADER_MANIFEST_FILENAME,
 };
diff --git a/actions/setup/js/generate_usage_activity_summary.cjs b/actions/setup/js/generate_usage_activity_summary.cjs
@@ -14,6 +14,7 @@
 
 const fs = require("fs");
 const path = require("path");
+const { GRADERS_DIR_NAME, GRADER_RESULTS_FILENAME, TMP_GH_AW_PATH } = require("./constants.cjs");
 const { readExperimentAssignments } = require("./experiment_helpers.cjs");
 const { calculateWorkingSetFromJSONL } = require("./working_set_metrics.cjs");
 
@@ -29,7 +30,7 @@ const PLACEHOLDER_DOMAIN_KEY = "-";
 const PLACEHOLDER_DEST_KEY = "-:-";
 const ERROR_DOMAIN_PREFIX = "error:";
 const AGENT_TOKEN_USAGE_PATH = "/tmp/gh-aw/usage/agent/token_usage.jsonl";
-const GRADER_RESULTS_PATHS = ["/tmp/gh-aw/usage/graders/grader_results.json", "/tmp/gh-aw/agent/graders/grader_results.json"];
+const GRADER_RESULTS_PATHS = [path.join(TMP_GH_AW_PATH, "usage", GRADERS_DIR_NAME, GRADER_RESULTS_FILENAME), path.join(TMP_GH_AW_PATH, "agent", GRADERS_DIR_NAME, GRADER_RESULTS_FILENAME)];
 const RPC_EVENT_TO_TYPE = { rpc_request: "REQUEST", rpc_response: "RESPONSE", difc_filtered: "DIFC_FILTERED" };
 
 function findFiles(rootDir, shouldIncludeFile, maxDepth = Number.POSITIVE_INFINITY, currentDepth = 0) {
@@ -793,20 +794,26 @@ function parseExperimentsData() {
  * @returns {{ version?: number, results: any[] } | null} grader document, or null when unavailable
  */
 function parseGraderResults(candidatePaths = GRADER_RESULTS_PATHS) {
+  /** @type {Error | null} */
+  let lastError = null;
   for (const candidate of candidatePaths) {
     if (!fs.existsSync(candidate)) {
       continue;
     }
-    let parsed;
     try {
-      parsed = JSON.parse(fs.readFileSync(candidate, "utf-8"));
+      const parsed = JSON.parse(fs.readFileSync(candidate, "utf-8"));
+      if (!parsed || !Array.isArray(parsed.results)) {
+        throw new Error(`Grader results ${candidate} do not contain a results array`);
+      }
+      return parsed;
     } catch (err) {
-      throw new Error(`Failed to read grader results ${candidate}`, { cause: err });
-    }
-    if (!parsed || !Array.isArray(parsed.results)) {
-      throw new Error(`Grader results ${candidate} do not contain a results array`);
+      // Keep trying the remaining candidates: a truncated copy in one artifact
+      // must not hide a usable copy in another.
+      lastError = new Error(`Failed to read grader results ${candidate}`, { cause: err });
     }
-    return parsed;
+  }
+  if (lastError) {
+    throw lastError;
   }
   return null;
 }

diff --git a/actions/setup/js/generate_usage_activity_summary.test.cjs b/actions/setup/js/generate_usage_activity_summary.test.cjs
@@ -422,6 +422,15 @@ describe("generate_usage_activity_summary.cjs", () => {
       expect(parseGraderResults([resultsPath])).toEqual(document);
     });
 
+    it("falls back to the next candidate when the first file is malformed", () => {
+      const brokenPath = path.join(gradersDir, "broken.json");
+      const goodPath = path.join(gradersDir, "good.json");
+      fs.writeFileSync(brokenPath, "not json");
+      fs.writeFileSync(goodPath, JSON.stringify({ version: 1, results: [{ id: "agent" }] }));
+
+      expect(parseGraderResults([brokenPath, goodPath]).results[0].id).toBe("agent");
+    });
+
     it("prefers the first existing candidate path", () => {
       const usagePath = path.join(gradersDir, "usage_results.json");
       const agentPath = path.join(gradersDir, "agent_results.json");
@@ -438,7 +447,7 @@ describe("generate_usage_activity_summary.cjs", () => {
       expect(() => parseGraderResults([resultsPath])).toThrow(/Failed to read grader results/);
 
       fs.writeFileSync(resultsPath, JSON.stringify({ version: 1 }));
-      expect(() => parseGraderResults([resultsPath])).toThrow(/do not contain a results array/);
+      expect(() => parseGraderResults([resultsPath])).toThrow(/Failed to read grader results/);
     });
   });
 

diff --git a/actions/setup/js/trace_graders.cjs b/actions/setup/js/trace_graders.cjs
@@ -5,6 +5,7 @@ const fs = require("fs");
 const path = require("path");
 const cp = require("child_process");
 const crypto = require("crypto");
+const { GRADERS_DIR_NAME, GRADER_RESULTS_FILENAME, GRADER_MANIFEST_FILENAME } = require("./constants.cjs");
 const { getErrorMessage } = require("./error_helpers.cjs");
 const { readExperimentAssignments } = require("./experiment_helpers.cjs");
 const { calculateWorkingSetFromEntries } = require("./working_set_metrics.cjs");
@@ -13,10 +14,10 @@ const { resolveRunCreatedAtForGrading } = require("./run_created_at.cjs");
 
 // --- Constants ---
 const TMP_GH_AW = "/tmp/gh-aw";
-const GRADERS_DIR = path.join(TMP_GH_AW, "agent", "graders");
-const MANIFEST_PATH = path.join(GRADERS_DIR, "grader_manifest.json");
+const GRADERS_DIR = path.join(TMP_GH_AW, "agent", GRADERS_DIR_NAME);
+const MANIFEST_PATH = path.join(GRADERS_DIR, GRADER_MANIFEST_FILENAME);
 const PAYLOAD_PATH = path.join(GRADERS_DIR, "grader_payload.json");
-const RESULTS_PATH = path.join(GRADERS_DIR, "grader_results.json");
+const RESULTS_PATH = path.join(GRADERS_DIR, GRADER_RESULTS_FILENAME);
 const OPERATIONAL_VALUE_EVALUATOR_PATH = path.join(GRADERS_DIR, "operational_value_evaluator.sh");
 
 // Trace source file paths