Skip to content
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 57 additions & 9 deletions actions/setup/js/check_daily_aic_workflow_guardrail.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
const { scanDailyAIC } = require("./daily_aic_scan.cjs");
const { createAPIBudget, retryNotBefore, safeResponseHeaders } = require("./daily_aic_api_budget.cjs");
const { loadBillableJobs, allBillableJobsSkipped, sumCoveredComponents } = require("./daily_aic_component_coverage.cjs");
const { readLedgerEntries } = require("./daily_aic_repo_memory_ledger.cjs");

const PRIMARY_GUARDRAIL_ARTIFACT_NAMES = ["usage"];
const MAX_WORKFLOW_RUN_PAGES = 10;
Expand All @@ -24,6 +25,7 @@
const MAX_LEGACY_AGENT_LOG_BYTES = 10 * 1024 * 1024;
const ENGINE_HARNESS_MARKER = /\[[^\]\r\n]+-harness\]/i;
const AWF_STARTUP_FAILURE_MARKER = /Fatal error:|Process exiting with code:|Refusing to use symlink as bind mountpoint|mcp gateway[^\r\n]{0,80}(?:startup failed|failed to start|startup error)/i;
const REPO_MEMORY_BACKEND = "repo-memory";

/**
* @returns {Promise<any>}
Expand Down Expand Up @@ -82,6 +84,10 @@
return normalized === "true" || normalized === "1" || normalized === "yes";
}

function dailyAICBackend() {
return (process.env.GH_AW_MAX_DAILY_AI_CREDITS_BACKEND || "").trim().toLowerCase();
}

/**
* @returns {boolean}
*/
Expand Down Expand Up @@ -387,7 +393,7 @@
});
return aic;
} finally {
fs.rmSync(downloadRoot, { recursive: true, force: true });

Check warning on line 396 in actions/setup/js/check_daily_aic_workflow_guardrail.cjs

View workflow job for this annotation

GitHub Actions / lint-js

Wrap fs.rmSync(downloadRoot) in try/catch — rmSync throws on permission denied, invalid path, or filesystem errors; without a call-site try/catch, you lose the original error context and get a generic engine-level stack instead of a specific message with `{ cause }`

Check warning on line 396 in actions/setup/js/check_daily_aic_workflow_guardrail.cjs

View workflow job for this annotation

GitHub Actions / lint-js

Wrap fs.rmSync(downloadRoot) in try/catch — rmSync throws on permission denied, invalid path, or filesystem errors; without a call-site try/catch, you lose the original error context and get a generic engine-level stack instead of a specific message with `{ cause }`
}
}

Expand Down Expand Up @@ -540,7 +546,7 @@
* @param {string} actorLogin
* @param {number} threshold
* @param {Array<{id:number, html_url:string, created_at:string, conclusion:string, aic:number}>} countedRuns
* @param {{remaining:number,limit:number,used:number,reset:string}} rateLimit
* @param {{remaining:number,limit:number,used:number,reset:string} | null} rateLimit
* @param {{candidateRunsCount:number,inspectedRunsCount:number,truncatedByRateLimit:boolean}} meta
* @returns {string}
*/
Expand All @@ -564,12 +570,13 @@
const minMaxAICFormatted = noRunData ? "— / —" : `${formatAICCredits(stats.min)} / ${formatAICCredits(stats.max)}`;

const noteLines = [];
if (meta.truncatedByRateLimit) {
if (meta.truncatedByRateLimit && rateLimit) {
noteLines.push(`- Stopped early to preserve GitHub API rate limit headroom (${rateLimit.remaining} remaining, reserve ${RATE_LIMIT_RESERVE}).`);
}
if (meta.candidateRunsCount > meta.inspectedRunsCount) {
noteLines.push(`- Considered ${meta.candidateRunsCount} prior runs in the 24h window and inspected ${meta.inspectedRunsCount}.`);
}
const apiRows = rateLimit ? [`| API remaining | ${formatInteger(rateLimit.remaining)} / ${formatInteger(rateLimit.limit)} |`, `| API used | ${formatInteger(rateLimit.used)} |`, `| API reset | ${rateLimit.reset || "unknown"} |`] : [];
return [
`**Workflow:** ${workflowName || "workflow"}`,
`**Actor:** ${actorLogin || "unknown"}`,
Expand All @@ -584,9 +591,7 @@
`| Avg AIC / run | ${avgAICFormatted} |`,
`| Std dev AIC | ${stddevAICFormatted} |`,
`| Min / Max AIC | ${minMaxAICFormatted} |`,
`| API remaining | ${formatInteger(rateLimit.remaining)} / ${formatInteger(rateLimit.limit)} |`,
`| API used | ${formatInteger(rateLimit.used)} |`,
`| API reset | ${rateLimit.reset || "unknown"} |`,
...apiRows,
"",
"Previous runs counted in the last 24 hours:",
"",
Expand All @@ -602,7 +607,7 @@
* @param {string} actorLogin
* @param {number} threshold
* @param {Array<{id:number, html_url:string, created_at:string, conclusion:string, aic:number}>} countedRuns
* @param {{remaining:number,limit:number,used:number,reset:string}} rateLimit
* @param {{remaining:number,limit:number,used:number,reset:string} | null} rateLimit
* @param {{candidateRunsCount:number,inspectedRunsCount:number,truncatedByRateLimit:boolean}} meta
* @returns {Promise<void>}
*/
Expand Down Expand Up @@ -644,7 +649,8 @@
}

const token = process.env.GH_AW_GITHUB_TOKEN || process.env.GITHUB_TOKEN || process.env.GH_TOKEN || "";
if (!token) {
const backend = dailyAICBackend();
if (!token && backend !== REPO_MEMORY_BACKEND) {
const message = "Daily workflow AI Credits are unknown: no artifact lookup token.";
core.setOutput("daily_ai_credits_guardrail_status", "structural_error");
core.setOutput("daily_ai_credits_guardrail_error", message);
Expand All @@ -656,8 +662,50 @@
try {
const githubClient = createRateLimitAwareGithub(github);
const budget = createAPIBudget();
const artifactClient = await module.exports.getArtifactClient(budget.observe);
const workflowName = process.env.GH_AW_WORKFLOW_NAME || process.env.GH_AW_WORKFLOW_ID || "workflow";
let actorLogin = process.env.GITHUB_TRIGGERING_ACTOR || process.env.GITHUB_ACTOR || "";
if (backend === REPO_MEMORY_BACKEND) {
const repository = `${context.repo.owner}/${context.repo.repo}`;
const countedRuns = readLedgerEntries({
repoMemoryDir: options.repoMemoryDir || process.env.GH_AW_DAILY_AIC_REPO_MEMORY_DIR,
repository,
workflowId: process.env.GH_AW_WORKFLOW_ID || workflowName,
actor: actorLogin,
}).map(entry => ({
id: entry.run_id,
html_url: entry.run_url || "",
created_at: entry.timestamp,
conclusion: "completed",
aic: entry.aic,
}));
const totalAIC = countedRuns.reduce((sum, run) => sum + run.aic, 0);
Comment on lines +674 to +685

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The repo-memory backend derives the entire 24h AI-credit total from JSONL files under the agent-writable repo-memory directory (/tmp/gh-aw/repo-memory/..., synced from a branch that agentic runs push to). Since agent output is untrusted (prompt-injectable) and entries carry no integrity protection, an attacker can cause the ledger files to be truncated/rewritten so that totalAIC reads as 0, fully bypassing the max-daily-ai-credits guardrail on every subsequent run. Consider keeping an authoritative, non-agent-writable source (e.g. Actions cache/artifact scan) as a cross-check, or signing/validating ledger files against data the agent cannot modify before trusting them for the guardrail decision.


Best fix without changing core functionality: fail closed for enforcement when using repo-memory backend unless an integrity signal is present, instead of trusting raw ledger contents unconditionally. Given only this file can be edited, the safest minimal remediation is to require an explicit integrity acknowledgement/env gate before honoring repo-memory totals; otherwise mark status as structural_error and stop. This preserves existing behavior when operators intentionally enable trusted repo-memory mode, while preventing silent bypass by default in untrusted setups.

Concretely, in actions/setup/js/check_daily_aic_workflow_guardrail.cjs, inside the if (backend === REPO_MEMORY_BACKEND) { ... } block (before calling readLedgerEntries), add a check for an env var such as GH_AW_TRUST_REPO_MEMORY_LEDGER === "true". If absent, set:

  • daily_ai_credits_guardrail_status = structural_error
  • daily_ai_credits_guardrail_error with a clear message explaining repo-memory ledger is untrusted unless explicitly trusted
  • core.setFailed(message) and return

No new imports or dependencies are needed.

Comment on lines +674 to +685
const rateLimit = null;
const summaryMeta = {
candidateRunsCount: countedRuns.length,
inspectedRunsCount: countedRuns.length,
truncatedByRateLimit: false,
};
core.setOutput("daily_ai_credits_total", String(totalAIC));
core.setOutput("daily_ai_credits_threshold", String(threshold));
logDailyGuardrail("Completed repo-memory AIC ledger read", {
countedRuns: countedRuns.length,
currentAIC: totalAIC,
threshold,
exceeded: totalAIC >= threshold,
});
if (totalAIC < threshold) {
core.setOutput("daily_ai_credits_guardrail_status", "under_budget");
await appendDailyAICSummary(workflowName, actorLogin, threshold, countedRuns, rateLimit, summaryMeta);
core.info(`Daily workflow AIC guardrail not exceeded (${totalAIC}/${threshold}).`);
return;
}
core.setOutput("daily_ai_credits_exceeded", "true");
core.setOutput("daily_ai_credits_guardrail_status", "exceeded");
await appendDailyAICSummary(workflowName, actorLogin, threshold, countedRuns, rateLimit, summaryMeta);
core.info(`Daily workflow AIC guardrail exceeded for ${workflowName}: ${totalAIC}/${threshold}.`);
return;
}
const artifactClient = await module.exports.getArtifactClient(budget.observe);
const { countedRuns, candidateRunsCount, cacheHits, current } = await scanDailyAIC({
github: githubClient,
context,
Expand All @@ -670,7 +718,7 @@
cachePath: options.cachePath,
});
const totalAIC = countedRuns.reduce((sum, run) => sum + run.aic, 0);
const actorLogin = process.env.GITHUB_TRIGGERING_ACTOR || current.triggering_actor?.login || current.actor?.login || process.env.GITHUB_ACTOR || "";
actorLogin = process.env.GITHUB_TRIGGERING_ACTOR || current.triggering_actor?.login || current.actor?.login || process.env.GITHUB_ACTOR || "";
const rateLimit = budget.snapshot();

core.setOutput("daily_ai_credits_total", String(totalAIC));
Expand Down
132 changes: 132 additions & 0 deletions actions/setup/js/daily_aic_repo_memory_ledger.cjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
// @ts-check

const fs = require("fs");
const path = require("path");
const { findJSONLFiles, isDailyAICUsageJSONLFile, sumAICFromUsageJSONLFiles } = require("./daily_aic_workflow_helpers.cjs");
const { getErrorMessage } = require("./error_helpers.cjs");

const LEDGER_SUBDIR = "daily-aic-ledger";
const WINDOW_MS = 24 * 60 * 60 * 1000;
const USAGE_ROOT = "/tmp/gh-aw";

function repoMemoryDirOrThrow(repoMemoryDir = process.env.GH_AW_DAILY_AIC_REPO_MEMORY_DIR) {
const resolved = typeof repoMemoryDir === "string" ? repoMemoryDir.trim() : "";
if (!resolved) {
throw new Error("GH_AW_DAILY_AIC_REPO_MEMORY_DIR is required for the daily AIC repo-memory backend.");
}
return resolved;
}

function ledgerRoot(repoMemoryDir) {
return path.join(repoMemoryDirOrThrow(repoMemoryDir), LEDGER_SUBDIR);
}

function utcDay(ms) {
return new Date(ms).toISOString().slice(0, 10);
}

function ledgerPathForDay(root, day) {
return path.join(root, `${day}.jsonl`);
}

function ledgerReadPaths(root, now = Date.now()) {
return [ledgerPathForDay(root, utcDay(now)), ledgerPathForDay(root, utcDay(now - WINDOW_MS))].filter((value, index, array) => array.indexOf(value) === index);
}

function validEntry(entry, repository, workflowId, actor, now = Date.now()) {
if (entry?.version !== 1) return false;
if (entry.repository !== repository || entry.workflow_id !== workflowId) return false;
if (actor && entry.actor !== actor) return false;
const timestamp = Date.parse(entry.timestamp);
return Number.isFinite(timestamp) && timestamp >= now - WINDOW_MS && timestamp <= now && Number.isSafeInteger(entry.run_id) && entry.run_id > 0 && Number.isFinite(entry.aic) && entry.aic >= 0;
}

function readLedgerEntries({ repoMemoryDir, repository, workflowId, actor, now = Date.now() }) {
const root = ledgerRoot(repoMemoryDir);
const entriesByRun = new Map();
for (const filePath of ledgerReadPaths(root, now)) {
let content = "";
try {
content = fs.readFileSync(filePath, "utf8");
} catch {
continue;
}
for (const line of content.split("\n")) {
const trimmed = line.trim();
if (!trimmed) continue;
try {
const entry = JSON.parse(trimmed);
if (!validEntry(entry, repository, workflowId, actor, now)) continue;
const prior = entriesByRun.get(entry.run_id);
const entryTimestamp = Date.parse(entry.timestamp);
const priorTimestamp = prior ? Date.parse(prior.timestamp) : NaN;
if (Number.isFinite(entryTimestamp) && (!prior || !Number.isFinite(priorTimestamp) || entryTimestamp >= priorTimestamp)) {
entriesByRun.set(entry.run_id, entry);
}
} catch {
// Ignore malformed ledger lines.
}
}
}
return [...entriesByRun.values()].sort((a, b) => Date.parse(b.timestamp) - Date.parse(a.timestamp));
}

function appendLedgerEntry(entry, repoMemoryDir, now = Date.now()) {
const root = ledgerRoot(repoMemoryDir);
try {
fs.mkdirSync(root, { recursive: true });
} catch (error) {
throw new Error(`Failed to create daily AIC repo-memory ledger directory ${root}: ${getErrorMessage(error)}`, { cause: error });
}
const ledgerPath = ledgerPathForDay(root, utcDay(now));
try {
fs.appendFileSync(ledgerPath, `${JSON.stringify(entry)}\n`, "utf8");
Comment on lines +81 to +83
} catch (error) {
throw new Error(`Failed to append daily AIC repo-memory ledger entry to ${ledgerPath}: ${getErrorMessage(error)}`, { cause: error });
}
}

/**
* @param {{ repoMemoryDir?: string, usageRoot?: string, now?: number }} [options]
*/
function appendCurrentRunLedgerEntry({ repoMemoryDir, usageRoot = USAGE_ROOT, now = Date.now() } = {}) {
const runId = Number(process.env.GITHUB_RUN_ID || 0);
const repository = process.env.GITHUB_REPOSITORY || "";
const workflowId = process.env.GH_AW_WORKFLOW_ID || process.env.GITHUB_WORKFLOW || "";
if (!Number.isFinite(runId) || runId <= 0 || !repository || !workflowId) {
core.warning("[daily-workflow-aic] Skipping repo-memory ledger write: missing run, repository, or workflow identity.");
return;
}
const usageFiles = findJSONLFiles(usageRoot).filter(isDailyAICUsageJSONLFile);
const aic = sumAICFromUsageJSONLFiles(usageFiles);
if (!Number.isFinite(aic) || aic < 0) {
core.warning(`[daily-workflow-aic] Skipping repo-memory ledger write: computed AIC is ${aic}.`);
return;
}
appendLedgerEntry(
{
version: 1,
repository,
workflow_id: workflowId,
run_id: runId,
run_url: process.env.GH_AW_RUN_URL || "",
actor: process.env.GITHUB_TRIGGERING_ACTOR || process.env.GITHUB_ACTOR || "",
aic,
timestamp: new Date(now).toISOString(),
},
repoMemoryDir,
now
);
core.info(`[daily-workflow-aic] Appended repo-memory ledger entry: ${JSON.stringify({ runId, workflowId, aic, files: usageFiles.length })}`);
}

module.exports = {
LEDGER_SUBDIR,
WINDOW_MS,
appendCurrentRunLedgerEntry,
appendLedgerEntry,
ledgerReadPaths,
ledgerRoot,
repoMemoryDirOrThrow,
readLedgerEntries,
};
88 changes: 88 additions & 0 deletions actions/setup/js/daily_aic_repo_memory_ledger.test.cjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";

let ledger;
let directory;
const now = Date.parse("2026-09-23T13:02:21.000Z");

describe("daily_aic_repo_memory_ledger", () => {
beforeEach(async () => {
vi.resetModules();
directory = fs.mkdtempSync(path.join(os.tmpdir(), "daily-aic-ledger-"));
global.core = { info: vi.fn(), warning: vi.fn() };
const mod = await import("./daily_aic_repo_memory_ledger.cjs");
ledger = mod.default || mod;
});

afterEach(() => {
delete global.core;
fs.rmSync(directory, { recursive: true, force: true });
});

it("reads current and previous UTC day buckets and filters by repository, workflow, actor, and window", () => {
const repoMemoryDir = path.join(directory, "memory");
const root = ledger.ledgerRoot(repoMemoryDir);
fs.mkdirSync(root, { recursive: true });
const currentEntry = {
version: 1,
repository: "owner/repo",
workflow_id: "triage",
run_id: 10,
run_url: "https://example.test/10",
actor: "monalisa",
aic: 5,
timestamp: "2026-09-23T12:00:00.000Z",
};
const previousEntry = { ...currentEntry, run_id: 9, aic: 7, timestamp: "2026-09-22T20:00:00.000Z" };
const staleEntry = { ...currentEntry, run_id: 8, aic: 99, timestamp: "2026-09-22T12:00:00.000Z" };
const otherActor = { ...currentEntry, run_id: 11, actor: "octocat", aic: 99 };
fs.writeFileSync(path.join(root, "2026-09-23.jsonl"), [JSON.stringify(currentEntry), JSON.stringify(otherActor)].join("\n"), "utf8");
fs.writeFileSync(path.join(root, "2026-09-22.jsonl"), [JSON.stringify(previousEntry), JSON.stringify(staleEntry)].join("\n"), "utf8");

expect(
ledger.readLedgerEntries({
repoMemoryDir,
repository: "owner/repo",
workflowId: "triage",
actor: "monalisa",
now,
})
).toEqual([currentEntry, previousEntry]);
});

it("appends current run AIC to the current UTC bucket", () => {
const repoMemoryDir = path.join(directory, "memory");
const usageDir = path.join(directory, "usage");
fs.mkdirSync(usageDir, { recursive: true });
fs.writeFileSync(path.join(usageDir, "agent_usage.jsonl"), `${JSON.stringify({ aic: 3 })}\n${JSON.stringify({ usage: { aic: 4 } })}\n`, "utf8");
process.env.GITHUB_RUN_ID = "42";
process.env.GITHUB_REPOSITORY = "owner/repo";
process.env.GH_AW_WORKFLOW_ID = "triage";
process.env.GITHUB_ACTOR = "monalisa";
delete process.env.GITHUB_TRIGGERING_ACTOR;
process.env.GH_AW_RUN_URL = "https://example.test/runs/42";
try {
ledger.appendCurrentRunLedgerEntry({ repoMemoryDir, usageRoot: usageDir, now });
} finally {
delete process.env.GITHUB_RUN_ID;
delete process.env.GITHUB_REPOSITORY;
delete process.env.GH_AW_WORKFLOW_ID;
delete process.env.GITHUB_ACTOR;
delete process.env.GH_AW_RUN_URL;
}

const content = fs.readFileSync(path.join(ledger.ledgerRoot(repoMemoryDir), "2026-09-23.jsonl"), "utf8").trim();
expect(JSON.parse(content)).toMatchObject({
version: 1,
repository: "owner/repo",
workflow_id: "triage",
run_id: 42,
run_url: "https://example.test/runs/42",
actor: "monalisa",
aic: 7,
timestamp: "2026-09-23T13:02:21.000Z",
});
});
});
10 changes: 10 additions & 0 deletions actions/setup/js/daily_aic_workflow_helpers.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,15 @@ const path = require("path");
const { computeInferenceAIC, findModelPricing, formatAIC } = require("./model_costs.cjs");

const TOKEN_USAGE_FILENAME = "token-usage.jsonl";
const DAILY_AIC_USAGE_JSONL_FILE = /(?:^|[/\\])(?:agent_usage|detection_usage|evals|token_usage)[^/\\]*\.jsonl$/;

/**
* @param {string} filePath
* @returns {boolean}
*/
function isDailyAICUsageJSONLFile(filePath) {
return DAILY_AIC_USAGE_JSONL_FILE.test(filePath);
}

/**
* @param {string} root
Expand Down Expand Up @@ -300,6 +309,7 @@ function formatAICCredits(value) {

module.exports = {
findJSONLFiles,
isDailyAICUsageJSONLFile,
sumAICFromUsageJSONLFiles,
calculateDailyAICStats,
formatAICCredits,
Expand Down
Loading
Loading