Skip to content

Add payload-based grader execution command #58634

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pelikhan merged 7 commits into from
Sep 5, 2026
Merged

Add payload-based grader execution command #58634

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
63c45e4
Add graders run command
Copilot Sep 4, 2026
0941a63
Harden grader payload replay
Copilot Sep 4, 2026
9d77579
Plan grader script tests
Copilot Sep 5, 2026
b18f6dc
Test script and inline graders
Copilot Sep 5, 2026
7240734
Add ADR for graders run payload replay
github-actions[bot] Sep 5, 2026
d34da01
Plan grader review follow-up
Copilot Sep 5, 2026
ea9c921
Honor repo override for grader replay
Copilot Sep 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Test script and inline graders 
Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>
  • Loading branch information
@pelikhan
Copilot and pelikhan authored Sep 5, 2026
commit b18f6dc289acf54ad99f6d7c6639f877346e23d0
54 changes: 51 additions & 3 deletions pkg/cli/graders_run_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ func TestRunGraderFromStdin(t *testing.T) {
}`, output.String())
}

func TestRunCustomGraderFromStdin(t *testing.T) {
func TestRunInlineScriptGraderFromStdin(t *testing.T) {
workflowID := writeGraderRunWorkflow(t, `---
graders:
custom-score:
Expand All @@ -73,8 +73,56 @@ graders:
Output: &output,
})
require.NoError(t, err)
assert.Contains(t, output.String(), `"value": 0.75`)
assert.Contains(t, output.String(), `"message": "computed"`)
assert.JSONEq(t, `{
"id":"custom-score",
"name":"custom-score",
"value":0.75,
"unit":"ratio",
"passed":true,
"status":"pass",
"source":"inline",
"implementation":{
"id":"gh-aw/graders",
"version":1,
"digest":"518c37ee83a83874d2added478398c3b391bdbaa7b92f6ea9567a016dd888640"
},
"message":"computed"
}`, output.String())
}

func TestRunScriptFileGraderFromStdin(t *testing.T) {
workflowID := writeGraderRunWorkflow(t, `---
graders:
operational-value:
run: .github/graders/test-operational-value.sh
---
`)
require.NoError(t, os.Mkdir(".git", 0o755))
require.NoError(t, os.MkdirAll(filepath.Join(".github", "graders"), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(".github", "graders", "test-operational-value.sh"), []byte(`#!/usr/bin/env bash
set -euo pipefail
case "${1:-}" in
--definition)
printf '%s\n' '{"schemaVersion":4,"grader":"operational-value","repository":"example/repo","workflowName":"Test","sourcePath":".github/workflows/test.md","adoption":{"commit":"abc","adoptedAt":"2026-01-01T00:00:00Z"},"operationalValue":"Test direct script execution.","evidence":{"opportunity":"test","assignment":"payload","accepted":"stdin","repositories":["example/repo"],"collection":"test","maturation":"immediate","zeroRule":"none","missingRule":"null"},"primaryMetric":{"id":"score","formula":"payload score","direction":"higher_is_better"},"baseline":{"mode":"attainment-only","value":null,"evidenceCutoff":null,"provenance":[]},"validationExamples":{"sample":{"valid":true}}}'
;;
--grade-run)
payload=$(cat)
[[ "$payload" == '{"score":0.8}' ]]
printf '%s\n' '{"value":0.8,"source":"script-file"}'
;;
*) exit 1 ;;
esac
`), 0o700))

var output bytes.Buffer
err := runGrader(context.Background(), graderRunConfig{
Workflow: workflowID,
GraderID: "operational-value",
Input: bytes.NewBufferString(`{"score":0.8}`),
Output: &output,
})
require.NoError(t, err)
assert.JSONEq(t, `{"value":0.8,"source":"script-file"}`, output.String())
}

func TestReadGraderPayloadValidation(t *testing.T) {
Expand Down
175 changes: 84 additions & 91 deletions pkg/workflow/schemas/github-workflow.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1944,117 +1944,110 @@
},
"workflow_call": {
"$comment": "https://docs.github.com/en/actions/learn-github-actions/events-that-trigger-workflows#workflow_call",
"type": "object",
"description": "Allows workflows to be reused by other workflows.",
"oneOf": [
{
"type": "null"
},
{
"properties": {
"inputs": {
"$comment": "https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#onworkflow_callinputs",
"description": "When using the workflow_call keyword, you can optionally specify inputs that are passed to the called workflow from the caller workflow.",
"type": "object",
"properties": {
"inputs": {
"$comment": "https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#onworkflow_callinputs",
"description": "When using the workflow_call keyword, you can optionally specify inputs that are passed to the called workflow from the caller workflow.",
"patternProperties": {
"^[_a-zA-Z][a-zA-Z0-9_-]*$": {
"$comment": "https://docs.github.com/en/actions/creating-actions/metadata-syntax-for-github-actions#inputsinput_id",
"description": "A string identifier to associate with the input. The value of <input_id> is a map of the input's metadata. The <input_id> must be a unique identifier within the inputs object. The <input_id> must start with a letter or _ and contain only alphanumeric characters, -, or _.",
"type": "object",
"patternProperties": {
"^[_a-zA-Z][a-zA-Z0-9_-]*$": {
"$comment": "https://docs.github.com/en/actions/creating-actions/metadata-syntax-for-github-actions#inputsinput_id",
"description": "A string identifier to associate with the input. The value of <input_id> is a map of the input's metadata. The <input_id> must be a unique identifier within the inputs object. The <input_id> must start with a letter or _ and contain only alphanumeric characters, -, or _.",
"type": "object",
"properties": {
"description": {
"$comment": "https://help.github.com/en/github/automating-your-workflow-with-github-actions/metadata-syntax-for-github-actions#inputsinput_iddescription",
"description": "A string description of the input parameter.",
"type": "string"
},
"required": {
"$comment": "https://help.github.com/en/github/automating-your-workflow-with-github-actions/metadata-syntax-for-github-actions#inputsinput_idrequired",
"description": "A boolean to indicate whether the action requires the input parameter. Set to true when the parameter is required.",
"properties": {
"description": {
"$comment": "https://help.github.com/en/github/automating-your-workflow-with-github-actions/metadata-syntax-for-github-actions#inputsinput_iddescription",
"description": "A string description of the input parameter.",
"type": "string"
},
"required": {
"$comment": "https://help.github.com/en/github/automating-your-workflow-with-github-actions/metadata-syntax-for-github-actions#inputsinput_idrequired",
"description": "A boolean to indicate whether the action requires the input parameter. Set to true when the parameter is required.",
"type": "boolean"
},
"type": {
"$comment": "https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_callinput_idtype",
"description": "Required if input is defined for the on.workflow_call keyword. The value of this parameter is a string specifying the data type of the input. This must be one of: boolean, number, or string.",
"type": "string",
"enum": ["boolean", "number", "string"]
},
"default": {
"$comment": "https://help.github.com/en/github/automating-your-workflow-with-github-actions/metadata-syntax-for-github-actions#inputsinput_iddefault",
"description": "The default value is used when an input parameter isn't specified in a workflow file.",
"anyOf": [
{
"type": "boolean"
},
"type": {
"$comment": "https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_callinput_idtype",
"description": "Required if input is defined for the on.workflow_call keyword. The value of this parameter is a string specifying the data type of the input. This must be one of: boolean, number, or string.",
"type": "string",
"enum": ["boolean", "number", "string"]
{
"type": "number"
},
"default": {
"$comment": "https://help.github.com/en/github/automating-your-workflow-with-github-actions/metadata-syntax-for-github-actions#inputsinput_iddefault",
"description": "The default value is used when an input parameter isn't specified in a workflow file.",
"anyOf": [
{
"type": "boolean"
},
{
"type": "number"
},
{
"type": "string"
}
]
{
"type": "string"
}
},
"required": ["type"],
"additionalProperties": false
]
}
},
"required": ["type"],
"additionalProperties": false
},
"outputs": {
"$comment": "https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#onworkflow_calloutputs",
"description": "When using the workflow_call keyword, you can optionally specify inputs that are passed to the called workflow from the caller workflow.",
}
},
"additionalProperties": false
},
"outputs": {
"$comment": "https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#onworkflow_calloutputs",
"description": "When using the workflow_call keyword, you can optionally specify inputs that are passed to the called workflow from the caller workflow.",
"type": "object",
"patternProperties": {
"^[_a-zA-Z][a-zA-Z0-9_-]*$": {
"$comment": "https://docs.github.com/en/actions/sharing-automations/creating-actions/metadata-syntax-for-github-actions#outputsoutput_id",
"description": "A string identifier to associate with the output. The value of <output_id> is a map of the output's metadata. The <output_id> must be a unique identifier within the outputs object. The <output_id> must start with a letter or _ and contain only alphanumeric characters, -, or _.",
"type": "object",
"patternProperties": {
"^[_a-zA-Z][a-zA-Z0-9_-]*$": {
"$comment": "https://docs.github.com/en/actions/sharing-automations/creating-actions/metadata-syntax-for-github-actions#outputsoutput_id",
"description": "A string identifier to associate with the output. The value of <output_id> is a map of the output's metadata. The <output_id> must be a unique identifier within the outputs object. The <output_id> must start with a letter or _ and contain only alphanumeric characters, -, or _.",
"type": "object",
"properties": {
"description": {
"$comment": "https://docs.github.com/en/actions/sharing-automations/creating-actions/metadata-syntax-for-github-actions#outputsoutput_iddescription",
"description": "A string description of the output parameter.",
"type": "string"
},
"value": {
"$comment": "https://docs.github.com/en/actions/sharing-automations/creating-actions/metadata-syntax-for-github-actions#outputsoutput_idvalue",
"description": "The value that the output parameter will be mapped to. You can set this to a string or an expression with context. For example, you can use the steps context to set the value of an output to the output value of a step.",
"type": "string"
}
},
"required": ["value"],
"additionalProperties": false
"properties": {
"description": {
"$comment": "https://docs.github.com/en/actions/sharing-automations/creating-actions/metadata-syntax-for-github-actions#outputsoutput_iddescription",
"description": "A string description of the output parameter.",
"type": "string"
},
"value": {
"$comment": "https://docs.github.com/en/actions/sharing-automations/creating-actions/metadata-syntax-for-github-actions#outputsoutput_idvalue",
"description": "The value that the output parameter will be mapped to. You can set this to a string or an expression with context. For example, you can use the steps context to set the value of an output to the output value of a step.",
"type": "string"
}
},
"required": ["value"],
"additionalProperties": false
},
"secrets": {
"$comment": "https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_callsecrets",
}
},
"additionalProperties": false
},
"secrets": {
"$comment": "https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_callsecrets",
"type": "object",
"description": "A map of the secrets that can be used in the called workflow. Within the called workflow, you can use the secrets context to refer to a secret.",
"patternProperties": {
"^[_a-zA-Z][a-zA-Z0-9_-]*$": {
"$comment": "https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_callsecretssecret_id",
"type": "object",
"description": "A map of the secrets that can be used in the called workflow. Within the called workflow, you can use the secrets context to refer to a secret.",
"patternProperties": {
"^[_a-zA-Z][a-zA-Z0-9_-]*$": {
"$comment": "https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_callsecretssecret_id",
"type": "object",
"description": "A string identifier to associate with the secret.",
"properties": {
"description": {
"description": "A string description of the secret parameter.",
"type": "string"
},
"required": {
"$comment": "https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_callsecretssecret_idrequired",
"description": "A boolean specifying whether the secret must be supplied.",
"type": "boolean"
}
},
"additionalProperties": false
"description": "A string identifier to associate with the secret.",
"properties": {
"description": {
"description": "A string description of the secret parameter.",
"type": "string"
},
"required": {
"$comment": "https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_callsecretssecret_idrequired",
"description": "A boolean specifying whether the secret must be supplied.",
"type": "boolean"
}
},
"additionalProperties": false
}
}
},
"additionalProperties": false
}
]
}
},
"workflow_dispatch": {
"$comment": "https://github.blog/changelog/2020-07-06-github-actions-manual-triggers-with-workflow_dispatch/",
Expand Down
Loading