Skip to content

Commit b18f6dc

Browse files
Copilotpelikhan
Copilot
and
pelikhan
authored
Test script and inline graders
Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>
1 parent 9d77579 commit b18f6dc

2 files changed

Lines changed: 135 additions & 94 deletions

File tree

‎pkg/cli/graders_run_test.go‎

Lines changed: 51 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ func TestRunGraderFromStdin(t *testing.T) {
5454
}`, output.String())
5555
}
5656

57-
func TestRunCustomGraderFromStdin(t *testing.T) {
57+
func TestRunInlineScriptGraderFromStdin(t *testing.T) {
5858
workflowID := writeGraderRunWorkflow(t, `---
5959
graders:
6060
custom-score:
@@ -73,8 +73,56 @@ graders:
7373
Output: &output,
7474
})
7575
require.NoError(t, err)
76-
assert.Contains(t, output.String(), `"value": 0.75`)
77-
assert.Contains(t, output.String(), `"message": "computed"`)
76+
assert.JSONEq(t, `{
77+
"id":"custom-score",
78+
"name":"custom-score",
79+
"value":0.75,
80+
"unit":"ratio",
81+
"passed":true,
82+
"status":"pass",
83+
"source":"inline",
84+
"implementation":{
85+
"id":"gh-aw/graders",
86+
"version":1,
87+
"digest":"518c37ee83a83874d2added478398c3b391bdbaa7b92f6ea9567a016dd888640"
88+
},
89+
"message":"computed"
90+
}`, output.String())
91+
}
92+
93+
func TestRunScriptFileGraderFromStdin(t *testing.T) {
94+
workflowID := writeGraderRunWorkflow(t, `---
95+
graders:
96+
operational-value:
97+
run: .github/graders/test-operational-value.sh
98+
---
99+
`)
100+
require.NoError(t, os.Mkdir(".git", 0o755))
101+
require.NoError(t, os.MkdirAll(filepath.Join(".github", "graders"), 0o755))
102+
require.NoError(t, os.WriteFile(filepath.Join(".github", "graders", "test-operational-value.sh"), []byte(`#!/usr/bin/env bash
103+
set -euo pipefail
104+
case "${1:-}" in
105+
--definition)
106+
printf '%s\n' '{"schemaVersion":4,"grader":"operational-value","repository":"example/repo","workflowName":"Test","sourcePath":".github/workflows/test.md","adoption":{"commit":"abc","adoptedAt":"2026-01-01T00:00:00Z"},"operationalValue":"Test direct script execution.","evidence":{"opportunity":"test","assignment":"payload","accepted":"stdin","repositories":["example/repo"],"collection":"test","maturation":"immediate","zeroRule":"none","missingRule":"null"},"primaryMetric":{"id":"score","formula":"payload score","direction":"higher_is_better"},"baseline":{"mode":"attainment-only","value":null,"evidenceCutoff":null,"provenance":[]},"validationExamples":{"sample":{"valid":true}}}'
107+
;;
108+
--grade-run)
109+
payload=$(cat)
110+
[[ "$payload" == '{"score":0.8}' ]]
111+
printf '%s\n' '{"value":0.8,"source":"script-file"}'
112+
;;
113+
*) exit 1 ;;
114+
esac
115+
`), 0o700))
116+
117+
var output bytes.Buffer
118+
err := runGrader(context.Background(), graderRunConfig{
119+
Workflow: workflowID,
120+
GraderID: "operational-value",
121+
Input: bytes.NewBufferString(`{"score":0.8}`),
122+
Output: &output,
123+
})
124+
require.NoError(t, err)
125+
assert.JSONEq(t, `{"value":0.8,"source":"script-file"}`, output.String())
78126
}
79127

80128
func TestReadGraderPayloadValidation(t *testing.T) {

‎pkg/workflow/schemas/github-workflow.json‎

Lines changed: 84 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -1944,117 +1944,110 @@
19441944
},
19451945
"workflow_call": {
19461946
"$comment": "https://docs.github.com/en/actions/learn-github-actions/events-that-trigger-workflows#workflow_call",
1947+
"type": "object",
19471948
"description": "Allows workflows to be reused by other workflows.",
1948-
"oneOf": [
1949-
{
1950-
"type": "null"
1951-
},
1952-
{
1949+
"properties": {
1950+
"inputs": {
1951+
"$comment": "https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#onworkflow_callinputs",
1952+
"description": "When using the workflow_call keyword, you can optionally specify inputs that are passed to the called workflow from the caller workflow.",
19531953
"type": "object",
1954-
"properties": {
1955-
"inputs": {
1956-
"$comment": "https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#onworkflow_callinputs",
1957-
"description": "When using the workflow_call keyword, you can optionally specify inputs that are passed to the called workflow from the caller workflow.",
1954+
"patternProperties": {
1955+
"^[_a-zA-Z][a-zA-Z0-9_-]*$": {
1956+
"$comment": "https://docs.github.com/en/actions/creating-actions/metadata-syntax-for-github-actions#inputsinput_id",
1957+
"description": "A string identifier to associate with the input. The value of <input_id> is a map of the input's metadata. The <input_id> must be a unique identifier within the inputs object. The <input_id> must start with a letter or _ and contain only alphanumeric characters, -, or _.",
19581958
"type": "object",
1959-
"patternProperties": {
1960-
"^[_a-zA-Z][a-zA-Z0-9_-]*$": {
1961-
"$comment": "https://docs.github.com/en/actions/creating-actions/metadata-syntax-for-github-actions#inputsinput_id",
1962-
"description": "A string identifier to associate with the input. The value of <input_id> is a map of the input's metadata. The <input_id> must be a unique identifier within the inputs object. The <input_id> must start with a letter or _ and contain only alphanumeric characters, -, or _.",
1963-
"type": "object",
1964-
"properties": {
1965-
"description": {
1966-
"$comment": "https://help.github.com/en/github/automating-your-workflow-with-github-actions/metadata-syntax-for-github-actions#inputsinput_iddescription",
1967-
"description": "A string description of the input parameter.",
1968-
"type": "string"
1969-
},
1970-
"required": {
1971-
"$comment": "https://help.github.com/en/github/automating-your-workflow-with-github-actions/metadata-syntax-for-github-actions#inputsinput_idrequired",
1972-
"description": "A boolean to indicate whether the action requires the input parameter. Set to true when the parameter is required.",
1959+
"properties": {
1960+
"description": {
1961+
"$comment": "https://help.github.com/en/github/automating-your-workflow-with-github-actions/metadata-syntax-for-github-actions#inputsinput_iddescription",
1962+
"description": "A string description of the input parameter.",
1963+
"type": "string"
1964+
},
1965+
"required": {
1966+
"$comment": "https://help.github.com/en/github/automating-your-workflow-with-github-actions/metadata-syntax-for-github-actions#inputsinput_idrequired",
1967+
"description": "A boolean to indicate whether the action requires the input parameter. Set to true when the parameter is required.",
1968+
"type": "boolean"
1969+
},
1970+
"type": {
1971+
"$comment": "https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_callinput_idtype",
1972+
"description": "Required if input is defined for the on.workflow_call keyword. The value of this parameter is a string specifying the data type of the input. This must be one of: boolean, number, or string.",
1973+
"type": "string",
1974+
"enum": ["boolean", "number", "string"]
1975+
},
1976+
"default": {
1977+
"$comment": "https://help.github.com/en/github/automating-your-workflow-with-github-actions/metadata-syntax-for-github-actions#inputsinput_iddefault",
1978+
"description": "The default value is used when an input parameter isn't specified in a workflow file.",
1979+
"anyOf": [
1980+
{
19731981
"type": "boolean"
19741982
},
1975-
"type": {
1976-
"$comment": "https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_callinput_idtype",
1977-
"description": "Required if input is defined for the on.workflow_call keyword. The value of this parameter is a string specifying the data type of the input. This must be one of: boolean, number, or string.",
1978-
"type": "string",
1979-
"enum": ["boolean", "number", "string"]
1983+
{
1984+
"type": "number"
19801985
},
1981-
"default": {
1982-
"$comment": "https://help.github.com/en/github/automating-your-workflow-with-github-actions/metadata-syntax-for-github-actions#inputsinput_iddefault",
1983-
"description": "The default value is used when an input parameter isn't specified in a workflow file.",
1984-
"anyOf": [
1985-
{
1986-
"type": "boolean"
1987-
},
1988-
{
1989-
"type": "number"
1990-
},
1991-
{
1992-
"type": "string"
1993-
}
1994-
]
1986+
{
1987+
"type": "string"
19951988
}
1996-
},
1997-
"required": ["type"],
1998-
"additionalProperties": false
1989+
]
19991990
}
20001991
},
1992+
"required": ["type"],
20011993
"additionalProperties": false
2002-
},
2003-
"outputs": {
2004-
"$comment": "https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#onworkflow_calloutputs",
2005-
"description": "When using the workflow_call keyword, you can optionally specify inputs that are passed to the called workflow from the caller workflow.",
1994+
}
1995+
},
1996+
"additionalProperties": false
1997+
},
1998+
"outputs": {
1999+
"$comment": "https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#onworkflow_calloutputs",
2000+
"description": "When using the workflow_call keyword, you can optionally specify inputs that are passed to the called workflow from the caller workflow.",
2001+
"type": "object",
2002+
"patternProperties": {
2003+
"^[_a-zA-Z][a-zA-Z0-9_-]*$": {
2004+
"$comment": "https://docs.github.com/en/actions/sharing-automations/creating-actions/metadata-syntax-for-github-actions#outputsoutput_id",
2005+
"description": "A string identifier to associate with the output. The value of <output_id> is a map of the output's metadata. The <output_id> must be a unique identifier within the outputs object. The <output_id> must start with a letter or _ and contain only alphanumeric characters, -, or _.",
20062006
"type": "object",
2007-
"patternProperties": {
2008-
"^[_a-zA-Z][a-zA-Z0-9_-]*$": {
2009-
"$comment": "https://docs.github.com/en/actions/sharing-automations/creating-actions/metadata-syntax-for-github-actions#outputsoutput_id",
2010-
"description": "A string identifier to associate with the output. The value of <output_id> is a map of the output's metadata. The <output_id> must be a unique identifier within the outputs object. The <output_id> must start with a letter or _ and contain only alphanumeric characters, -, or _.",
2011-
"type": "object",
2012-
"properties": {
2013-
"description": {
2014-
"$comment": "https://docs.github.com/en/actions/sharing-automations/creating-actions/metadata-syntax-for-github-actions#outputsoutput_iddescription",
2015-
"description": "A string description of the output parameter.",
2016-
"type": "string"
2017-
},
2018-
"value": {
2019-
"$comment": "https://docs.github.com/en/actions/sharing-automations/creating-actions/metadata-syntax-for-github-actions#outputsoutput_idvalue",
2020-
"description": "The value that the output parameter will be mapped to. You can set this to a string or an expression with context. For example, you can use the steps context to set the value of an output to the output value of a step.",
2021-
"type": "string"
2022-
}
2023-
},
2024-
"required": ["value"],
2025-
"additionalProperties": false
2007+
"properties": {
2008+
"description": {
2009+
"$comment": "https://docs.github.com/en/actions/sharing-automations/creating-actions/metadata-syntax-for-github-actions#outputsoutput_iddescription",
2010+
"description": "A string description of the output parameter.",
2011+
"type": "string"
2012+
},
2013+
"value": {
2014+
"$comment": "https://docs.github.com/en/actions/sharing-automations/creating-actions/metadata-syntax-for-github-actions#outputsoutput_idvalue",
2015+
"description": "The value that the output parameter will be mapped to. You can set this to a string or an expression with context. For example, you can use the steps context to set the value of an output to the output value of a step.",
2016+
"type": "string"
20262017
}
20272018
},
2019+
"required": ["value"],
20282020
"additionalProperties": false
2029-
},
2030-
"secrets": {
2031-
"$comment": "https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_callsecrets",
2021+
}
2022+
},
2023+
"additionalProperties": false
2024+
},
2025+
"secrets": {
2026+
"$comment": "https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_callsecrets",
2027+
"type": "object",
2028+
"description": "A map of the secrets that can be used in the called workflow. Within the called workflow, you can use the secrets context to refer to a secret.",
2029+
"patternProperties": {
2030+
"^[_a-zA-Z][a-zA-Z0-9_-]*$": {
2031+
"$comment": "https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_callsecretssecret_id",
20322032
"type": "object",
2033-
"description": "A map of the secrets that can be used in the called workflow. Within the called workflow, you can use the secrets context to refer to a secret.",
2034-
"patternProperties": {
2035-
"^[_a-zA-Z][a-zA-Z0-9_-]*$": {
2036-
"$comment": "https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_callsecretssecret_id",
2037-
"type": "object",
2038-
"description": "A string identifier to associate with the secret.",
2039-
"properties": {
2040-
"description": {
2041-
"description": "A string description of the secret parameter.",
2042-
"type": "string"
2043-
},
2044-
"required": {
2045-
"$comment": "https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_callsecretssecret_idrequired",
2046-
"description": "A boolean specifying whether the secret must be supplied.",
2047-
"type": "boolean"
2048-
}
2049-
},
2050-
"additionalProperties": false
2033+
"description": "A string identifier to associate with the secret.",
2034+
"properties": {
2035+
"description": {
2036+
"description": "A string description of the secret parameter.",
2037+
"type": "string"
2038+
},
2039+
"required": {
2040+
"$comment": "https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_callsecretssecret_idrequired",
2041+
"description": "A boolean specifying whether the secret must be supplied.",
2042+
"type": "boolean"
20512043
}
20522044
},
20532045
"additionalProperties": false
20542046
}
2055-
}
2047+
},
2048+
"additionalProperties": false
20562049
}
2057-
]
2050+
}
20582051
},
20592052
"workflow_dispatch": {
20602053
"$comment": "https://github.blog/changelog/2020-07-06-github-actions-manual-triggers-with-workflow_dispatch/",

0 commit comments

Comments
 (0)