Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/smoke-copilot-aoai-apikey.lock.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion .github/workflows/smoke-copilot-aoai-entra.lock.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion .github/workflows/smoke-copilot.lock.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 6 additions & 0 deletions docs/src/content/docs/experimental/experiments.md
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,12 @@ Read the given file and return a concise summary.
See [Inline Sub-Agents](/gh-aw/reference/inline-sub-agents/) for the full
syntax.

With Copilot, you can also declare `subagent_model: [small, large]` in
`experiments` and set the sub-agent's `model` to
`${{ experiments.subagent_model }}`. The selected variant is used in the
sub-agent definition and recorded in the run's declared-model metadata.
Pi sub-agents require literal models or aliases, not expressions.

### Try different subskills

This pattern compares two reusable instruction blocks, sometimes called
Expand Down
4 changes: 4 additions & 0 deletions docs/src/content/docs/reference/permissions.md
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,10 @@ permissions:

This permission does not require safe-outputs.

The compiler emits an informational reminder to validate OIDC audiences and
cloud-provider trust policies. The reminder does not count as a warning or
block warnings-as-errors validation, including `gh aw compile --dry-run`.

### Special Permission: `copilot-requests: write`

The `copilot-requests: write` permission enables Copilot inference using the built-in GitHub Actions token. This is the recommended way to authenticate Copilot for workflows running in organizations with a Copilot subscription.
Expand Down
30 changes: 30 additions & 0 deletions pkg/cli/compile_development_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
Expand Down Expand Up @@ -182,6 +184,34 @@ func TestEnforceDevelopmentDiagnostics(t *testing.T) {
}
}

func TestDevelopmentCompileAllowsIDTokenInfo(t *testing.T) {
source := filepath.Join(t.TempDir(), "oidc.md")
content := `---
on: workflow_dispatch
engine: copilot
permissions:
contents: read
id-token: write
safe-outputs:
noop:
---
# OIDC workflow
Say Done.
`
require.NoError(t, os.WriteFile(source, []byte(content), 0600))
config := applyDevelopmentCompileMode(CompileConfig{
DryRun: true, activeModels: &activeModelInventory{},
})
compiler := createAndConfigureCompiler(config)
require.NoError(t, compiler.CompileWorkflow(source))
require.Zero(t, compiler.GetWarningCount())
stats := &CompilationStats{Total: 1, Succeeded: 1}
results := []ValidationResult{{Workflow: "oidc.md", Valid: true}}
require.NoError(t, enforceDevelopmentDiagnostics(config, compiler, stats, &results))
assert.True(t, results[0].Valid)
assert.Zero(t, stats.Errors)
}

func TestDevelopmentExplicitlyDisabledChecks(t *testing.T) {
t.Parallel()
for _, name := range DryRunRequiredBoolFlags() {
Expand Down
80 changes: 80 additions & 0 deletions pkg/workflow/compiler_experiments_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,14 @@
package workflow

import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"

"github.com/github/gh-aw/pkg/parser"
"github.com/goccy/go-yaml"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
Expand Down Expand Up @@ -225,6 +230,81 @@ func TestRewriteActivationOutputsToLocalStepOutputs(t *testing.T) {
assert.Equal(t, "${{ needs.activation.outputs.unknown }}", RewriteActivationOutputsToLocalStepOutputs("${{ needs.activation.outputs.unknown }}", experiments))
}

func TestWriteSubagentModelDeclarationsExperimentExpressions(t *testing.T) {
for _, test := range []struct {
name string
model string
wantModel string
wantPatterns []string
}{
{"experiment", "${{ experiments.subagent_model }}", "${{ steps.pick-experiment.outputs.subagent_model }}", nil},
{"fallback", "${{ experiments.subagent_model || 'small' }}", "${{ steps.pick-experiment.outputs.subagent_model || 'small' }}", nil},
Comment thread
pelikhan marked this conversation as resolved.
Resolved
{"literal in expression", "${{ format('experiments.subagent_model-{0}', experiments.subagent_model) }}", "${{ format('experiments.subagent_model-{0}', steps.pick-experiment.outputs.subagent_model) }}", nil},
{"downstream output", "${{ needs.activation.outputs.subagent_model }}", "${{ steps.pick-experiment.outputs.subagent_model }}", nil},
{"other context", "${{ inputs.model }}", "${{ inputs.model }}", nil},
{"undeclared experiment", "${{ experiments.unknown }}", "${{ experiments.unknown }}", nil},
{"static model", "gpt-5.4", "gpt-5.4", []string{"github-copilot/gpt-5.4"}},
} {
t.Run(test.name, func(t *testing.T) {
data := &WorkflowData{
Experiments: map[string][]string{"subagent_model": {"small", "large"}},
SubAgentModels: []parser.SubAgentModel{{Name: "reader", Model: test.model}},
}
var generated strings.Builder
writeSubagentModelDeclarations(&generated, data, "copilot")
var env map[string]string
require.NoError(t, yaml.Unmarshal([]byte(generated.String()), &env))
var declarations []struct {
Name string `json:"name"`
Model string `json:"model"`
Patterns []string `json:"patterns"`
}
require.NoError(t, json.Unmarshal([]byte(env["GH_AW_INFO_SUB_AGENT_MODELS"]), &declarations))
require.Len(t, declarations, 1)
assert.Equal(t, "reader", declarations[0].Name)
assert.Equal(t, test.wantModel, declarations[0].Model)
assert.Equal(t, test.wantPatterns, declarations[0].Patterns)
assert.Equal(t, test.model, data.SubAgentModels[0].Model, "metadata generation must not mutate the runtime declaration")
})
}
}

func TestCompileCopilotWithExperimentalSubagentModels(t *testing.T) {
agent := "## agent: `reader`\n---\ndescription: Summarize a file\nmodel: ${{ experiments.subagent_model }}\n---\nRead the file.\n"
for _, imported := range []bool{false, true} {
name := "inline"
if imported {
name = "imported"
}
t.Run(name, func(t *testing.T) {
source := filepath.Join(t.TempDir(), "workflow.md")
content := "---\non: workflow_dispatch\nstrict: false\npermissions:\n contents: read\nengine: copilot\nexperiments:\n subagent_model: [small, large]\n"
if imported {
require.NoError(t, os.WriteFile(filepath.Join(filepath.Dir(source), "shared.md"), []byte("# Shared\n\n"+agent), 0600))
content += "imports:\n - shared.md\n"
}
content += "---\n# Work\nSummarize README.md using reader.\n\n"
if !imported {
content += agent
}
require.NoError(t, os.WriteFile(source, []byte(content), 0600))
compiler := NewCompiler()
data, err := compiler.ParseWorkflowFile(source)
require.NoError(t, err)
require.Equal(t, []parser.SubAgentModel{{Name: "reader", Model: "${{ experiments.subagent_model }}"}}, data.SubAgentModels)
generated, err := compiler.CompileToYAML(data, source)
require.NoError(t, err)
assert.NotContains(t, generated, "${{ experiments.subagent_model }}", "compiled YAML must not contain the raw experiment expression")
assert.Contains(t, generated, `GH_AW_INFO_SUB_AGENT_MODELS: "[{\"model\":\"${{ steps.pick-experiment.outputs.subagent_model }}\",\"name\":\"reader\",\"patterns\":null}]"`, "sub-agent metadata must use the activation-local experiment output")
pickIndex := strings.Index(generated, "id: pick-experiment")
infoIndex := strings.Index(generated, "id: generate_aw_info")
require.NotEqual(t, -1, pickIndex)
require.NotEqual(t, -1, infoIndex)
assert.Less(t, pickIndex, infoIndex, "experiment selection must precede sub-agent model metadata")
})
}
}

// TestRewriteExperimentPrefixCollision guards against one declared experiment name being a
// prefix of another (e.g. "model" and "model_variant"); the word-boundary-anchored regex
// rewrite must not partially corrupt the longer name's reference while rewriting the shorter
Expand Down
23 changes: 23 additions & 0 deletions pkg/workflow/compiler_threat_detection_formal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package workflow
import (
"testing"

"github.com/github/gh-aw/pkg/testutil"
"github.com/stretchr/testify/require"
)

Expand Down Expand Up @@ -122,6 +123,28 @@ func TestFormal_CTR001_WritePermissionsRejected(t *testing.T) {
}
}

func TestFormal_CTR001_IDTokenWriteIsInformational(t *testing.T) {
for name, strict := range map[string]bool{"strict": true, "non-strict": false} {
t.Run(name, func(t *testing.T) {
compiler := NewCompiler()
compiler.SetStrictMode(strict)
stderr := testutil.CaptureStderr(t, func() {
perms, err := compiler.validatePermissions(&WorkflowData{
Permissions: "permissions:\n contents: read\n id-token: write\n",
}, "workflow.md")
require.NoError(t, err)
level, exists := perms.Get(PermissionIdToken)
require.True(t, exists)
require.Equal(t, PermissionWrite, level)
})
require.Contains(t, stderr, "info: This workflow grants id-token: write permission")
require.Contains(t, stderr, "Ensure proper audience validation and trust policies are configured.")
require.NotContains(t, stderr, "warning: This workflow grants id-token: write permission")
require.Zero(t, compiler.GetWarningCount())
})
}
}

func TestFormal_CTR001_ReadOnlyPermissionsAllowed(t *testing.T) {
perms := NewPermissions()
for _, scope := range GetAllPermissionScopes() {
Expand Down
4 changes: 3 additions & 1 deletion pkg/workflow/compiler_yaml_step_lifecycle.go
Original file line number Diff line number Diff line change
Expand Up @@ -279,8 +279,10 @@ func writeSubagentModelDeclarations(yaml *strings.Builder, data *WorkflowData, e
}
}
for _, agent := range data.SubAgentModels {
model := RewriteExperimentsReferenceForDownstreamJobs(agent.Model, data.Experiments)
model = RewriteActivationOutputsToLocalStepOutputs(model, data.Experiments)
declarations = append(declarations, map[string]any{
"name": agent.Name, "model": agent.Model,
"name": agent.Name, "model": model,
Comment thread
pelikhan marked this conversation as resolved.
Resolved
"patterns": expandModelPatterns(agent.Model, data.ModelMappings, provider),
Comment thread
pelikhan marked this conversation as resolved.
Resolved
})
}
Expand Down
Loading
Loading