|
3 | 3 | package workflow |
4 | 4 |
|
5 | 5 | import ( |
| 6 | + "bytes" |
6 | 7 | "encoding/json" |
7 | 8 | "os" |
8 | 9 | "path/filepath" |
@@ -171,12 +172,75 @@ func TestGenerateExperimentSteps_SpecJSON(t *testing.T) { |
171 | 172 |
|
172 | 173 | func TestExperimentHarnessVersionUsesOnlyFrontmatterHash(t *testing.T) { |
173 | 174 | assert.Equal(t, "frontmatter-hash", experimentHarnessVersion(&WorkflowData{FrontmatterHash: "frontmatter-hash", BodyHash: "body-hash"})) |
174 | | - assert.Equal(t, "frontmatter-hash", experimentHarnessVersion(&WorkflowData{FrontmatterHash: "frontmatter-hash", BodyHash: "different-body"})) |
| 175 | + frontmatter := "experiments:\n style: [short, long]" |
| 176 | + assert.Equal(t, |
| 177 | + experimentHarnessVersion(&WorkflowData{FrontmatterYAML: frontmatter, FrontmatterHash: "frontmatter-hash", BodyHash: "body-hash"}), |
| 178 | + experimentHarnessVersion(&WorkflowData{FrontmatterYAML: frontmatter, FrontmatterHash: "changed-by-inline-body", BodyHash: "different-body"}), |
| 179 | + "pure frontmatter identity must ignore compiler freshness and body hashes", |
| 180 | + ) |
175 | 181 | assert.Equal(t, "frontmatter-hash", experimentHarnessVersion(&WorkflowData{FrontmatterHash: "frontmatter-hash"})) |
176 | 182 | assert.Equal(t, "unknown", experimentHarnessVersion(&WorkflowData{BodyHash: "body-hash"})) |
177 | 183 | assert.Equal(t, "unknown", experimentHarnessVersion(&WorkflowData{})) |
178 | 184 | } |
179 | 185 |
|
| 186 | +func TestExperimentHarnessVersionIdentityModes(t *testing.T) { |
| 187 | + base := &WorkflowData{ |
| 188 | + WorkflowID: "daily-fact", |
| 189 | + FrontmatterYAML: "experiments:\n style: [short, long]", |
| 190 | + FrontmatterHash: "compiler-freshness-hash", |
| 191 | + BodyHash: "body-hash", |
| 192 | + ExperimentsIdentity: ExperimentsIdentityFrontmatter, |
| 193 | + } |
| 194 | + frontmatterVersion := experimentHarnessVersion(base) |
| 195 | + bodyEdit := *base |
| 196 | + bodyEdit.FrontmatterHash = "freshness-hash-changed-by-body" |
| 197 | + bodyEdit.BodyHash = "different-body-hash" |
| 198 | + assert.Equal(t, frontmatterVersion, experimentHarnessVersion(&bodyEdit)) |
| 199 | + |
| 200 | + full := *base |
| 201 | + full.ExperimentsIdentity = ExperimentsIdentityFull |
| 202 | + fullVersion := experimentHarnessVersion(&full) |
| 203 | + full.BodyHash = "different-body-hash" |
| 204 | + assert.NotEqual(t, fullVersion, experimentHarnessVersion(&full)) |
| 205 | + |
| 206 | + workflow := *base |
| 207 | + workflow.ExperimentsIdentity = ExperimentsIdentityWorkflow |
| 208 | + assert.Equal(t, "daily-fact", experimentHarnessVersion(&workflow)) |
| 209 | + workflow.BodyHash = "different-body-hash" |
| 210 | + workflow.FrontmatterHash = "different-frontmatter-hash" |
| 211 | + assert.Equal(t, "daily-fact", experimentHarnessVersion(&workflow)) |
| 212 | +} |
| 213 | + |
| 214 | +func TestCompileWorkflowFrontmatterIdentityIgnoresInlinedBodyEdits(t *testing.T) { |
| 215 | + compile := func(body string) string { |
| 216 | + t.Helper() |
| 217 | + dir := t.TempDir() |
| 218 | + workflowPath := filepath.Join(dir, "harness.md") |
| 219 | + content := `--- |
| 220 | +on: workflow_dispatch |
| 221 | +strict: false |
| 222 | +engine: copilot |
| 223 | +inlined-imports: true |
| 224 | +experiments: |
| 225 | + style: [short, long] |
| 226 | +--- |
| 227 | +
|
| 228 | +` + body + "\n" |
| 229 | + require.NoError(t, os.WriteFile(workflowPath, []byte(content), 0600)) |
| 230 | + require.NoError(t, NewCompiler().CompileWorkflow(workflowPath)) |
| 231 | + lock, err := os.ReadFile(filepath.Join(dir, "harness.lock.yml")) |
| 232 | + require.NoError(t, err) |
| 233 | + const marker = "GH_AW_HARNESS_VERSION: " |
| 234 | + lineStart := bytes.Index(lock, []byte(marker)) |
| 235 | + require.NotEqual(t, -1, lineStart, "compiled workflow should expose the experiment identity") |
| 236 | + lineEnd := bytes.IndexByte(lock[lineStart:], '\n') |
| 237 | + require.NotEqual(t, -1, lineEnd, "compiled identity should end at a newline") |
| 238 | + return strings.TrimSpace(string(lock[lineStart+len(marker) : lineStart+lineEnd])) |
| 239 | + } |
| 240 | + |
| 241 | + assert.Equal(t, compile("First prompt body."), compile("Revised prompt body.")) |
| 242 | +} |
| 243 | + |
180 | 244 | func TestGenerateExperimentSteps_SingleQuoteEscaping(t *testing.T) { |
181 | 245 | c := &Compiler{} |
182 | 246 | data := &WorkflowData{ |
@@ -691,17 +755,38 @@ func TestExperimentsBranchName(t *testing.T) { |
691 | 755 |
|
692 | 756 | // ── storage key is excluded from experiment configs ─────────────────────── |
693 | 757 |
|
694 | | -func TestExtractExperimentConfigsFromFrontmatter_StorageKeyIsSkipped(t *testing.T) { |
| 758 | +func TestExtractExperimentConfigsFromFrontmatter_ReservedKeysAreSkipped(t *testing.T) { |
695 | 759 | frontmatter := map[string]any{ |
696 | 760 | "experiments": map[string]any{ |
697 | | - "storage": "repo", |
698 | | - "my_exp": []any{"A", "B"}, |
| 761 | + "storage": "repo", |
| 762 | + "identity": "frontmatter", |
| 763 | + "my_exp": []any{"A", "B"}, |
699 | 764 | }, |
700 | 765 | } |
701 | 766 | got := extractExperimentConfigsFromFrontmatter(frontmatter) |
702 | 767 | require.NotNil(t, got, "experiment configs should not be nil") |
703 | 768 | assert.Contains(t, got, "my_exp", "my_exp should be present") |
704 | 769 | assert.NotContains(t, got, "storage", "storage key should be excluded from experiment configs") |
| 770 | + assert.NotContains(t, got, "identity", "identity key should be excluded from experiment configs") |
| 771 | +} |
| 772 | + |
| 773 | +func TestExtractExperimentsIdentityFromFrontmatter(t *testing.T) { |
| 774 | + tests := []struct { |
| 775 | + name string |
| 776 | + frontmatter map[string]any |
| 777 | + want ExperimentIdentityMode |
| 778 | + }{ |
| 779 | + {name: "defaults to frontmatter", frontmatter: map[string]any{}, want: ExperimentsIdentityFrontmatter}, |
| 780 | + {name: "full", frontmatter: map[string]any{"experiments": map[string]any{"identity": "full"}}, want: ExperimentsIdentityFull}, |
| 781 | + {name: "frontmatter", frontmatter: map[string]any{"experiments": map[string]any{"identity": "frontmatter"}}, want: ExperimentsIdentityFrontmatter}, |
| 782 | + {name: "workflow", frontmatter: map[string]any{"experiments": map[string]any{"identity": "workflow"}}, want: ExperimentsIdentityWorkflow}, |
| 783 | + {name: "unknown defaults to frontmatter", frontmatter: map[string]any{"experiments": map[string]any{"identity": "other"}}, want: ExperimentsIdentityFrontmatter}, |
| 784 | + } |
| 785 | + for _, tt := range tests { |
| 786 | + t.Run(tt.name, func(t *testing.T) { |
| 787 | + assert.Equal(t, tt.want, extractExperimentsIdentityFromFrontmatter(tt.frontmatter)) |
| 788 | + }) |
| 789 | + } |
705 | 790 | } |
706 | 791 |
|
707 | 792 | func TestParseExperimentMetricEvalReference(t *testing.T) { |
|
0 commit comments