Repository navigation
Expand file tree
/
Copy pathcouncil.go
More file actions
220 lines (191 loc) · 6.61 KB
/
Copy pathcouncil.go
File metadata and controls
220 lines (191 loc) · 6.61 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
package engine
import (
"context"
"fmt"
"strings"
"sync"
routing "github.com/GrayCodeAI/reflect/internal/provider/routing"
)
// CouncilConfig controls the Karpathy LLM Council pattern.
type CouncilConfig struct {
Models []string // council member model names
Chairman string // chairman model (synthesizer)
}
// CouncilResponse holds one model's contribution.
type CouncilResponse struct {
Model string
Response string
}
// CouncilRanking holds one model's ranking of responses.
type CouncilRanking struct {
Model string
Ranking string
}
// CouncilResult holds the full council output.
type CouncilResult struct {
Responses []CouncilResponse
Rankings []CouncilRanking
Synthesis string
}
// RunCouncil implements Karpathy's 3-stage LLM Council pattern:
// 1. Send query to all models in parallel, collect responses
// 2. Anonymize responses, send ranking prompt to all models in parallel
// 3. Send all responses + rankings to chairman for synthesis
func RunCouncil(ctx context.Context, query string, cfg CouncilConfig, sess *Session) (*CouncilResult, error) {
if len(cfg.Models) == 0 {
return nil, fmt.Errorf("council: no models specified")
}
if cfg.Chairman == "" {
cfg.Chairman = sess.Model()
}
// Stage 1: parallel query to all models
responses, err := councilStage1(ctx, sess, query, cfg.Models)
if err != nil {
return nil, err
}
// Stage 2: parallel ranking by all models
rankings, err := councilStage2(ctx, sess, query, responses, cfg.Models)
if err != nil {
return nil, err
}
// Stage 3: chairman synthesis
synthesis, err := councilStage3(ctx, sess, query, responses, rankings, cfg.Chairman)
if err != nil {
return nil, err
}
return &CouncilResult{Responses: responses, Rankings: rankings, Synthesis: synthesis}, nil
}
// councilStage1 sends the query to all models in parallel.
func councilStage1(ctx context.Context, sess *Session, query string, models []string) ([]CouncilResponse, error) {
responses := make([]CouncilResponse, len(models))
var mu sync.Mutex
var wg sync.WaitGroup
var firstErr error
for i, model := range models {
wg.Add(1)
go func(idx int, m string) {
defer wg.Done()
resp, err := councilQuery(ctx, sess, m, query)
mu.Lock()
defer mu.Unlock()
if err != nil {
if firstErr == nil {
firstErr = fmt.Errorf("council: model %s: %w", m, err)
}
responses[idx] = CouncilResponse{Model: m, Response: fmt.Sprintf("(error: %v)", err)}
} else {
responses[idx] = CouncilResponse{Model: m, Response: resp}
}
}(i, model)
}
wg.Wait()
var valid int
for _, r := range responses {
if !strings.HasPrefix(r.Response, "(error:") {
valid++
}
}
if valid == 0 {
if firstErr != nil {
return nil, firstErr
}
return nil, fmt.Errorf("council: all models failed")
}
return responses, nil
}
// councilStage2 sends anonymized responses to all models for ranking.
func councilStage2(ctx context.Context, sess *Session, query string, responses []CouncilResponse, models []string) ([]CouncilRanking, error) {
rankPrompt := buildRankingPrompt(query, responses)
rankings := make([]CouncilRanking, len(models))
var wg sync.WaitGroup
var mu sync.Mutex
for i, model := range models {
wg.Add(1)
go func(idx int, m string) {
defer wg.Done()
resp, err := councilQuery(ctx, sess, m, rankPrompt)
mu.Lock()
defer mu.Unlock()
if err != nil {
rankings[idx] = CouncilRanking{Model: m, Ranking: "(error)"}
} else {
rankings[idx] = CouncilRanking{Model: m, Ranking: resp}
}
}(i, model)
}
wg.Wait()
return rankings, nil
}
// councilStage3 sends everything to the chairman for final synthesis.
func councilStage3(ctx context.Context, sess *Session, query string, responses []CouncilResponse, rankings []CouncilRanking, chairman string) (string, error) {
prompt := buildChairmanPrompt(query, responses, rankings)
return councilQuery(ctx, sess, chairman, prompt)
}
// buildRankingPrompt creates the Stage 2 ranking prompt (Karpathy's format).
func buildRankingPrompt(query string, responses []CouncilResponse) string {
var b strings.Builder
b.WriteString("You are evaluating multiple AI responses to the following question:\n\n")
b.WriteString("QUESTION: " + query + "\n\n")
b.WriteString("Here are the responses:\n\n")
for i, r := range responses {
label := string(rune('A' + i))
b.WriteString(fmt.Sprintf("=== Response %s ===\n%s\n\n", label, r.Response))
}
b.WriteString("Please rank these responses from best to worst. Consider accuracy, completeness, clarity, and helpfulness.\n\n")
b.WriteString("Provide a brief justification for your ranking, then end with your final ranking in this exact format:\n\n")
b.WriteString("FINAL RANKING: [best to worst, e.g. B, A, C]\n")
return b.String()
}
// buildChairmanPrompt creates the Stage 3 chairman synthesis prompt (Karpathy's format).
func buildChairmanPrompt(query string, responses []CouncilResponse, rankings []CouncilRanking) string {
var b strings.Builder
b.WriteString("You are the chairman of an LLM council. Your job is to synthesize the best possible answer.\n\n")
b.WriteString("ORIGINAL QUESTION: " + query + "\n\n")
b.WriteString("=== COUNCIL RESPONSES ===\n\n")
for i, r := range responses {
label := string(rune('A' + i))
b.WriteString(fmt.Sprintf("--- Response %s (from %s) ---\n%s\n\n", label, r.Model, r.Response))
}
b.WriteString("=== COUNCIL RANKINGS ===\n\n")
for _, r := range rankings {
b.WriteString(fmt.Sprintf("--- Ranking by %s ---\n%s\n\n", r.Model, r.Ranking))
}
b.WriteString("Based on the responses and rankings above, synthesize the best possible answer to the original question. ")
b.WriteString("Take the strongest elements from the highest-ranked responses. Be thorough and accurate.\n")
return b.String()
}
// councilQuery queries a specific model using the session's client infrastructure.
func councilQuery(ctx context.Context, sess *Session, modelName, prompt string) (string, error) {
sub := sess.SubSession(modelName, sess.Persistence().System(), sess.Tools().Registry())
if sess.LifecycleSvc() != nil {
sess.LifecycleSvc().Limits().SetMaxTurns(1)
}
sub.AddUser(prompt)
ch, err := sub.Stream(ctx)
if err != nil {
return "", err
}
var b strings.Builder
for ev := range ch {
switch ev.Type {
case "content":
b.WriteString(ev.Content)
case "error":
return b.String(), fmt.Errorf("%s", ev.Content)
}
}
return b.String(), nil
}
// DefaultCouncilModels returns one catalog-backed default model per provider (dynamic).
func DefaultCouncilModels() []string {
var out []string
for _, p := range routing.AllProviders() {
if m := routing.DefaultModel(p); m != "" {
out = append(out, m)
}
if len(out) >= 3 {
break
}
}
return out
}