Skip to content

Commit 1cc404d

Browse files
MrFlounderclaude
andauthored
feat(crab-pf): add --reasoning flag for GPT-5 reasoning_effort (#34)
## Summary - Add `--reasoning low|medium|high` CLI flag for GPT-5/5.1/5.2 reasoning models - Maps to OpenAI Chat Completions `reasoning_effort` parameter - Stored on the provider instance so every chat call includes it automatically ## Test plan - [x] `crab pf --provider openai:gpt-5 --reasoning low` against app 07 — 7 turns, 2/2 eval pass - [ ] Full suite across all 11 apps with `openai:gpt-5 --reasoning low` 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
1 parent b5017cb commit 1cc404d

2 files changed

Lines changed: 11 additions & 5 deletions

File tree

‎plugins/promptfoo/src/agent/providers.ts‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ export interface ChatOptions {
2222
tools: unknown[];
2323
maxTokens?: number;
2424
temperature?: number;
25+
reasoningEffort?: 'low' | 'medium' | 'high';
2526
}
2627

2728
export interface ChatResponse {
@@ -41,11 +42,13 @@ export class OpenAIProvider implements LLMProvider {
4142
private apiKey: string;
4243
private model: string;
4344
private baseUrl: string;
45+
reasoningEffort?: string;
4446

45-
constructor(options: { apiKey?: string; model?: string; baseUrl?: string }) {
47+
constructor(options: { apiKey?: string; model?: string; baseUrl?: string; reasoningEffort?: string }) {
4648
this.apiKey = options.apiKey || process.env.OPENAI_API_KEY || '';
4749
this.model = options.model || 'gpt-4o';
4850
this.baseUrl = options.baseUrl || 'https://api.openai.com/v1';
51+
this.reasoningEffort = options.reasoningEffort;
4952

5053
if (!this.apiKey) {
5154
throw new Error('OpenAI API key required. Set OPENAI_API_KEY env var.');
@@ -69,6 +72,7 @@ export class OpenAIProvider implements LLMProvider {
6972
...(this.model.startsWith('gpt-5') || this.model.startsWith('o1') || this.model.startsWith('o3')
7073
? {}
7174
: { temperature: options.temperature ?? 0.7 }),
75+
...(options.reasoningEffort || this.reasoningEffort ? { reasoning_effort: options.reasoningEffort || this.reasoningEffort } : {}),
7276
}),
7377
});
7478

@@ -257,12 +261,12 @@ export class AnthropicProvider implements LLMProvider {
257261
/**
258262
* Create provider from string identifier
259263
*/
260-
export function createProvider(provider: string): LLMProvider {
264+
export function createProvider(provider: string, options?: { reasoningEffort?: string }): LLMProvider {
261265
const [type, model] = provider.split(':');
262266

263267
switch (type) {
264268
case 'openai':
265-
return new OpenAIProvider({ model: model || 'gpt-4o' });
269+
return new OpenAIProvider({ model: model || 'gpt-4o', reasoningEffort: options?.reasoningEffort });
266270
case 'anthropic':
267271
return new AnthropicProvider({ model: model || 'claude-sonnet-4-20250514' });
268272
default:

‎plugins/promptfoo/src/cli.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ async function main() {
2727
const outputDir = getArg('--output') || getArg('-o') || '.';
2828
const verbose = args.includes('--verbose') || args.includes('-v');
2929
const maxTurns = parseInt(getArg('--max-turns') || '30', 10);
30+
const reasoningEffort = getArg('--reasoning');
3031

3132
let context: string;
3233

@@ -52,7 +53,7 @@ async function main() {
5253
// Detect format
5354
const format = detectFormat(context);
5455
console.log(`Detected format: ${format === 'unknown' ? 'text/description' : format}`);
55-
console.log(`Provider: ${providerStr}`);
56+
console.log(`Provider: ${providerStr}${reasoningEffort ? ` (reasoning: ${reasoningEffort})` : ''}`);
5657
console.log(`Output: ${outputDir}`);
5758
console.log('');
5859

@@ -62,7 +63,7 @@ async function main() {
6263
}
6364

6465
// Create provider
65-
const provider = createProvider(providerStr);
66+
const provider = createProvider(providerStr, reasoningEffort ? { reasoningEffort } : undefined);
6667

6768
// Run discovery agent
6869
console.log('Starting target discovery agent...');
@@ -134,6 +135,7 @@ Options:
134135
--output, -o <dir> Output directory (default: current dir)
135136
--provider <provider> LLM provider (default: openai:gpt-4o)
136137
--max-turns <n> Max agent turns (default: 30)
138+
--reasoning <effort> Reasoning effort for GPT-5/o-series (low, medium, high)
137139
--verbose, -v Show detailed output
138140
139141
Supported input formats:

0 commit comments

Comments
 (0)