Skip to content

Commit daab8c9

Browse files
Gerome-ElassaadGerome-Elassaad
authored
Merge pull request #71 from codinit-dev/69-local-model-server-failure
Fix: #69 local model server failure
2 parents 8d19153 + be51577 commit daab8c9

8 files changed

Lines changed: 19898 additions & 63 deletions

File tree

‎app/components/ui/Button.tsx‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,7 @@ const buttonVariants = cva(
3232
);
3333

3434
export interface ButtonProps
35-
extends React.ButtonHTMLAttributes<HTMLButtonElement>,
36-
VariantProps<typeof buttonVariants> {
35+
extends React.ButtonHTMLAttributes<HTMLButtonElement>, VariantProps<typeof buttonVariants> {
3736
_asChild?: boolean;
3837
}
3938

‎app/lib/.server/llm/stream-text.ts‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ import { allowedHTMLElements } from '~/utils/markdown';
88
import { LLMManager } from '~/lib/modules/llm/manager';
99
import { createScopedLogger } from '~/utils/logger';
1010
import { createFilesContext, extractPropertiesFromMessage } from './utils';
11-
import { BuiltInToolService } from '~/lib/services/builtInToolService';
1211

1312
export type Messages = Message[];
1413

@@ -210,16 +209,16 @@ Use these preferences when creating UI components, styling code, or suggesting d
210209

211210
logger.info(`Sending llm call to ${provider.name} with model ${modelDetails.name}`);
212211

213-
let allTools = { ...options?.tools };
214-
const builtInToolService = BuiltInToolService.getInstance();
215-
const builtInTools = builtInToolService.toolsWithoutExecute;
212+
/*
213+
* Only pass tools that are properly implemented with valid Zod schemas.
214+
* Built-in tools from JSON don't have Zod validation, so we don't pass any tools
215+
* to avoid zod-to-json-schema conversion errors.
216+
*/
217+
const allTools = {};
216218

217-
if (Object.keys(builtInTools).length > 0) {
218-
allTools = { ...allTools, ...builtInTools };
219-
logger.info(`Added ${Object.keys(builtInTools).length} built-in tools:`, Object.keys(builtInTools));
220-
}
219+
logger.info(`Skipping all tool passing to AI SDK - tools are processed server-side only`);
221220

222-
const hasTools = Object.keys(allTools).length > 0;
221+
const hasTools = false;
223222

224223
return await _streamText({
225224
model: provider.getModelInstance({
@@ -232,6 +231,5 @@ Use these preferences when creating UI components, styling code, or suggesting d
232231
maxTokens: dynamicMaxTokens,
233232
messages: convertToCoreMessages(processedMessages as any),
234233
...(hasTools ? { tools: allTools, toolChoice: 'auto' } : {}),
235-
...options,
236234
});
237235
}

‎app/lib/hooks/StickToBottom.tsx‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,7 @@ export interface StickToBottomContext {
3333
const StickToBottomContext = createContext<StickToBottomContext | null>(null);
3434

3535
export interface StickToBottomProps
36-
extends Omit<React.HTMLAttributes<HTMLDivElement>, 'children'>,
37-
StickToBottomOptions {
36+
extends Omit<React.HTMLAttributes<HTMLDivElement>, 'children'>, StickToBottomOptions {
3837
contextRef?: React.Ref<StickToBottomContext>;
3938
instance?: ReturnType<typeof useStickToBottom>;
4039
children: ((context: StickToBottomContext) => ReactNode) | ReactNode;

‎app/lib/hooks/useLocalProviders.ts‎

Lines changed: 98 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,114 @@
1-
import { useCallback, useState } from 'react';
1+
import { useCallback, useState, useEffect } from 'react';
22
import type { IProviderConfig } from '~/types/model';
3+
import { updateProviderSettings } from '~/lib/stores/settings';
34

45
export interface UseLocalProvidersReturn {
56
localProviders: IProviderConfig[];
67
refreshLocalProviders: () => void;
8+
isChecking: boolean;
79
}
810

911
export function useLocalProviders(): UseLocalProvidersReturn {
1012
const [localProviders, setLocalProviders] = useState<IProviderConfig[]>([]);
13+
const [isChecking, setIsChecking] = useState(false);
1114

12-
const refreshLocalProviders = useCallback(() => {
13-
/*
14-
* Refresh logic for local providers
15-
* This would typically involve checking the status of Ollama and LMStudio
16-
* For now, we'll just return an empty array
17-
*/
18-
setLocalProviders([]);
19-
}, []);
15+
const checkLocalProvider = useCallback(
16+
async (name: string, baseUrl: string, icon: string): Promise<IProviderConfig | null> => {
17+
try {
18+
// Try to connect to the local provider
19+
const response = await fetch(`${baseUrl}/api/tags`, {
20+
method: 'GET',
21+
signal: AbortSignal.timeout(5000), // 5 second timeout
22+
});
23+
24+
if (response.ok) {
25+
return {
26+
name,
27+
icon,
28+
getApiKeyLink: `https://${name.toLowerCase()}.com/download`,
29+
labelForGetApiKey: `Download ${name}`,
30+
staticModels: [],
31+
settings: {
32+
enabled: true,
33+
baseUrl,
34+
},
35+
};
36+
}
37+
} catch {
38+
// Provider not available, return null
39+
}
40+
return null;
41+
},
42+
[],
43+
);
44+
45+
const refreshLocalProviders = useCallback(async () => {
46+
setIsChecking(true);
47+
48+
try {
49+
const providers: IProviderConfig[] = [];
50+
51+
// Check Ollama
52+
53+
const ollamaProvider = await checkLocalProvider(
54+
'Ollama',
55+
'http://localhost:11434',
56+
'/thirdparty/logos/ollama.svg',
57+
);
58+
59+
if (ollamaProvider) {
60+
providers.push(ollamaProvider);
61+
}
62+
63+
// Check LMStudio
64+
65+
const lmStudioProvider = await checkLocalProvider(
66+
'LMStudio',
67+
'http://localhost:1234',
68+
'/thirdparty/logos/lmstudio.svg',
69+
);
70+
71+
if (lmStudioProvider) {
72+
providers.push(lmStudioProvider);
73+
}
74+
75+
setLocalProviders(providers);
76+
77+
// Enable detected providers in settings
78+
providers.forEach((provider) => {
79+
updateProviderSettings(provider.name, { enabled: true, baseUrl: provider.settings.baseUrl });
80+
});
81+
82+
// Disable providers that are no longer available
83+
const availableProviderNames = providers.map((p) => p.name);
84+
const allLocalProviderNames = ['Ollama', 'LMStudio'];
85+
86+
allLocalProviderNames.forEach((providerName) => {
87+
if (!availableProviderNames.includes(providerName)) {
88+
updateProviderSettings(providerName, { enabled: false });
89+
}
90+
});
91+
} catch (error) {
92+
console.error('Error checking local providers:', error);
93+
setLocalProviders([]);
94+
95+
// Disable all local providers if there's an error
96+
['Ollama', 'LMStudio'].forEach((providerName) => {
97+
updateProviderSettings(providerName, { enabled: false });
98+
});
99+
} finally {
100+
setIsChecking(false);
101+
}
102+
}, [checkLocalProvider]);
103+
104+
// Auto-refresh on mount
105+
useEffect(() => {
106+
refreshLocalProviders();
107+
}, [refreshLocalProviders]);
20108

21109
return {
22110
localProviders,
23111
refreshLocalProviders,
112+
isChecking,
24113
};
25114
}

‎app/lib/modules/llm/providers/ollama.ts‎

Lines changed: 34 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -109,14 +109,40 @@ export default class OllamaProvider extends BaseProvider {
109109
throw new Error('Invalid response from Ollama API: missing models array');
110110
}
111111

112-
return data.models.map((model: OllamaModel) => ({
113-
name: model.name,
114-
label: `${model.name} (${model.details?.parameter_size || 'unknown'})`,
115-
provider: this.name,
116-
maxTokenAllowed: 8000,
117-
maxCompletionTokens: 8000,
118-
icon: '/thirdparty/logos/ollama.svg',
119-
}));
112+
return data.models.map((model: OllamaModel) => {
113+
// Use proper context window based on model family and parameter size
114+
let contextWindow = 4096; // default
115+
116+
// Larger context windows for modern models
117+
if (model.details?.parameter_size) {
118+
const paramSize = parseInt(model.details.parameter_size.replace(/[^0-9]/g, ''));
119+
120+
// Models with larger parameter sizes generally support larger contexts
121+
if (paramSize >= 70) {
122+
contextWindow = 32768; // 32k for 70B+ models
123+
} else if (paramSize >= 30) {
124+
contextWindow = 16384; // 16k for 30B+ models
125+
} else if (paramSize >= 7) {
126+
contextWindow = 8192; // 8k for 7B+ models
127+
}
128+
}
129+
130+
// Special handling for specific model families
131+
if (model.details?.family?.includes('llama') && model.details?.parameter_size?.includes('70')) {
132+
contextWindow = 32768; // Llama 70B models
133+
} else if (model.details?.family?.includes('llama') && model.details?.parameter_size?.includes('405')) {
134+
contextWindow = 128000; // Llama 405B models
135+
}
136+
137+
return {
138+
name: model.name,
139+
label: `${model.name} (${model.details?.parameter_size || 'unknown'}, ${contextWindow >= 1000 ? Math.floor(contextWindow / 1000) + 'k' : contextWindow} ctx)`,
140+
provider: this.name,
141+
maxTokenAllowed: contextWindow,
142+
maxCompletionTokens: Math.min(contextWindow, 4096), // Cap completion tokens
143+
icon: '/thirdparty/logos/ollama.svg',
144+
};
145+
});
120146
}
121147

122148
getModelInstance: (options: {

‎app/lib/modules/llm/providers/open-router.ts‎

Lines changed: 12 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -33,23 +33,23 @@ export default class OpenRouterProvider extends BaseProvider {
3333
* Claude Opus 4.5 via OpenRouter: 200k context, maximum intelligence
3434
*/
3535
{
36-
name: 'anthropic/claude-opus-4-5',
36+
name: 'anthropic/claude-opus-4.5',
3737
label: 'Claude Opus 4.5',
3838
provider: 'OpenRouter',
3939
maxTokenAllowed: 200000,
4040
maxCompletionTokens: 64000,
4141
},
4242

43-
// Claude Sonnet 4.5 via OpenRouter: 200k context
43+
// Claude Sonnet 4.5 via OpenRouter: 1M context, highest intelligence
4444
{
45-
name: 'anthropic/claude-sonnet-4-5',
45+
name: 'anthropic/claude-sonnet-4.5',
4646
label: 'Claude Sonnet 4.5',
4747
provider: 'OpenRouter',
48-
maxTokenAllowed: 200000,
48+
maxTokenAllowed: 1000000,
4949
maxCompletionTokens: 64000,
5050
},
5151

52-
// GPT-5.2 Pro via OpenRouter: 400k context, highest accuracy
52+
// GPT-5.2 Pro via OpenRouter: 400k context, latest GPT
5353
{
5454
name: 'openai/gpt-5.2-pro',
5555
label: 'GPT-5.2 Pro',
@@ -58,39 +58,21 @@ export default class OpenRouterProvider extends BaseProvider {
5858
maxCompletionTokens: 128000,
5959
},
6060

61-
// GPT-5.2 Thinking via OpenRouter: 400k context, complex reasoning
62-
{
63-
name: 'openai/gpt-5.2-thinking',
64-
label: 'GPT-5.2 Thinking',
65-
provider: 'OpenRouter',
66-
maxTokenAllowed: 400000,
67-
maxCompletionTokens: 128000,
68-
},
69-
70-
// GPT-5.2 Instant via OpenRouter: 400k context, optimized for speed
71-
{
72-
name: 'openai/gpt-5.2-instant',
73-
label: 'GPT-5.2 Instant',
74-
provider: 'OpenRouter',
75-
maxTokenAllowed: 400000,
76-
maxCompletionTokens: 128000,
77-
},
78-
79-
// GPT-5.1 via OpenRouter: 128k context
61+
// GPT-4o via OpenRouter: 128k context, reliable fallback
8062
{
81-
name: 'openai/gpt-5.1',
82-
label: 'GPT-5.1',
63+
name: 'openai/gpt-4o',
64+
label: 'GPT-4o',
8365
provider: 'OpenRouter',
8466
maxTokenAllowed: 128000,
8567
maxCompletionTokens: 16384,
8668
},
8769

88-
// DeepSeek-R1 via OpenRouter: 128k context
70+
// DeepSeek R1 via OpenRouter: 163k context, free tier available
8971
{
90-
name: 'deepseek/deepseek-r1',
91-
label: 'DeepSeek-R1',
72+
name: 'deepseek/deepseek-r1-0528:free',
73+
label: 'DeepSeek R1 (Free)',
9274
provider: 'OpenRouter',
93-
maxTokenAllowed: 131072,
75+
maxTokenAllowed: 163840,
9476
maxCompletionTokens: 32768,
9577
},
9678
];

‎app/lib/stores/settings.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { atom, map } from 'nanostores';
22
import { PROVIDER_LIST } from '~/utils/constants';
3-
import type { IProviderConfig } from '~/types/model';
3+
import type { IProviderConfig, IProviderSetting } from '~/types/model';
44
import type {
55
TabVisibilityConfig,
66
TabWindowConfig,
@@ -101,7 +101,7 @@ const getInitialProviderSettings = (): ProviderSetting => {
101101
export const providersStore = map<ProviderSetting>(getInitialProviderSettings());
102102

103103
// Create a function to update provider settings that handles both store and persistence
104-
export const updateProviderSettings = (provider: string, settings: ProviderSetting) => {
104+
export const updateProviderSettings = (provider: string, settings: Partial<IProviderSetting>) => {
105105
const currentSettings = providersStore.get();
106106

107107
// Create new provider config with updated settings

0 commit comments

Comments
 (0)