UniqAI/bolt.new
1
0
Fork 0
mirror of https://github.com/stackblitz/bolt.new synced 2025-02-06 04:48:04 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

max token is now dynamically handle for each model

Browse Source
This commit is contained in:
Raiyan Hasan 2024-11-20 21:41:51 +05:30
parent d41a0ac2c2
commit eabfbb2220
4 changed files with 77 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/lib/.server/llm/constants.ts
View File

@ -1,5 +1,5 @@
// see https://docs.anthropic.com/en/docs/about-claude/models // see https://docs.anthropic.com/en/docs/about-claude/models
export const MAX_TOKENS = 4096; export const MAX_TOKENS = 8000;
// limits the number of model responses that can be returned in a single request // limits the number of model responses that can be returned in a single request
export const MAX_RESPONSE_SEGMENTS = 2; export const MAX_RESPONSE_SEGMENTS = 2;

13
app/lib/.server/llm/stream-text.ts
View File

@ -41,7 +41,6 @@ function extractPropertiesFromMessage(message: Message): { model: string; provid
return { model, provider, content: cleanedContent }; return { model, provider, content: cleanedContent };
} }
export function streamText( export function streamText(
messages: Messages, messages: Messages,
env: Env, env: Env,
@ -64,13 +63,21 @@ export function streamText(
return { ...message, content }; return { ...message, content };
} }
return message; // No changes for non-user messages return message;
}); });
const modelDetails = MODEL_LIST.find((m) => m.name === currentModel);
const dynamicMaxTokens =
modelDetails && modelDetails.maxTokenAllowed
? Math.min(MAX_TOKENS, modelDetails.maxTokenAllowed)
: MAX_TOKENS;
console.log(dynamicMaxTokens)
return _streamText({ return _streamText({
model: getModel(currentProvider, currentModel, env, apiKeys), model: getModel(currentProvider, currentModel, env, apiKeys),
system: getSystemPrompt(), system: getSystemPrompt(),
maxTokens: MAX_TOKENS, maxTokens: dynamicMaxTokens,
messages: convertToCoreMessages(processedMessages), messages: convertToCoreMessages(processedMessages),
...options, ...options,
}); });

119
app/utils/constants.ts
View File

@ -12,12 +12,12 @@ const PROVIDER_LIST: ProviderInfo[] = [
{ {
name: 'Anthropic', name: 'Anthropic',
staticModels: [ staticModels: [
{ name: 'claude-3-5-sonnet-latest', label: 'Claude 3.5 Sonnet (new)', provider: 'Anthropic' }, { name: 'claude-3-5-sonnet-latest', label: 'Claude 3.5 Sonnet (new)', provider: 'Anthropic', maxTokenAllowed: 8000 },
{ name: 'claude-3-5-sonnet-20240620', label: 'Claude 3.5 Sonnet (old)', provider: 'Anthropic' }, { name: 'claude-3-5-sonnet-20240620', label: 'Claude 3.5 Sonnet (old)', provider: 'Anthropic', maxTokenAllowed: 8000 },
{ name: 'claude-3-5-haiku-latest', label: 'Claude 3.5 Haiku (new)', provider: 'Anthropic' }, { name: 'claude-3-5-haiku-latest', label: 'Claude 3.5 Haiku (new)', provider: 'Anthropic', maxTokenAllowed: 8000 },
{ name: 'claude-3-opus-latest', label: 'Claude 3 Opus', provider: 'Anthropic' }, { name: 'claude-3-opus-latest', label: 'Claude 3 Opus', provider: 'Anthropic', maxTokenAllowed: 8000 },
{ name: 'claude-3-sonnet-20240229', label: 'Claude 3 Sonnet', provider: 'Anthropic' }, { name: 'claude-3-sonnet-20240229', label: 'Claude 3 Sonnet', provider: 'Anthropic', maxTokenAllowed: 8000 },
{ name: 'claude-3-haiku-20240307', label: 'Claude 3 Haiku', provider: 'Anthropic' } { name: 'claude-3-haiku-20240307', label: 'Claude 3 Haiku', provider: 'Anthropic', maxTokenAllowed: 8000 }
], ],
getApiKeyLink: "https://console.anthropic.com/settings/keys", getApiKeyLink: "https://console.anthropic.com/settings/keys",
}, },
@ -36,36 +36,37 @@ const PROVIDER_LIST: ProviderInfo[] = [
{ {
name: 'Cohere', name: 'Cohere',
staticModels: [ staticModels: [
{ name: 'command-r-plus-08-2024', label: 'Command R plus Latest', provider: 'Cohere' }, { name: 'command-r-plus-08-2024', label: 'Command R plus Latest', provider: 'Cohere', maxTokenAllowed: 4096 },
{ name: 'command-r-08-2024', label: 'Command R Latest', provider: 'Cohere' }, { name: 'command-r-08-2024', label: 'Command R Latest', provider: 'Cohere', maxTokenAllowed: 4096 },
{ name: 'command-r-plus', label: 'Command R plus', provider: 'Cohere' }, { name: 'command-r-plus', label: 'Command R plus', provider: 'Cohere', maxTokenAllowed: 4096 },
{ name: 'command-r', label: 'Command R', provider: 'Cohere' }, { name: 'command-r', label: 'Command R', provider: 'Cohere', maxTokenAllowed: 4096 },
{ name: 'command', label: 'Command', provider: 'Cohere' }, { name: 'command', label: 'Command', provider: 'Cohere', maxTokenAllowed: 4096 },
{ name: 'command-nightly', label: 'Command Nightly', provider: 'Cohere' }, { name: 'command-nightly', label: 'Command Nightly', provider: 'Cohere', maxTokenAllowed: 4096 },
{ name: 'command-light', label: 'Command Light', provider: 'Cohere' }, { name: 'command-light', label: 'Command Light', provider: 'Cohere', maxTokenAllowed: 4096 },
{ name: 'command-light-nightly', label: 'Command Light Nightly', provider: 'Cohere' }, { name: 'command-light-nightly', label: 'Command Light Nightly', provider: 'Cohere', maxTokenAllowed: 4096 },
{ name: 'c4ai-aya-expanse-8b', label: 'c4AI Aya Expanse 8b', provider: 'Cohere' }, { name: 'c4ai-aya-expanse-8b', label: 'c4AI Aya Expanse 8b', provider: 'Cohere', maxTokenAllowed: 4096 },
{ name: 'c4ai-aya-expanse-32b', label: 'c4AI Aya Expanse 32b', provider: 'Cohere' }, { name: 'c4ai-aya-expanse-32b', label: 'c4AI Aya Expanse 32b', provider: 'Cohere', maxTokenAllowed: 4096 },
], ],
getApiKeyLink: 'https://dashboard.cohere.com/api-keys' getApiKeyLink: 'https://dashboard.cohere.com/api-keys'
}, },
{ {
name: 'OpenRouter', name: 'OpenRouter',
staticModels: [ staticModels: [
{ name: 'gpt-4o', label: 'GPT-4o', provider: 'OpenAI' }, { name: 'gpt-4o', label: 'GPT-4o', provider: 'OpenAI', maxTokenAllowed: 8000 },
{ {
name: 'anthropic/claude-3.5-sonnet', name: 'anthropic/claude-3.5-sonnet',
label: 'Anthropic: Claude 3.5 Sonnet (OpenRouter)', label: 'Anthropic: Claude 3.5 Sonnet (OpenRouter)',
provider: 'OpenRouter' provider: 'OpenRouter'
, maxTokenAllowed: 8000
}, },
{ name: 'anthropic/claude-3-haiku', label: 'Anthropic: Claude 3 Haiku (OpenRouter)', provider: 'OpenRouter' }, { name: 'anthropic/claude-3-haiku', label: 'Anthropic: Claude 3 Haiku (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 },
{ name: 'deepseek/deepseek-coder', label: 'Deepseek-Coder V2 236B (OpenRouter)', provider: 'OpenRouter' }, { name: 'deepseek/deepseek-coder', label: 'Deepseek-Coder V2 236B (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 },
{ name: 'google/gemini-flash-1.5', label: 'Google Gemini Flash 1.5 (OpenRouter)', provider: 'OpenRouter' }, { name: 'google/gemini-flash-1.5', label: 'Google Gemini Flash 1.5 (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 },
{ name: 'google/gemini-pro-1.5', label: 'Google Gemini Pro 1.5 (OpenRouter)', provider: 'OpenRouter' }, { name: 'google/gemini-pro-1.5', label: 'Google Gemini Pro 1.5 (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 },
{ name: 'x-ai/grok-beta', label: 'xAI Grok Beta (OpenRouter)', provider: 'OpenRouter' }, { name: 'x-ai/grok-beta', label: 'xAI Grok Beta (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 },
{ name: 'mistralai/mistral-nemo', label: 'OpenRouter Mistral Nemo (OpenRouter)', provider: 'OpenRouter' }, { name: 'mistralai/mistral-nemo', label: 'OpenRouter Mistral Nemo (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 },
{ name: 'qwen/qwen-110b-chat', label: 'OpenRouter Qwen 110b Chat (OpenRouter)', provider: 'OpenRouter' }, { name: 'qwen/qwen-110b-chat', label: 'OpenRouter Qwen 110b Chat (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 },
{ name: 'cohere/command', label: 'Cohere Command (OpenRouter)', provider: 'OpenRouter' } { name: 'cohere/command', label: 'Cohere Command (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 4096 }
], ],
getDynamicModels: getOpenRouterModels, getDynamicModels: getOpenRouterModels,
getApiKeyLink: 'https://openrouter.ai/settings/keys', getApiKeyLink: 'https://openrouter.ai/settings/keys',
@ -73,32 +74,32 @@ const PROVIDER_LIST: ProviderInfo[] = [
}, { }, {
name: 'Google', name: 'Google',
staticModels: [ staticModels: [
{ name: 'gemini-1.5-flash-latest', label: 'Gemini 1.5 Flash', provider: 'Google' }, { name: 'gemini-1.5-flash-latest', label: 'Gemini 1.5 Flash', provider: 'Google', maxTokenAllowed: 8192 },
{ name: 'gemini-1.5-flash-002', label: 'Gemini 1.5 Flash-002', provider: 'Google' }, { name: 'gemini-1.5-flash-002', label: 'Gemini 1.5 Flash-002', provider: 'Google', maxTokenAllowed: 8192 },
{ name: 'gemini-1.5-flash-8b', label: 'Gemini 1.5 Flash-8b', provider: 'Google' }, { name: 'gemini-1.5-flash-8b', label: 'Gemini 1.5 Flash-8b', provider: 'Google', maxTokenAllowed: 8192 },
{ name: 'gemini-1.5-pro-latest', label: 'Gemini 1.5 Pro', provider: 'Google' }, { name: 'gemini-1.5-pro-latest', label: 'Gemini 1.5 Pro', provider: 'Google', maxTokenAllowed: 8192 },
{ name: 'gemini-1.5-pro-002', label: 'Gemini 1.5 Pro-002', provider: 'Google' }, { name: 'gemini-1.5-pro-002', label: 'Gemini 1.5 Pro-002', provider: 'Google', maxTokenAllowed: 8192 },
{ name: 'gemini-exp-1114', label: 'Gemini exp-1114', provider: 'Google' } { name: 'gemini-exp-1114', label: 'Gemini exp-1114', provider: 'Google', maxTokenAllowed: 8192 }
], ],
getApiKeyLink: 'https://aistudio.google.com/app/apikey' getApiKeyLink: 'https://aistudio.google.com/app/apikey'
}, { }, {
name: 'Groq', name: 'Groq',
staticModels: [ staticModels: [
{ name: 'llama-3.1-70b-versatile', label: 'Llama 3.1 70b (Groq)', provider: 'Groq' }, { name: 'llama-3.1-70b-versatile', label: 'Llama 3.1 70b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
{ name: 'llama-3.1-8b-instant', label: 'Llama 3.1 8b (Groq)', provider: 'Groq' }, { name: 'llama-3.1-8b-instant', label: 'Llama 3.1 8b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
{ name: 'llama-3.2-11b-vision-preview', label: 'Llama 3.2 11b (Groq)', provider: 'Groq' }, { name: 'llama-3.2-11b-vision-preview', label: 'Llama 3.2 11b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
{ name: 'llama-3.2-3b-preview', label: 'Llama 3.2 3b (Groq)', provider: 'Groq' }, { name: 'llama-3.2-3b-preview', label: 'Llama 3.2 3b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
{ name: 'llama-3.2-1b-preview', label: 'Llama 3.2 1b (Groq)', provider: 'Groq' } { name: 'llama-3.2-1b-preview', label: 'Llama 3.2 1b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }
], ],
getApiKeyLink: 'https://console.groq.com/keys' getApiKeyLink: 'https://console.groq.com/keys'
}, },
{ {
name: 'HuggingFace', name: 'HuggingFace',
staticModels: [ staticModels: [
{ name: 'Qwen/Qwen2.5-Coder-32B-Instruct', label: 'Qwen2.5-Coder-32B-Instruct (HuggingFace)', provider: 'HuggingFace' }, { name: 'Qwen/Qwen2.5-Coder-32B-Instruct', label: 'Qwen2.5-Coder-32B-Instruct (HuggingFace)', provider: 'HuggingFace', maxTokenAllowed: 8000 },
{ name: '01-ai/Yi-1.5-34B-Chat', label: 'Yi-1.5-34B-Chat (HuggingFace)', provider: 'HuggingFace' }, { name: '01-ai/Yi-1.5-34B-Chat', label: 'Yi-1.5-34B-Chat (HuggingFace)', provider: 'HuggingFace', maxTokenAllowed: 8000 },
{ name: 'codellama/CodeLlama-34b-Instruct-hf', label: 'CodeLlama-34b-Instruct (HuggingFace)', provider: 'HuggingFace' }, { name: 'codellama/CodeLlama-34b-Instruct-hf', label: 'CodeLlama-34b-Instruct (HuggingFace)', provider: 'HuggingFace', maxTokenAllowed: 8000 },
{ name: 'NousResearch/Hermes-3-Llama-3.1-8B', label: 'Hermes-3-Llama-3.1-8B (HuggingFace)', provider: 'HuggingFace' } { name: 'NousResearch/Hermes-3-Llama-3.1-8B', label: 'Hermes-3-Llama-3.1-8B (HuggingFace)', provider: 'HuggingFace', maxTokenAllowed: 8000 }
], ],
getApiKeyLink: 'https://huggingface.co/settings/tokens' getApiKeyLink: 'https://huggingface.co/settings/tokens'
}, },
@ -106,37 +107,37 @@ const PROVIDER_LIST: ProviderInfo[] = [
{ {
name: 'OpenAI', name: 'OpenAI',
staticModels: [ staticModels: [
{ name: 'gpt-4o-mini', label: 'GPT-4o Mini', provider: 'OpenAI' }, { name: 'gpt-4o-mini', label: 'GPT-4o Mini', provider: 'OpenAI', maxTokenAllowed: 8000 },
{ name: 'gpt-4-turbo', label: 'GPT-4 Turbo', provider: 'OpenAI' }, { name: 'gpt-4-turbo', label: 'GPT-4 Turbo', provider: 'OpenAI', maxTokenAllowed: 8000 },
{ name: 'gpt-4', label: 'GPT-4', provider: 'OpenAI' }, { name: 'gpt-4', label: 'GPT-4', provider: 'OpenAI', maxTokenAllowed: 8000 },
{ name: 'gpt-3.5-turbo', label: 'GPT-3.5 Turbo', provider: 'OpenAI' } { name: 'gpt-3.5-turbo', label: 'GPT-3.5 Turbo', provider: 'OpenAI', maxTokenAllowed: 8000 }
], ],
getApiKeyLink: "https://platform.openai.com/api-keys", getApiKeyLink: "https://platform.openai.com/api-keys",
}, { }, {
name: 'xAI', name: 'xAI',
staticModels: [ staticModels: [
{ name: 'grok-beta', label: 'xAI Grok Beta', provider: 'xAI' } { name: 'grok-beta', label: 'xAI Grok Beta', provider: 'xAI', maxTokenAllowed: 8000 }
], ],
getApiKeyLink: 'https://docs.x.ai/docs/quickstart#creating-an-api-key' getApiKeyLink: 'https://docs.x.ai/docs/quickstart#creating-an-api-key'
}, { }, {
name: 'Deepseek', name: 'Deepseek',
staticModels: [ staticModels: [
{ name: 'deepseek-coder', label: 'Deepseek-Coder', provider: 'Deepseek' }, { name: 'deepseek-coder', label: 'Deepseek-Coder', provider: 'Deepseek', maxTokenAllowed: 8000 },
{ name: 'deepseek-chat', label: 'Deepseek-Chat', provider: 'Deepseek' } { name: 'deepseek-chat', label: 'Deepseek-Chat', provider: 'Deepseek', maxTokenAllowed: 8000 }
], ],
getApiKeyLink: 'https://platform.deepseek.com/api_keys' getApiKeyLink: 'https://platform.deepseek.com/api_keys'
}, { }, {
name: 'Mistral', name: 'Mistral',
staticModels: [ staticModels: [
{ name: 'open-mistral-7b', label: 'Mistral 7B', provider: 'Mistral' }, { name: 'open-mistral-7b', label: 'Mistral 7B', provider: 'Mistral', maxTokenAllowed: 8000 },
{ name: 'open-mixtral-8x7b', label: 'Mistral 8x7B', provider: 'Mistral' }, { name: 'open-mixtral-8x7b', label: 'Mistral 8x7B', provider: 'Mistral', maxTokenAllowed: 8000 },
{ name: 'open-mixtral-8x22b', label: 'Mistral 8x22B', provider: 'Mistral' }, { name: 'open-mixtral-8x22b', label: 'Mistral 8x22B', provider: 'Mistral', maxTokenAllowed: 8000 },
{ name: 'open-codestral-mamba', label: 'Codestral Mamba', provider: 'Mistral' }, { name: 'open-codestral-mamba', label: 'Codestral Mamba', provider: 'Mistral', maxTokenAllowed: 8000 },
{ name: 'open-mistral-nemo', label: 'Mistral Nemo', provider: 'Mistral' }, { name: 'open-mistral-nemo', label: 'Mistral Nemo', provider: 'Mistral', maxTokenAllowed: 8000 },
{ name: 'ministral-8b-latest', label: 'Mistral 8B', provider: 'Mistral' }, { name: 'ministral-8b-latest', label: 'Mistral 8B', provider: 'Mistral', maxTokenAllowed: 8000 },
{ name: 'mistral-small-latest', label: 'Mistral Small', provider: 'Mistral' }, { name: 'mistral-small-latest', label: 'Mistral Small', provider: 'Mistral', maxTokenAllowed: 8000 },
{ name: 'codestral-latest', label: 'Codestral', provider: 'Mistral' }, { name: 'codestral-latest', label: 'Codestral', provider: 'Mistral', maxTokenAllowed: 8000 },
{ name: 'mistral-large-latest', label: 'Mistral Large Latest', provider: 'Mistral' } { name: 'mistral-large-latest', label: 'Mistral Large Latest', provider: 'Mistral', maxTokenAllowed: 8000 }
], ],
getApiKeyLink: 'https://console.mistral.ai/api-keys/' getApiKeyLink: 'https://console.mistral.ai/api-keys/'
}, { }, {
@ -180,7 +181,8 @@ async function getOllamaModels(): Promise<ModelInfo[]> {
return data.models.map((model: OllamaModel) => ({ return data.models.map((model: OllamaModel) => ({
name: model.name, name: model.name,
label: `${model.name} (${model.details.parameter_size})`, label: `${model.name} (${model.details.parameter_size})`,
provider: 'Ollama' provider: 'Ollama',
maxTokenAllowed:8000,
})); }));
} catch (e) { } catch (e) {
return []; return [];
@ -234,7 +236,8 @@ async function getOpenRouterModels(): Promise<ModelInfo[]> {
label: `${m.name} - in:$${(m.pricing.prompt * 1_000_000).toFixed( label: `${m.name} - in:$${(m.pricing.prompt * 1_000_000).toFixed(
2)} out:$${(m.pricing.completion * 1_000_000).toFixed(2)} - context ${Math.floor( 2)} out:$${(m.pricing.completion * 1_000_000).toFixed(2)} - context ${Math.floor(
m.context_length / 1000)}k`, m.context_length / 1000)}k`,
provider: 'OpenRouter' provider: 'OpenRouter',
maxTokenAllowed:8000,
})); }));
} }

1
app/utils/types.ts
View File

@ -25,6 +25,7 @@ export interface ModelInfo {
name: string; name: string;
label: string; label: string;
provider: string; provider: string;
maxTokenAllowed: number;
} }
export interface ProviderInfo { export interface ProviderInfo {