From 8c34bcf0efe8e757e868a6cf1a4b17dc6dca756b Mon Sep 17 00:00:00 2001 From: KevIsDev Date: Thu, 15 May 2025 11:21:37 +0100 Subject: [PATCH] refactor: remove hardcoded static models and add token formatting - Remove hardcoded static models from all provider classes to simplify maintenance and improve flexibility. Introduce a new `formatTokens` utility function to standardize token count display across providers. This change ensures consistency and reduces redundancy in model definitions. - add anthropic-beta header for increased output limit --- app/lib/modules/llm/providers/anthropic.ts | 33 ++----------- app/lib/modules/llm/providers/google.ts | 32 +++++-------- app/lib/modules/llm/providers/groq.ts | 18 ++----- app/lib/modules/llm/providers/hyperbolic.ts | 36 ++------------ app/lib/modules/llm/providers/open-router.ts | 50 ++------------------ app/lib/modules/llm/providers/openai.ts | 8 +--- app/lib/modules/llm/providers/together.ts | 24 ++-------- app/lib/modules/llm/providers/tokenFormat.ts | 11 +++++ 8 files changed, 38 insertions(+), 174 deletions(-) create mode 100644 app/lib/modules/llm/providers/tokenFormat.ts diff --git a/app/lib/modules/llm/providers/anthropic.ts b/app/lib/modules/llm/providers/anthropic.ts index 70f93c07..6c0b2791 100644 --- a/app/lib/modules/llm/providers/anthropic.ts +++ b/app/lib/modules/llm/providers/anthropic.ts @@ -12,35 +12,7 @@ export default class AnthropicProvider extends BaseProvider { apiTokenKey: 'ANTHROPIC_API_KEY', }; - staticModels: ModelInfo[] = [ - { - name: 'claude-3-7-sonnet-20250219', - label: 'Claude 3.7 Sonnet', - provider: 'Anthropic', - maxTokenAllowed: 8000, - }, - { - name: 'claude-3-5-sonnet-latest', - label: 'Claude 3.5 Sonnet (new)', - provider: 'Anthropic', - maxTokenAllowed: 8000, - }, - { - name: 'claude-3-5-sonnet-20240620', - label: 'Claude 3.5 Sonnet (old)', - provider: 'Anthropic', - maxTokenAllowed: 8000, - }, - { - name: 'claude-3-5-haiku-latest', - label: 'Claude 3.5 Haiku (new)', - provider: 'Anthropic', - maxTokenAllowed: 8000, - }, - { name: 'claude-3-opus-latest', label: 'Claude 3 Opus', provider: 'Anthropic', maxTokenAllowed: 8000 }, - { name: 'claude-3-sonnet-20240229', label: 'Claude 3 Sonnet', provider: 'Anthropic', maxTokenAllowed: 8000 }, - { name: 'claude-3-haiku-20240307', label: 'Claude 3 Haiku', provider: 'Anthropic', maxTokenAllowed: 8000 }, - ]; + staticModels: ModelInfo[] = []; async getDynamicModels( apiKeys?: Record, @@ -63,6 +35,7 @@ export default class AnthropicProvider extends BaseProvider { headers: { 'x-api-key': `${apiKey}`, 'anthropic-version': '2023-06-01', + 'anthropic-beta': 'output-128k-2025-02-19', }, }); @@ -75,7 +48,7 @@ export default class AnthropicProvider extends BaseProvider { name: m.id, label: `${m.display_name}`, provider: this.name, - maxTokenAllowed: 32000, + maxTokenAllowed: 128000, })); } diff --git a/app/lib/modules/llm/providers/google.ts b/app/lib/modules/llm/providers/google.ts index 67043bad..ef590d87 100644 --- a/app/lib/modules/llm/providers/google.ts +++ b/app/lib/modules/llm/providers/google.ts @@ -3,6 +3,7 @@ import type { ModelInfo } from '~/lib/modules/llm/types'; import type { IProviderSetting } from '~/types/model'; import type { LanguageModelV1 } from 'ai'; import { createGoogleGenerativeAI } from '@ai-sdk/google'; +import { formatTokens } from './tokenFormat'; export default class GoogleProvider extends BaseProvider { name = 'Google'; @@ -12,21 +13,7 @@ export default class GoogleProvider extends BaseProvider { apiTokenKey: 'GOOGLE_GENERATIVE_AI_API_KEY', }; - staticModels: ModelInfo[] = [ - { name: 'gemini-1.5-flash-latest', label: 'Gemini 1.5 Flash', provider: 'Google', maxTokenAllowed: 8192 }, - { - name: 'gemini-2.0-flash-thinking-exp-01-21', - label: 'Gemini 2.0 Flash-thinking-exp-01-21', - provider: 'Google', - maxTokenAllowed: 65536, - }, - { name: 'gemini-2.0-flash-exp', label: 'Gemini 2.0 Flash', provider: 'Google', maxTokenAllowed: 8192 }, - { name: 'gemini-1.5-flash-002', label: 'Gemini 1.5 Flash-002', provider: 'Google', maxTokenAllowed: 8192 }, - { name: 'gemini-1.5-flash-8b', label: 'Gemini 1.5 Flash-8b', provider: 'Google', maxTokenAllowed: 8192 }, - { name: 'gemini-1.5-pro-latest', label: 'Gemini 1.5 Pro', provider: 'Google', maxTokenAllowed: 8192 }, - { name: 'gemini-1.5-pro-002', label: 'Gemini 1.5 Pro-002', provider: 'Google', maxTokenAllowed: 8192 }, - { name: 'gemini-exp-1206', label: 'Gemini exp-1206', provider: 'Google', maxTokenAllowed: 8192 }, - ]; + staticModels: ModelInfo[] = []; async getDynamicModels( apiKeys?: Record, @@ -55,12 +42,15 @@ export default class GoogleProvider extends BaseProvider { const data = res.models.filter((model: any) => model.outputTokenLimit > 8000); - return data.map((m: any) => ({ - name: m.name.replace('models/', ''), - label: `${m.displayName} - context ${Math.floor((m.inputTokenLimit + m.outputTokenLimit) / 1000) + 'k'}`, - provider: this.name, - maxTokenAllowed: m.inputTokenLimit + m.outputTokenLimit || 8000, - })); + return data.map((m: any) => { + const totalTokens = m.inputTokenLimit + m.outputTokenLimit; + return { + name: m.name.replace('models/', ''), + label: `${m.displayName} - context: ${formatTokens(totalTokens)} tokens`, + provider: this.name, + maxTokenAllowed: totalTokens || 8000, + }; + }); } getModelInstance(options: { diff --git a/app/lib/modules/llm/providers/groq.ts b/app/lib/modules/llm/providers/groq.ts index e9d2b0bd..3cb6058e 100644 --- a/app/lib/modules/llm/providers/groq.ts +++ b/app/lib/modules/llm/providers/groq.ts @@ -3,6 +3,7 @@ import type { ModelInfo } from '~/lib/modules/llm/types'; import type { IProviderSetting } from '~/types/model'; import type { LanguageModelV1 } from 'ai'; import { createOpenAI } from '@ai-sdk/openai'; +import { formatTokens } from './tokenFormat'; export default class GroqProvider extends BaseProvider { name = 'Groq'; @@ -12,20 +13,7 @@ export default class GroqProvider extends BaseProvider { apiTokenKey: 'GROQ_API_KEY', }; - staticModels: ModelInfo[] = [ - { name: 'llama-3.1-8b-instant', label: 'Llama 3.1 8b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, - { name: 'llama-3.2-11b-vision-preview', label: 'Llama 3.2 11b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, - { name: 'llama-3.2-90b-vision-preview', label: 'Llama 3.2 90b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, - { name: 'llama-3.2-3b-preview', label: 'Llama 3.2 3b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, - { name: 'llama-3.2-1b-preview', label: 'Llama 3.2 1b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, - { name: 'llama-3.3-70b-versatile', label: 'Llama 3.3 70b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, - { - name: 'deepseek-r1-distill-llama-70b', - label: 'Deepseek R1 Distill Llama 70b (Groq)', - provider: 'Groq', - maxTokenAllowed: 131072, - }, - ]; + staticModels: ModelInfo[] = []; async getDynamicModels( apiKeys?: Record, @@ -58,7 +46,7 @@ export default class GroqProvider extends BaseProvider { return data.map((m: any) => ({ name: m.id, - label: `${m.id} - context ${m.context_window ? Math.floor(m.context_window / 1000) + 'k' : 'N/A'} [ by ${m.owned_by}]`, + label: `${m.id} - context: ${formatTokens(m.context_window)} tokens`, provider: this.name, maxTokenAllowed: m.context_window || 8000, })); diff --git a/app/lib/modules/llm/providers/hyperbolic.ts b/app/lib/modules/llm/providers/hyperbolic.ts index b91a81bc..e96e1d1e 100644 --- a/app/lib/modules/llm/providers/hyperbolic.ts +++ b/app/lib/modules/llm/providers/hyperbolic.ts @@ -3,6 +3,7 @@ import type { ModelInfo } from '~/lib/modules/llm/types'; import type { IProviderSetting } from '~/types/model'; import type { LanguageModelV1 } from 'ai'; import { createOpenAI } from '@ai-sdk/openai'; +import { formatTokens } from './tokenFormat'; export default class HyperbolicProvider extends BaseProvider { name = 'Hyperbolic'; @@ -12,38 +13,7 @@ export default class HyperbolicProvider extends BaseProvider { apiTokenKey: 'HYPERBOLIC_API_KEY', }; - staticModels: ModelInfo[] = [ - { - name: 'Qwen/Qwen2.5-Coder-32B-Instruct', - label: 'Qwen 2.5 Coder 32B Instruct', - provider: 'Hyperbolic', - maxTokenAllowed: 8192, - }, - { - name: 'Qwen/Qwen2.5-72B-Instruct', - label: 'Qwen2.5-72B-Instruct', - provider: 'Hyperbolic', - maxTokenAllowed: 8192, - }, - { - name: 'deepseek-ai/DeepSeek-V2.5', - label: 'DeepSeek-V2.5', - provider: 'Hyperbolic', - maxTokenAllowed: 8192, - }, - { - name: 'Qwen/QwQ-32B-Preview', - label: 'QwQ-32B-Preview', - provider: 'Hyperbolic', - maxTokenAllowed: 8192, - }, - { - name: 'Qwen/Qwen2-VL-72B-Instruct', - label: 'Qwen2-VL-72B-Instruct', - provider: 'Hyperbolic', - maxTokenAllowed: 8192, - }, - ]; + staticModels: ModelInfo[] = []; async getDynamicModels( apiKeys?: Record, @@ -75,7 +45,7 @@ export default class HyperbolicProvider extends BaseProvider { return data.map((m: any) => ({ name: m.id, - label: `${m.id} - context ${m.context_length ? Math.floor(m.context_length / 1000) + 'k' : 'N/A'}`, + label: `${m.id} - context ${formatTokens(m.context_length)} : 'N/A'}`, provider: this.name, maxTokenAllowed: m.context_length || 8000, })); diff --git a/app/lib/modules/llm/providers/open-router.ts b/app/lib/modules/llm/providers/open-router.ts index 242f8f7d..cfe1fbfc 100644 --- a/app/lib/modules/llm/providers/open-router.ts +++ b/app/lib/modules/llm/providers/open-router.ts @@ -3,6 +3,7 @@ import type { ModelInfo } from '~/lib/modules/llm/types'; import type { IProviderSetting } from '~/types/model'; import type { LanguageModelV1 } from 'ai'; import { createOpenRouter } from '@openrouter/ai-sdk-provider'; +import { formatTokens } from './tokenFormat'; interface OpenRouterModel { name: string; @@ -26,52 +27,7 @@ export default class OpenRouterProvider extends BaseProvider { apiTokenKey: 'OPEN_ROUTER_API_KEY', }; - staticModels: ModelInfo[] = [ - { - name: 'anthropic/claude-3.5-sonnet', - label: 'Anthropic: Claude 3.5 Sonnet (OpenRouter)', - provider: 'OpenRouter', - maxTokenAllowed: 8000, - }, - { - name: 'anthropic/claude-3-haiku', - label: 'Anthropic: Claude 3 Haiku (OpenRouter)', - provider: 'OpenRouter', - maxTokenAllowed: 8000, - }, - { - name: 'deepseek/deepseek-coder', - label: 'Deepseek-Coder V2 236B (OpenRouter)', - provider: 'OpenRouter', - maxTokenAllowed: 8000, - }, - { - name: 'google/gemini-flash-1.5', - label: 'Google Gemini Flash 1.5 (OpenRouter)', - provider: 'OpenRouter', - maxTokenAllowed: 8000, - }, - { - name: 'google/gemini-pro-1.5', - label: 'Google Gemini Pro 1.5 (OpenRouter)', - provider: 'OpenRouter', - maxTokenAllowed: 8000, - }, - { name: 'x-ai/grok-beta', label: 'xAI Grok Beta (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 }, - { - name: 'mistralai/mistral-nemo', - label: 'OpenRouter Mistral Nemo (OpenRouter)', - provider: 'OpenRouter', - maxTokenAllowed: 8000, - }, - { - name: 'qwen/qwen-110b-chat', - label: 'OpenRouter Qwen 110b Chat (OpenRouter)', - provider: 'OpenRouter', - maxTokenAllowed: 8000, - }, - { name: 'cohere/command', label: 'Cohere Command (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 4096 }, - ]; + staticModels: ModelInfo[] = []; async getDynamicModels( _apiKeys?: Record, @@ -91,7 +47,7 @@ export default class OpenRouterProvider extends BaseProvider { .sort((a, b) => a.name.localeCompare(b.name)) .map((m) => ({ name: m.id, - label: `${m.name} - in:$${(m.pricing.prompt * 1_000_000).toFixed(2)} out:$${(m.pricing.completion * 1_000_000).toFixed(2)} - context ${Math.floor(m.context_length / 1000)}k`, + label: `${m.name} - in:$${(m.pricing.prompt * 1_000_000).toFixed(2)} out:$${(m.pricing.completion * 1_000_000).toFixed(2)} - context ${formatTokens(m.context_length)}`, provider: this.name, maxTokenAllowed: 8000, })); diff --git a/app/lib/modules/llm/providers/openai.ts b/app/lib/modules/llm/providers/openai.ts index b6193b91..403638d1 100644 --- a/app/lib/modules/llm/providers/openai.ts +++ b/app/lib/modules/llm/providers/openai.ts @@ -12,13 +12,7 @@ export default class OpenAIProvider extends BaseProvider { apiTokenKey: 'OPENAI_API_KEY', }; - staticModels: ModelInfo[] = [ - { name: 'gpt-4o', label: 'GPT-4o', provider: 'OpenAI', maxTokenAllowed: 8000 }, - { name: 'gpt-4o-mini', label: 'GPT-4o Mini', provider: 'OpenAI', maxTokenAllowed: 8000 }, - { name: 'gpt-4-turbo', label: 'GPT-4 Turbo', provider: 'OpenAI', maxTokenAllowed: 8000 }, - { name: 'gpt-4', label: 'GPT-4', provider: 'OpenAI', maxTokenAllowed: 8000 }, - { name: 'gpt-3.5-turbo', label: 'GPT-3.5 Turbo', provider: 'OpenAI', maxTokenAllowed: 8000 }, - ]; + staticModels: ModelInfo[] = []; async getDynamicModels( apiKeys?: Record, diff --git a/app/lib/modules/llm/providers/together.ts b/app/lib/modules/llm/providers/together.ts index 2e11f64b..474be550 100644 --- a/app/lib/modules/llm/providers/together.ts +++ b/app/lib/modules/llm/providers/together.ts @@ -2,6 +2,7 @@ import { BaseProvider, getOpenAILikeModel } from '~/lib/modules/llm/base-provide import type { ModelInfo } from '~/lib/modules/llm/types'; import type { IProviderSetting } from '~/types/model'; import type { LanguageModelV1 } from 'ai'; +import { formatTokens } from './tokenFormat'; export default class TogetherProvider extends BaseProvider { name = 'Together'; @@ -12,26 +13,7 @@ export default class TogetherProvider extends BaseProvider { apiTokenKey: 'TOGETHER_API_KEY', }; - staticModels: ModelInfo[] = [ - { - name: 'Qwen/Qwen2.5-Coder-32B-Instruct', - label: 'Qwen/Qwen2.5-Coder-32B-Instruct', - provider: 'Together', - maxTokenAllowed: 8000, - }, - { - name: 'meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo', - label: 'meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo', - provider: 'Together', - maxTokenAllowed: 8000, - }, - { - name: 'mistralai/Mixtral-8x7B-Instruct-v0.1', - label: 'Mixtral 8x7B Instruct', - provider: 'Together', - maxTokenAllowed: 8192, - }, - ]; + staticModels: ModelInfo[] = []; async getDynamicModels( apiKeys?: Record, @@ -64,7 +46,7 @@ export default class TogetherProvider extends BaseProvider { return data.map((m: any) => ({ name: m.id, - label: `${m.display_name} - in:$${m.pricing.input.toFixed(2)} out:$${m.pricing.output.toFixed(2)} - context ${Math.floor(m.context_length / 1000)}k`, + label: `${m.display_name} - in:$${m.pricing.input.toFixed(2)} out:$${m.pricing.output.toFixed(2)} - context ${formatTokens(m.context_length)}`, provider: this.name, maxTokenAllowed: 8000, })); diff --git a/app/lib/modules/llm/providers/tokenFormat.ts b/app/lib/modules/llm/providers/tokenFormat.ts new file mode 100644 index 00000000..8da5d63c --- /dev/null +++ b/app/lib/modules/llm/providers/tokenFormat.ts @@ -0,0 +1,11 @@ +export function formatTokens(num: number): string { + if (num >= 1_000_000) { + return (num / 1_000_000).toFixed(1).replace(/\.0$/, '') + 'M'; + } + + if (num >= 1_000) { + return (num / 1_000).toFixed(1).replace(/\.0$/, '') + 'K'; + } + + return num.toString(); +}