From 7295352a9802d4328cc8cb2bed5de5b874ce35e8 Mon Sep 17 00:00:00 2001 From: Anirban Kar Date: Sat, 21 Dec 2024 11:45:17 +0530 Subject: [PATCH] refactor: refactored LLM Providers: Adapting Modular Approach (#832) * refactor: Refactoring Providers to have providers as modules * updated package and lock file * added grok model back * updated registry system --- app/components/chat/BaseChat.tsx | 3 +- app/components/chat/Chat.client.tsx | 2 +- app/components/chat/ModelSelector.tsx | 2 +- app/lib/.server/llm/api-key.ts | 111 --- app/lib/.server/llm/model.ts | 190 ---- app/lib/.server/llm/stream-text.ts | 11 +- app/lib/modules/llm/base-provider.ts | 72 ++ app/lib/modules/llm/manager.ts | 116 +++ app/lib/modules/llm/providers/anthropic.ts | 58 ++ app/lib/modules/llm/providers/cohere.ts | 54 + app/lib/modules/llm/providers/deepseek.ts | 47 + app/lib/modules/llm/providers/google.ts | 51 + app/lib/modules/llm/providers/groq.ts | 51 + app/lib/modules/llm/providers/huggingface.ts | 69 ++ app/lib/modules/llm/providers/lmstudio.ts | 73 ++ app/lib/modules/llm/providers/mistral.ts | 53 + app/lib/modules/llm/providers/ollama.ts | 99 ++ app/lib/modules/llm/providers/open-router.ts | 132 +++ app/lib/modules/llm/providers/openai-like.ts | 77 ++ app/lib/modules/llm/providers/openai.ts | 48 + app/lib/modules/llm/providers/perplexity.ts | 63 ++ app/lib/modules/llm/providers/together.ts | 100 ++ app/lib/modules/llm/providers/xai.ts | 47 + app/lib/modules/llm/registry.ts | 33 + app/lib/modules/llm/types.ts | 32 + app/types/model.ts | 2 +- app/utils/constants.ts | 976 +++++++------------ app/utils/types.ts | 7 - pnpm-lock.yaml | 2 +- worker-configuration.d.ts | 1 + 30 files changed, 1621 insertions(+), 961 deletions(-) delete mode 100644 app/lib/.server/llm/api-key.ts delete mode 100644 app/lib/.server/llm/model.ts create mode 100644 app/lib/modules/llm/base-provider.ts create mode 100644 app/lib/modules/llm/manager.ts create mode 100644 app/lib/modules/llm/providers/anthropic.ts create mode 100644 app/lib/modules/llm/providers/cohere.ts create mode 100644 app/lib/modules/llm/providers/deepseek.ts create mode 100644 app/lib/modules/llm/providers/google.ts create mode 100644 app/lib/modules/llm/providers/groq.ts create mode 100644 app/lib/modules/llm/providers/huggingface.ts create mode 100644 app/lib/modules/llm/providers/lmstudio.ts create mode 100644 app/lib/modules/llm/providers/mistral.ts create mode 100644 app/lib/modules/llm/providers/ollama.ts create mode 100644 app/lib/modules/llm/providers/open-router.ts create mode 100644 app/lib/modules/llm/providers/openai-like.ts create mode 100644 app/lib/modules/llm/providers/openai.ts create mode 100644 app/lib/modules/llm/providers/perplexity.ts create mode 100644 app/lib/modules/llm/providers/together.ts create mode 100644 app/lib/modules/llm/providers/xai.ts create mode 100644 app/lib/modules/llm/registry.ts create mode 100644 app/lib/modules/llm/types.ts diff --git a/app/components/chat/BaseChat.tsx b/app/components/chat/BaseChat.tsx index 5db6653..7c2a9da 100644 --- a/app/components/chat/BaseChat.tsx +++ b/app/components/chat/BaseChat.tsx @@ -160,6 +160,7 @@ export const BaseChat = React.forwardRef( } initializeModelList({ apiKeys: parsedApiKeys, providerSettings }).then((modelList) => { + console.log('Model List: ', modelList); setModelList(modelList); }); @@ -359,7 +360,7 @@ export const BaseChat = React.forwardRef( modelList={modelList} provider={provider} setProvider={setProvider} - providerList={providerList || PROVIDER_LIST} + providerList={providerList || (PROVIDER_LIST as ProviderInfo[])} apiKeys={apiKeys} /> {(providerList || []).length > 0 && provider && ( diff --git a/app/components/chat/Chat.client.tsx b/app/components/chat/Chat.client.tsx index 28be779..1b830fd 100644 --- a/app/components/chat/Chat.client.tsx +++ b/app/components/chat/Chat.client.tsx @@ -122,7 +122,7 @@ export const ChatImpl = memo( }); const [provider, setProvider] = useState(() => { const savedProvider = Cookies.get('selectedProvider'); - return PROVIDER_LIST.find((p) => p.name === savedProvider) || DEFAULT_PROVIDER; + return (PROVIDER_LIST.find((p) => p.name === savedProvider) || DEFAULT_PROVIDER) as ProviderInfo; }); const { showChat } = useStore(chatStore); diff --git a/app/components/chat/ModelSelector.tsx b/app/components/chat/ModelSelector.tsx index 7571d63..ec4da63 100644 --- a/app/components/chat/ModelSelector.tsx +++ b/app/components/chat/ModelSelector.tsx @@ -1,6 +1,6 @@ import type { ProviderInfo } from '~/types/model'; -import type { ModelInfo } from '~/utils/types'; import { useEffect } from 'react'; +import type { ModelInfo } from '~/lib/modules/llm/types'; interface ModelSelectorProps { model?: string; diff --git a/app/lib/.server/llm/api-key.ts b/app/lib/.server/llm/api-key.ts deleted file mode 100644 index 4b0fc53..0000000 --- a/app/lib/.server/llm/api-key.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { env } from 'node:process'; -import type { IProviderSetting } from '~/types/model'; -import { getProviderBaseUrlAndKey } from '~/utils/constants'; - -export function getAPIKey(cloudflareEnv: Env, provider: string, userApiKeys?: Record) { - /** - * The `cloudflareEnv` is only used when deployed or when previewing locally. - * In development the environment variables are available through `env`. - */ - - // First check user-provided API keys - if (userApiKeys?.[provider]) { - return userApiKeys[provider]; - } - - const { apiKey } = getProviderBaseUrlAndKey({ - provider, - apiKeys: userApiKeys, - providerSettings: undefined, - serverEnv: cloudflareEnv as any, - defaultBaseUrlKey: '', - defaultApiTokenKey: '', - }); - - if (apiKey) { - return apiKey; - } - - // Fall back to hardcoded environment variables names - switch (provider) { - case 'Anthropic': - return env.ANTHROPIC_API_KEY || cloudflareEnv.ANTHROPIC_API_KEY; - case 'OpenAI': - return env.OPENAI_API_KEY || cloudflareEnv.OPENAI_API_KEY; - case 'Google': - return env.GOOGLE_GENERATIVE_AI_API_KEY || cloudflareEnv.GOOGLE_GENERATIVE_AI_API_KEY; - case 'Groq': - return env.GROQ_API_KEY || cloudflareEnv.GROQ_API_KEY; - case 'HuggingFace': - return env.HuggingFace_API_KEY || cloudflareEnv.HuggingFace_API_KEY; - case 'OpenRouter': - return env.OPEN_ROUTER_API_KEY || cloudflareEnv.OPEN_ROUTER_API_KEY; - case 'Deepseek': - return env.DEEPSEEK_API_KEY || cloudflareEnv.DEEPSEEK_API_KEY; - case 'Mistral': - return env.MISTRAL_API_KEY || cloudflareEnv.MISTRAL_API_KEY; - case 'OpenAILike': - return env.OPENAI_LIKE_API_KEY || cloudflareEnv.OPENAI_LIKE_API_KEY; - case 'Together': - return env.TOGETHER_API_KEY || cloudflareEnv.TOGETHER_API_KEY; - case 'xAI': - return env.XAI_API_KEY || cloudflareEnv.XAI_API_KEY; - case 'Perplexity': - return env.PERPLEXITY_API_KEY || cloudflareEnv.PERPLEXITY_API_KEY; - case 'Cohere': - return env.COHERE_API_KEY; - case 'AzureOpenAI': - return env.AZURE_OPENAI_API_KEY; - default: - return ''; - } -} - -export function getBaseURL(cloudflareEnv: Env, provider: string, providerSettings?: Record) { - const { baseUrl } = getProviderBaseUrlAndKey({ - provider, - apiKeys: {}, - providerSettings, - serverEnv: cloudflareEnv as any, - defaultBaseUrlKey: '', - defaultApiTokenKey: '', - }); - - if (baseUrl) { - return baseUrl; - } - - let settingBaseUrl = providerSettings?.[provider].baseUrl; - - if (settingBaseUrl && settingBaseUrl.length == 0) { - settingBaseUrl = undefined; - } - - switch (provider) { - case 'Together': - return ( - settingBaseUrl || - env.TOGETHER_API_BASE_URL || - cloudflareEnv.TOGETHER_API_BASE_URL || - 'https://api.together.xyz/v1' - ); - case 'OpenAILike': - return settingBaseUrl || env.OPENAI_LIKE_API_BASE_URL || cloudflareEnv.OPENAI_LIKE_API_BASE_URL; - case 'LMStudio': - return ( - settingBaseUrl || env.LMSTUDIO_API_BASE_URL || cloudflareEnv.LMSTUDIO_API_BASE_URL || 'http://localhost:1234' - ); - case 'Ollama': { - let baseUrl = - settingBaseUrl || env.OLLAMA_API_BASE_URL || cloudflareEnv.OLLAMA_API_BASE_URL || 'http://localhost:11434'; - - if (env.RUNNING_IN_DOCKER === 'true') { - baseUrl = baseUrl.replace('localhost', 'host.docker.internal'); - } - - return baseUrl; - } - default: - return ''; - } -} diff --git a/app/lib/.server/llm/model.ts b/app/lib/.server/llm/model.ts deleted file mode 100644 index 308e27d..0000000 --- a/app/lib/.server/llm/model.ts +++ /dev/null @@ -1,190 +0,0 @@ -/* - * @ts-nocheck - * Preventing TS checks with files presented in the video for a better presentation. - */ -import { getAPIKey, getBaseURL } from '~/lib/.server/llm/api-key'; -import { createAnthropic } from '@ai-sdk/anthropic'; -import { createOpenAI } from '@ai-sdk/openai'; -import { createGoogleGenerativeAI } from '@ai-sdk/google'; -import { ollama } from 'ollama-ai-provider'; -import { createOpenRouter } from '@openrouter/ai-sdk-provider'; -import { createMistral } from '@ai-sdk/mistral'; -import { createCohere } from '@ai-sdk/cohere'; -import type { LanguageModelV1 } from 'ai'; -import type { IProviderSetting } from '~/types/model'; - -export const DEFAULT_NUM_CTX = process.env.DEFAULT_NUM_CTX ? parseInt(process.env.DEFAULT_NUM_CTX, 10) : 32768; - -type OptionalApiKey = string | undefined; - -export function getAnthropicModel(apiKey: OptionalApiKey, model: string) { - const anthropic = createAnthropic({ - apiKey, - }); - - return anthropic(model); -} -export function getOpenAILikeModel(baseURL: string, apiKey: OptionalApiKey, model: string) { - const openai = createOpenAI({ - baseURL, - apiKey, - }); - - return openai(model); -} - -export function getCohereAIModel(apiKey: OptionalApiKey, model: string) { - const cohere = createCohere({ - apiKey, - }); - - return cohere(model); -} - -export function getOpenAIModel(apiKey: OptionalApiKey, model: string) { - const openai = createOpenAI({ - apiKey, - }); - - return openai(model); -} - -export function getMistralModel(apiKey: OptionalApiKey, model: string) { - const mistral = createMistral({ - apiKey, - }); - - return mistral(model); -} - -export function getGoogleModel(apiKey: OptionalApiKey, model: string) { - const google = createGoogleGenerativeAI({ - apiKey, - }); - - return google(model); -} - -export function getGroqModel(apiKey: OptionalApiKey, model: string) { - const openai = createOpenAI({ - baseURL: 'https://api.groq.com/openai/v1', - apiKey, - }); - - return openai(model); -} - -export function getHuggingFaceModel(apiKey: OptionalApiKey, model: string) { - const openai = createOpenAI({ - baseURL: 'https://api-inference.huggingface.co/v1/', - apiKey, - }); - - return openai(model); -} - -export function getOllamaModel(baseURL: string, model: string) { - const ollamaInstance = ollama(model, { - numCtx: DEFAULT_NUM_CTX, - }) as LanguageModelV1 & { config: any }; - - ollamaInstance.config.baseURL = `${baseURL}/api`; - - return ollamaInstance; -} - -export function getDeepseekModel(apiKey: OptionalApiKey, model: string) { - const openai = createOpenAI({ - baseURL: 'https://api.deepseek.com/beta', - apiKey, - }); - - return openai(model); -} - -export function getOpenRouterModel(apiKey: OptionalApiKey, model: string) { - const openRouter = createOpenRouter({ - apiKey, - }); - - return openRouter.chat(model); -} - -export function getLMStudioModel(baseURL: string, model: string) { - const lmstudio = createOpenAI({ - baseUrl: `${baseURL}/v1`, - apiKey: '', - }); - - return lmstudio(model); -} - -export function getXAIModel(apiKey: OptionalApiKey, model: string) { - const openai = createOpenAI({ - baseURL: 'https://api.x.ai/v1', - apiKey, - }); - - return openai(model); -} - -export function getPerplexityModel(apiKey: OptionalApiKey, model: string) { - const perplexity = createOpenAI({ - baseURL: 'https://api.perplexity.ai/', - apiKey, - }); - - return perplexity(model); -} - -export function getModel( - provider: string, - model: string, - serverEnv: Env, - apiKeys?: Record, - providerSettings?: Record, -) { - /* - * let apiKey; // Declare first - * let baseURL; - */ - // console.log({provider,model}); - - const apiKey = getAPIKey(serverEnv, provider, apiKeys); // Then assign - const baseURL = getBaseURL(serverEnv, provider, providerSettings); - - // console.log({apiKey,baseURL}); - - switch (provider) { - case 'Anthropic': - return getAnthropicModel(apiKey, model); - case 'OpenAI': - return getOpenAIModel(apiKey, model); - case 'Groq': - return getGroqModel(apiKey, model); - case 'HuggingFace': - return getHuggingFaceModel(apiKey, model); - case 'OpenRouter': - return getOpenRouterModel(apiKey, model); - case 'Google': - return getGoogleModel(apiKey, model); - case 'OpenAILike': - return getOpenAILikeModel(baseURL, apiKey, model); - case 'Together': - return getOpenAILikeModel(baseURL, apiKey, model); - case 'Deepseek': - return getDeepseekModel(apiKey, model); - case 'Mistral': - return getMistralModel(apiKey, model); - case 'LMStudio': - return getLMStudioModel(baseURL, model); - case 'xAI': - return getXAIModel(apiKey, model); - case 'Cohere': - return getCohereAIModel(apiKey, model); - case 'Perplexity': - return getPerplexityModel(apiKey, model); - default: - return getOllamaModel(baseURL, model); - } -} diff --git a/app/lib/.server/llm/stream-text.ts b/app/lib/.server/llm/stream-text.ts index 6bbf568..a717922 100644 --- a/app/lib/.server/llm/stream-text.ts +++ b/app/lib/.server/llm/stream-text.ts @@ -1,5 +1,4 @@ import { convertToCoreMessages, streamText as _streamText } from 'ai'; -import { getModel } from '~/lib/.server/llm/model'; import { MAX_TOKENS } from './constants'; import { getSystemPrompt } from '~/lib/common/prompts/prompts'; import { @@ -8,6 +7,7 @@ import { getModelList, MODEL_REGEX, MODIFICATIONS_TAG_NAME, + PROVIDER_LIST, PROVIDER_REGEX, WORK_DIR, } from '~/utils/constants'; @@ -184,6 +184,8 @@ export async function streamText(props: { const dynamicMaxTokens = modelDetails && modelDetails.maxTokenAllowed ? modelDetails.maxTokenAllowed : MAX_TOKENS; + const provider = PROVIDER_LIST.find((p) => p.name === currentProvider) || DEFAULT_PROVIDER; + let systemPrompt = PromptLibrary.getPropmtFromLibrary(promptId || 'default', { cwd: WORK_DIR, @@ -199,7 +201,12 @@ export async function streamText(props: { } return _streamText({ - model: getModel(currentProvider, currentModel, serverEnv, apiKeys, providerSettings) as any, + model: provider.getModelInstance({ + model: currentModel, + serverEnv, + apiKeys, + providerSettings, + }), system: systemPrompt, maxTokens: dynamicMaxTokens, messages: convertToCoreMessages(processedMessages as any), diff --git a/app/lib/modules/llm/base-provider.ts b/app/lib/modules/llm/base-provider.ts new file mode 100644 index 0000000..ebc2a3a --- /dev/null +++ b/app/lib/modules/llm/base-provider.ts @@ -0,0 +1,72 @@ +import type { LanguageModelV1 } from 'ai'; +import type { ProviderInfo, ProviderConfig, ModelInfo } from './types'; +import type { IProviderSetting } from '~/types/model'; +import { createOpenAI } from '@ai-sdk/openai'; +import { LLMManager } from './manager'; + +export abstract class BaseProvider implements ProviderInfo { + abstract name: string; + abstract staticModels: ModelInfo[]; + abstract config: ProviderConfig; + + getApiKeyLink?: string; + labelForGetApiKey?: string; + icon?: string; + + getProviderBaseUrlAndKey(options: { + apiKeys?: Record; + providerSettings?: IProviderSetting; + serverEnv?: Record; + defaultBaseUrlKey: string; + defaultApiTokenKey: string; + }) { + const { apiKeys, providerSettings, serverEnv, defaultBaseUrlKey, defaultApiTokenKey } = options; + let settingsBaseUrl = providerSettings?.baseUrl; + const manager = LLMManager.getInstance(); + + if (settingsBaseUrl && settingsBaseUrl.length == 0) { + settingsBaseUrl = undefined; + } + + const baseUrlKey = this.config.baseUrlKey || defaultBaseUrlKey; + let baseUrl = settingsBaseUrl || serverEnv?.[baseUrlKey] || process?.env?.[baseUrlKey] || manager.env?.[baseUrlKey]; + + if (baseUrl && baseUrl.endsWith('/')) { + baseUrl = baseUrl.slice(0, -1); + } + + const apiTokenKey = this.config.apiTokenKey || defaultApiTokenKey; + const apiKey = + apiKeys?.[this.name] || serverEnv?.[apiTokenKey] || process?.env?.[apiTokenKey] || manager.env?.[baseUrlKey]; + + return { + baseUrl, + apiKey, + }; + } + + // Declare the optional getDynamicModels method + getDynamicModels?( + apiKeys?: Record, + settings?: IProviderSetting, + serverEnv?: Record, + ): Promise; + + abstract getModelInstance(options: { + model: string; + serverEnv: Env; + apiKeys?: Record; + providerSettings?: Record; + }): LanguageModelV1; +} + +type OptionalApiKey = string | undefined; + +export function getOpenAILikeModel(baseURL: string, apiKey: OptionalApiKey, model: string) { + const openai = createOpenAI({ + baseURL, + apiKey, + }); + + return openai(model); +} diff --git a/app/lib/modules/llm/manager.ts b/app/lib/modules/llm/manager.ts new file mode 100644 index 0000000..38dc825 --- /dev/null +++ b/app/lib/modules/llm/manager.ts @@ -0,0 +1,116 @@ +import type { IProviderSetting } from '~/types/model'; +import { BaseProvider } from './base-provider'; +import type { ModelInfo, ProviderInfo } from './types'; +import * as providers from './registry'; + +export class LLMManager { + private static _instance: LLMManager; + private _providers: Map = new Map(); + private _modelList: ModelInfo[] = []; + private readonly _env: any = {}; + + private constructor(_env: Record) { + this._registerProvidersFromDirectory(); + this._env = _env; + } + + static getInstance(env: Record = {}): LLMManager { + if (!LLMManager._instance) { + LLMManager._instance = new LLMManager(env); + } + + return LLMManager._instance; + } + get env() { + return this._env; + } + + private async _registerProvidersFromDirectory() { + try { + /* + * Dynamically import all files from the providers directory + * const providerModules = import.meta.glob('./providers/*.ts', { eager: true }); + */ + + // Look for exported classes that extend BaseProvider + for (const exportedItem of Object.values(providers)) { + if (typeof exportedItem === 'function' && exportedItem.prototype instanceof BaseProvider) { + const provider = new exportedItem(); + + try { + this.registerProvider(provider); + } catch (error: any) { + console.log('Failed To Register Provider: ', provider.name, 'error:', error.message); + } + } + } + } catch (error) { + console.error('Error registering providers:', error); + } + } + + registerProvider(provider: BaseProvider) { + if (this._providers.has(provider.name)) { + console.warn(`Provider ${provider.name} is already registered. Skipping.`); + return; + } + + console.log('Registering Provider: ', provider.name); + this._providers.set(provider.name, provider); + this._modelList = [...this._modelList, ...provider.staticModels]; + } + + getProvider(name: string): BaseProvider | undefined { + return this._providers.get(name); + } + + getAllProviders(): BaseProvider[] { + return Array.from(this._providers.values()); + } + + getModelList(): ModelInfo[] { + return this._modelList; + } + + async updateModelList(options: { + apiKeys?: Record; + providerSettings?: Record; + serverEnv?: Record; + }): Promise { + const { apiKeys, providerSettings, serverEnv } = options; + + // Get dynamic models from all providers that support them + const dynamicModels = await Promise.all( + Array.from(this._providers.values()) + .filter( + (provider): provider is BaseProvider & Required> => + !!provider.getDynamicModels, + ) + .map((provider) => + provider.getDynamicModels(apiKeys, providerSettings?.[provider.name], serverEnv).catch((err) => { + console.error(`Error getting dynamic models ${provider.name} :`, err); + return []; + }), + ), + ); + + // Combine static and dynamic models + const modelList = [ + ...dynamicModels.flat(), + ...Array.from(this._providers.values()).flatMap((p) => p.staticModels || []), + ]; + this._modelList = modelList; + + return modelList; + } + + getDefaultProvider(): BaseProvider { + const firstProvider = this._providers.values().next().value; + + if (!firstProvider) { + throw new Error('No providers registered'); + } + + return firstProvider; + } +} diff --git a/app/lib/modules/llm/providers/anthropic.ts b/app/lib/modules/llm/providers/anthropic.ts new file mode 100644 index 0000000..099f02d --- /dev/null +++ b/app/lib/modules/llm/providers/anthropic.ts @@ -0,0 +1,58 @@ +import { BaseProvider } from '~/lib/modules/llm/base-provider'; +import type { ModelInfo } from '~/lib/modules/llm/types'; +import type { LanguageModelV1 } from 'ai'; +import type { IProviderSetting } from '~/types/model'; +import { createAnthropic } from '@ai-sdk/anthropic'; + +export default class AnthropicProvider extends BaseProvider { + name = 'Anthropic'; + getApiKeyLink = 'https://console.anthropic.com/settings/keys'; + + config = { + apiTokenKey: 'ANTHROPIC_API_KEY', + }; + + staticModels: ModelInfo[] = [ + { + name: 'claude-3-5-sonnet-latest', + label: 'Claude 3.5 Sonnet (new)', + provider: 'Anthropic', + maxTokenAllowed: 8000, + }, + { + name: 'claude-3-5-sonnet-20240620', + label: 'Claude 3.5 Sonnet (old)', + provider: 'Anthropic', + maxTokenAllowed: 8000, + }, + { + name: 'claude-3-5-haiku-latest', + label: 'Claude 3.5 Haiku (new)', + provider: 'Anthropic', + maxTokenAllowed: 8000, + }, + { name: 'claude-3-opus-latest', label: 'Claude 3 Opus', provider: 'Anthropic', maxTokenAllowed: 8000 }, + { name: 'claude-3-sonnet-20240229', label: 'Claude 3 Sonnet', provider: 'Anthropic', maxTokenAllowed: 8000 }, + { name: 'claude-3-haiku-20240307', label: 'Claude 3 Haiku', provider: 'Anthropic', maxTokenAllowed: 8000 }, + ]; + getModelInstance: (options: { + model: string; + serverEnv: Env; + apiKeys?: Record; + providerSettings?: Record; + }) => LanguageModelV1 = (options) => { + const { apiKeys, providerSettings, serverEnv, model } = options; + const { apiKey } = this.getProviderBaseUrlAndKey({ + apiKeys, + providerSettings, + serverEnv: serverEnv as any, + defaultBaseUrlKey: '', + defaultApiTokenKey: 'ANTHROPIC_API_KEY', + }); + const anthropic = createAnthropic({ + apiKey, + }); + + return anthropic(model); + }; +} diff --git a/app/lib/modules/llm/providers/cohere.ts b/app/lib/modules/llm/providers/cohere.ts new file mode 100644 index 0000000..9233b96 --- /dev/null +++ b/app/lib/modules/llm/providers/cohere.ts @@ -0,0 +1,54 @@ +import { BaseProvider } from '~/lib/modules/llm/base-provider'; +import type { ModelInfo } from '~/lib/modules/llm/types'; +import type { IProviderSetting } from '~/types/model'; +import type { LanguageModelV1 } from 'ai'; +import { createCohere } from '@ai-sdk/cohere'; + +export default class CohereProvider extends BaseProvider { + name = 'Cohere'; + getApiKeyLink = 'https://dashboard.cohere.com/api-keys'; + + config = { + apiTokenKey: 'COHERE_API_KEY', + }; + + staticModels: ModelInfo[] = [ + { name: 'command-r-plus-08-2024', label: 'Command R plus Latest', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command-r-08-2024', label: 'Command R Latest', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command-r-plus', label: 'Command R plus', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command-r', label: 'Command R', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command', label: 'Command', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command-nightly', label: 'Command Nightly', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command-light', label: 'Command Light', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command-light-nightly', label: 'Command Light Nightly', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'c4ai-aya-expanse-8b', label: 'c4AI Aya Expanse 8b', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'c4ai-aya-expanse-32b', label: 'c4AI Aya Expanse 32b', provider: 'Cohere', maxTokenAllowed: 4096 }, + ]; + + getModelInstance(options: { + model: string; + serverEnv: Env; + apiKeys?: Record; + providerSettings?: Record; + }): LanguageModelV1 { + const { model, serverEnv, apiKeys, providerSettings } = options; + + const { apiKey } = this.getProviderBaseUrlAndKey({ + apiKeys, + providerSettings: providerSettings?.[this.name], + serverEnv: serverEnv as any, + defaultBaseUrlKey: '', + defaultApiTokenKey: 'COHERE_API_KEY', + }); + + if (!apiKey) { + throw new Error(`Missing API key for ${this.name} provider`); + } + + const cohere = createCohere({ + apiKey, + }); + + return cohere(model); + } +} diff --git a/app/lib/modules/llm/providers/deepseek.ts b/app/lib/modules/llm/providers/deepseek.ts new file mode 100644 index 0000000..dac0d23 --- /dev/null +++ b/app/lib/modules/llm/providers/deepseek.ts @@ -0,0 +1,47 @@ +import { BaseProvider } from '~/lib/modules/llm/base-provider'; +import type { ModelInfo } from '~/lib/modules/llm/types'; +import type { IProviderSetting } from '~/types/model'; +import type { LanguageModelV1 } from 'ai'; +import { createOpenAI } from '@ai-sdk/openai'; + +export default class DeepseekProvider extends BaseProvider { + name = 'Deepseek'; + getApiKeyLink = 'https://platform.deepseek.com/apiKeys'; + + config = { + apiTokenKey: 'DEEPSEEK_API_KEY', + }; + + staticModels: ModelInfo[] = [ + { name: 'deepseek-coder', label: 'Deepseek-Coder', provider: 'Deepseek', maxTokenAllowed: 8000 }, + { name: 'deepseek-chat', label: 'Deepseek-Chat', provider: 'Deepseek', maxTokenAllowed: 8000 }, + ]; + + getModelInstance(options: { + model: string; + serverEnv: Env; + apiKeys?: Record; + providerSettings?: Record; + }): LanguageModelV1 { + const { model, serverEnv, apiKeys, providerSettings } = options; + + const { apiKey } = this.getProviderBaseUrlAndKey({ + apiKeys, + providerSettings: providerSettings?.[this.name], + serverEnv: serverEnv as any, + defaultBaseUrlKey: '', + defaultApiTokenKey: 'DEEPSEEK_API_KEY', + }); + + if (!apiKey) { + throw new Error(`Missing API key for ${this.name} provider`); + } + + const openai = createOpenAI({ + baseURL: 'https://api.deepseek.com/beta', + apiKey, + }); + + return openai(model); + } +} diff --git a/app/lib/modules/llm/providers/google.ts b/app/lib/modules/llm/providers/google.ts new file mode 100644 index 0000000..edc8e1d --- /dev/null +++ b/app/lib/modules/llm/providers/google.ts @@ -0,0 +1,51 @@ +import { BaseProvider } from '~/lib/modules/llm/base-provider'; +import type { ModelInfo } from '~/lib/modules/llm/types'; +import type { IProviderSetting } from '~/types/model'; +import type { LanguageModelV1 } from 'ai'; +import { createGoogleGenerativeAI } from '@ai-sdk/google'; + +export default class GoogleProvider extends BaseProvider { + name = 'Google'; + getApiKeyLink = 'https://aistudio.google.com/app/apikey'; + + config = { + apiTokenKey: 'GOOGLE_GENERATIVE_AI_API_KEY', + }; + + staticModels: ModelInfo[] = [ + { name: 'gemini-1.5-flash-latest', label: 'Gemini 1.5 Flash', provider: 'Google', maxTokenAllowed: 8192 }, + { name: 'gemini-2.0-flash-exp', label: 'Gemini 2.0 Flash', provider: 'Google', maxTokenAllowed: 8192 }, + { name: 'gemini-1.5-flash-002', label: 'Gemini 1.5 Flash-002', provider: 'Google', maxTokenAllowed: 8192 }, + { name: 'gemini-1.5-flash-8b', label: 'Gemini 1.5 Flash-8b', provider: 'Google', maxTokenAllowed: 8192 }, + { name: 'gemini-1.5-pro-latest', label: 'Gemini 1.5 Pro', provider: 'Google', maxTokenAllowed: 8192 }, + { name: 'gemini-1.5-pro-002', label: 'Gemini 1.5 Pro-002', provider: 'Google', maxTokenAllowed: 8192 }, + { name: 'gemini-exp-1206', label: 'Gemini exp-1206', provider: 'Google', maxTokenAllowed: 8192 }, + ]; + + getModelInstance(options: { + model: string; + serverEnv: any; + apiKeys?: Record; + providerSettings?: Record; + }): LanguageModelV1 { + const { model, serverEnv, apiKeys, providerSettings } = options; + + const { apiKey } = this.getProviderBaseUrlAndKey({ + apiKeys, + providerSettings: providerSettings?.[this.name], + serverEnv: serverEnv as any, + defaultBaseUrlKey: '', + defaultApiTokenKey: 'GOOGLE_GENERATIVE_AI_API_KEY', + }); + + if (!apiKey) { + throw new Error(`Missing API key for ${this.name} provider`); + } + + const google = createGoogleGenerativeAI({ + apiKey, + }); + + return google(model); + } +} diff --git a/app/lib/modules/llm/providers/groq.ts b/app/lib/modules/llm/providers/groq.ts new file mode 100644 index 0000000..034dab1 --- /dev/null +++ b/app/lib/modules/llm/providers/groq.ts @@ -0,0 +1,51 @@ +import { BaseProvider } from '~/lib/modules/llm/base-provider'; +import type { ModelInfo } from '~/lib/modules/llm/types'; +import type { IProviderSetting } from '~/types/model'; +import type { LanguageModelV1 } from 'ai'; +import { createOpenAI } from '@ai-sdk/openai'; + +export default class GroqProvider extends BaseProvider { + name = 'Groq'; + getApiKeyLink = 'https://console.groq.com/keys'; + + config = { + apiTokenKey: 'GROQ_API_KEY', + }; + + staticModels: ModelInfo[] = [ + { name: 'llama-3.1-8b-instant', label: 'Llama 3.1 8b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, + { name: 'llama-3.2-11b-vision-preview', label: 'Llama 3.2 11b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, + { name: 'llama-3.2-90b-vision-preview', label: 'Llama 3.2 90b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, + { name: 'llama-3.2-3b-preview', label: 'Llama 3.2 3b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, + { name: 'llama-3.2-1b-preview', label: 'Llama 3.2 1b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, + { name: 'llama-3.3-70b-versatile', label: 'Llama 3.3 70b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, + ]; + + getModelInstance(options: { + model: string; + serverEnv: Env; + apiKeys?: Record; + providerSettings?: Record; + }): LanguageModelV1 { + const { model, serverEnv, apiKeys, providerSettings } = options; + + const { apiKey } = this.getProviderBaseUrlAndKey({ + apiKeys, + providerSettings: providerSettings?.[this.name], + serverEnv: serverEnv as any, + defaultBaseUrlKey: '', + defaultApiTokenKey: 'GROQ_API_KEY', + }); + + if (!apiKey) { + throw new Error(`Missing API key for ${this.name} provider`); + } + + const openai = createOpenAI({ + baseURL: 'https://api.groq.com/openai/v1', + apiKey, + }); + + return openai(model); + } +} diff --git a/app/lib/modules/llm/providers/huggingface.ts b/app/lib/modules/llm/providers/huggingface.ts new file mode 100644 index 0000000..5ae1d1b --- /dev/null +++ b/app/lib/modules/llm/providers/huggingface.ts @@ -0,0 +1,69 @@ +import { BaseProvider } from '~/lib/modules/llm/base-provider'; +import type { ModelInfo } from '~/lib/modules/llm/types'; +import type { IProviderSetting } from '~/types/model'; +import type { LanguageModelV1 } from 'ai'; +import { createOpenAI } from '@ai-sdk/openai'; + +export default class HuggingFaceProvider extends BaseProvider { + name = 'HuggingFace'; + getApiKeyLink = 'https://huggingface.co/settings/tokens'; + + config = { + apiTokenKey: 'HuggingFace_API_KEY', + }; + + staticModels: ModelInfo[] = [ + { + name: 'Qwen/Qwen2.5-Coder-32B-Instruct', + label: 'Qwen2.5-Coder-32B-Instruct (HuggingFace)', + provider: 'HuggingFace', + maxTokenAllowed: 8000, + }, + { + name: '01-ai/Yi-1.5-34B-Chat', + label: 'Yi-1.5-34B-Chat (HuggingFace)', + provider: 'HuggingFace', + maxTokenAllowed: 8000, + }, + { + name: 'meta-llama/Llama-3.1-70B-Instruct', + label: 'Llama-3.1-70B-Instruct (HuggingFace)', + provider: 'HuggingFace', + maxTokenAllowed: 8000, + }, + { + name: 'meta-llama/Llama-3.1-405B', + label: 'Llama-3.1-405B (HuggingFace)', + provider: 'HuggingFace', + maxTokenAllowed: 8000, + }, + ]; + + getModelInstance(options: { + model: string; + serverEnv: Env; + apiKeys?: Record; + providerSettings?: Record; + }): LanguageModelV1 { + const { model, serverEnv, apiKeys, providerSettings } = options; + + const { apiKey } = this.getProviderBaseUrlAndKey({ + apiKeys, + providerSettings: providerSettings?.[this.name], + serverEnv: serverEnv as any, + defaultBaseUrlKey: '', + defaultApiTokenKey: 'HuggingFace_API_KEY', + }); + + if (!apiKey) { + throw new Error(`Missing API key for ${this.name} provider`); + } + + const openai = createOpenAI({ + baseURL: 'https://api-inference.huggingface.co/v1/', + apiKey, + }); + + return openai(model); + } +} diff --git a/app/lib/modules/llm/providers/lmstudio.ts b/app/lib/modules/llm/providers/lmstudio.ts new file mode 100644 index 0000000..4040453 --- /dev/null +++ b/app/lib/modules/llm/providers/lmstudio.ts @@ -0,0 +1,73 @@ +import { BaseProvider } from '~/lib/modules/llm/base-provider'; +import type { ModelInfo } from '~/lib/modules/llm/types'; +import type { IProviderSetting } from '~/types/model'; +import { createOpenAI } from '@ai-sdk/openai'; +import type { LanguageModelV1 } from 'ai'; + +export default class LMStudioProvider extends BaseProvider { + name = 'LMStudio'; + getApiKeyLink = 'https://lmstudio.ai/'; + labelForGetApiKey = 'Get LMStudio'; + icon = 'i-ph:cloud-arrow-down'; + + config = { + baseUrlKey: 'LMSTUDIO_API_BASE_URL', + }; + + staticModels: ModelInfo[] = []; + + async getDynamicModels( + apiKeys?: Record, + settings?: IProviderSetting, + serverEnv: Record = {}, + ): Promise { + try { + const { baseUrl } = this.getProviderBaseUrlAndKey({ + apiKeys, + providerSettings: settings, + serverEnv, + defaultBaseUrlKey: 'LMSTUDIO_API_BASE_URL', + defaultApiTokenKey: '', + }); + + if (!baseUrl) { + return []; + } + + const response = await fetch(`${baseUrl}/v1/models`); + const data = (await response.json()) as { data: Array<{ id: string }> }; + + return data.data.map((model) => ({ + name: model.id, + label: model.id, + provider: this.name, + maxTokenAllowed: 8000, + })); + } catch (error: any) { + console.log('Error getting LMStudio models:', error.message); + + return []; + } + } + getModelInstance: (options: { + model: string; + serverEnv: Env; + apiKeys?: Record; + providerSettings?: Record; + }) => LanguageModelV1 = (options) => { + const { apiKeys, providerSettings, serverEnv, model } = options; + const { baseUrl } = this.getProviderBaseUrlAndKey({ + apiKeys, + providerSettings, + serverEnv: serverEnv as any, + defaultBaseUrlKey: 'OLLAMA_API_BASE_URL', + defaultApiTokenKey: '', + }); + const lmstudio = createOpenAI({ + baseUrl: `${baseUrl}/v1`, + apiKey: '', + }); + + return lmstudio(model); + }; +} diff --git a/app/lib/modules/llm/providers/mistral.ts b/app/lib/modules/llm/providers/mistral.ts new file mode 100644 index 0000000..c5becee --- /dev/null +++ b/app/lib/modules/llm/providers/mistral.ts @@ -0,0 +1,53 @@ +import { BaseProvider } from '~/lib/modules/llm/base-provider'; +import type { ModelInfo } from '~/lib/modules/llm/types'; +import type { IProviderSetting } from '~/types/model'; +import type { LanguageModelV1 } from 'ai'; +import { createMistral } from '@ai-sdk/mistral'; + +export default class MistralProvider extends BaseProvider { + name = 'Mistral'; + getApiKeyLink = 'https://console.mistral.ai/api-keys/'; + + config = { + apiTokenKey: 'MISTRAL_API_KEY', + }; + + staticModels: ModelInfo[] = [ + { name: 'open-mistral-7b', label: 'Mistral 7B', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'open-mixtral-8x7b', label: 'Mistral 8x7B', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'open-mixtral-8x22b', label: 'Mistral 8x22B', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'open-codestral-mamba', label: 'Codestral Mamba', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'open-mistral-nemo', label: 'Mistral Nemo', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'ministral-8b-latest', label: 'Mistral 8B', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'mistral-small-latest', label: 'Mistral Small', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'codestral-latest', label: 'Codestral', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'mistral-large-latest', label: 'Mistral Large Latest', provider: 'Mistral', maxTokenAllowed: 8000 }, + ]; + + getModelInstance(options: { + model: string; + serverEnv: Env; + apiKeys?: Record; + providerSettings?: Record; + }): LanguageModelV1 { + const { model, serverEnv, apiKeys, providerSettings } = options; + + const { apiKey } = this.getProviderBaseUrlAndKey({ + apiKeys, + providerSettings: providerSettings?.[this.name], + serverEnv: serverEnv as any, + defaultBaseUrlKey: '', + defaultApiTokenKey: 'MISTRAL_API_KEY', + }); + + if (!apiKey) { + throw new Error(`Missing API key for ${this.name} provider`); + } + + const mistral = createMistral({ + apiKey, + }); + + return mistral(model); + } +} diff --git a/app/lib/modules/llm/providers/ollama.ts b/app/lib/modules/llm/providers/ollama.ts new file mode 100644 index 0000000..ebe6b1b --- /dev/null +++ b/app/lib/modules/llm/providers/ollama.ts @@ -0,0 +1,99 @@ +import { BaseProvider } from '~/lib/modules/llm/base-provider'; +import type { ModelInfo } from '~/lib/modules/llm/types'; +import type { IProviderSetting } from '~/types/model'; +import type { LanguageModelV1 } from 'ai'; +import { ollama } from 'ollama-ai-provider'; + +interface OllamaModelDetails { + parent_model: string; + format: string; + family: string; + families: string[]; + parameter_size: string; + quantization_level: string; +} + +export interface OllamaModel { + name: string; + model: string; + modified_at: string; + size: number; + digest: string; + details: OllamaModelDetails; +} + +export interface OllamaApiResponse { + models: OllamaModel[]; +} + +export const DEFAULT_NUM_CTX = process?.env?.DEFAULT_NUM_CTX ? parseInt(process.env.DEFAULT_NUM_CTX, 10) : 32768; + +export default class OllamaProvider extends BaseProvider { + name = 'Ollama'; + getApiKeyLink = 'https://ollama.com/download'; + labelForGetApiKey = 'Download Ollama'; + icon = 'i-ph:cloud-arrow-down'; + + config = { + baseUrlKey: 'OLLAMA_API_BASE_URL', + }; + + staticModels: ModelInfo[] = []; + + async getDynamicModels( + apiKeys?: Record, + settings?: IProviderSetting, + serverEnv: Record = {}, + ): Promise { + try { + const { baseUrl } = this.getProviderBaseUrlAndKey({ + apiKeys, + providerSettings: settings, + serverEnv, + defaultBaseUrlKey: 'OLLAMA_API_BASE_URL', + defaultApiTokenKey: '', + }); + + if (!baseUrl) { + return []; + } + + const response = await fetch(`${baseUrl}/api/tags`); + const data = (await response.json()) as OllamaApiResponse; + + // console.log({ ollamamodels: data.models }); + + return data.models.map((model: OllamaModel) => ({ + name: model.name, + label: `${model.name} (${model.details.parameter_size})`, + provider: this.name, + maxTokenAllowed: 8000, + })); + } catch (e) { + console.error('Failed to get Ollama models:', e); + return []; + } + } + getModelInstance: (options: { + model: string; + serverEnv: Env; + apiKeys?: Record; + providerSettings?: Record; + }) => LanguageModelV1 = (options) => { + const { apiKeys, providerSettings, serverEnv, model } = options; + const { baseUrl } = this.getProviderBaseUrlAndKey({ + apiKeys, + providerSettings, + serverEnv: serverEnv as any, + defaultBaseUrlKey: 'OLLAMA_API_BASE_URL', + defaultApiTokenKey: '', + }); + const ollamaInstance = ollama(model, { + numCtx: DEFAULT_NUM_CTX, + }) as LanguageModelV1 & { config: any }; + + ollamaInstance.config.baseURL = `${baseUrl}/api`; + + return ollamaInstance; + }; +} diff --git a/app/lib/modules/llm/providers/open-router.ts b/app/lib/modules/llm/providers/open-router.ts new file mode 100644 index 0000000..45defb2 --- /dev/null +++ b/app/lib/modules/llm/providers/open-router.ts @@ -0,0 +1,132 @@ +import { BaseProvider } from '~/lib/modules/llm/base-provider'; +import type { ModelInfo } from '~/lib/modules/llm/types'; +import type { IProviderSetting } from '~/types/model'; +import type { LanguageModelV1 } from 'ai'; +import { createOpenRouter } from '@openrouter/ai-sdk-provider'; + +interface OpenRouterModel { + name: string; + id: string; + context_length: number; + pricing: { + prompt: number; + completion: number; + }; +} + +interface OpenRouterModelsResponse { + data: OpenRouterModel[]; +} + +export default class OpenRouterProvider extends BaseProvider { + name = 'OpenRouter'; + getApiKeyLink = 'https://openrouter.ai/settings/keys'; + + config = { + apiTokenKey: 'OPEN_ROUTER_API_KEY', + }; + + staticModels: ModelInfo[] = [ + { name: 'gpt-4o', label: 'GPT-4o', provider: 'OpenAI', maxTokenAllowed: 8000 }, + { + name: 'anthropic/claude-3.5-sonnet', + label: 'Anthropic: Claude 3.5 Sonnet (OpenRouter)', + provider: 'OpenRouter', + maxTokenAllowed: 8000, + }, + { + name: 'anthropic/claude-3-haiku', + label: 'Anthropic: Claude 3 Haiku (OpenRouter)', + provider: 'OpenRouter', + maxTokenAllowed: 8000, + }, + { + name: 'deepseek/deepseek-coder', + label: 'Deepseek-Coder V2 236B (OpenRouter)', + provider: 'OpenRouter', + maxTokenAllowed: 8000, + }, + { + name: 'google/gemini-flash-1.5', + label: 'Google Gemini Flash 1.5 (OpenRouter)', + provider: 'OpenRouter', + maxTokenAllowed: 8000, + }, + { + name: 'google/gemini-pro-1.5', + label: 'Google Gemini Pro 1.5 (OpenRouter)', + provider: 'OpenRouter', + maxTokenAllowed: 8000, + }, + { name: 'x-ai/grok-beta', label: 'xAI Grok Beta (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 }, + { + name: 'mistralai/mistral-nemo', + label: 'OpenRouter Mistral Nemo (OpenRouter)', + provider: 'OpenRouter', + maxTokenAllowed: 8000, + }, + { + name: 'qwen/qwen-110b-chat', + label: 'OpenRouter Qwen 110b Chat (OpenRouter)', + provider: 'OpenRouter', + maxTokenAllowed: 8000, + }, + { name: 'cohere/command', label: 'Cohere Command (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 4096 }, + ]; + + async getDynamicModels( + _apiKeys?: Record, + _settings?: IProviderSetting, + _serverEnv: Record = {}, + ): Promise { + try { + const response = await fetch('https://openrouter.ai/api/v1/models', { + headers: { + 'Content-Type': 'application/json', + }, + }); + + const data = (await response.json()) as OpenRouterModelsResponse; + + return data.data + .sort((a, b) => a.name.localeCompare(b.name)) + .map((m) => ({ + name: m.id, + label: `${m.name} - in:$${(m.pricing.prompt * 1_000_000).toFixed(2)} out:$${(m.pricing.completion * 1_000_000).toFixed(2)} - context ${Math.floor(m.context_length / 1000)}k`, + provider: this.name, + maxTokenAllowed: 8000, + })); + } catch (error) { + console.error('Error getting OpenRouter models:', error); + return []; + } + } + + getModelInstance(options: { + model: string; + serverEnv: Env; + apiKeys?: Record; + providerSettings?: Record; + }): LanguageModelV1 { + const { model, serverEnv, apiKeys, providerSettings } = options; + + const { apiKey } = this.getProviderBaseUrlAndKey({ + apiKeys, + providerSettings: providerSettings?.[this.name], + serverEnv: serverEnv as any, + defaultBaseUrlKey: '', + defaultApiTokenKey: 'OPEN_ROUTER_API_KEY', + }); + + if (!apiKey) { + throw new Error(`Missing API key for ${this.name} provider`); + } + + const openRouter = createOpenRouter({ + apiKey, + }); + const instance = openRouter.chat(model) as LanguageModelV1; + + return instance; + } +} diff --git a/app/lib/modules/llm/providers/openai-like.ts b/app/lib/modules/llm/providers/openai-like.ts new file mode 100644 index 0000000..44fb1ab --- /dev/null +++ b/app/lib/modules/llm/providers/openai-like.ts @@ -0,0 +1,77 @@ +import { BaseProvider, getOpenAILikeModel } from '~/lib/modules/llm/base-provider'; +import type { ModelInfo } from '~/lib/modules/llm/types'; +import type { IProviderSetting } from '~/types/model'; +import type { LanguageModelV1 } from 'ai'; + +export default class OpenAILikeProvider extends BaseProvider { + name = 'OpenAILike'; + getApiKeyLink = undefined; + + config = { + baseUrlKey: 'OPENAI_LIKE_API_BASE_URL', + apiTokenKey: 'OPENAI_LIKE_API_KEY', + }; + + staticModels: ModelInfo[] = []; + + async getDynamicModels( + apiKeys?: Record, + settings?: IProviderSetting, + serverEnv: Record = {}, + ): Promise { + try { + const { baseUrl, apiKey } = this.getProviderBaseUrlAndKey({ + apiKeys, + providerSettings: settings, + serverEnv, + defaultBaseUrlKey: 'OPENAI_LIKE_API_BASE_URL', + defaultApiTokenKey: 'OPENAI_LIKE_API_KEY', + }); + + if (!baseUrl || !apiKey) { + return []; + } + + const response = await fetch(`${baseUrl}/models`, { + headers: { + Authorization: `Bearer ${apiKey}`, + }, + }); + + const res = (await response.json()) as any; + + return res.data.map((model: any) => ({ + name: model.id, + label: model.id, + provider: this.name, + maxTokenAllowed: 8000, + })); + } catch (error) { + console.error('Error getting OpenAILike models:', error); + return []; + } + } + + getModelInstance(options: { + model: string; + serverEnv: Env; + apiKeys?: Record; + providerSettings?: Record; + }): LanguageModelV1 { + const { model, serverEnv, apiKeys, providerSettings } = options; + + const { baseUrl, apiKey } = this.getProviderBaseUrlAndKey({ + apiKeys, + providerSettings: providerSettings?.[this.name], + serverEnv: serverEnv as any, + defaultBaseUrlKey: 'OPENAI_LIKE_API_BASE_URL', + defaultApiTokenKey: 'OPENAI_LIKE_API_KEY', + }); + + if (!baseUrl || !apiKey) { + throw new Error(`Missing configuration for ${this.name} provider`); + } + + return getOpenAILikeModel(baseUrl, apiKey, model); + } +} diff --git a/app/lib/modules/llm/providers/openai.ts b/app/lib/modules/llm/providers/openai.ts new file mode 100644 index 0000000..9a54118 --- /dev/null +++ b/app/lib/modules/llm/providers/openai.ts @@ -0,0 +1,48 @@ +import { BaseProvider } from '~/lib/modules/llm/base-provider'; +import type { ModelInfo } from '~/lib/modules/llm/types'; +import type { IProviderSetting } from '~/types/model'; +import type { LanguageModelV1 } from 'ai'; +import { createOpenAI } from '@ai-sdk/openai'; + +export default class OpenAIProvider extends BaseProvider { + name = 'OpenAI'; + getApiKeyLink = 'https://platform.openai.com/api-keys'; + + config = { + apiTokenKey: 'OPENAI_API_KEY', + }; + + staticModels: ModelInfo[] = [ + { name: 'gpt-4o-mini', label: 'GPT-4o Mini', provider: 'OpenAI', maxTokenAllowed: 8000 }, + { name: 'gpt-4-turbo', label: 'GPT-4 Turbo', provider: 'OpenAI', maxTokenAllowed: 8000 }, + { name: 'gpt-4', label: 'GPT-4', provider: 'OpenAI', maxTokenAllowed: 8000 }, + { name: 'gpt-3.5-turbo', label: 'GPT-3.5 Turbo', provider: 'OpenAI', maxTokenAllowed: 8000 }, + ]; + + getModelInstance(options: { + model: string; + serverEnv: Env; + apiKeys?: Record; + providerSettings?: Record; + }): LanguageModelV1 { + const { model, serverEnv, apiKeys, providerSettings } = options; + + const { apiKey } = this.getProviderBaseUrlAndKey({ + apiKeys, + providerSettings: providerSettings?.[this.name], + serverEnv: serverEnv as any, + defaultBaseUrlKey: '', + defaultApiTokenKey: 'OPENAI_API_KEY', + }); + + if (!apiKey) { + throw new Error(`Missing API key for ${this.name} provider`); + } + + const openai = createOpenAI({ + apiKey, + }); + + return openai(model); + } +} diff --git a/app/lib/modules/llm/providers/perplexity.ts b/app/lib/modules/llm/providers/perplexity.ts new file mode 100644 index 0000000..eb58d74 --- /dev/null +++ b/app/lib/modules/llm/providers/perplexity.ts @@ -0,0 +1,63 @@ +import { BaseProvider } from '~/lib/modules/llm/base-provider'; +import type { ModelInfo } from '~/lib/modules/llm/types'; +import type { IProviderSetting } from '~/types/model'; +import type { LanguageModelV1 } from 'ai'; +import { createOpenAI } from '@ai-sdk/openai'; + +export default class PerplexityProvider extends BaseProvider { + name = 'Perplexity'; + getApiKeyLink = 'https://www.perplexity.ai/settings/api'; + + config = { + apiTokenKey: 'PERPLEXITY_API_KEY', + }; + + staticModels: ModelInfo[] = [ + { + name: 'llama-3.1-sonar-small-128k-online', + label: 'Sonar Small Online', + provider: 'Perplexity', + maxTokenAllowed: 8192, + }, + { + name: 'llama-3.1-sonar-large-128k-online', + label: 'Sonar Large Online', + provider: 'Perplexity', + maxTokenAllowed: 8192, + }, + { + name: 'llama-3.1-sonar-huge-128k-online', + label: 'Sonar Huge Online', + provider: 'Perplexity', + maxTokenAllowed: 8192, + }, + ]; + + getModelInstance(options: { + model: string; + serverEnv: Env; + apiKeys?: Record; + providerSettings?: Record; + }): LanguageModelV1 { + const { model, serverEnv, apiKeys, providerSettings } = options; + + const { apiKey } = this.getProviderBaseUrlAndKey({ + apiKeys, + providerSettings: providerSettings?.[this.name], + serverEnv: serverEnv as any, + defaultBaseUrlKey: '', + defaultApiTokenKey: 'PERPLEXITY_API_KEY', + }); + + if (!apiKey) { + throw new Error(`Missing API key for ${this.name} provider`); + } + + const perplexity = createOpenAI({ + baseURL: 'https://api.perplexity.ai/', + apiKey, + }); + + return perplexity(model); + } +} diff --git a/app/lib/modules/llm/providers/together.ts b/app/lib/modules/llm/providers/together.ts new file mode 100644 index 0000000..1a908b8 --- /dev/null +++ b/app/lib/modules/llm/providers/together.ts @@ -0,0 +1,100 @@ +import { BaseProvider, getOpenAILikeModel } from '~/lib/modules/llm/base-provider'; +import type { ModelInfo } from '~/lib/modules/llm/types'; +import type { IProviderSetting } from '~/types/model'; +import type { LanguageModelV1 } from 'ai'; + +export default class TogetherProvider extends BaseProvider { + name = 'Together'; + getApiKeyLink = 'https://api.together.xyz/settings/api-keys'; + + config = { + baseUrlKey: 'TOGETHER_API_BASE_URL', + apiTokenKey: 'TOGETHER_API_KEY', + }; + + staticModels: ModelInfo[] = [ + { + name: 'Qwen/Qwen2.5-Coder-32B-Instruct', + label: 'Qwen/Qwen2.5-Coder-32B-Instruct', + provider: 'Together', + maxTokenAllowed: 8000, + }, + { + name: 'meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo', + label: 'meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo', + provider: 'Together', + maxTokenAllowed: 8000, + }, + { + name: 'mistralai/Mixtral-8x7B-Instruct-v0.1', + label: 'Mixtral 8x7B Instruct', + provider: 'Together', + maxTokenAllowed: 8192, + }, + ]; + + async getDynamicModels( + apiKeys?: Record, + settings?: IProviderSetting, + serverEnv: Record = {}, + ): Promise { + try { + const { baseUrl: fetchBaseUrl, apiKey } = this.getProviderBaseUrlAndKey({ + apiKeys, + providerSettings: settings, + serverEnv, + defaultBaseUrlKey: 'TOGETHER_API_BASE_URL', + defaultApiTokenKey: 'TOGETHER_API_KEY', + }); + const baseUrl = fetchBaseUrl || 'https://api.together.xyz/v1'; + + if (!baseUrl || !apiKey) { + return []; + } + + // console.log({ baseUrl, apiKey }); + + const response = await fetch(`${baseUrl}/models`, { + headers: { + Authorization: `Bearer ${apiKey}`, + }, + }); + + const res = (await response.json()) as any; + const data = (res || []).filter((model: any) => model.type === 'chat'); + + return data.map((m: any) => ({ + name: m.id, + label: `${m.display_name} - in:$${m.pricing.input.toFixed(2)} out:$${m.pricing.output.toFixed(2)} - context ${Math.floor(m.context_length / 1000)}k`, + provider: this.name, + maxTokenAllowed: 8000, + })); + } catch (error: any) { + console.error('Error getting Together models:', error.message); + return []; + } + } + + getModelInstance(options: { + model: string; + serverEnv: Env; + apiKeys?: Record; + providerSettings?: Record; + }): LanguageModelV1 { + const { model, serverEnv, apiKeys, providerSettings } = options; + + const { baseUrl, apiKey } = this.getProviderBaseUrlAndKey({ + apiKeys, + providerSettings: providerSettings?.[this.name], + serverEnv: serverEnv as any, + defaultBaseUrlKey: 'TOGETHER_API_BASE_URL', + defaultApiTokenKey: 'TOGETHER_API_KEY', + }); + + if (!baseUrl || !apiKey) { + throw new Error(`Missing configuration for ${this.name} provider`); + } + + return getOpenAILikeModel(baseUrl, apiKey, model); + } +} diff --git a/app/lib/modules/llm/providers/xai.ts b/app/lib/modules/llm/providers/xai.ts new file mode 100644 index 0000000..032b01b --- /dev/null +++ b/app/lib/modules/llm/providers/xai.ts @@ -0,0 +1,47 @@ +import { BaseProvider } from '~/lib/modules/llm/base-provider'; +import type { ModelInfo } from '~/lib/modules/llm/types'; +import type { IProviderSetting } from '~/types/model'; +import type { LanguageModelV1 } from 'ai'; +import { createOpenAI } from '@ai-sdk/openai'; + +export default class XAIProvider extends BaseProvider { + name = 'xAI'; + getApiKeyLink = 'https://docs.x.ai/docs/quickstart#creating-an-api-key'; + + config = { + apiTokenKey: 'XAI_API_KEY', + }; + + staticModels: ModelInfo[] = [ + { name: 'grok-beta', label: 'xAI Grok Beta', provider: 'xAI', maxTokenAllowed: 8000 }, + { name: 'grok-2-1212', label: 'xAI Grok2 1212', provider: 'xAI', maxTokenAllowed: 8000 }, + ]; + + getModelInstance(options: { + model: string; + serverEnv: Env; + apiKeys?: Record; + providerSettings?: Record; + }): LanguageModelV1 { + const { model, serverEnv, apiKeys, providerSettings } = options; + + const { apiKey } = this.getProviderBaseUrlAndKey({ + apiKeys, + providerSettings: providerSettings?.[this.name], + serverEnv: serverEnv as any, + defaultBaseUrlKey: '', + defaultApiTokenKey: 'XAI_API_KEY', + }); + + if (!apiKey) { + throw new Error(`Missing API key for ${this.name} provider`); + } + + const openai = createOpenAI({ + baseURL: 'https://api.x.ai/v1', + apiKey, + }); + + return openai(model); + } +} diff --git a/app/lib/modules/llm/registry.ts b/app/lib/modules/llm/registry.ts new file mode 100644 index 0000000..fb5a31f --- /dev/null +++ b/app/lib/modules/llm/registry.ts @@ -0,0 +1,33 @@ +import AnthropicProvider from './providers/anthropic'; +import CohereProvider from './providers/cohere'; +import DeepseekProvider from './providers/deepseek'; +import GoogleProvider from './providers/google'; +import GroqProvider from './providers/groq'; +import HuggingFaceProvider from './providers/huggingface'; +import LMStudioProvider from './providers/lmstudio'; +import MistralProvider from './providers/mistral'; +import OllamaProvider from './providers/ollama'; +import OpenRouterProvider from './providers/open-router'; +import OpenAILikeProvider from './providers/openai-like'; +import OpenAIProvider from './providers/openai'; +import PerplexityProvider from './providers/perplexity'; +import TogetherProvider from './providers/together'; +import XAIProvider from './providers/xai'; + +export { + AnthropicProvider, + CohereProvider, + DeepseekProvider, + GoogleProvider, + GroqProvider, + HuggingFaceProvider, + MistralProvider, + OllamaProvider, + OpenAIProvider, + OpenRouterProvider, + OpenAILikeProvider, + PerplexityProvider, + XAIProvider, + TogetherProvider, + LMStudioProvider, +}; diff --git a/app/lib/modules/llm/types.ts b/app/lib/modules/llm/types.ts new file mode 100644 index 0000000..337b110 --- /dev/null +++ b/app/lib/modules/llm/types.ts @@ -0,0 +1,32 @@ +import type { LanguageModelV1 } from 'ai'; +import type { IProviderSetting } from '~/types/model'; + +export interface ModelInfo { + name: string; + label: string; + provider: string; + maxTokenAllowed: number; +} + +export interface ProviderInfo { + name: string; + staticModels: ModelInfo[]; + getDynamicModels?: ( + apiKeys?: Record, + settings?: IProviderSetting, + serverEnv?: Record, + ) => Promise; + getModelInstance: (options: { + model: string; + serverEnv: Env; + apiKeys?: Record; + providerSettings?: Record; + }) => LanguageModelV1; + getApiKeyLink?: string; + labelForGetApiKey?: string; + icon?: string; +} +export interface ProviderConfig { + baseUrlKey?: string; + apiTokenKey?: string; +} diff --git a/app/types/model.ts b/app/types/model.ts index b449363..d16b10a 100644 --- a/app/types/model.ts +++ b/app/types/model.ts @@ -1,4 +1,4 @@ -import type { ModelInfo } from '~/utils/types'; +import type { ModelInfo } from '~/lib/modules/llm/types'; export type ProviderInfo = { staticModels: ModelInfo[]; diff --git a/app/utils/constants.ts b/app/utils/constants.ts index dca3320..64e08a7 100644 --- a/app/utils/constants.ts +++ b/app/utils/constants.ts @@ -1,8 +1,7 @@ -import Cookies from 'js-cookie'; -import type { ModelInfo, OllamaApiResponse, OllamaModel } from './types'; -import type { ProviderInfo, IProviderSetting } from '~/types/model'; -import { createScopedLogger } from './logger'; -import { logStore } from '~/lib/stores/logs'; +import type { IProviderSetting } from '~/types/model'; + +import { LLMManager } from '~/lib/modules/llm/manager'; +import type { ModelInfo } from '~/lib/modules/llm/types'; export const WORK_DIR_NAME = 'project'; export const WORK_DIR = `/home/${WORK_DIR_NAME}`; @@ -12,619 +11,334 @@ export const PROVIDER_REGEX = /\[Provider: (.*?)\]\n\n/; export const DEFAULT_MODEL = 'claude-3-5-sonnet-latest'; export const PROMPT_COOKIE_KEY = 'cachedPrompt'; -const logger = createScopedLogger('Constants'); +const llmManager = LLMManager.getInstance(import.meta.env); -const PROVIDER_LIST: ProviderInfo[] = [ - { - name: 'Anthropic', - staticModels: [ - { - name: 'claude-3-5-sonnet-latest', - label: 'Claude 3.5 Sonnet (new)', - provider: 'Anthropic', - maxTokenAllowed: 8000, - }, - { - name: 'claude-3-5-sonnet-20240620', - label: 'Claude 3.5 Sonnet (old)', - provider: 'Anthropic', - maxTokenAllowed: 8000, - }, - { - name: 'claude-3-5-haiku-latest', - label: 'Claude 3.5 Haiku (new)', - provider: 'Anthropic', - maxTokenAllowed: 8000, - }, - { name: 'claude-3-opus-latest', label: 'Claude 3 Opus', provider: 'Anthropic', maxTokenAllowed: 8000 }, - { name: 'claude-3-sonnet-20240229', label: 'Claude 3 Sonnet', provider: 'Anthropic', maxTokenAllowed: 8000 }, - { name: 'claude-3-haiku-20240307', label: 'Claude 3 Haiku', provider: 'Anthropic', maxTokenAllowed: 8000 }, - ], - getApiKeyLink: 'https://console.anthropic.com/settings/keys', - }, - { - name: 'Ollama', - staticModels: [], - getDynamicModels: getOllamaModels, - getApiKeyLink: 'https://ollama.com/download', - labelForGetApiKey: 'Download Ollama', - icon: 'i-ph:cloud-arrow-down', - }, - { - name: 'OpenAILike', - staticModels: [], - getDynamicModels: getOpenAILikeModels, - }, - { - name: 'Cohere', - staticModels: [ - { name: 'command-r-plus-08-2024', label: 'Command R plus Latest', provider: 'Cohere', maxTokenAllowed: 4096 }, - { name: 'command-r-08-2024', label: 'Command R Latest', provider: 'Cohere', maxTokenAllowed: 4096 }, - { name: 'command-r-plus', label: 'Command R plus', provider: 'Cohere', maxTokenAllowed: 4096 }, - { name: 'command-r', label: 'Command R', provider: 'Cohere', maxTokenAllowed: 4096 }, - { name: 'command', label: 'Command', provider: 'Cohere', maxTokenAllowed: 4096 }, - { name: 'command-nightly', label: 'Command Nightly', provider: 'Cohere', maxTokenAllowed: 4096 }, - { name: 'command-light', label: 'Command Light', provider: 'Cohere', maxTokenAllowed: 4096 }, - { name: 'command-light-nightly', label: 'Command Light Nightly', provider: 'Cohere', maxTokenAllowed: 4096 }, - { name: 'c4ai-aya-expanse-8b', label: 'c4AI Aya Expanse 8b', provider: 'Cohere', maxTokenAllowed: 4096 }, - { name: 'c4ai-aya-expanse-32b', label: 'c4AI Aya Expanse 32b', provider: 'Cohere', maxTokenAllowed: 4096 }, - ], - getApiKeyLink: 'https://dashboard.cohere.com/api-keys', - }, - { - name: 'OpenRouter', - staticModels: [ - { name: 'gpt-4o', label: 'GPT-4o', provider: 'OpenAI', maxTokenAllowed: 8000 }, - { - name: 'anthropic/claude-3.5-sonnet', - label: 'Anthropic: Claude 3.5 Sonnet (OpenRouter)', - provider: 'OpenRouter', - maxTokenAllowed: 8000, - }, - { - name: 'anthropic/claude-3-haiku', - label: 'Anthropic: Claude 3 Haiku (OpenRouter)', - provider: 'OpenRouter', - maxTokenAllowed: 8000, - }, - { - name: 'deepseek/deepseek-coder', - label: 'Deepseek-Coder V2 236B (OpenRouter)', - provider: 'OpenRouter', - maxTokenAllowed: 8000, - }, - { - name: 'google/gemini-flash-1.5', - label: 'Google Gemini Flash 1.5 (OpenRouter)', - provider: 'OpenRouter', - maxTokenAllowed: 8000, - }, - { - name: 'google/gemini-pro-1.5', - label: 'Google Gemini Pro 1.5 (OpenRouter)', - provider: 'OpenRouter', - maxTokenAllowed: 8000, - }, - { name: 'x-ai/grok-beta', label: 'xAI Grok Beta (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 }, - { - name: 'mistralai/mistral-nemo', - label: 'OpenRouter Mistral Nemo (OpenRouter)', - provider: 'OpenRouter', - maxTokenAllowed: 8000, - }, - { - name: 'qwen/qwen-110b-chat', - label: 'OpenRouter Qwen 110b Chat (OpenRouter)', - provider: 'OpenRouter', - maxTokenAllowed: 8000, - }, - { name: 'cohere/command', label: 'Cohere Command (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 4096 }, - ], - getDynamicModels: getOpenRouterModels, - getApiKeyLink: 'https://openrouter.ai/settings/keys', - }, - { - name: 'Google', - staticModels: [ - { name: 'gemini-1.5-flash-latest', label: 'Gemini 1.5 Flash', provider: 'Google', maxTokenAllowed: 8192 }, - { name: 'gemini-2.0-flash-exp', label: 'Gemini 2.0 Flash', provider: 'Google', maxTokenAllowed: 8192 }, - { name: 'gemini-1.5-flash-002', label: 'Gemini 1.5 Flash-002', provider: 'Google', maxTokenAllowed: 8192 }, - { name: 'gemini-1.5-flash-8b', label: 'Gemini 1.5 Flash-8b', provider: 'Google', maxTokenAllowed: 8192 }, - { name: 'gemini-1.5-pro-latest', label: 'Gemini 1.5 Pro', provider: 'Google', maxTokenAllowed: 8192 }, - { name: 'gemini-1.5-pro-002', label: 'Gemini 1.5 Pro-002', provider: 'Google', maxTokenAllowed: 8192 }, - { name: 'gemini-exp-1206', label: 'Gemini exp-1206', provider: 'Google', maxTokenAllowed: 8192 }, - ], - getApiKeyLink: 'https://aistudio.google.com/app/apikey', - }, - { - name: 'Groq', - staticModels: [ - { name: 'llama-3.1-8b-instant', label: 'Llama 3.1 8b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, - { name: 'llama-3.2-11b-vision-preview', label: 'Llama 3.2 11b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, - { name: 'llama-3.2-90b-vision-preview', label: 'Llama 3.2 90b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, - { name: 'llama-3.2-3b-preview', label: 'Llama 3.2 3b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, - { name: 'llama-3.2-1b-preview', label: 'Llama 3.2 1b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, - { name: 'llama-3.3-70b-versatile', label: 'Llama 3.3 70b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, - ], - getApiKeyLink: 'https://console.groq.com/keys', - }, - { - name: 'HuggingFace', - staticModels: [ - { - name: 'Qwen/Qwen2.5-Coder-32B-Instruct', - label: 'Qwen2.5-Coder-32B-Instruct (HuggingFace)', - provider: 'HuggingFace', - maxTokenAllowed: 8000, - }, - { - name: '01-ai/Yi-1.5-34B-Chat', - label: 'Yi-1.5-34B-Chat (HuggingFace)', - provider: 'HuggingFace', - maxTokenAllowed: 8000, - }, - { - name: 'codellama/CodeLlama-34b-Instruct-hf', - label: 'CodeLlama-34b-Instruct (HuggingFace)', - provider: 'HuggingFace', - maxTokenAllowed: 8000, - }, - { - name: 'NousResearch/Hermes-3-Llama-3.1-8B', - label: 'Hermes-3-Llama-3.1-8B (HuggingFace)', - provider: 'HuggingFace', - maxTokenAllowed: 8000, - }, - { - name: 'Qwen/Qwen2.5-Coder-32B-Instruct', - label: 'Qwen2.5-Coder-32B-Instruct (HuggingFace)', - provider: 'HuggingFace', - maxTokenAllowed: 8000, - }, - { - name: 'Qwen/Qwen2.5-72B-Instruct', - label: 'Qwen2.5-72B-Instruct (HuggingFace)', - provider: 'HuggingFace', - maxTokenAllowed: 8000, - }, - { - name: 'meta-llama/Llama-3.1-70B-Instruct', - label: 'Llama-3.1-70B-Instruct (HuggingFace)', - provider: 'HuggingFace', - maxTokenAllowed: 8000, - }, - { - name: 'meta-llama/Llama-3.1-405B', - label: 'Llama-3.1-405B (HuggingFace)', - provider: 'HuggingFace', - maxTokenAllowed: 8000, - }, - { - name: '01-ai/Yi-1.5-34B-Chat', - label: 'Yi-1.5-34B-Chat (HuggingFace)', - provider: 'HuggingFace', - maxTokenAllowed: 8000, - }, - { - name: 'codellama/CodeLlama-34b-Instruct-hf', - label: 'CodeLlama-34b-Instruct (HuggingFace)', - provider: 'HuggingFace', - maxTokenAllowed: 8000, - }, - { - name: 'NousResearch/Hermes-3-Llama-3.1-8B', - label: 'Hermes-3-Llama-3.1-8B (HuggingFace)', - provider: 'HuggingFace', - maxTokenAllowed: 8000, - }, - ], - getApiKeyLink: 'https://huggingface.co/settings/tokens', - }, - { - name: 'OpenAI', - staticModels: [ - { name: 'gpt-4o-mini', label: 'GPT-4o Mini', provider: 'OpenAI', maxTokenAllowed: 8000 }, - { name: 'gpt-4-turbo', label: 'GPT-4 Turbo', provider: 'OpenAI', maxTokenAllowed: 8000 }, - { name: 'gpt-4', label: 'GPT-4', provider: 'OpenAI', maxTokenAllowed: 8000 }, - { name: 'gpt-3.5-turbo', label: 'GPT-3.5 Turbo', provider: 'OpenAI', maxTokenAllowed: 8000 }, - ], - getApiKeyLink: 'https://platform.openai.com/api-keys', - }, - { - name: 'xAI', - staticModels: [ - { name: 'grok-beta', label: 'xAI Grok Beta', provider: 'xAI', maxTokenAllowed: 8000 }, - { name: 'grok-2-1212', label: 'xAI Grok2 1212', provider: 'xAI', maxTokenAllowed: 8000 }, - ], - getApiKeyLink: 'https://docs.x.ai/docs/quickstart#creating-an-api-key', - }, - { - name: 'Deepseek', - staticModels: [ - { name: 'deepseek-coder', label: 'Deepseek-Coder', provider: 'Deepseek', maxTokenAllowed: 8000 }, - { name: 'deepseek-chat', label: 'Deepseek-Chat', provider: 'Deepseek', maxTokenAllowed: 8000 }, - ], - getApiKeyLink: 'https://platform.deepseek.com/apiKeys', - }, - { - name: 'Mistral', - staticModels: [ - { name: 'open-mistral-7b', label: 'Mistral 7B', provider: 'Mistral', maxTokenAllowed: 8000 }, - { name: 'open-mixtral-8x7b', label: 'Mistral 8x7B', provider: 'Mistral', maxTokenAllowed: 8000 }, - { name: 'open-mixtral-8x22b', label: 'Mistral 8x22B', provider: 'Mistral', maxTokenAllowed: 8000 }, - { name: 'open-codestral-mamba', label: 'Codestral Mamba', provider: 'Mistral', maxTokenAllowed: 8000 }, - { name: 'open-mistral-nemo', label: 'Mistral Nemo', provider: 'Mistral', maxTokenAllowed: 8000 }, - { name: 'ministral-8b-latest', label: 'Mistral 8B', provider: 'Mistral', maxTokenAllowed: 8000 }, - { name: 'mistral-small-latest', label: 'Mistral Small', provider: 'Mistral', maxTokenAllowed: 8000 }, - { name: 'codestral-latest', label: 'Codestral', provider: 'Mistral', maxTokenAllowed: 8000 }, - { name: 'mistral-large-latest', label: 'Mistral Large Latest', provider: 'Mistral', maxTokenAllowed: 8000 }, - ], - getApiKeyLink: 'https://console.mistral.ai/api-keys/', - }, - { - name: 'LMStudio', - staticModels: [], - getDynamicModels: getLMStudioModels, - getApiKeyLink: 'https://lmstudio.ai/', - labelForGetApiKey: 'Get LMStudio', - icon: 'i-ph:cloud-arrow-down', - }, - { - name: 'Together', - getDynamicModels: getTogetherModels, - staticModels: [ - { - name: 'Qwen/Qwen2.5-Coder-32B-Instruct', - label: 'Qwen/Qwen2.5-Coder-32B-Instruct', - provider: 'Together', - maxTokenAllowed: 8000, - }, - { - name: 'meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo', - label: 'meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo', - provider: 'Together', - maxTokenAllowed: 8000, - }, +export const PROVIDER_LIST = llmManager.getAllProviders(); +export const DEFAULT_PROVIDER = llmManager.getDefaultProvider(); - { - name: 'mistralai/Mixtral-8x7B-Instruct-v0.1', - label: 'Mixtral 8x7B Instruct', - provider: 'Together', - maxTokenAllowed: 8192, - }, - ], - getApiKeyLink: 'https://api.together.xyz/settings/api-keys', - }, - { - name: 'Perplexity', - staticModels: [ - { - name: 'llama-3.1-sonar-small-128k-online', - label: 'Sonar Small Online', - provider: 'Perplexity', - maxTokenAllowed: 8192, - }, - { - name: 'llama-3.1-sonar-large-128k-online', - label: 'Sonar Large Online', - provider: 'Perplexity', - maxTokenAllowed: 8192, - }, - { - name: 'llama-3.1-sonar-huge-128k-online', - label: 'Sonar Huge Online', - provider: 'Perplexity', - maxTokenAllowed: 8192, - }, - ], - getApiKeyLink: 'https://www.perplexity.ai/settings/api', - }, -]; +let MODEL_LIST = llmManager.getModelList(); -export const providerBaseUrlEnvKeys: Record = { - Anthropic: { - apiTokenKey: 'ANTHROPIC_API_KEY', - }, - OpenAI: { - apiTokenKey: 'OPENAI_API_KEY', - }, - Groq: { - apiTokenKey: 'GROQ_API_KEY', - }, - HuggingFace: { - apiTokenKey: 'HuggingFace_API_KEY', - }, - OpenRouter: { - apiTokenKey: 'OPEN_ROUTER_API_KEY', - }, - Google: { - apiTokenKey: 'GOOGLE_GENERATIVE_AI_API_KEY', - }, - OpenAILike: { - baseUrlKey: 'OPENAI_LIKE_API_BASE_URL', - apiTokenKey: 'OPENAI_LIKE_API_KEY', - }, - Together: { - baseUrlKey: 'TOGETHER_API_BASE_URL', - apiTokenKey: 'TOGETHER_API_KEY', - }, - Deepseek: { - apiTokenKey: 'DEEPSEEK_API_KEY', - }, - Mistral: { - apiTokenKey: 'MISTRAL_API_KEY', - }, - LMStudio: { - baseUrlKey: 'LMSTUDIO_API_BASE_URL', - }, - xAI: { - apiTokenKey: 'XAI_API_KEY', - }, - Cohere: { - apiTokenKey: 'COHERE_API_KEY', - }, - Perplexity: { - apiTokenKey: 'PERPLEXITY_API_KEY', - }, - Ollama: { - baseUrlKey: 'OLLAMA_API_BASE_URL', - }, -}; +/* + *const PROVIDER_LIST_OLD: ProviderInfo[] = [ + * { + * name: 'Anthropic', + * staticModels: [ + * { + * name: 'claude-3-5-sonnet-latest', + * label: 'Claude 3.5 Sonnet (new)', + * provider: 'Anthropic', + * maxTokenAllowed: 8000, + * }, + * { + * name: 'claude-3-5-sonnet-20240620', + * label: 'Claude 3.5 Sonnet (old)', + * provider: 'Anthropic', + * maxTokenAllowed: 8000, + * }, + * { + * name: 'claude-3-5-haiku-latest', + * label: 'Claude 3.5 Haiku (new)', + * provider: 'Anthropic', + * maxTokenAllowed: 8000, + * }, + * { name: 'claude-3-opus-latest', label: 'Claude 3 Opus', provider: 'Anthropic', maxTokenAllowed: 8000 }, + * { name: 'claude-3-sonnet-20240229', label: 'Claude 3 Sonnet', provider: 'Anthropic', maxTokenAllowed: 8000 }, + * { name: 'claude-3-haiku-20240307', label: 'Claude 3 Haiku', provider: 'Anthropic', maxTokenAllowed: 8000 }, + * ], + * getApiKeyLink: 'https://console.anthropic.com/settings/keys', + * }, + * { + * name: 'Ollama', + * staticModels: [], + * getDynamicModels: getOllamaModels, + * getApiKeyLink: 'https://ollama.com/download', + * labelForGetApiKey: 'Download Ollama', + * icon: 'i-ph:cloud-arrow-down', + * }, + * { + * name: 'OpenAILike', + * staticModels: [], + * getDynamicModels: getOpenAILikeModels, + * }, + * { + * name: 'Cohere', + * staticModels: [ + * { name: 'command-r-plus-08-2024', label: 'Command R plus Latest', provider: 'Cohere', maxTokenAllowed: 4096 }, + * { name: 'command-r-08-2024', label: 'Command R Latest', provider: 'Cohere', maxTokenAllowed: 4096 }, + * { name: 'command-r-plus', label: 'Command R plus', provider: 'Cohere', maxTokenAllowed: 4096 }, + * { name: 'command-r', label: 'Command R', provider: 'Cohere', maxTokenAllowed: 4096 }, + * { name: 'command', label: 'Command', provider: 'Cohere', maxTokenAllowed: 4096 }, + * { name: 'command-nightly', label: 'Command Nightly', provider: 'Cohere', maxTokenAllowed: 4096 }, + * { name: 'command-light', label: 'Command Light', provider: 'Cohere', maxTokenAllowed: 4096 }, + * { name: 'command-light-nightly', label: 'Command Light Nightly', provider: 'Cohere', maxTokenAllowed: 4096 }, + * { name: 'c4ai-aya-expanse-8b', label: 'c4AI Aya Expanse 8b', provider: 'Cohere', maxTokenAllowed: 4096 }, + * { name: 'c4ai-aya-expanse-32b', label: 'c4AI Aya Expanse 32b', provider: 'Cohere', maxTokenAllowed: 4096 }, + * ], + * getApiKeyLink: 'https://dashboard.cohere.com/api-keys', + * }, + * { + * name: 'OpenRouter', + * staticModels: [ + * { name: 'gpt-4o', label: 'GPT-4o', provider: 'OpenAI', maxTokenAllowed: 8000 }, + * { + * name: 'anthropic/claude-3.5-sonnet', + * label: 'Anthropic: Claude 3.5 Sonnet (OpenRouter)', + * provider: 'OpenRouter', + * maxTokenAllowed: 8000, + * }, + * { + * name: 'anthropic/claude-3-haiku', + * label: 'Anthropic: Claude 3 Haiku (OpenRouter)', + * provider: 'OpenRouter', + * maxTokenAllowed: 8000, + * }, + * { + * name: 'deepseek/deepseek-coder', + * label: 'Deepseek-Coder V2 236B (OpenRouter)', + * provider: 'OpenRouter', + * maxTokenAllowed: 8000, + * }, + * { + * name: 'google/gemini-flash-1.5', + * label: 'Google Gemini Flash 1.5 (OpenRouter)', + * provider: 'OpenRouter', + * maxTokenAllowed: 8000, + * }, + * { + * name: 'google/gemini-pro-1.5', + * label: 'Google Gemini Pro 1.5 (OpenRouter)', + * provider: 'OpenRouter', + * maxTokenAllowed: 8000, + * }, + * { name: 'x-ai/grok-beta', label: 'xAI Grok Beta (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 }, + * { + * name: 'mistralai/mistral-nemo', + * label: 'OpenRouter Mistral Nemo (OpenRouter)', + * provider: 'OpenRouter', + * maxTokenAllowed: 8000, + * }, + * { + * name: 'qwen/qwen-110b-chat', + * label: 'OpenRouter Qwen 110b Chat (OpenRouter)', + * provider: 'OpenRouter', + * maxTokenAllowed: 8000, + * }, + * { name: 'cohere/command', label: 'Cohere Command (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 4096 }, + * ], + * getDynamicModels: getOpenRouterModels, + * getApiKeyLink: 'https://openrouter.ai/settings/keys', + * }, + * { + * name: 'Google', + * staticModels: [ + * { name: 'gemini-1.5-flash-latest', label: 'Gemini 1.5 Flash', provider: 'Google', maxTokenAllowed: 8192 }, + * { name: 'gemini-2.0-flash-exp', label: 'Gemini 2.0 Flash', provider: 'Google', maxTokenAllowed: 8192 }, + * { name: 'gemini-1.5-flash-002', label: 'Gemini 1.5 Flash-002', provider: 'Google', maxTokenAllowed: 8192 }, + * { name: 'gemini-1.5-flash-8b', label: 'Gemini 1.5 Flash-8b', provider: 'Google', maxTokenAllowed: 8192 }, + * { name: 'gemini-1.5-pro-latest', label: 'Gemini 1.5 Pro', provider: 'Google', maxTokenAllowed: 8192 }, + * { name: 'gemini-1.5-pro-002', label: 'Gemini 1.5 Pro-002', provider: 'Google', maxTokenAllowed: 8192 }, + * { name: 'gemini-exp-1206', label: 'Gemini exp-1206', provider: 'Google', maxTokenAllowed: 8192 }, + * ], + * getApiKeyLink: 'https://aistudio.google.com/app/apikey', + * }, + * { + * name: 'Groq', + * staticModels: [ + * { name: 'llama-3.1-8b-instant', label: 'Llama 3.1 8b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, + * { name: 'llama-3.2-11b-vision-preview', label: 'Llama 3.2 11b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, + * { name: 'llama-3.2-90b-vision-preview', label: 'Llama 3.2 90b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, + * { name: 'llama-3.2-3b-preview', label: 'Llama 3.2 3b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, + * { name: 'llama-3.2-1b-preview', label: 'Llama 3.2 1b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, + * { name: 'llama-3.3-70b-versatile', label: 'Llama 3.3 70b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, + * ], + * getApiKeyLink: 'https://console.groq.com/keys', + * }, + * { + * name: 'HuggingFace', + * staticModels: [ + * { + * name: 'Qwen/Qwen2.5-Coder-32B-Instruct', + * label: 'Qwen2.5-Coder-32B-Instruct (HuggingFace)', + * provider: 'HuggingFace', + * maxTokenAllowed: 8000, + * }, + * { + * name: '01-ai/Yi-1.5-34B-Chat', + * label: 'Yi-1.5-34B-Chat (HuggingFace)', + * provider: 'HuggingFace', + * maxTokenAllowed: 8000, + * }, + * { + * name: 'codellama/CodeLlama-34b-Instruct-hf', + * label: 'CodeLlama-34b-Instruct (HuggingFace)', + * provider: 'HuggingFace', + * maxTokenAllowed: 8000, + * }, + * { + * name: 'NousResearch/Hermes-3-Llama-3.1-8B', + * label: 'Hermes-3-Llama-3.1-8B (HuggingFace)', + * provider: 'HuggingFace', + * maxTokenAllowed: 8000, + * }, + * { + * name: 'Qwen/Qwen2.5-Coder-32B-Instruct', + * label: 'Qwen2.5-Coder-32B-Instruct (HuggingFace)', + * provider: 'HuggingFace', + * maxTokenAllowed: 8000, + * }, + * { + * name: 'Qwen/Qwen2.5-72B-Instruct', + * label: 'Qwen2.5-72B-Instruct (HuggingFace)', + * provider: 'HuggingFace', + * maxTokenAllowed: 8000, + * }, + * { + * name: 'meta-llama/Llama-3.1-70B-Instruct', + * label: 'Llama-3.1-70B-Instruct (HuggingFace)', + * provider: 'HuggingFace', + * maxTokenAllowed: 8000, + * }, + * { + * name: 'meta-llama/Llama-3.1-405B', + * label: 'Llama-3.1-405B (HuggingFace)', + * provider: 'HuggingFace', + * maxTokenAllowed: 8000, + * }, + * { + * name: '01-ai/Yi-1.5-34B-Chat', + * label: 'Yi-1.5-34B-Chat (HuggingFace)', + * provider: 'HuggingFace', + * maxTokenAllowed: 8000, + * }, + * { + * name: 'codellama/CodeLlama-34b-Instruct-hf', + * label: 'CodeLlama-34b-Instruct (HuggingFace)', + * provider: 'HuggingFace', + * maxTokenAllowed: 8000, + * }, + * { + * name: 'NousResearch/Hermes-3-Llama-3.1-8B', + * label: 'Hermes-3-Llama-3.1-8B (HuggingFace)', + * provider: 'HuggingFace', + * maxTokenAllowed: 8000, + * }, + * ], + * getApiKeyLink: 'https://huggingface.co/settings/tokens', + * }, + * { + * name: 'OpenAI', + * staticModels: [ + * { name: 'gpt-4o-mini', label: 'GPT-4o Mini', provider: 'OpenAI', maxTokenAllowed: 8000 }, + * { name: 'gpt-4-turbo', label: 'GPT-4 Turbo', provider: 'OpenAI', maxTokenAllowed: 8000 }, + * { name: 'gpt-4', label: 'GPT-4', provider: 'OpenAI', maxTokenAllowed: 8000 }, + * { name: 'gpt-3.5-turbo', label: 'GPT-3.5 Turbo', provider: 'OpenAI', maxTokenAllowed: 8000 }, + * ], + * getApiKeyLink: 'https://platform.openai.com/api-keys', + * }, + * { + * name: 'xAI', + * staticModels: [{ name: 'grok-beta', label: 'xAI Grok Beta', provider: 'xAI', maxTokenAllowed: 8000 }], + * getApiKeyLink: 'https://docs.x.ai/docs/quickstart#creating-an-api-key', + * }, + * { + * name: 'Deepseek', + * staticModels: [ + * { name: 'deepseek-coder', label: 'Deepseek-Coder', provider: 'Deepseek', maxTokenAllowed: 8000 }, + * { name: 'deepseek-chat', label: 'Deepseek-Chat', provider: 'Deepseek', maxTokenAllowed: 8000 }, + * ], + * getApiKeyLink: 'https://platform.deepseek.com/apiKeys', + * }, + * { + * name: 'Mistral', + * staticModels: [ + * { name: 'open-mistral-7b', label: 'Mistral 7B', provider: 'Mistral', maxTokenAllowed: 8000 }, + * { name: 'open-mixtral-8x7b', label: 'Mistral 8x7B', provider: 'Mistral', maxTokenAllowed: 8000 }, + * { name: 'open-mixtral-8x22b', label: 'Mistral 8x22B', provider: 'Mistral', maxTokenAllowed: 8000 }, + * { name: 'open-codestral-mamba', label: 'Codestral Mamba', provider: 'Mistral', maxTokenAllowed: 8000 }, + * { name: 'open-mistral-nemo', label: 'Mistral Nemo', provider: 'Mistral', maxTokenAllowed: 8000 }, + * { name: 'ministral-8b-latest', label: 'Mistral 8B', provider: 'Mistral', maxTokenAllowed: 8000 }, + * { name: 'mistral-small-latest', label: 'Mistral Small', provider: 'Mistral', maxTokenAllowed: 8000 }, + * { name: 'codestral-latest', label: 'Codestral', provider: 'Mistral', maxTokenAllowed: 8000 }, + * { name: 'mistral-large-latest', label: 'Mistral Large Latest', provider: 'Mistral', maxTokenAllowed: 8000 }, + * ], + * getApiKeyLink: 'https://console.mistral.ai/api-keys/', + * }, + * { + * name: 'LMStudio', + * staticModels: [], + * getDynamicModels: getLMStudioModels, + * getApiKeyLink: 'https://lmstudio.ai/', + * labelForGetApiKey: 'Get LMStudio', + * icon: 'i-ph:cloud-arrow-down', + * }, + * { + * name: 'Together', + * getDynamicModels: getTogetherModels, + * staticModels: [ + * { + * name: 'Qwen/Qwen2.5-Coder-32B-Instruct', + * label: 'Qwen/Qwen2.5-Coder-32B-Instruct', + * provider: 'Together', + * maxTokenAllowed: 8000, + * }, + * { + * name: 'meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo', + * label: 'meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo', + * provider: 'Together', + * maxTokenAllowed: 8000, + * }, + * + * { + * name: 'mistralai/Mixtral-8x7B-Instruct-v0.1', + * label: 'Mixtral 8x7B Instruct', + * provider: 'Together', + * maxTokenAllowed: 8192, + * }, + * ], + * getApiKeyLink: 'https://api.together.xyz/settings/api-keys', + * }, + * { + * name: 'Perplexity', + * staticModels: [ + * { + * name: 'llama-3.1-sonar-small-128k-online', + * label: 'Sonar Small Online', + * provider: 'Perplexity', + * maxTokenAllowed: 8192, + * }, + * { + * name: 'llama-3.1-sonar-large-128k-online', + * label: 'Sonar Large Online', + * provider: 'Perplexity', + * maxTokenAllowed: 8192, + * }, + * { + * name: 'llama-3.1-sonar-huge-128k-online', + * label: 'Sonar Huge Online', + * provider: 'Perplexity', + * maxTokenAllowed: 8192, + * }, + * ], + * getApiKeyLink: 'https://www.perplexity.ai/settings/api', + * }, + *]; + */ -export const getProviderBaseUrlAndKey = (options: { - provider: string; - apiKeys?: Record; - providerSettings?: IProviderSetting; - serverEnv?: Record; - defaultBaseUrlKey: string; - defaultApiTokenKey: string; -}) => { - const { provider, apiKeys, providerSettings, serverEnv, defaultBaseUrlKey, defaultApiTokenKey } = options; - let settingsBaseUrl = providerSettings?.baseUrl; - - if (settingsBaseUrl && settingsBaseUrl.length == 0) { - settingsBaseUrl = undefined; - } - - const baseUrlKey = providerBaseUrlEnvKeys[provider]?.baseUrlKey || defaultBaseUrlKey; - const baseUrl = settingsBaseUrl || serverEnv?.[baseUrlKey] || process.env[baseUrlKey] || import.meta.env[baseUrlKey]; - - const apiTokenKey = providerBaseUrlEnvKeys[provider]?.apiTokenKey || defaultApiTokenKey; - const apiKey = - apiKeys?.[provider] || serverEnv?.[apiTokenKey] || process.env[apiTokenKey] || import.meta.env[apiTokenKey]; - - return { - baseUrl, - apiKey, +const providerBaseUrlEnvKeys: Record = {}; +PROVIDER_LIST.forEach((provider) => { + providerBaseUrlEnvKeys[provider.name] = { + baseUrlKey: provider.config.baseUrlKey, + apiTokenKey: provider.config.apiTokenKey, }; -}; -export const DEFAULT_PROVIDER = PROVIDER_LIST[0]; - -const staticModels: ModelInfo[] = PROVIDER_LIST.map((p) => p.staticModels).flat(); - -export let MODEL_LIST: ModelInfo[] = [...staticModels]; +}); +// Export the getModelList function using the manager export async function getModelList(options: { apiKeys?: Record; providerSettings?: Record; serverEnv?: Record; }) { - const { apiKeys, providerSettings, serverEnv } = options; - - MODEL_LIST = [ - ...( - await Promise.all( - PROVIDER_LIST.filter( - (p): p is ProviderInfo & { getDynamicModels: () => Promise } => !!p.getDynamicModels, - ).map((p) => p.getDynamicModels(p.name, apiKeys, providerSettings?.[p.name], serverEnv)), - ) - ).flat(), - ...staticModels, - ]; - - return MODEL_LIST; -} - -async function getTogetherModels( - name: string, - apiKeys?: Record, - settings?: IProviderSetting, - serverEnv: Record = {}, -): Promise { - try { - const { baseUrl, apiKey } = getProviderBaseUrlAndKey({ - provider: name, - apiKeys, - providerSettings: settings, - serverEnv, - defaultBaseUrlKey: 'TOGETHER_API_BASE_URL', - defaultApiTokenKey: 'TOGETHER_API_KEY', - }); - - if (!baseUrl) { - return []; - } - - if (!apiKey) { - return []; - } - - const response = await fetch(`${baseUrl}/models`, { - headers: { - Authorization: `Bearer ${apiKey}`, - }, - }); - const res = (await response.json()) as any; - const data: any[] = (res || []).filter((model: any) => model.type == 'chat'); - - return data.map((m: any) => ({ - name: m.id, - label: `${m.display_name} - in:$${m.pricing.input.toFixed( - 2, - )} out:$${m.pricing.output.toFixed(2)} - context ${Math.floor(m.context_length / 1000)}k`, - provider: name, - maxTokenAllowed: 8000, - })); - } catch (e) { - console.error('Error getting OpenAILike models:', e); - return []; - } -} - -const getOllamaBaseUrl = (name: string, settings?: IProviderSetting, serverEnv: Record = {}) => { - const { baseUrl } = getProviderBaseUrlAndKey({ - provider: name, - providerSettings: settings, - serverEnv, - defaultBaseUrlKey: 'OLLAMA_API_BASE_URL', - defaultApiTokenKey: '', - }); - - // Check if we're in the browser - if (typeof window !== 'undefined') { - // Frontend always uses localhost - return baseUrl; - } - - // Backend: Check if we're running in Docker - const isDocker = process.env.RUNNING_IN_DOCKER === 'true'; - - return isDocker ? baseUrl.replace('localhost', 'host.docker.internal') : baseUrl; -}; - -async function getOllamaModels( - name: string, - _apiKeys?: Record, - settings?: IProviderSetting, - serverEnv: Record = {}, -): Promise { - try { - const baseUrl = getOllamaBaseUrl(name, settings, serverEnv); - - if (!baseUrl) { - return []; - } - - const response = await fetch(`${baseUrl}/api/tags`); - const data = (await response.json()) as OllamaApiResponse; - - return data.models.map((model: OllamaModel) => ({ - name: model.name, - label: `${model.name} (${model.details.parameter_size})`, - provider: 'Ollama', - maxTokenAllowed: 8000, - })); - } catch (e: any) { - logStore.logError('Failed to get Ollama models', e, { baseUrl: settings?.baseUrl }); - logger.warn('Failed to get Ollama models: ', e.message || ''); - - return []; - } -} - -async function getOpenAILikeModels( - name: string, - apiKeys?: Record, - settings?: IProviderSetting, - serverEnv: Record = {}, -): Promise { - try { - const { baseUrl, apiKey } = getProviderBaseUrlAndKey({ - provider: name, - apiKeys, - providerSettings: settings, - serverEnv, - defaultBaseUrlKey: 'OPENAI_LIKE_API_BASE_URL', - defaultApiTokenKey: 'OPENAI_LIKE_API_KEY', - }); - - if (!baseUrl) { - return []; - } - - const response = await fetch(`${baseUrl}/models`, { - headers: { - Authorization: `Bearer ${apiKey}`, - }, - }); - const res = (await response.json()) as any; - - return res.data.map((model: any) => ({ - name: model.id, - label: model.id, - provider: name, - })); - } catch (e) { - console.error('Error getting OpenAILike models:', e); - return []; - } -} - -type OpenRouterModelsResponse = { - data: { - name: string; - id: string; - context_length: number; - pricing: { - prompt: number; - completion: number; - }; - }[]; -}; - -async function getOpenRouterModels(): Promise { - const data: OpenRouterModelsResponse = await ( - await fetch('https://openrouter.ai/api/v1/models', { - headers: { - 'Content-Type': 'application/json', - }, - }) - ).json(); - - return data.data - .sort((a, b) => a.name.localeCompare(b.name)) - .map((m) => ({ - name: m.id, - label: `${m.name} - in:$${(m.pricing.prompt * 1_000_000).toFixed( - 2, - )} out:$${(m.pricing.completion * 1_000_000).toFixed(2)} - context ${Math.floor(m.context_length / 1000)}k`, - provider: 'OpenRouter', - maxTokenAllowed: 8000, - })); -} - -async function getLMStudioModels( - name: string, - apiKeys?: Record, - settings?: IProviderSetting, - serverEnv: Record = {}, -): Promise { - try { - const { baseUrl } = getProviderBaseUrlAndKey({ - provider: name, - apiKeys, - providerSettings: settings, - serverEnv, - defaultBaseUrlKey: 'LMSTUDIO_API_BASE_URL', - defaultApiTokenKey: '', - }); - - if (!baseUrl) { - return []; - } - - const response = await fetch(`${baseUrl}/v1/models`); - const data = (await response.json()) as any; - - return data.data.map((model: any) => ({ - name: model.id, - label: model.id, - provider: 'LMStudio', - })); - } catch (e: any) { - logStore.logError('Failed to get LMStudio models', e, { baseUrl: settings?.baseUrl }); - return []; - } + return await llmManager.updateModelList(options); } async function initializeModelList(options: { @@ -632,46 +346,16 @@ async function initializeModelList(options: { providerSettings?: Record; apiKeys?: Record; }): Promise { - const { providerSettings, apiKeys: providedApiKeys, env } = options; - let apiKeys: Record = providedApiKeys || {}; + const { providerSettings, apiKeys, env } = options; + const list = await getModelList({ + apiKeys, + providerSettings, + serverEnv: env, + }); + MODEL_LIST = list || MODEL_LIST; - if (!providedApiKeys) { - try { - const storedApiKeys = Cookies.get('apiKeys'); - - if (storedApiKeys) { - const parsedKeys = JSON.parse(storedApiKeys); - - if (typeof parsedKeys === 'object' && parsedKeys !== null) { - apiKeys = parsedKeys; - } - } - } catch (error: any) { - logStore.logError('Failed to fetch API keys from cookies', error); - logger.warn(`Failed to fetch apikeys from cookies: ${error?.message}`); - } - } - - MODEL_LIST = [ - ...( - await Promise.all( - PROVIDER_LIST.filter( - (p): p is ProviderInfo & { getDynamicModels: () => Promise } => !!p.getDynamicModels, - ).map((p) => p.getDynamicModels(p.name, apiKeys, providerSettings?.[p.name], env)), - ) - ).flat(), - ...staticModels, - ]; - - return MODEL_LIST; + return list; } // initializeModelList({}) -export { - getOllamaModels, - getOpenAILikeModels, - getLMStudioModels, - initializeModelList, - getOpenRouterModels, - PROVIDER_LIST, -}; +export { initializeModelList, providerBaseUrlEnvKeys, MODEL_LIST }; diff --git a/app/utils/types.ts b/app/utils/types.ts index 1fa253f..5f8bc39 100644 --- a/app/utils/types.ts +++ b/app/utils/types.ts @@ -19,10 +19,3 @@ export interface OllamaModel { export interface OllamaApiResponse { models: OllamaModel[]; } - -export interface ModelInfo { - name: string; - label: string; - provider: string; - maxTokenAllowed: number; -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index efec898..da73295 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11957,4 +11957,4 @@ snapshots: zod@3.23.8: {} - zwitch@2.0.4: {} + zwitch@2.0.4: {} \ No newline at end of file diff --git a/worker-configuration.d.ts b/worker-configuration.d.ts index ed2afca..fb5157d 100644 --- a/worker-configuration.d.ts +++ b/worker-configuration.d.ts @@ -1,4 +1,5 @@ interface Env { + DEFAULT_NUM_CTX:Settings; ANTHROPIC_API_KEY: string; OPENAI_API_KEY: string; GROQ_API_KEY: string;