diff --git a/app/commit.json b/app/commit.json index b9c669a..4ff5294 100644 --- a/app/commit.json +++ b/app/commit.json @@ -1 +1 @@ -{ "commit": "eb6d4353565be31c6e20bfca2c5aea29e4f45b6d", "version": "0.0.3" } +{ "commit": "fce8999f27c0affbc762dc90de992b5a759ab325" } diff --git a/app/components/chat/BaseChat.tsx b/app/components/chat/BaseChat.tsx index 2084cbb..5db6653 100644 --- a/app/components/chat/BaseChat.tsx +++ b/app/components/chat/BaseChat.tsx @@ -119,6 +119,9 @@ export const BaseChat = React.forwardRef( useEffect(() => { // Load API keys from cookies on component mount + + let parsedApiKeys: Record | undefined = {}; + try { const storedApiKeys = Cookies.get('apiKeys'); @@ -127,6 +130,7 @@ export const BaseChat = React.forwardRef( if (typeof parsedKeys === 'object' && parsedKeys !== null) { setApiKeys(parsedKeys); + parsedApiKeys = parsedKeys; } } } catch (error) { @@ -155,7 +159,7 @@ export const BaseChat = React.forwardRef( Cookies.remove('providers'); } - initializeModelList(providerSettings).then((modelList) => { + initializeModelList({ apiKeys: parsedApiKeys, providerSettings }).then((modelList) => { setModelList(modelList); }); diff --git a/app/components/settings/providers/ProvidersTab.tsx b/app/components/settings/providers/ProvidersTab.tsx index 281b4c8..49a16f6 100644 --- a/app/components/settings/providers/ProvidersTab.tsx +++ b/app/components/settings/providers/ProvidersTab.tsx @@ -87,7 +87,12 @@ export default function ProvidersTab() { type="text" value={provider.settings.baseUrl || ''} onChange={(e) => { - const newBaseUrl = e.target.value; + let newBaseUrl: string | undefined = e.target.value; + + if (newBaseUrl && newBaseUrl.trim().length === 0) { + newBaseUrl = undefined; + } + updateProviderSettings(provider.name, { ...provider.settings, baseUrl: newBaseUrl }); logStore.logProvider(`Base URL updated for ${provider.name}`, { provider: provider.name, diff --git a/app/entry.server.tsx b/app/entry.server.tsx index a44917f..5e92d21 100644 --- a/app/entry.server.tsx +++ b/app/entry.server.tsx @@ -14,7 +14,7 @@ export default async function handleRequest( remixContext: EntryContext, _loadContext: AppLoadContext, ) { - await initializeModelList(); + await initializeModelList({}); const readable = await renderToReadableStream(, { signal: request.signal, diff --git a/app/lib/.server/llm/api-key.ts b/app/lib/.server/llm/api-key.ts index e82d08e..d21f070 100644 --- a/app/lib/.server/llm/api-key.ts +++ b/app/lib/.server/llm/api-key.ts @@ -3,6 +3,7 @@ * Preventing TS checks with files presented in the video for a better presentation. */ import { env } from 'node:process'; +import type { IProviderSetting } from '~/types/model'; export function getAPIKey(cloudflareEnv: Env, provider: string, userApiKeys?: Record) { /** @@ -50,16 +51,30 @@ export function getAPIKey(cloudflareEnv: Env, provider: string, userApiKeys?: Re } } -export function getBaseURL(cloudflareEnv: Env, provider: string) { +export function getBaseURL(cloudflareEnv: Env, provider: string, providerSettings?: Record) { + let settingBaseUrl = providerSettings?.[provider].baseUrl; + + if (settingBaseUrl && settingBaseUrl.length == 0) { + settingBaseUrl = undefined; + } + switch (provider) { case 'Together': - return env.TOGETHER_API_BASE_URL || cloudflareEnv.TOGETHER_API_BASE_URL || 'https://api.together.xyz/v1'; + return ( + settingBaseUrl || + env.TOGETHER_API_BASE_URL || + cloudflareEnv.TOGETHER_API_BASE_URL || + 'https://api.together.xyz/v1' + ); case 'OpenAILike': - return env.OPENAI_LIKE_API_BASE_URL || cloudflareEnv.OPENAI_LIKE_API_BASE_URL; + return settingBaseUrl || env.OPENAI_LIKE_API_BASE_URL || cloudflareEnv.OPENAI_LIKE_API_BASE_URL; case 'LMStudio': - return env.LMSTUDIO_API_BASE_URL || cloudflareEnv.LMSTUDIO_API_BASE_URL || 'http://localhost:1234'; + return ( + settingBaseUrl || env.LMSTUDIO_API_BASE_URL || cloudflareEnv.LMSTUDIO_API_BASE_URL || 'http://localhost:1234' + ); case 'Ollama': { - let baseUrl = env.OLLAMA_API_BASE_URL || cloudflareEnv.OLLAMA_API_BASE_URL || 'http://localhost:11434'; + let baseUrl = + settingBaseUrl || env.OLLAMA_API_BASE_URL || cloudflareEnv.OLLAMA_API_BASE_URL || 'http://localhost:11434'; if (env.RUNNING_IN_DOCKER === 'true') { baseUrl = baseUrl.replace('localhost', 'host.docker.internal'); diff --git a/app/lib/.server/llm/model.ts b/app/lib/.server/llm/model.ts index 1a5aab7..1feb499 100644 --- a/app/lib/.server/llm/model.ts +++ b/app/lib/.server/llm/model.ts @@ -84,6 +84,8 @@ export function getHuggingFaceModel(apiKey: OptionalApiKey, model: string) { } export function getOllamaModel(baseURL: string, model: string) { + console.log({ baseURL, model }); + const ollamaInstance = ollama(model, { numCtx: DEFAULT_NUM_CTX, }) as LanguageModelV1 & { config: any }; @@ -140,7 +142,7 @@ export function getPerplexityModel(apiKey: OptionalApiKey, model: string) { export function getModel( provider: string, model: string, - env: Env, + serverEnv: Env, apiKeys?: Record, providerSettings?: Record, ) { @@ -148,9 +150,12 @@ export function getModel( * let apiKey; // Declare first * let baseURL; */ + // console.log({provider,model}); - const apiKey = getAPIKey(env, provider, apiKeys); // Then assign - const baseURL = providerSettings?.[provider].baseUrl || getBaseURL(env, provider); + const apiKey = getAPIKey(serverEnv, provider, apiKeys); // Then assign + const baseURL = getBaseURL(serverEnv, provider, providerSettings); + + // console.log({apiKey,baseURL}); switch (provider) { case 'Anthropic': diff --git a/app/lib/.server/llm/stream-text.ts b/app/lib/.server/llm/stream-text.ts index 74cdd9d..6bbf568 100644 --- a/app/lib/.server/llm/stream-text.ts +++ b/app/lib/.server/llm/stream-text.ts @@ -151,10 +151,13 @@ export async function streamText(props: { providerSettings?: Record; promptId?: string; }) { - const { messages, env, options, apiKeys, files, providerSettings, promptId } = props; + const { messages, env: serverEnv, options, apiKeys, files, providerSettings, promptId } = props; + + // console.log({serverEnv}); + let currentModel = DEFAULT_MODEL; let currentProvider = DEFAULT_PROVIDER.name; - const MODEL_LIST = await getModelList(apiKeys || {}, providerSettings); + const MODEL_LIST = await getModelList({ apiKeys, providerSettings, serverEnv: serverEnv as any }); const processedMessages = messages.map((message) => { if (message.role === 'user') { const { model, provider, content } = extractPropertiesFromMessage(message); @@ -196,7 +199,7 @@ export async function streamText(props: { } return _streamText({ - model: getModel(currentProvider, currentModel, env, apiKeys, providerSettings) as any, + model: getModel(currentProvider, currentModel, serverEnv, apiKeys, providerSettings) as any, system: systemPrompt, maxTokens: dynamicMaxTokens, messages: convertToCoreMessages(processedMessages as any), diff --git a/app/types/model.ts b/app/types/model.ts index 3bfbfde..a747a3f 100644 --- a/app/types/model.ts +++ b/app/types/model.ts @@ -3,7 +3,11 @@ import type { ModelInfo } from '~/utils/types'; export type ProviderInfo = { staticModels: ModelInfo[]; name: string; - getDynamicModels?: (apiKeys?: Record, providerSettings?: IProviderSetting) => Promise; + getDynamicModels?: ( + apiKeys?: Record, + providerSettings?: IProviderSetting, + serverEnv?: Record, + ) => Promise; getApiKeyLink?: string; labelForGetApiKey?: string; icon?: string; diff --git a/app/utils/constants.ts b/app/utils/constants.ts index 6425995..6595d9c 100644 --- a/app/utils/constants.ts +++ b/app/utils/constants.ts @@ -220,7 +220,6 @@ const PROVIDER_LIST: ProviderInfo[] = [ ], getApiKeyLink: 'https://huggingface.co/settings/tokens', }, - { name: 'OpenAI', staticModels: [ @@ -325,26 +324,46 @@ const staticModels: ModelInfo[] = PROVIDER_LIST.map((p) => p.staticModels).flat( export let MODEL_LIST: ModelInfo[] = [...staticModels]; -export async function getModelList( - apiKeys: Record, - providerSettings?: Record, -) { +export async function getModelList(options: { + apiKeys?: Record; + providerSettings?: Record; + serverEnv?: Record; +}) { + const { apiKeys, providerSettings, serverEnv } = options; + + // console.log({ providerSettings, serverEnv,env:process.env }); MODEL_LIST = [ ...( await Promise.all( PROVIDER_LIST.filter( (p): p is ProviderInfo & { getDynamicModels: () => Promise } => !!p.getDynamicModels, - ).map((p) => p.getDynamicModels(apiKeys, providerSettings?.[p.name])), + ).map((p) => p.getDynamicModels(apiKeys, providerSettings?.[p.name], serverEnv)), ) ).flat(), ...staticModels, ]; + return MODEL_LIST; } -async function getTogetherModels(apiKeys?: Record, settings?: IProviderSetting): Promise { +async function getTogetherModels( + apiKeys?: Record, + settings?: IProviderSetting, + serverEnv: Record = {}, +): Promise { try { - const baseUrl = settings?.baseUrl || import.meta.env.TOGETHER_API_BASE_URL || ''; + let settingsBaseUrl = settings?.baseUrl; + + if (settingsBaseUrl && settingsBaseUrl.length == 0) { + settingsBaseUrl = undefined; + } + + const baseUrl = + settingsBaseUrl || + serverEnv?.TOGETHER_API_BASE_URL || + process.env.TOGETHER_API_BASE_URL || + import.meta.env.TOGETHER_API_BASE_URL || + ''; const provider = 'Together'; if (!baseUrl) { @@ -383,8 +402,19 @@ async function getTogetherModels(apiKeys?: Record, settings?: IP } } -const getOllamaBaseUrl = (settings?: IProviderSetting) => { - const defaultBaseUrl = settings?.baseUrl || import.meta.env.OLLAMA_API_BASE_URL || 'http://localhost:11434'; +const getOllamaBaseUrl = (settings?: IProviderSetting, serverEnv: Record = {}) => { + let settingsBaseUrl = settings?.baseUrl; + + if (settingsBaseUrl && settingsBaseUrl.length == 0) { + settingsBaseUrl = undefined; + } + + const defaultBaseUrl = + settings?.baseUrl || + serverEnv?.OLLAMA_API_BASE_URL || + process.env.OLLAMA_API_BASE_URL || + import.meta.env.OLLAMA_API_BASE_URL || + 'http://localhost:11434'; // Check if we're in the browser if (typeof window !== 'undefined') { @@ -398,9 +428,13 @@ const getOllamaBaseUrl = (settings?: IProviderSetting) => { return isDocker ? defaultBaseUrl.replace('localhost', 'host.docker.internal') : defaultBaseUrl; }; -async function getOllamaModels(apiKeys?: Record, settings?: IProviderSetting): Promise { +async function getOllamaModels( + apiKeys?: Record, + settings?: IProviderSetting, + serverEnv: Record = {}, +): Promise { try { - const baseUrl = getOllamaBaseUrl(settings); + const baseUrl = getOllamaBaseUrl(settings, serverEnv); const response = await fetch(`${baseUrl}/api/tags`); const data = (await response.json()) as OllamaApiResponse; @@ -421,9 +455,21 @@ async function getOllamaModels(apiKeys?: Record, settings?: IPro async function getOpenAILikeModels( apiKeys?: Record, settings?: IProviderSetting, + serverEnv: Record = {}, ): Promise { try { - const baseUrl = settings?.baseUrl || import.meta.env.OPENAI_LIKE_API_BASE_URL || ''; + let settingsBaseUrl = settings?.baseUrl; + + if (settingsBaseUrl && settingsBaseUrl.length == 0) { + settingsBaseUrl = undefined; + } + + const baseUrl = + settingsBaseUrl || + serverEnv.OPENAI_LIKE_API_BASE_URL || + process.env.OPENAI_LIKE_API_BASE_URL || + import.meta.env.OPENAI_LIKE_API_BASE_URL || + ''; if (!baseUrl) { return []; @@ -486,9 +532,24 @@ async function getOpenRouterModels(): Promise { })); } -async function getLMStudioModels(_apiKeys?: Record, settings?: IProviderSetting): Promise { +async function getLMStudioModels( + _apiKeys?: Record, + settings?: IProviderSetting, + serverEnv: Record = {}, +): Promise { try { - const baseUrl = settings?.baseUrl || import.meta.env.LMSTUDIO_API_BASE_URL || 'http://localhost:1234'; + let settingsBaseUrl = settings?.baseUrl; + + if (settingsBaseUrl && settingsBaseUrl.length == 0) { + settingsBaseUrl = undefined; + } + + const baseUrl = + settingsBaseUrl || + serverEnv.LMSTUDIO_API_BASE_URL || + process.env.LMSTUDIO_API_BASE_URL || + import.meta.env.LMSTUDIO_API_BASE_URL || + 'http://localhost:1234'; const response = await fetch(`${baseUrl}/v1/models`); const data = (await response.json()) as any; @@ -503,29 +564,37 @@ async function getLMStudioModels(_apiKeys?: Record, settings?: I } } -async function initializeModelList(providerSettings?: Record): Promise { - let apiKeys: Record = {}; +async function initializeModelList(options: { + env?: Record; + providerSettings?: Record; + apiKeys?: Record; +}): Promise { + const { providerSettings, apiKeys: providedApiKeys, env } = options; + let apiKeys: Record = providedApiKeys || {}; - try { - const storedApiKeys = Cookies.get('apiKeys'); + if (!providedApiKeys) { + try { + const storedApiKeys = Cookies.get('apiKeys'); - if (storedApiKeys) { - const parsedKeys = JSON.parse(storedApiKeys); + if (storedApiKeys) { + const parsedKeys = JSON.parse(storedApiKeys); - if (typeof parsedKeys === 'object' && parsedKeys !== null) { - apiKeys = parsedKeys; + if (typeof parsedKeys === 'object' && parsedKeys !== null) { + apiKeys = parsedKeys; + } } + } catch (error: any) { + logStore.logError('Failed to fetch API keys from cookies', error); + logger.warn(`Failed to fetch apikeys from cookies: ${error?.message}`); } - } catch (error: any) { - logStore.logError('Failed to fetch API keys from cookies', error); - logger.warn(`Failed to fetch apikeys from cookies: ${error?.message}`); } + MODEL_LIST = [ ...( await Promise.all( PROVIDER_LIST.filter( (p): p is ProviderInfo & { getDynamicModels: () => Promise } => !!p.getDynamicModels, - ).map((p) => p.getDynamicModels(apiKeys, providerSettings?.[p.name])), + ).map((p) => p.getDynamicModels(apiKeys, providerSettings?.[p.name], env)), ) ).flat(), ...staticModels, @@ -534,6 +603,7 @@ async function initializeModelList(providerSettings?: Record