diff --git a/app/lib/.server/llm/stream-text.ts b/app/lib/.server/llm/stream-text.ts index 4b2b5b4..f408ba2 100644 --- a/app/lib/.server/llm/stream-text.ts +++ b/app/lib/.server/llm/stream-text.ts @@ -58,10 +58,15 @@ function extractPropertiesFromMessage(message: Message): { model: string; provid return { model, provider, content: cleanedContent }; } -export async function streamText(messages: Messages, env: Env, options?: StreamingOptions,apiKeys?: Record) { +export async function streamText( + messages: Messages, + env: Env, + options?: StreamingOptions, + apiKeys?: Record, +) { let currentModel = DEFAULT_MODEL; let currentProvider = DEFAULT_PROVIDER.name; - const MODEL_LIST = await getModelList(apiKeys||{}); + const MODEL_LIST = await getModelList(apiKeys || {}); const processedMessages = messages.map((message) => { if (message.role === 'user') { const { model, provider, content } = extractPropertiesFromMessage(message); @@ -69,6 +74,7 @@ export async function streamText(messages: Messages, env: Env, options?: Streami if (MODEL_LIST.find((m) => m.name === model)) { currentModel = model; } + currentProvider = provider; return { ...message, content }; diff --git a/app/routes/api.chat.ts b/app/routes/api.chat.ts index 81d3035..0073274 100644 --- a/app/routes/api.chat.ts +++ b/app/routes/api.chat.ts @@ -8,8 +8,8 @@ export async function action(args: ActionFunctionArgs) { return chatAction(args); } -function parseCookies(cookieHeader:string) { - const cookies:any = {}; +function parseCookies(cookieHeader: string) { + const cookies: any = {}; // Split the cookie string by semicolons and spaces const items = cookieHeader.split(';').map((cookie) => cookie.trim()); @@ -29,7 +29,7 @@ function parseCookies(cookieHeader:string) { } async function chatAction({ context, request }: ActionFunctionArgs) { - const { messages, model } = await request.json<{ + const { messages } = await request.json<{ messages: Messages; model: string; }>(); @@ -37,7 +37,7 @@ async function chatAction({ context, request }: ActionFunctionArgs) { const cookieHeader = request.headers.get('Cookie'); // Parse the cookie's value (returns an object or null if no cookie exists) - const apiKeys = JSON.parse(parseCookies(cookieHeader||"").apiKeys || '{}'); + const apiKeys = JSON.parse(parseCookies(cookieHeader || '').apiKeys || '{}'); const stream = new SwitchableStream(); @@ -60,7 +60,7 @@ async function chatAction({ context, request }: ActionFunctionArgs) { messages.push({ role: 'assistant', content }); messages.push({ role: 'user', content: CONTINUE_PROMPT }); - const result = await streamText(messages, context.cloudflare.env, options,apiKeys); + const result = await streamText(messages, context.cloudflare.env, options, apiKeys); return stream.switchSource(result.toAIStream()); }, @@ -76,7 +76,7 @@ async function chatAction({ context, request }: ActionFunctionArgs) { contentType: 'text/plain; charset=utf-8', }, }); - } catch (error:any) { + } catch (error: any) { console.log(error); if (error.message?.includes('API key')) { diff --git a/app/utils/constants.ts b/app/utils/constants.ts index 7109a93..26db65a 100644 --- a/app/utils/constants.ts +++ b/app/utils/constants.ts @@ -263,7 +263,7 @@ const PROVIDER_LIST: ProviderInfo[] = [ }, { name: 'Together', - getDynamicModels: getTogetherModels, + getDynamicModels: getTogetherModels, staticModels: [ { name: 'Qwen/Qwen2.5-Coder-32B-Instruct', @@ -295,7 +295,6 @@ const staticModels: ModelInfo[] = PROVIDER_LIST.map((p) => p.staticModels).flat( export let MODEL_LIST: ModelInfo[] = [...staticModels]; - export async function getModelList(apiKeys: Record) { MODEL_LIST = [ ...( @@ -312,43 +311,44 @@ export async function getModelList(apiKeys: Record) { async function getTogetherModels(apiKeys?: Record): Promise { try { - let baseUrl = import.meta.env.TOGETHER_API_BASE_URL || ''; - let provider='Together' + const baseUrl = import.meta.env.TOGETHER_API_BASE_URL || ''; + const provider = 'Together'; - if (!baseUrl) { + if (!baseUrl) { + return []; + } + + let apiKey = import.meta.env.OPENAI_LIKE_API_KEY ?? ''; + + if (apiKeys && apiKeys[provider]) { + apiKey = apiKeys[provider]; + } + + if (!apiKey) { + return []; + } + + const response = await fetch(`${baseUrl}/models`, { + headers: { + Authorization: `Bearer ${apiKey}`, + }, + }); + const res = (await response.json()) as any; + const data: any[] = (res || []).filter((model: any) => model.type == 'chat'); + + return data.map((m: any) => ({ + name: m.id, + label: `${m.display_name} - in:$${m.pricing.input.toFixed( + 2, + )} out:$${m.pricing.output.toFixed(2)} - context ${Math.floor(m.context_length / 1000)}k`, + provider, + maxTokenAllowed: 8000, + })); + } catch (e) { + console.error('Error getting OpenAILike models:', e); return []; } - let apiKey = import.meta.env.OPENAI_LIKE_API_KEY ?? '' - - if (apiKeys && apiKeys[provider]) { - apiKey = apiKeys[provider]; - } - - if (!apiKey) { - return []; - } - - const response = await fetch(`${baseUrl}/models`, { - headers: { - Authorization: `Bearer ${apiKey}`, - }, - }); - const res = (await response.json()) as any; - let data: any[] = (res || []).filter((model: any) => model.type=='chat') - return data.map((m: any) => ({ - name: m.id, - label: `${m.display_name} - in:$${(m.pricing.input).toFixed( - 2, - )} out:$${(m.pricing.output).toFixed(2)} - context ${Math.floor(m.context_length / 1000)}k`, - provider: provider, - maxTokenAllowed: 8000, - })); -} catch (e) { - console.error('Error getting OpenAILike models:', e); - return []; } -} - const getOllamaBaseUrl = () => { const defaultBaseUrl = import.meta.env.OLLAMA_API_BASE_URL || 'http://localhost:11434'; @@ -396,11 +396,13 @@ async function getOpenAILikeModels(): Promise { if (!baseUrl) { return []; } + let apiKey = import.meta.env.OPENAI_LIKE_API_KEY ?? ''; - - let apikeys = JSON.parse(Cookies.get('apiKeys')||'{}') - if (apikeys && apikeys['OpenAILike']){ - apiKey = apikeys['OpenAILike']; + + const apikeys = JSON.parse(Cookies.get('apiKeys') || '{}'); + + if (apikeys && apikeys.OpenAILike) { + apiKey = apikeys.OpenAILike; } const response = await fetch(`${baseUrl}/models`, { @@ -458,6 +460,7 @@ async function getLMStudioModels(): Promise { if (typeof window === 'undefined') { return []; } + try { const baseUrl = import.meta.env.LMSTUDIO_API_BASE_URL || 'http://localhost:1234'; const response = await fetch(`${baseUrl}/v1/models`); @@ -476,6 +479,7 @@ async function getLMStudioModels(): Promise { async function initializeModelList(): Promise { let apiKeys: Record = {}; + try { const storedApiKeys = Cookies.get('apiKeys'); @@ -486,9 +490,8 @@ async function initializeModelList(): Promise { apiKeys = parsedKeys; } } - - } catch (error) { - + } catch (error: any) { + console.warn(`Failed to fetch apikeys from cookies:${error?.message}`); } MODEL_LIST = [ ...( @@ -500,6 +503,7 @@ async function initializeModelList(): Promise { ).flat(), ...staticModels, ]; + return MODEL_LIST; }