From 1589d2a8f57143a05b7d2c9e7a03fc7fc1eca27c Mon Sep 17 00:00:00 2001 From: Anirban Kar Date: Tue, 3 Dec 2024 02:13:33 +0530 Subject: [PATCH 1/3] feat(Dynamic Models): together AI Dynamic Models --- app/lib/.server/llm/stream-text.ts | 18 +++---- app/routes/api.chat.ts | 14 ++--- app/types/model.ts | 2 +- app/utils/constants.ts | 83 ++++++++++++++++++++++++++++-- app/utils/parseCookies.ts | 19 +++++++ vite.config.ts | 2 +- 6 files changed, 113 insertions(+), 25 deletions(-) create mode 100644 app/utils/parseCookies.ts diff --git a/app/lib/.server/llm/stream-text.ts b/app/lib/.server/llm/stream-text.ts index 3ef8792..bfb88c2 100644 --- a/app/lib/.server/llm/stream-text.ts +++ b/app/lib/.server/llm/stream-text.ts @@ -1,11 +1,8 @@ -// eslint-disable-next-line @typescript-eslint/ban-ts-comment -// @ts-nocheck – TODO: Provider proper types - import { convertToCoreMessages, streamText as _streamText } from 'ai'; import { getModel } from '~/lib/.server/llm/model'; import { MAX_TOKENS } from './constants'; import { getSystemPrompt } from './prompts'; -import { DEFAULT_MODEL, DEFAULT_PROVIDER, MODEL_LIST, MODEL_REGEX, PROVIDER_REGEX } from '~/utils/constants'; +import { DEFAULT_MODEL, DEFAULT_PROVIDER, getModelList, MODEL_REGEX, PROVIDER_REGEX } from '~/utils/constants'; interface ToolResult { toolCallId: string; @@ -32,7 +29,7 @@ function extractPropertiesFromMessage(message: Message): { model: string; provid // Extract provider const providerMatch = message.content.match(PROVIDER_REGEX); - const provider = providerMatch ? providerMatch[1] : DEFAULT_PROVIDER; + const provider = providerMatch ? providerMatch[1] : DEFAULT_PROVIDER.name; // Remove model and provider lines from content const cleanedContent = message.content.replace(MODEL_REGEX, '').replace(PROVIDER_REGEX, '').trim(); @@ -40,10 +37,10 @@ function extractPropertiesFromMessage(message: Message): { model: string; provid return { model, provider, content: cleanedContent }; } -export function streamText(messages: Messages, env: Env, options?: StreamingOptions, apiKeys?: Record) { +export async function streamText(messages: Messages, env: Env, options?: StreamingOptions,apiKeys?: Record) { let currentModel = DEFAULT_MODEL; - let currentProvider = DEFAULT_PROVIDER; - + let currentProvider = DEFAULT_PROVIDER.name; + const MODEL_LIST = await getModelList(apiKeys||{}); const processedMessages = messages.map((message) => { if (message.role === 'user') { const { model, provider, content } = extractPropertiesFromMessage(message); @@ -51,7 +48,6 @@ export function streamText(messages: Messages, env: Env, options?: StreamingOpti if (MODEL_LIST.find((m) => m.name === model)) { currentModel = model; } - currentProvider = provider; return { ...message, content }; @@ -65,10 +61,10 @@ export function streamText(messages: Messages, env: Env, options?: StreamingOpti const dynamicMaxTokens = modelDetails && modelDetails.maxTokenAllowed ? modelDetails.maxTokenAllowed : MAX_TOKENS; return _streamText({ - model: getModel(currentProvider, currentModel, env, apiKeys), + model: getModel(currentProvider, currentModel, env, apiKeys) as any, system: getSystemPrompt(), maxTokens: dynamicMaxTokens, - messages: convertToCoreMessages(processedMessages), + messages: convertToCoreMessages(processedMessages as any), ...options, }); } diff --git a/app/routes/api.chat.ts b/app/routes/api.chat.ts index ac35a22..017ae92 100644 --- a/app/routes/api.chat.ts +++ b/app/routes/api.chat.ts @@ -1,6 +1,3 @@ -// eslint-disable-next-line @typescript-eslint/ban-ts-comment -// @ts-nocheck – TODO: Provider proper types - import { type ActionFunctionArgs } from '@remix-run/cloudflare'; import { MAX_RESPONSE_SEGMENTS, MAX_TOKENS } from '~/lib/.server/llm/constants'; import { CONTINUE_PROMPT } from '~/lib/.server/llm/prompts'; @@ -11,8 +8,8 @@ export async function action(args: ActionFunctionArgs) { return chatAction(args); } -function parseCookies(cookieHeader) { - const cookies = {}; +function parseCookies(cookieHeader:string) { + const cookies:any = {}; // Split the cookie string by semicolons and spaces const items = cookieHeader.split(';').map((cookie) => cookie.trim()); @@ -39,14 +36,13 @@ async function chatAction({ context, request }: ActionFunctionArgs) { const cookieHeader = request.headers.get('Cookie'); // Parse the cookie's value (returns an object or null if no cookie exists) - const apiKeys = JSON.parse(parseCookies(cookieHeader).apiKeys || '{}'); + const apiKeys = JSON.parse(parseCookies(cookieHeader||"").apiKeys || '{}'); const stream = new SwitchableStream(); try { const options: StreamingOptions = { toolChoice: 'none', - apiKeys, onFinish: async ({ text: content, finishReason }) => { if (finishReason !== 'length') { return stream.close(); @@ -63,7 +59,7 @@ async function chatAction({ context, request }: ActionFunctionArgs) { messages.push({ role: 'assistant', content }); messages.push({ role: 'user', content: CONTINUE_PROMPT }); - const result = await streamText(messages, context.cloudflare.env, options); + const result = await streamText(messages, context.cloudflare.env, options,apiKeys); return stream.switchSource(result.toAIStream()); }, @@ -79,7 +75,7 @@ async function chatAction({ context, request }: ActionFunctionArgs) { contentType: 'text/plain; charset=utf-8', }, }); - } catch (error) { + } catch (error:any) { console.log(error); if (error.message?.includes('API key')) { diff --git a/app/types/model.ts b/app/types/model.ts index 32522c6..29bff2e 100644 --- a/app/types/model.ts +++ b/app/types/model.ts @@ -3,7 +3,7 @@ import type { ModelInfo } from '~/utils/types'; export type ProviderInfo = { staticModels: ModelInfo[]; name: string; - getDynamicModels?: () => Promise; + getDynamicModels?: (apiKeys?: Record) => Promise; getApiKeyLink?: string; labelForGetApiKey?: string; icon?: string; diff --git a/app/utils/constants.ts b/app/utils/constants.ts index 1120bc1..81a6457 100644 --- a/app/utils/constants.ts +++ b/app/utils/constants.ts @@ -1,3 +1,5 @@ +import Cookies from 'js-cookie'; +import { parseCookies } from './parseCookies'; import type { ModelInfo, OllamaApiResponse, OllamaModel } from './types'; import type { ProviderInfo } from '~/types/model'; @@ -262,6 +264,7 @@ const PROVIDER_LIST: ProviderInfo[] = [ }, { name: 'Together', + getDynamicModels: getTogetherModels, staticModels: [ { name: 'Qwen/Qwen2.5-Coder-32B-Instruct', @@ -293,6 +296,61 @@ const staticModels: ModelInfo[] = PROVIDER_LIST.map((p) => p.staticModels).flat( export let MODEL_LIST: ModelInfo[] = [...staticModels]; + +export async function getModelList(apiKeys: Record) { + MODEL_LIST = [ + ...( + await Promise.all( + PROVIDER_LIST.filter( + (p): p is ProviderInfo & { getDynamicModels: () => Promise } => !!p.getDynamicModels, + ).map((p) => p.getDynamicModels(apiKeys)), + ) + ).flat(), + ...staticModels, + ]; + return MODEL_LIST; +} + +async function getTogetherModels(apiKeys?: Record): Promise { + try { + let baseUrl = import.meta.env.TOGETHER_API_BASE_URL || ''; + let provider='Together' + + if (!baseUrl) { + return []; + } + let apiKey = import.meta.env.OPENAI_LIKE_API_KEY ?? '' + + if (apiKeys && apiKeys[provider]) { + apiKey = apiKeys[provider]; + } + + if (!apiKey) { + return []; + } + + const response = await fetch(`${baseUrl}/models`, { + headers: { + Authorization: `Bearer ${apiKey}`, + }, + }); + const res = (await response.json()) as any; + let data: any[] = (res || []).filter((model: any) => model.type=='chat') + return data.map((m: any) => ({ + name: m.id, + label: `${m.display_name} - in:$${(m.pricing.input).toFixed( + 2, + )} out:$${(m.pricing.output).toFixed(2)} - context ${Math.floor(m.context_length / 1000)}k`, + provider: provider, + maxTokenAllowed: 8000, + })); +} catch (e) { + console.error('Error getting OpenAILike models:', e); + return []; +} +} + + const getOllamaBaseUrl = () => { const defaultBaseUrl = import.meta.env.OLLAMA_API_BASE_URL || 'http://localhost:11434'; @@ -339,8 +397,13 @@ async function getOpenAILikeModels(): Promise { if (!baseUrl) { return []; } + let apiKey = import.meta.env.OPENAI_LIKE_API_KEY ?? ''; + + let apikeys = JSON.parse(Cookies.get('apiKeys')||'{}') + if (apikeys && apikeys['OpenAILike']){ + apiKey = apikeys['OpenAILike']; + } - const apiKey = import.meta.env.OPENAI_LIKE_API_KEY ?? ''; const response = await fetch(`${baseUrl}/models`, { headers: { Authorization: `Bearer ${apiKey}`, @@ -396,7 +459,6 @@ async function getLMStudioModels(): Promise { if (typeof window === 'undefined') { return []; } - try { const baseUrl = import.meta.env.LMSTUDIO_API_BASE_URL || 'http://localhost:1234'; const response = await fetch(`${baseUrl}/v1/models`); @@ -414,12 +476,27 @@ async function getLMStudioModels(): Promise { } async function initializeModelList(): Promise { + let apiKeys: Record = {}; + try { + const storedApiKeys = Cookies.get('apiKeys'); + + if (storedApiKeys) { + const parsedKeys = JSON.parse(storedApiKeys); + + if (typeof parsedKeys === 'object' && parsedKeys !== null) { + apiKeys = parsedKeys; + } + } + + } catch (error) { + + } MODEL_LIST = [ ...( await Promise.all( PROVIDER_LIST.filter( (p): p is ProviderInfo & { getDynamicModels: () => Promise } => !!p.getDynamicModels, - ).map((p) => p.getDynamicModels()), + ).map((p) => p.getDynamicModels(apiKeys)), ) ).flat(), ...staticModels, diff --git a/app/utils/parseCookies.ts b/app/utils/parseCookies.ts new file mode 100644 index 0000000..235adc3 --- /dev/null +++ b/app/utils/parseCookies.ts @@ -0,0 +1,19 @@ +export function parseCookies(cookieHeader: string) { + const cookies: any = {}; + + // Split the cookie string by semicolons and spaces + const items = cookieHeader.split(';').map((cookie) => cookie.trim()); + + items.forEach((item) => { + const [name, ...rest] = item.split('='); + + if (name && rest) { + // Decode the name and value, and join value parts in case it contains '=' + const decodedName = decodeURIComponent(name.trim()); + const decodedValue = decodeURIComponent(rest.join('=').trim()); + cookies[decodedName] = decodedValue; + } + }); + + return cookies; +} \ No newline at end of file diff --git a/vite.config.ts b/vite.config.ts index 5f86830..1bb08d3 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -28,7 +28,7 @@ export default defineConfig((config) => { chrome129IssuePlugin(), config.mode === 'production' && optimizeCssModules({ apply: 'build' }), ], - envPrefix:["VITE_","OPENAI_LIKE_API_","OLLAMA_API_BASE_URL","LMSTUDIO_API_BASE_URL"], + envPrefix: ["VITE_", "OPENAI_LIKE_API_", "OLLAMA_API_BASE_URL", "LMSTUDIO_API_BASE_URL","TOGETHER_API_BASE_URL"], css: { preprocessorOptions: { scss: { From 7192690c1cec4a87275d031df05c6861e43f84d9 Mon Sep 17 00:00:00 2001 From: Anirban Kar Date: Tue, 3 Dec 2024 02:19:30 +0530 Subject: [PATCH 2/3] clean up --- app/utils/constants.ts | 1 - app/utils/parseCookies.ts | 19 ------------------- 2 files changed, 20 deletions(-) delete mode 100644 app/utils/parseCookies.ts diff --git a/app/utils/constants.ts b/app/utils/constants.ts index 81a6457..7109a93 100644 --- a/app/utils/constants.ts +++ b/app/utils/constants.ts @@ -1,5 +1,4 @@ import Cookies from 'js-cookie'; -import { parseCookies } from './parseCookies'; import type { ModelInfo, OllamaApiResponse, OllamaModel } from './types'; import type { ProviderInfo } from '~/types/model'; diff --git a/app/utils/parseCookies.ts b/app/utils/parseCookies.ts deleted file mode 100644 index 235adc3..0000000 --- a/app/utils/parseCookies.ts +++ /dev/null @@ -1,19 +0,0 @@ -export function parseCookies(cookieHeader: string) { - const cookies: any = {}; - - // Split the cookie string by semicolons and spaces - const items = cookieHeader.split(';').map((cookie) => cookie.trim()); - - items.forEach((item) => { - const [name, ...rest] = item.split('='); - - if (name && rest) { - // Decode the name and value, and join value parts in case it contains '=' - const decodedName = decodeURIComponent(name.trim()); - const decodedValue = decodeURIComponent(rest.join('=').trim()); - cookies[decodedName] = decodedValue; - } - }); - - return cookies; -} \ No newline at end of file From 7efad132844218aee3b03a7ddf3f878fcf4cf9cf Mon Sep 17 00:00:00 2001 From: Anirban Kar Date: Fri, 6 Dec 2024 16:58:04 +0530 Subject: [PATCH 3/3] list fix --- app/lib/.server/llm/stream-text.ts | 10 +++- app/routes/api.chat.ts | 12 ++-- app/utils/constants.ts | 88 ++++++++++++++++-------------- 3 files changed, 60 insertions(+), 50 deletions(-) diff --git a/app/lib/.server/llm/stream-text.ts b/app/lib/.server/llm/stream-text.ts index 4b2b5b4..f408ba2 100644 --- a/app/lib/.server/llm/stream-text.ts +++ b/app/lib/.server/llm/stream-text.ts @@ -58,10 +58,15 @@ function extractPropertiesFromMessage(message: Message): { model: string; provid return { model, provider, content: cleanedContent }; } -export async function streamText(messages: Messages, env: Env, options?: StreamingOptions,apiKeys?: Record) { +export async function streamText( + messages: Messages, + env: Env, + options?: StreamingOptions, + apiKeys?: Record, +) { let currentModel = DEFAULT_MODEL; let currentProvider = DEFAULT_PROVIDER.name; - const MODEL_LIST = await getModelList(apiKeys||{}); + const MODEL_LIST = await getModelList(apiKeys || {}); const processedMessages = messages.map((message) => { if (message.role === 'user') { const { model, provider, content } = extractPropertiesFromMessage(message); @@ -69,6 +74,7 @@ export async function streamText(messages: Messages, env: Env, options?: Streami if (MODEL_LIST.find((m) => m.name === model)) { currentModel = model; } + currentProvider = provider; return { ...message, content }; diff --git a/app/routes/api.chat.ts b/app/routes/api.chat.ts index 81d3035..0073274 100644 --- a/app/routes/api.chat.ts +++ b/app/routes/api.chat.ts @@ -8,8 +8,8 @@ export async function action(args: ActionFunctionArgs) { return chatAction(args); } -function parseCookies(cookieHeader:string) { - const cookies:any = {}; +function parseCookies(cookieHeader: string) { + const cookies: any = {}; // Split the cookie string by semicolons and spaces const items = cookieHeader.split(';').map((cookie) => cookie.trim()); @@ -29,7 +29,7 @@ function parseCookies(cookieHeader:string) { } async function chatAction({ context, request }: ActionFunctionArgs) { - const { messages, model } = await request.json<{ + const { messages } = await request.json<{ messages: Messages; model: string; }>(); @@ -37,7 +37,7 @@ async function chatAction({ context, request }: ActionFunctionArgs) { const cookieHeader = request.headers.get('Cookie'); // Parse the cookie's value (returns an object or null if no cookie exists) - const apiKeys = JSON.parse(parseCookies(cookieHeader||"").apiKeys || '{}'); + const apiKeys = JSON.parse(parseCookies(cookieHeader || '').apiKeys || '{}'); const stream = new SwitchableStream(); @@ -60,7 +60,7 @@ async function chatAction({ context, request }: ActionFunctionArgs) { messages.push({ role: 'assistant', content }); messages.push({ role: 'user', content: CONTINUE_PROMPT }); - const result = await streamText(messages, context.cloudflare.env, options,apiKeys); + const result = await streamText(messages, context.cloudflare.env, options, apiKeys); return stream.switchSource(result.toAIStream()); }, @@ -76,7 +76,7 @@ async function chatAction({ context, request }: ActionFunctionArgs) { contentType: 'text/plain; charset=utf-8', }, }); - } catch (error:any) { + } catch (error: any) { console.log(error); if (error.message?.includes('API key')) { diff --git a/app/utils/constants.ts b/app/utils/constants.ts index 7109a93..26db65a 100644 --- a/app/utils/constants.ts +++ b/app/utils/constants.ts @@ -263,7 +263,7 @@ const PROVIDER_LIST: ProviderInfo[] = [ }, { name: 'Together', - getDynamicModels: getTogetherModels, + getDynamicModels: getTogetherModels, staticModels: [ { name: 'Qwen/Qwen2.5-Coder-32B-Instruct', @@ -295,7 +295,6 @@ const staticModels: ModelInfo[] = PROVIDER_LIST.map((p) => p.staticModels).flat( export let MODEL_LIST: ModelInfo[] = [...staticModels]; - export async function getModelList(apiKeys: Record) { MODEL_LIST = [ ...( @@ -312,43 +311,44 @@ export async function getModelList(apiKeys: Record) { async function getTogetherModels(apiKeys?: Record): Promise { try { - let baseUrl = import.meta.env.TOGETHER_API_BASE_URL || ''; - let provider='Together' + const baseUrl = import.meta.env.TOGETHER_API_BASE_URL || ''; + const provider = 'Together'; - if (!baseUrl) { + if (!baseUrl) { + return []; + } + + let apiKey = import.meta.env.OPENAI_LIKE_API_KEY ?? ''; + + if (apiKeys && apiKeys[provider]) { + apiKey = apiKeys[provider]; + } + + if (!apiKey) { + return []; + } + + const response = await fetch(`${baseUrl}/models`, { + headers: { + Authorization: `Bearer ${apiKey}`, + }, + }); + const res = (await response.json()) as any; + const data: any[] = (res || []).filter((model: any) => model.type == 'chat'); + + return data.map((m: any) => ({ + name: m.id, + label: `${m.display_name} - in:$${m.pricing.input.toFixed( + 2, + )} out:$${m.pricing.output.toFixed(2)} - context ${Math.floor(m.context_length / 1000)}k`, + provider, + maxTokenAllowed: 8000, + })); + } catch (e) { + console.error('Error getting OpenAILike models:', e); return []; } - let apiKey = import.meta.env.OPENAI_LIKE_API_KEY ?? '' - - if (apiKeys && apiKeys[provider]) { - apiKey = apiKeys[provider]; - } - - if (!apiKey) { - return []; - } - - const response = await fetch(`${baseUrl}/models`, { - headers: { - Authorization: `Bearer ${apiKey}`, - }, - }); - const res = (await response.json()) as any; - let data: any[] = (res || []).filter((model: any) => model.type=='chat') - return data.map((m: any) => ({ - name: m.id, - label: `${m.display_name} - in:$${(m.pricing.input).toFixed( - 2, - )} out:$${(m.pricing.output).toFixed(2)} - context ${Math.floor(m.context_length / 1000)}k`, - provider: provider, - maxTokenAllowed: 8000, - })); -} catch (e) { - console.error('Error getting OpenAILike models:', e); - return []; } -} - const getOllamaBaseUrl = () => { const defaultBaseUrl = import.meta.env.OLLAMA_API_BASE_URL || 'http://localhost:11434'; @@ -396,11 +396,13 @@ async function getOpenAILikeModels(): Promise { if (!baseUrl) { return []; } + let apiKey = import.meta.env.OPENAI_LIKE_API_KEY ?? ''; - - let apikeys = JSON.parse(Cookies.get('apiKeys')||'{}') - if (apikeys && apikeys['OpenAILike']){ - apiKey = apikeys['OpenAILike']; + + const apikeys = JSON.parse(Cookies.get('apiKeys') || '{}'); + + if (apikeys && apikeys.OpenAILike) { + apiKey = apikeys.OpenAILike; } const response = await fetch(`${baseUrl}/models`, { @@ -458,6 +460,7 @@ async function getLMStudioModels(): Promise { if (typeof window === 'undefined') { return []; } + try { const baseUrl = import.meta.env.LMSTUDIO_API_BASE_URL || 'http://localhost:1234'; const response = await fetch(`${baseUrl}/v1/models`); @@ -476,6 +479,7 @@ async function getLMStudioModels(): Promise { async function initializeModelList(): Promise { let apiKeys: Record = {}; + try { const storedApiKeys = Cookies.get('apiKeys'); @@ -486,9 +490,8 @@ async function initializeModelList(): Promise { apiKeys = parsedKeys; } } - - } catch (error) { - + } catch (error: any) { + console.warn(`Failed to fetch apikeys from cookies:${error?.message}`); } MODEL_LIST = [ ...( @@ -500,6 +503,7 @@ async function initializeModelList(): Promise { ).flat(), ...staticModels, ]; + return MODEL_LIST; }