From bfaaf86c69aa558dda35dc638e30108e5d58dc69 Mon Sep 17 00:00:00 2001 From: Aaron Bolton Date: Mon, 18 Nov 2024 20:48:35 +0000 Subject: [PATCH 01/15] Created DEFAULT_NUM_CTX VAR with a deafult of 32768 --- .env.example | 7 +++++++ app/lib/.server/llm/model.ts | 6 +++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 46a21e89..386d4079 100644 --- a/.env.example +++ b/.env.example @@ -56,3 +56,10 @@ XAI_API_KEY= # Include this environment variable if you want more logging for debugging locally VITE_LOG_LEVEL=debug + +# Example Context Values for qwen2.5-coder:32b +# +# DEFAULT_NUM_CTX=32768 # Consumes 36GB of VRAM +# DEFAULT_NUM_CTX=24576 # Consumes 32GB of VRAM +# DEFAULT_NUM_CTX=12288 # Consumes 26GB of VRAM +# DEFAULT_NUM_CTX=6144 # Consumes 24GB of VRAM \ No newline at end of file diff --git a/app/lib/.server/llm/model.ts b/app/lib/.server/llm/model.ts index 6be9d117..266dd69d 100644 --- a/app/lib/.server/llm/model.ts +++ b/app/lib/.server/llm/model.ts @@ -8,6 +8,10 @@ import { ollama } from 'ollama-ai-provider'; import { createOpenRouter } from "@openrouter/ai-sdk-provider"; import { createMistral } from '@ai-sdk/mistral'; +export const DEFAULT_NUM_CTX = process.env.DEFAULT_NUM_CTX ? + parseInt(process.env.DEFAULT_NUM_CTX, 10) : + 32768; + export function getAnthropicModel(apiKey: string, model: string) { const anthropic = createAnthropic({ apiKey, @@ -58,7 +62,7 @@ export function getGroqModel(apiKey: string, model: string) { export function getOllamaModel(baseURL: string, model: string) { let Ollama = ollama(model, { - numCtx: 32768, + numCtx: DEFAULT_NUM_CTX, }); Ollama.config.baseURL = `${baseURL}/api`; From 53594234819346a3d8940b9101ebd9ec69b25641 Mon Sep 17 00:00:00 2001 From: Aaron Bolton Date: Tue, 19 Nov 2024 07:46:51 +0000 Subject: [PATCH 02/15] DEFAULT_NUM_CTX additions adding further changes for DEFAULT_NUM_CTX, including docs --- .env.example | 3 ++- CONTRIBUTING.md | 16 ++++++++++++++++ Dockerfile | 8 ++++++-- docker-compose.yaml | 2 ++ 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 386d4079..9710a8eb 100644 --- a/.env.example +++ b/.env.example @@ -62,4 +62,5 @@ VITE_LOG_LEVEL=debug # DEFAULT_NUM_CTX=32768 # Consumes 36GB of VRAM # DEFAULT_NUM_CTX=24576 # Consumes 32GB of VRAM # DEFAULT_NUM_CTX=12288 # Consumes 26GB of VRAM -# DEFAULT_NUM_CTX=6144 # Consumes 24GB of VRAM \ No newline at end of file +# DEFAULT_NUM_CTX=6144 # Consumes 24GB of VRAM +DEFAULT_NUM_CTX= \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1bf3bfb7..23f2b8d0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,7 @@ # Contributing to Bolt.new Fork +## DEFAULT_NUM_CTX + +The `DEFAULT_NUM_CTX` environment variable can be used to limit the maximum number of context values used by the qwen2.5-coder model. For example, to limit the context to 24576 values (which uses 32GB of VRAM), set `DEFAULT_NUM_CTX=24576` in your `.env.local` file. First off, thank you for considering contributing to Bolt.new! This fork aims to expand the capabilities of the original project by integrating multiple LLM providers and enhancing functionality. Every contribution helps make Bolt.new a better tool for developers worldwide. @@ -80,6 +83,19 @@ ANTHROPIC_API_KEY=XXX ```bash VITE_LOG_LEVEL=debug ``` + + - Optionally set context size: +```bash +DEFAULT_NUM_CTX=32768 +``` + +Some Example Context Values for the qwen2.5-coder:32b models are. + +* DEFAULT_NUM_CTX=32768 - Consumes 36GB of VRAM +* DEFAULT_NUM_CTX=24576 - Consumes 32GB of VRAM +* DEFAULT_NUM_CTX=12288 - Consumes 26GB of VRAM +* DEFAULT_NUM_CTX=6144 - Consumes 24GB of VRAM + **Important**: Never commit your `.env.local` file to version control. It's already included in .gitignore. ### 🚀 Running the Development Server diff --git a/Dockerfile b/Dockerfile index 3b5a74cd..1d686737 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,7 @@ ARG OPEN_ROUTER_API_KEY ARG GOOGLE_GENERATIVE_AI_API_KEY ARG OLLAMA_API_BASE_URL ARG VITE_LOG_LEVEL=debug +ARG DEFAULT_NUM_CTX ENV WRANGLER_SEND_METRICS=false \ GROQ_API_KEY=${GROQ_API_KEY} \ @@ -33,7 +34,8 @@ ENV WRANGLER_SEND_METRICS=false \ OPEN_ROUTER_API_KEY=${OPEN_ROUTER_API_KEY} \ GOOGLE_GENERATIVE_AI_API_KEY=${GOOGLE_GENERATIVE_AI_API_KEY} \ OLLAMA_API_BASE_URL=${OLLAMA_API_BASE_URL} \ - VITE_LOG_LEVEL=${VITE_LOG_LEVEL} + VITE_LOG_LEVEL=${VITE_LOG_LEVEL} \ + DEFAULT_NUM_CTX=${DEFAULT_NUM_CTX} # Pre-configure wrangler to disable metrics RUN mkdir -p /root/.config/.wrangler && \ @@ -54,6 +56,7 @@ ARG OPEN_ROUTER_API_KEY ARG GOOGLE_GENERATIVE_AI_API_KEY ARG OLLAMA_API_BASE_URL ARG VITE_LOG_LEVEL=debug +ARG DEFAULT_NUM_CTX ENV GROQ_API_KEY=${GROQ_API_KEY} \ OPENAI_API_KEY=${OPENAI_API_KEY} \ @@ -61,7 +64,8 @@ ENV GROQ_API_KEY=${GROQ_API_KEY} \ OPEN_ROUTER_API_KEY=${OPEN_ROUTER_API_KEY} \ GOOGLE_GENERATIVE_AI_API_KEY=${GOOGLE_GENERATIVE_AI_API_KEY} \ OLLAMA_API_BASE_URL=${OLLAMA_API_BASE_URL} \ - VITE_LOG_LEVEL=${VITE_LOG_LEVEL} + VITE_LOG_LEVEL=${VITE_LOG_LEVEL} \ + DEFAULT_NUM_CTX=${DEFAULT_NUM_CTX} RUN mkdir -p ${WORKDIR}/run CMD pnpm run dev --host diff --git a/docker-compose.yaml b/docker-compose.yaml index c391dd73..6fbd704a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -20,6 +20,7 @@ services: - GOOGLE_GENERATIVE_AI_API_KEY=${GOOGLE_GENERATIVE_AI_API_KEY} - OLLAMA_API_BASE_URL=${OLLAMA_API_BASE_URL} - VITE_LOG_LEVEL=${VITE_LOG_LEVEL:-debug} + - DEFAULT_NUM_CTX=${DEFAULT_NUM_CTX:-32768} - RUNNING_IN_DOCKER=true extra_hosts: - "host.docker.internal:host-gateway" @@ -46,6 +47,7 @@ services: - GOOGLE_GENERATIVE_AI_API_KEY=${GOOGLE_GENERATIVE_AI_API_KEY} - OLLAMA_API_BASE_URL=${OLLAMA_API_BASE_URL} - VITE_LOG_LEVEL=${VITE_LOG_LEVEL:-debug} + - DEFAULT_NUM_CTX=${DEFAULT_NUM_CTX:-32768} - RUNNING_IN_DOCKER=true extra_hosts: - "host.docker.internal:host-gateway" From 7a03b2647375844c076d0a4c72923b9deffc857c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stig-=C3=98rjan=20Smelror?= Date: Tue, 19 Nov 2024 20:21:27 +0100 Subject: [PATCH 03/15] Update the Google Gemini models list --- app/utils/constants.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/utils/constants.ts b/app/utils/constants.ts index fbcf226e..42441b3e 100644 --- a/app/utils/constants.ts +++ b/app/utils/constants.ts @@ -58,7 +58,11 @@ const PROVIDER_LIST: ProviderInfo[] = [ name: 'Google', staticModels: [ { name: 'gemini-1.5-flash-latest', label: 'Gemini 1.5 Flash', provider: 'Google' }, - { name: 'gemini-1.5-pro-latest', label: 'Gemini 1.5 Pro', provider: 'Google' } + { name: 'gemini-1.5-flash-002', label: 'Gemini 1.5 Flash', provider: 'Google' }, + { name: 'gemini-1.5-flash-8b', label: 'Gemini 1.5 Flash 8b', provider: 'Google' }, + { name: 'gemini-1.5-pro-latest', label: 'Gemini 1.5 Pro', provider: 'Google' }, + { name: 'gemini-1.5-pro-002', label: 'Gemini 1.5 Pro', provider: 'Google' }, + { name: 'gemini-exp-1114', label: 'Gemini exp-1114', provider: 'Google' } ], getApiKeyLink: 'https://aistudio.google.com/app/apikey' }, { From b945ec8d2ca57996685e74a85da4e965daab7ca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stig-=C3=98rjan=20Smelror?= Date: Tue, 19 Nov 2024 20:24:52 +0100 Subject: [PATCH 04/15] Fix the list of names to include the correct model --- app/utils/constants.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/utils/constants.ts b/app/utils/constants.ts index 42441b3e..b7aed65a 100644 --- a/app/utils/constants.ts +++ b/app/utils/constants.ts @@ -58,10 +58,10 @@ const PROVIDER_LIST: ProviderInfo[] = [ name: 'Google', staticModels: [ { name: 'gemini-1.5-flash-latest', label: 'Gemini 1.5 Flash', provider: 'Google' }, - { name: 'gemini-1.5-flash-002', label: 'Gemini 1.5 Flash', provider: 'Google' }, - { name: 'gemini-1.5-flash-8b', label: 'Gemini 1.5 Flash 8b', provider: 'Google' }, + { name: 'gemini-1.5-flash-002', label: 'Gemini 1.5 Flash-002', provider: 'Google' }, + { name: 'gemini-1.5-flash-8b', label: 'Gemini 1.5 Flash-8b', provider: 'Google' }, { name: 'gemini-1.5-pro-latest', label: 'Gemini 1.5 Pro', provider: 'Google' }, - { name: 'gemini-1.5-pro-002', label: 'Gemini 1.5 Pro', provider: 'Google' }, + { name: 'gemini-1.5-pro-002', label: 'Gemini 1.5 Pro-002', provider: 'Google' }, { name: 'gemini-exp-1114', label: 'Gemini exp-1114', provider: 'Google' } ], getApiKeyLink: 'https://aistudio.google.com/app/apikey' From 4ac0af4cdbbe998240f7497cb8b489e29c51c390 Mon Sep 17 00:00:00 2001 From: Sujal Shah Date: Wed, 20 Nov 2024 17:48:41 +0530 Subject: [PATCH 05/15] fix: enhance prompt "Invalid or missing provider" bad request error --- app/lib/hooks/usePromptEnhancer.ts | 37 +++++++++++++++--------------- app/routes/api.enhancer.ts | 33 +++++++++++++++----------- 2 files changed, 38 insertions(+), 32 deletions(-) diff --git a/app/lib/hooks/usePromptEnhancer.ts b/app/lib/hooks/usePromptEnhancer.ts index ee449992..6275ef37 100644 --- a/app/lib/hooks/usePromptEnhancer.ts +++ b/app/lib/hooks/usePromptEnhancer.ts @@ -1,4 +1,5 @@ import { useState } from 'react'; +import type { ProviderInfo } from '~/types/model'; import { createScopedLogger } from '~/utils/logger'; const logger = createScopedLogger('usePromptEnhancement'); @@ -13,54 +14,54 @@ export function usePromptEnhancer() { }; const enhancePrompt = async ( - input: string, + input: string, setInput: (value: string) => void, model: string, - provider: string, - apiKeys?: Record + provider: ProviderInfo, + apiKeys?: Record, ) => { setEnhancingPrompt(true); setPromptEnhanced(false); - + const requestBody: any = { message: input, model, provider, }; - + if (apiKeys) { requestBody.apiKeys = apiKeys; } - + const response = await fetch('/api/enhancer', { method: 'POST', body: JSON.stringify(requestBody), }); - + const reader = response.body?.getReader(); - + const originalInput = input; - + if (reader) { const decoder = new TextDecoder(); - + let _input = ''; let _error; - + try { setInput(''); - + while (true) { const { value, done } = await reader.read(); - + if (done) { break; } - + _input += decoder.decode(value); - + logger.trace('Set input', _input); - + setInput(_input); } } catch (error) { @@ -70,10 +71,10 @@ export function usePromptEnhancer() { if (_error) { logger.error(_error); } - + setEnhancingPrompt(false); setPromptEnhanced(true); - + setTimeout(() => { setInput(_input); }); diff --git a/app/routes/api.enhancer.ts b/app/routes/api.enhancer.ts index 7040b890..77e6f2fd 100644 --- a/app/routes/api.enhancer.ts +++ b/app/routes/api.enhancer.ts @@ -2,7 +2,7 @@ import { type ActionFunctionArgs } from '@remix-run/cloudflare'; import { StreamingTextResponse, parseStreamPart } from 'ai'; import { streamText } from '~/lib/.server/llm/stream-text'; import { stripIndents } from '~/utils/stripIndent'; -import type { StreamingOptions } from '~/lib/.server/llm/stream-text'; +import type { ProviderInfo } from '~/types/model'; const encoder = new TextEncoder(); const decoder = new TextDecoder(); @@ -12,25 +12,27 @@ export async function action(args: ActionFunctionArgs) { } async function enhancerAction({ context, request }: ActionFunctionArgs) { - const { message, model, provider, apiKeys } = await request.json<{ + const { message, model, provider, apiKeys } = await request.json<{ message: string; model: string; - provider: string; + provider: ProviderInfo; apiKeys?: Record; }>(); - // Validate 'model' and 'provider' fields + const { name: providerName } = provider; + + // validate 'model' and 'provider' fields if (!model || typeof model !== 'string') { throw new Response('Invalid or missing model', { status: 400, - statusText: 'Bad Request' + statusText: 'Bad Request', }); } - if (!provider || typeof provider !== 'string') { + if (!providerName || typeof providerName !== 'string') { throw new Response('Invalid or missing provider', { status: 400, - statusText: 'Bad Request' + statusText: 'Bad Request', }); } @@ -39,7 +41,9 @@ async function enhancerAction({ context, request }: ActionFunctionArgs) { [ { role: 'user', - content: `[Model: ${model}]\n\n[Provider: ${provider}]\n\n` + stripIndents` + content: + `[Model: ${model}]\n\n[Provider: ${providerName}]\n\n` + + stripIndents` I want you to improve the user prompt that is wrapped in \`\` tags. IMPORTANT: Only respond with the improved prompt and nothing else! @@ -52,23 +56,24 @@ async function enhancerAction({ context, request }: ActionFunctionArgs) { ], context.cloudflare.env, undefined, - apiKeys + apiKeys, ); const transformStream = new TransformStream({ transform(chunk, controller) { const text = decoder.decode(chunk); - const lines = text.split('\n').filter(line => line.trim() !== ''); - + const lines = text.split('\n').filter((line) => line.trim() !== ''); + for (const line of lines) { try { const parsed = parseStreamPart(line); + if (parsed.type === 'text') { controller.enqueue(encoder.encode(parsed.value)); } } catch (e) { - // Skip invalid JSON lines - console.warn('Failed to parse stream part:', line); + // skip invalid JSON lines + console.warn('Failed to parse stream part:', line, e); } } }, @@ -83,7 +88,7 @@ async function enhancerAction({ context, request }: ActionFunctionArgs) { if (error instanceof Error && error.message?.includes('API key')) { throw new Response('Invalid or missing API key', { status: 401, - statusText: 'Unauthorized' + statusText: 'Unauthorized', }); } From 50de8d0e1b26e3198427aab6b1fe1302d904a99f Mon Sep 17 00:00:00 2001 From: Cole Medin Date: Wed, 20 Nov 2024 08:27:21 -0600 Subject: [PATCH 06/15] Updating README with new features and a link to our community --- README.md | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 52889233..6bcacd4e 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,12 @@ [![Bolt.new: AI-Powered Full-Stack Web Development in the Browser](./public/social_preview_index.jpg)](https://bolt.new) -# Bolt.new Fork by Cole Medin +# Bolt.new Fork by Cole Medin - oTToDev -This fork of Bolt.new allows you to choose the LLM that you use for each prompt! Currently, you can use OpenAI, Anthropic, Ollama, OpenRouter, Gemini, or Groq models - and it is easily extended to use any other model supported by the Vercel AI SDK! See the instructions below for running this locally and extending it to include more models. +This fork of Bolt.new (oTToDev) allows you to choose the LLM that you use for each prompt! Currently, you can use OpenAI, Anthropic, Ollama, OpenRouter, Gemini, LMStudio, Mistral, xAI, HuggingFace, DeepSeek, or Groq models - and it is easily extended to use any other model supported by the Vercel AI SDK! See the instructions below for running this locally and extending it to include more models. + +Join the community for oTToDev! + +https://thinktank.ottomator.ai # Requested Additions to this Fork - Feel Free to Contribute!! @@ -20,21 +24,23 @@ This fork of Bolt.new allows you to choose the LLM that you use for each prompt! - ✅ Publish projects directly to GitHub (@goncaloalves) - ✅ Ability to enter API keys in the UI (@ali00209) - ✅ xAI Grok Beta Integration (@milutinke) +- ✅ LM Studio Integration (@karrot0) +- ✅ HuggingFace Integration (@ahsan3219) +- ✅ Bolt terminal to see the output of LLM run commands (@thecodacus) +- ✅ Streaming of code output (@thecodacus) +- ✅ Ability to revert code to earlier version (@wonderwhy-er) - ⬜ **HIGH PRIORITY** - Prevent Bolt from rewriting files as often (file locking and diffs) - ⬜ **HIGH PRIORITY** - Better prompting for smaller LLMs (code window sometimes doesn't start) -- ⬜ **HIGH PRIORITY** Load local projects into the app +- ⬜ **HIGH PRIORITY** - Load local projects into the app - ⬜ **HIGH PRIORITY** - Attach images to prompts - ⬜ **HIGH PRIORITY** - Run agents in the backend as opposed to a single model call - ⬜ Mobile friendly -- ⬜ LM Studio Integration - ⬜ Together Integration - ⬜ Azure Open AI API Integration -- ⬜ HuggingFace Integration - ⬜ Perplexity Integration - ⬜ Vertex AI Integration - ⬜ Cohere Integration - ⬜ Deploy directly to Vercel/Netlify/other similar platforms -- ⬜ Ability to revert code to earlier version - ⬜ Prompt caching - ⬜ Better prompt enhancing - ⬜ Have LLM plan the project in a MD file for better results/transparency From b39e48b7da7c49ddc838b583a60391a975dee23e Mon Sep 17 00:00:00 2001 From: eduardruzga Date: Sun, 17 Nov 2024 15:52:38 +0200 Subject: [PATCH 07/15] .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b43105b7..3303fba9 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ dist-ssr *.sln *.sw? +/.history /.cache /build .env.local From b0b6760136a4156ff73b900162a8c28de9f1a10d Mon Sep 17 00:00:00 2001 From: eduardruzga Date: Wed, 20 Nov 2024 16:47:19 +0200 Subject: [PATCH 08/15] Add background for chat window --- app/components/chat/BaseChat.tsx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/components/chat/BaseChat.tsx b/app/components/chat/BaseChat.tsx index 629c5cbe..902cb232 100644 --- a/app/components/chat/BaseChat.tsx +++ b/app/components/chat/BaseChat.tsx @@ -192,9 +192,11 @@ export const BaseChat = React.forwardRef( }}
Date: Wed, 20 Nov 2024 15:08:42 +0000 Subject: [PATCH 09/15] Cohere support added --- .env.example | 4 +++ README.md | 2 +- app/lib/.server/llm/api-key.ts | 2 ++ app/lib/.server/llm/constants.ts | 2 +- app/lib/.server/llm/model.ts | 14 ++++++++++ app/utils/constants.ts | 16 +++++++++++ package.json | 1 + pnpm-lock.yaml | 47 ++++++++++++++++++++++++++++++++ 8 files changed, 86 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 00a0fd94..473f05ac 100644 --- a/.env.example +++ b/.env.example @@ -49,6 +49,10 @@ OPENAI_LIKE_API_KEY= # You only need this environment variable set if you want to use Mistral models MISTRAL_API_KEY= +# Get the Cohere Api key by following these instructions - +# https://dashboard.cohere.com/api-keys +# You only need this environment variable set if you want to use Cohere models +COHERE_API_KEY= # Get LMStudio Base URL from LM Studio Developer Console # Make sure to enable CORS diff --git a/README.md b/README.md index 6bcacd4e..7a3957ac 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ https://thinktank.ottomator.ai - ⬜ Azure Open AI API Integration - ⬜ Perplexity Integration - ⬜ Vertex AI Integration -- ⬜ Cohere Integration +- ✅ Cohere Integration - ⬜ Deploy directly to Vercel/Netlify/other similar platforms - ⬜ Prompt caching - ⬜ Better prompt enhancing diff --git a/app/lib/.server/llm/api-key.ts b/app/lib/.server/llm/api-key.ts index f282aa80..7d8d2f93 100644 --- a/app/lib/.server/llm/api-key.ts +++ b/app/lib/.server/llm/api-key.ts @@ -35,6 +35,8 @@ export function getAPIKey(cloudflareEnv: Env, provider: string, userApiKeys?: Re return env.OPENAI_LIKE_API_KEY || cloudflareEnv.OPENAI_LIKE_API_KEY; case "xAI": return env.XAI_API_KEY || cloudflareEnv.XAI_API_KEY; + case "Cohere": + return env.COHERE_API_KEY; default: return ""; } diff --git a/app/lib/.server/llm/constants.ts b/app/lib/.server/llm/constants.ts index 7b3a0f24..1f993c98 100644 --- a/app/lib/.server/llm/constants.ts +++ b/app/lib/.server/llm/constants.ts @@ -1,5 +1,5 @@ // see https://docs.anthropic.com/en/docs/about-claude/models -export const MAX_TOKENS = 8000; +export const MAX_TOKENS = 4096; // limits the number of model responses that can be returned in a single request export const MAX_RESPONSE_SEGMENTS = 2; diff --git a/app/lib/.server/llm/model.ts b/app/lib/.server/llm/model.ts index e07f2bbc..2e7c5684 100644 --- a/app/lib/.server/llm/model.ts +++ b/app/lib/.server/llm/model.ts @@ -7,6 +7,7 @@ import { createGoogleGenerativeAI } from '@ai-sdk/google'; import { ollama } from 'ollama-ai-provider'; import { createOpenRouter } from "@openrouter/ai-sdk-provider"; import { createMistral } from '@ai-sdk/mistral'; +import { createCohere } from '@ai-sdk/cohere' export function getAnthropicModel(apiKey: string, model: string) { const anthropic = createAnthropic({ @@ -23,6 +24,15 @@ export function getOpenAILikeModel(baseURL:string,apiKey: string, model: string) return openai(model); } + +export function getCohereAIModel(apiKey:string, model: string){ + const cohere = createCohere({ + apiKey, + }); + + return cohere(model); +} + export function getOpenAIModel(apiKey: string, model: string) { const openai = createOpenAI({ apiKey, @@ -108,6 +118,8 @@ export function getXAIModel(apiKey: string, model: string) { return openai(model); } + + export function getModel(provider: string, model: string, env: Env, apiKeys?: Record) { const apiKey = getAPIKey(env, provider, apiKeys); const baseURL = getBaseURL(env, provider); @@ -135,6 +147,8 @@ export function getModel(provider: string, model: string, env: Env, apiKeys?: Re return getLMStudioModel(baseURL, model); case 'xAI': return getXAIModel(apiKey, model); + case 'Cohere': + return getCohereAIModel(apiKey, model); default: return getOllamaModel(baseURL, model); } diff --git a/app/utils/constants.ts b/app/utils/constants.ts index 672c7a81..fe769322 100644 --- a/app/utils/constants.ts +++ b/app/utils/constants.ts @@ -33,6 +33,22 @@ const PROVIDER_LIST: ProviderInfo[] = [ staticModels: [], getDynamicModels: getOpenAILikeModels }, + { + name: 'Cohere', + staticModels: [ + { name: 'command-r-plus-08-2024', label: 'Command R plus Latest', provider: 'Cohere' }, + { name: 'command-r-08-2024', label: 'Command R Latest', provider: 'Cohere' }, + { name: 'command-r-plus', label: 'Command R plus', provider: 'Cohere' }, + { name: 'command-r', label: 'Command R', provider: 'Cohere' }, + { name: 'command', label: 'Command', provider: 'Cohere' }, + { name: 'command-nightly', label: 'Command Nightly', provider: 'Cohere' }, + { name: 'command-light', label: 'Command Light', provider: 'Cohere' }, + { name: 'command-light-nightly', label: 'Command Light Nightly', provider: 'Cohere' }, + { name: 'c4ai-aya-expanse-8b', label: 'c4AI Aya Expanse 8b', provider: 'Cohere' }, + { name: 'c4ai-aya-expanse-32b', label: 'c4AI Aya Expanse 32b', provider: 'Cohere' }, + ], + getApiKeyLink: 'https://dashboard.cohere.com/api-keys' + }, { name: 'OpenRouter', staticModels: [ diff --git a/package.json b/package.json index 40ede0f7..cc1c256d 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ }, "dependencies": { "@ai-sdk/anthropic": "^0.0.39", + "@ai-sdk/cohere": "^1.0.1", "@ai-sdk/google": "^0.0.52", "@ai-sdk/mistral": "^0.0.43", "@ai-sdk/openai": "^0.0.66", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4158d19c..951d1a4f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,6 +14,9 @@ importers: '@ai-sdk/anthropic': specifier: ^0.0.39 version: 0.0.39(zod@3.23.8) + '@ai-sdk/cohere': + specifier: ^1.0.1 + version: 1.0.1(zod@3.23.8) '@ai-sdk/google': specifier: ^0.0.52 version: 0.0.52(zod@3.23.8) @@ -279,6 +282,12 @@ packages: peerDependencies: zod: ^3.0.0 + '@ai-sdk/cohere@1.0.1': + resolution: {integrity: sha512-xLaSYl/hs9EqfpvT9PvqZrDWjJPQPZBd0iT32T6812vN6kwuEQ6sSgQvqHWczIqxeej2GNRgMQwDL6Lh0L5pZw==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.0.0 + '@ai-sdk/google@0.0.52': resolution: {integrity: sha512-bfsA/1Ae0SQ6NfLwWKs5SU4MBwlzJjVhK6bTVBicYFjUxg9liK/W76P1Tq/qK9OlrODACz3i1STOIWsFPpIOuQ==} engines: {node: '>=18'} @@ -324,6 +333,15 @@ packages: zod: optional: true + '@ai-sdk/provider-utils@2.0.1': + resolution: {integrity: sha512-TNg7rPhRtETB2Z9F0JpOvpGii9Fs8EWM8nYy1jEkvSXkrPJ6b/9zVnDdaJsmLFDyrMbOsPJlkblYtmYEQou36w==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.0.0 + peerDependenciesMeta: + zod: + optional: true + '@ai-sdk/provider@0.0.12': resolution: {integrity: sha512-oOwPQD8i2Ynpn22cur4sk26FW3mSy6t6/X/K1Ay2yGBKYiSpRyLfObhOrZEGsXDx+3euKy4nEZ193R36NM+tpQ==} engines: {node: '>=18'} @@ -336,6 +354,10 @@ packages: resolution: {integrity: sha512-XMsNGJdGO+L0cxhhegtqZ8+T6nn4EoShS819OvCgI2kLbYTIvk0GWFGD0AXJmxkxs3DrpsJxKAFukFR7bvTkgQ==} engines: {node: '>=18'} + '@ai-sdk/provider@1.0.0': + resolution: {integrity: sha512-Sj29AzooJ7SYvhPd+AAWt/E7j63E9+AzRnoMHUaJPRYzOd/WDrVNxxv85prF9gDcQ7XPVlSk9j6oAZV9/DXYpA==} + engines: {node: '>=18'} + '@ai-sdk/react@0.0.62': resolution: {integrity: sha512-1asDpxgmeHWL0/EZPCLENxfOHT+0jce0z/zasRhascodm2S6f6/KZn5doLG9jdmarcb+GjMjFmmwyOVXz3W1xg==} engines: {node: '>=18'} @@ -3033,6 +3055,10 @@ packages: resolution: {integrity: sha512-v0eOBUbiaFojBu2s2NPBfYUoRR9GjcDNvCXVaqEf5vVfpIAh9f8RCo4vXTP8c63QRKCFwoLpMpTdPwwhEKVgzA==} engines: {node: '>=14.18'} + eventsource-parser@3.0.0: + resolution: {integrity: sha512-T1C0XCUimhxVQzW4zFipdx0SficT651NnkR0ZSH3yQwh+mFMdLfgjABVi4YtMTtaL4s168593DaoaRLMqryavA==} + engines: {node: '>=18.0.0'} + evp_bytestokey@1.0.3: resolution: {integrity: sha512-/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA==} @@ -5687,6 +5713,12 @@ snapshots: '@ai-sdk/provider-utils': 1.0.9(zod@3.23.8) zod: 3.23.8 + '@ai-sdk/cohere@1.0.1(zod@3.23.8)': + dependencies: + '@ai-sdk/provider': 1.0.0 + '@ai-sdk/provider-utils': 2.0.1(zod@3.23.8) + zod: 3.23.8 + '@ai-sdk/google@0.0.52(zod@3.23.8)': dependencies: '@ai-sdk/provider': 0.0.24 @@ -5733,6 +5765,15 @@ snapshots: optionalDependencies: zod: 3.23.8 + '@ai-sdk/provider-utils@2.0.1(zod@3.23.8)': + dependencies: + '@ai-sdk/provider': 1.0.0 + eventsource-parser: 3.0.0 + nanoid: 3.3.7 + secure-json-parse: 2.7.0 + optionalDependencies: + zod: 3.23.8 + '@ai-sdk/provider@0.0.12': dependencies: json-schema: 0.4.0 @@ -5745,6 +5786,10 @@ snapshots: dependencies: json-schema: 0.4.0 + '@ai-sdk/provider@1.0.0': + dependencies: + json-schema: 0.4.0 + '@ai-sdk/react@0.0.62(react@18.3.1)(zod@3.23.8)': dependencies: '@ai-sdk/provider-utils': 1.0.20(zod@3.23.8) @@ -8751,6 +8796,8 @@ snapshots: eventsource-parser@1.1.2: {} + eventsource-parser@3.0.0: {} + evp_bytestokey@1.0.3: dependencies: md5.js: 1.3.5 From eabfbb2220c58b9b54535f6167f3c8a60deaa8b4 Mon Sep 17 00:00:00 2001 From: Raiyan Hasan Date: Wed, 20 Nov 2024 21:41:51 +0530 Subject: [PATCH 10/15] max token is now dynamically handle for each model --- app/lib/.server/llm/constants.ts | 2 +- app/lib/.server/llm/stream-text.ts | 17 ++-- app/utils/constants.ts | 123 +++++++++++++++-------------- app/utils/types.ts | 1 + 4 files changed, 77 insertions(+), 66 deletions(-) diff --git a/app/lib/.server/llm/constants.ts b/app/lib/.server/llm/constants.ts index 1f993c98..7b3a0f24 100644 --- a/app/lib/.server/llm/constants.ts +++ b/app/lib/.server/llm/constants.ts @@ -1,5 +1,5 @@ // see https://docs.anthropic.com/en/docs/about-claude/models -export const MAX_TOKENS = 4096; +export const MAX_TOKENS = 8000; // limits the number of model responses that can be returned in a single request export const MAX_RESPONSE_SEGMENTS = 2; diff --git a/app/lib/.server/llm/stream-text.ts b/app/lib/.server/llm/stream-text.ts index 8bfd7978..dba5a610 100644 --- a/app/lib/.server/llm/stream-text.ts +++ b/app/lib/.server/llm/stream-text.ts @@ -41,10 +41,9 @@ function extractPropertiesFromMessage(message: Message): { model: string; provid return { model, provider, content: cleanedContent }; } - export function streamText( - messages: Messages, - env: Env, + messages: Messages, + env: Env, options?: StreamingOptions, apiKeys?: Record ) { @@ -64,13 +63,21 @@ export function streamText( return { ...message, content }; } - return message; // No changes for non-user messages + return message; }); + const modelDetails = MODEL_LIST.find((m) => m.name === currentModel); + + const dynamicMaxTokens = + modelDetails && modelDetails.maxTokenAllowed + ? Math.min(MAX_TOKENS, modelDetails.maxTokenAllowed) + : MAX_TOKENS; + console.log(dynamicMaxTokens) + return _streamText({ model: getModel(currentProvider, currentModel, env, apiKeys), system: getSystemPrompt(), - maxTokens: MAX_TOKENS, + maxTokens: dynamicMaxTokens, messages: convertToCoreMessages(processedMessages), ...options, }); diff --git a/app/utils/constants.ts b/app/utils/constants.ts index fe769322..942afc4f 100644 --- a/app/utils/constants.ts +++ b/app/utils/constants.ts @@ -12,12 +12,12 @@ const PROVIDER_LIST: ProviderInfo[] = [ { name: 'Anthropic', staticModels: [ - { name: 'claude-3-5-sonnet-latest', label: 'Claude 3.5 Sonnet (new)', provider: 'Anthropic' }, - { name: 'claude-3-5-sonnet-20240620', label: 'Claude 3.5 Sonnet (old)', provider: 'Anthropic' }, - { name: 'claude-3-5-haiku-latest', label: 'Claude 3.5 Haiku (new)', provider: 'Anthropic' }, - { name: 'claude-3-opus-latest', label: 'Claude 3 Opus', provider: 'Anthropic' }, - { name: 'claude-3-sonnet-20240229', label: 'Claude 3 Sonnet', provider: 'Anthropic' }, - { name: 'claude-3-haiku-20240307', label: 'Claude 3 Haiku', provider: 'Anthropic' } + { name: 'claude-3-5-sonnet-latest', label: 'Claude 3.5 Sonnet (new)', provider: 'Anthropic', maxTokenAllowed: 8000 }, + { name: 'claude-3-5-sonnet-20240620', label: 'Claude 3.5 Sonnet (old)', provider: 'Anthropic', maxTokenAllowed: 8000 }, + { name: 'claude-3-5-haiku-latest', label: 'Claude 3.5 Haiku (new)', provider: 'Anthropic', maxTokenAllowed: 8000 }, + { name: 'claude-3-opus-latest', label: 'Claude 3 Opus', provider: 'Anthropic', maxTokenAllowed: 8000 }, + { name: 'claude-3-sonnet-20240229', label: 'Claude 3 Sonnet', provider: 'Anthropic', maxTokenAllowed: 8000 }, + { name: 'claude-3-haiku-20240307', label: 'Claude 3 Haiku', provider: 'Anthropic', maxTokenAllowed: 8000 } ], getApiKeyLink: "https://console.anthropic.com/settings/keys", }, @@ -36,36 +36,37 @@ const PROVIDER_LIST: ProviderInfo[] = [ { name: 'Cohere', staticModels: [ - { name: 'command-r-plus-08-2024', label: 'Command R plus Latest', provider: 'Cohere' }, - { name: 'command-r-08-2024', label: 'Command R Latest', provider: 'Cohere' }, - { name: 'command-r-plus', label: 'Command R plus', provider: 'Cohere' }, - { name: 'command-r', label: 'Command R', provider: 'Cohere' }, - { name: 'command', label: 'Command', provider: 'Cohere' }, - { name: 'command-nightly', label: 'Command Nightly', provider: 'Cohere' }, - { name: 'command-light', label: 'Command Light', provider: 'Cohere' }, - { name: 'command-light-nightly', label: 'Command Light Nightly', provider: 'Cohere' }, - { name: 'c4ai-aya-expanse-8b', label: 'c4AI Aya Expanse 8b', provider: 'Cohere' }, - { name: 'c4ai-aya-expanse-32b', label: 'c4AI Aya Expanse 32b', provider: 'Cohere' }, + { name: 'command-r-plus-08-2024', label: 'Command R plus Latest', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command-r-08-2024', label: 'Command R Latest', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command-r-plus', label: 'Command R plus', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command-r', label: 'Command R', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command', label: 'Command', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command-nightly', label: 'Command Nightly', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command-light', label: 'Command Light', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command-light-nightly', label: 'Command Light Nightly', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'c4ai-aya-expanse-8b', label: 'c4AI Aya Expanse 8b', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'c4ai-aya-expanse-32b', label: 'c4AI Aya Expanse 32b', provider: 'Cohere', maxTokenAllowed: 4096 }, ], getApiKeyLink: 'https://dashboard.cohere.com/api-keys' }, { name: 'OpenRouter', staticModels: [ - { name: 'gpt-4o', label: 'GPT-4o', provider: 'OpenAI' }, + { name: 'gpt-4o', label: 'GPT-4o', provider: 'OpenAI', maxTokenAllowed: 8000 }, { name: 'anthropic/claude-3.5-sonnet', label: 'Anthropic: Claude 3.5 Sonnet (OpenRouter)', provider: 'OpenRouter' + , maxTokenAllowed: 8000 }, - { name: 'anthropic/claude-3-haiku', label: 'Anthropic: Claude 3 Haiku (OpenRouter)', provider: 'OpenRouter' }, - { name: 'deepseek/deepseek-coder', label: 'Deepseek-Coder V2 236B (OpenRouter)', provider: 'OpenRouter' }, - { name: 'google/gemini-flash-1.5', label: 'Google Gemini Flash 1.5 (OpenRouter)', provider: 'OpenRouter' }, - { name: 'google/gemini-pro-1.5', label: 'Google Gemini Pro 1.5 (OpenRouter)', provider: 'OpenRouter' }, - { name: 'x-ai/grok-beta', label: 'xAI Grok Beta (OpenRouter)', provider: 'OpenRouter' }, - { name: 'mistralai/mistral-nemo', label: 'OpenRouter Mistral Nemo (OpenRouter)', provider: 'OpenRouter' }, - { name: 'qwen/qwen-110b-chat', label: 'OpenRouter Qwen 110b Chat (OpenRouter)', provider: 'OpenRouter' }, - { name: 'cohere/command', label: 'Cohere Command (OpenRouter)', provider: 'OpenRouter' } + { name: 'anthropic/claude-3-haiku', label: 'Anthropic: Claude 3 Haiku (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 }, + { name: 'deepseek/deepseek-coder', label: 'Deepseek-Coder V2 236B (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 }, + { name: 'google/gemini-flash-1.5', label: 'Google Gemini Flash 1.5 (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 }, + { name: 'google/gemini-pro-1.5', label: 'Google Gemini Pro 1.5 (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 }, + { name: 'x-ai/grok-beta', label: 'xAI Grok Beta (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 }, + { name: 'mistralai/mistral-nemo', label: 'OpenRouter Mistral Nemo (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 }, + { name: 'qwen/qwen-110b-chat', label: 'OpenRouter Qwen 110b Chat (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 }, + { name: 'cohere/command', label: 'Cohere Command (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 4096 } ], getDynamicModels: getOpenRouterModels, getApiKeyLink: 'https://openrouter.ai/settings/keys', @@ -73,70 +74,70 @@ const PROVIDER_LIST: ProviderInfo[] = [ }, { name: 'Google', staticModels: [ - { name: 'gemini-1.5-flash-latest', label: 'Gemini 1.5 Flash', provider: 'Google' }, - { name: 'gemini-1.5-flash-002', label: 'Gemini 1.5 Flash-002', provider: 'Google' }, - { name: 'gemini-1.5-flash-8b', label: 'Gemini 1.5 Flash-8b', provider: 'Google' }, - { name: 'gemini-1.5-pro-latest', label: 'Gemini 1.5 Pro', provider: 'Google' }, - { name: 'gemini-1.5-pro-002', label: 'Gemini 1.5 Pro-002', provider: 'Google' }, - { name: 'gemini-exp-1114', label: 'Gemini exp-1114', provider: 'Google' } + { name: 'gemini-1.5-flash-latest', label: 'Gemini 1.5 Flash', provider: 'Google', maxTokenAllowed: 8192 }, + { name: 'gemini-1.5-flash-002', label: 'Gemini 1.5 Flash-002', provider: 'Google', maxTokenAllowed: 8192 }, + { name: 'gemini-1.5-flash-8b', label: 'Gemini 1.5 Flash-8b', provider: 'Google', maxTokenAllowed: 8192 }, + { name: 'gemini-1.5-pro-latest', label: 'Gemini 1.5 Pro', provider: 'Google', maxTokenAllowed: 8192 }, + { name: 'gemini-1.5-pro-002', label: 'Gemini 1.5 Pro-002', provider: 'Google', maxTokenAllowed: 8192 }, + { name: 'gemini-exp-1114', label: 'Gemini exp-1114', provider: 'Google', maxTokenAllowed: 8192 } ], getApiKeyLink: 'https://aistudio.google.com/app/apikey' }, { name: 'Groq', staticModels: [ - { name: 'llama-3.1-70b-versatile', label: 'Llama 3.1 70b (Groq)', provider: 'Groq' }, - { name: 'llama-3.1-8b-instant', label: 'Llama 3.1 8b (Groq)', provider: 'Groq' }, - { name: 'llama-3.2-11b-vision-preview', label: 'Llama 3.2 11b (Groq)', provider: 'Groq' }, - { name: 'llama-3.2-3b-preview', label: 'Llama 3.2 3b (Groq)', provider: 'Groq' }, - { name: 'llama-3.2-1b-preview', label: 'Llama 3.2 1b (Groq)', provider: 'Groq' } + { name: 'llama-3.1-70b-versatile', label: 'Llama 3.1 70b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, + { name: 'llama-3.1-8b-instant', label: 'Llama 3.1 8b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, + { name: 'llama-3.2-11b-vision-preview', label: 'Llama 3.2 11b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, + { name: 'llama-3.2-3b-preview', label: 'Llama 3.2 3b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, + { name: 'llama-3.2-1b-preview', label: 'Llama 3.2 1b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 } ], getApiKeyLink: 'https://console.groq.com/keys' }, { name: 'HuggingFace', staticModels: [ - { name: 'Qwen/Qwen2.5-Coder-32B-Instruct', label: 'Qwen2.5-Coder-32B-Instruct (HuggingFace)', provider: 'HuggingFace' }, - { name: '01-ai/Yi-1.5-34B-Chat', label: 'Yi-1.5-34B-Chat (HuggingFace)', provider: 'HuggingFace' }, - { name: 'codellama/CodeLlama-34b-Instruct-hf', label: 'CodeLlama-34b-Instruct (HuggingFace)', provider: 'HuggingFace' }, - { name: 'NousResearch/Hermes-3-Llama-3.1-8B', label: 'Hermes-3-Llama-3.1-8B (HuggingFace)', provider: 'HuggingFace' } + { name: 'Qwen/Qwen2.5-Coder-32B-Instruct', label: 'Qwen2.5-Coder-32B-Instruct (HuggingFace)', provider: 'HuggingFace', maxTokenAllowed: 8000 }, + { name: '01-ai/Yi-1.5-34B-Chat', label: 'Yi-1.5-34B-Chat (HuggingFace)', provider: 'HuggingFace', maxTokenAllowed: 8000 }, + { name: 'codellama/CodeLlama-34b-Instruct-hf', label: 'CodeLlama-34b-Instruct (HuggingFace)', provider: 'HuggingFace', maxTokenAllowed: 8000 }, + { name: 'NousResearch/Hermes-3-Llama-3.1-8B', label: 'Hermes-3-Llama-3.1-8B (HuggingFace)', provider: 'HuggingFace', maxTokenAllowed: 8000 } ], getApiKeyLink: 'https://huggingface.co/settings/tokens' }, - + { name: 'OpenAI', staticModels: [ - { name: 'gpt-4o-mini', label: 'GPT-4o Mini', provider: 'OpenAI' }, - { name: 'gpt-4-turbo', label: 'GPT-4 Turbo', provider: 'OpenAI' }, - { name: 'gpt-4', label: 'GPT-4', provider: 'OpenAI' }, - { name: 'gpt-3.5-turbo', label: 'GPT-3.5 Turbo', provider: 'OpenAI' } + { name: 'gpt-4o-mini', label: 'GPT-4o Mini', provider: 'OpenAI', maxTokenAllowed: 8000 }, + { name: 'gpt-4-turbo', label: 'GPT-4 Turbo', provider: 'OpenAI', maxTokenAllowed: 8000 }, + { name: 'gpt-4', label: 'GPT-4', provider: 'OpenAI', maxTokenAllowed: 8000 }, + { name: 'gpt-3.5-turbo', label: 'GPT-3.5 Turbo', provider: 'OpenAI', maxTokenAllowed: 8000 } ], getApiKeyLink: "https://platform.openai.com/api-keys", }, { name: 'xAI', staticModels: [ - { name: 'grok-beta', label: 'xAI Grok Beta', provider: 'xAI' } + { name: 'grok-beta', label: 'xAI Grok Beta', provider: 'xAI', maxTokenAllowed: 8000 } ], getApiKeyLink: 'https://docs.x.ai/docs/quickstart#creating-an-api-key' }, { name: 'Deepseek', staticModels: [ - { name: 'deepseek-coder', label: 'Deepseek-Coder', provider: 'Deepseek' }, - { name: 'deepseek-chat', label: 'Deepseek-Chat', provider: 'Deepseek' } + { name: 'deepseek-coder', label: 'Deepseek-Coder', provider: 'Deepseek', maxTokenAllowed: 8000 }, + { name: 'deepseek-chat', label: 'Deepseek-Chat', provider: 'Deepseek', maxTokenAllowed: 8000 } ], getApiKeyLink: 'https://platform.deepseek.com/api_keys' }, { name: 'Mistral', staticModels: [ - { name: 'open-mistral-7b', label: 'Mistral 7B', provider: 'Mistral' }, - { name: 'open-mixtral-8x7b', label: 'Mistral 8x7B', provider: 'Mistral' }, - { name: 'open-mixtral-8x22b', label: 'Mistral 8x22B', provider: 'Mistral' }, - { name: 'open-codestral-mamba', label: 'Codestral Mamba', provider: 'Mistral' }, - { name: 'open-mistral-nemo', label: 'Mistral Nemo', provider: 'Mistral' }, - { name: 'ministral-8b-latest', label: 'Mistral 8B', provider: 'Mistral' }, - { name: 'mistral-small-latest', label: 'Mistral Small', provider: 'Mistral' }, - { name: 'codestral-latest', label: 'Codestral', provider: 'Mistral' }, - { name: 'mistral-large-latest', label: 'Mistral Large Latest', provider: 'Mistral' } + { name: 'open-mistral-7b', label: 'Mistral 7B', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'open-mixtral-8x7b', label: 'Mistral 8x7B', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'open-mixtral-8x22b', label: 'Mistral 8x22B', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'open-codestral-mamba', label: 'Codestral Mamba', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'open-mistral-nemo', label: 'Mistral Nemo', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'ministral-8b-latest', label: 'Mistral 8B', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'mistral-small-latest', label: 'Mistral Small', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'codestral-latest', label: 'Codestral', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'mistral-large-latest', label: 'Mistral Large Latest', provider: 'Mistral', maxTokenAllowed: 8000 } ], getApiKeyLink: 'https://console.mistral.ai/api-keys/' }, { @@ -180,7 +181,8 @@ async function getOllamaModels(): Promise { return data.models.map((model: OllamaModel) => ({ name: model.name, label: `${model.name} (${model.details.parameter_size})`, - provider: 'Ollama' + provider: 'Ollama', + maxTokenAllowed:8000, })); } catch (e) { return []; @@ -233,8 +235,9 @@ async function getOpenRouterModels(): Promise { name: m.id, label: `${m.name} - in:$${(m.pricing.prompt * 1_000_000).toFixed( 2)} out:$${(m.pricing.completion * 1_000_000).toFixed(2)} - context ${Math.floor( - m.context_length / 1000)}k`, - provider: 'OpenRouter' + m.context_length / 1000)}k`, + provider: 'OpenRouter', + maxTokenAllowed:8000, })); } diff --git a/app/utils/types.ts b/app/utils/types.ts index a5f9fc18..171edc3d 100644 --- a/app/utils/types.ts +++ b/app/utils/types.ts @@ -25,6 +25,7 @@ export interface ModelInfo { name: string; label: string; provider: string; + maxTokenAllowed: number; } export interface ProviderInfo { From 934044f73549cbc2a94e7888af172cc6a4ab743c Mon Sep 17 00:00:00 2001 From: Raiyan Hasan Date: Wed, 20 Nov 2024 21:47:43 +0530 Subject: [PATCH 11/15] console message removed --- app/lib/.server/llm/stream-text.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/app/lib/.server/llm/stream-text.ts b/app/lib/.server/llm/stream-text.ts index dba5a610..0a9c8e2d 100644 --- a/app/lib/.server/llm/stream-text.ts +++ b/app/lib/.server/llm/stream-text.ts @@ -72,7 +72,6 @@ export function streamText( modelDetails && modelDetails.maxTokenAllowed ? Math.min(MAX_TOKENS, modelDetails.maxTokenAllowed) : MAX_TOKENS; - console.log(dynamicMaxTokens) return _streamText({ model: getModel(currentProvider, currentModel, env, apiKeys), From 4269d57ea8c1b0f8e3142936af74dbeb6e7b2e6c Mon Sep 17 00:00:00 2001 From: Raiyan Hasan Date: Wed, 20 Nov 2024 21:52:08 +0530 Subject: [PATCH 12/15] README.md updated --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7a3957ac..f952e0af 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,8 @@ https://thinktank.ottomator.ai - ⬜ Azure Open AI API Integration - ⬜ Perplexity Integration - ⬜ Vertex AI Integration -- ✅ Cohere Integration +- ✅ Cohere Integration (@hasanraiyan) +- ✅ Dynamic model max token length (@hasanraiyan) - ⬜ Deploy directly to Vercel/Netlify/other similar platforms - ⬜ Prompt caching - ⬜ Better prompt enhancing From e1c3d603dfa9d31d1aa955a86264916a1a56e28a Mon Sep 17 00:00:00 2001 From: Raiyan Hasan Date: Thu, 21 Nov 2024 11:49:09 +0530 Subject: [PATCH 13/15] Update stream-text.ts dynamic model max Token updated --- app/lib/.server/llm/stream-text.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app/lib/.server/llm/stream-text.ts b/app/lib/.server/llm/stream-text.ts index 0a9c8e2d..4fedccb8 100644 --- a/app/lib/.server/llm/stream-text.ts +++ b/app/lib/.server/llm/stream-text.ts @@ -68,10 +68,12 @@ export function streamText( const modelDetails = MODEL_LIST.find((m) => m.name === currentModel); - const dynamicMaxTokens = - modelDetails && modelDetails.maxTokenAllowed - ? Math.min(MAX_TOKENS, modelDetails.maxTokenAllowed) - : MAX_TOKENS; + + + const dynamicMaxTokens = +modelDetails && modelDetails.maxTokenAllowed + ? modelDetails.maxTokenAllowed + : MAX_TOKENS; return _streamText({ model: getModel(currentProvider, currentModel, env, apiKeys), From f644066189333512eec77b24b9c515f734c6b354 Mon Sep 17 00:00:00 2001 From: Qwikode Date: Thu, 21 Nov 2024 15:12:33 +0200 Subject: [PATCH 14/15] fix(ui): mobile friendly --- app/components/chat/APIKeyManager.tsx | 47 +++++---- app/components/chat/BaseChat.tsx | 36 +++---- app/components/chat/Messages.client.tsx | 97 ++++++++++--------- .../header/HeaderActionButtons.client.tsx | 5 +- app/components/workbench/Workbench.client.tsx | 7 +- app/lib/hooks/index.ts | 1 + app/lib/hooks/useViewport.ts | 18 ++++ 7 files changed, 124 insertions(+), 87 deletions(-) create mode 100644 app/lib/hooks/useViewport.ts diff --git a/app/components/chat/APIKeyManager.tsx b/app/components/chat/APIKeyManager.tsx index 5b2c85e4..c61e466d 100644 --- a/app/components/chat/APIKeyManager.tsx +++ b/app/components/chat/APIKeyManager.tsx @@ -10,11 +10,7 @@ interface APIKeyManagerProps { labelForGetApiKey?: string; } -export const APIKeyManager: React.FC = ({ - provider, - apiKey, - setApiKey, - }) => { +export const APIKeyManager: React.FC = ({ provider, apiKey, setApiKey }) => { const [isEditing, setIsEditing] = useState(false); const [tempKey, setTempKey] = useState(apiKey); @@ -24,15 +20,29 @@ export const APIKeyManager: React.FC = ({ }; return ( -
- {provider?.name} API Key: +
+
+ {provider?.name} API Key: + {!isEditing && ( +
+ + {apiKey ? '••••••••' : 'Not set (will still work if set in .env file)'} + + setIsEditing(true)} title="Edit API Key"> +
+ +
+ )} +
+ {isEditing ? ( - <> +
setTempKey(e.target.value)} - className="flex-1 p-1 text-sm rounded border border-bolt-elements-borderColor bg-bolt-elements-prompt-background text-bolt-elements-textPrimary focus:outline-none focus:ring-2 focus:ring-bolt-elements-focus" + className="flex-1 px-2 py-1 text-xs lg:text-sm rounded border border-bolt-elements-borderColor bg-bolt-elements-prompt-background text-bolt-elements-textPrimary focus:outline-none focus:ring-2 focus:ring-bolt-elements-focus" />
@@ -40,20 +50,15 @@ export const APIKeyManager: React.FC = ({ setIsEditing(false)} title="Cancel">
- +
) : ( <> - - {apiKey ? '••••••••' : 'Not set (will still work if set in .env file)'} - - setIsEditing(true)} title="Edit API Key"> -
- - - {provider?.getApiKeyLink && window.open(provider?.getApiKeyLink)} title="Edit API Key"> - {provider?.labelForGetApiKey || 'Get API Key'} -
- } + {provider?.getApiKeyLink && ( + window.open(provider?.getApiKeyLink)} title="Edit API Key"> + {provider?.labelForGetApiKey || 'Get API Key'} +
+ + )} )}
diff --git a/app/components/chat/BaseChat.tsx b/app/components/chat/BaseChat.tsx index 902cb232..396fb012 100644 --- a/app/components/chat/BaseChat.tsx +++ b/app/components/chat/BaseChat.tsx @@ -27,9 +27,9 @@ const EXAMPLE_PROMPTS = [ const providerList = PROVIDER_LIST; -const ModelSelector = ({ model, setModel, provider, setProvider, modelList, providerList }) => { +const ModelSelector = ({ model, setModel, provider, setProvider, modelList, providerList, apiKeys }) => { return ( -
+