diff --git a/.env.example b/.env.example index 00a0fd94..08774d2e 100644 --- a/.env.example +++ b/.env.example @@ -49,6 +49,10 @@ OPENAI_LIKE_API_KEY= # You only need this environment variable set if you want to use Mistral models MISTRAL_API_KEY= +# Get the Cohere Api key by following these instructions - +# https://dashboard.cohere.com/api-keys +# You only need this environment variable set if you want to use Cohere models +COHERE_API_KEY= # Get LMStudio Base URL from LM Studio Developer Console # Make sure to enable CORS @@ -61,4 +65,12 @@ LMSTUDIO_API_BASE_URL= XAI_API_KEY= # Include this environment variable if you want more logging for debugging locally -VITE_LOG_LEVEL=debug \ No newline at end of file +VITE_LOG_LEVEL=debug + +# Example Context Values for qwen2.5-coder:32b +# +# DEFAULT_NUM_CTX=32768 # Consumes 36GB of VRAM +# DEFAULT_NUM_CTX=24576 # Consumes 32GB of VRAM +# DEFAULT_NUM_CTX=12288 # Consumes 26GB of VRAM +# DEFAULT_NUM_CTX=6144 # Consumes 24GB of VRAM +DEFAULT_NUM_CTX= diff --git a/.gitignore b/.gitignore index b43105b7..3303fba9 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ dist-ssr *.sln *.sw? +/.history /.cache /build .env.local diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5bb82b8f..0f7744ab 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,7 @@ # Contributing to Bolt.new Fork +## DEFAULT_NUM_CTX + +The `DEFAULT_NUM_CTX` environment variable can be used to limit the maximum number of context values used by the qwen2.5-coder model. For example, to limit the context to 24576 values (which uses 32GB of VRAM), set `DEFAULT_NUM_CTX=24576` in your `.env.local` file. First off, thank you for considering contributing to Bolt.new! This fork aims to expand the capabilities of the original project by integrating multiple LLM providers and enhancing functionality. Every contribution helps make Bolt.new a better tool for developers worldwide. @@ -81,6 +84,19 @@ ANTHROPIC_API_KEY=XXX ```bash VITE_LOG_LEVEL=debug ``` + + - Optionally set context size: +```bash +DEFAULT_NUM_CTX=32768 +``` + +Some Example Context Values for the qwen2.5-coder:32b models are. + +* DEFAULT_NUM_CTX=32768 - Consumes 36GB of VRAM +* DEFAULT_NUM_CTX=24576 - Consumes 32GB of VRAM +* DEFAULT_NUM_CTX=12288 - Consumes 26GB of VRAM +* DEFAULT_NUM_CTX=6144 - Consumes 24GB of VRAM + **Important**: Never commit your `.env.local` file to version control. It's already included in .gitignore. ### 🚀 Running the Development Server diff --git a/Dockerfile b/Dockerfile index 82c07da0..c581f7f1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,7 @@ ARG OPEN_ROUTER_API_KEY ARG GOOGLE_GENERATIVE_AI_API_KEY ARG OLLAMA_API_BASE_URL ARG VITE_LOG_LEVEL=debug +ARG DEFAULT_NUM_CTX ENV WRANGLER_SEND_METRICS=false \ GROQ_API_KEY=${GROQ_API_KEY} \ @@ -35,7 +36,8 @@ ENV WRANGLER_SEND_METRICS=false \ OPEN_ROUTER_API_KEY=${OPEN_ROUTER_API_KEY} \ GOOGLE_GENERATIVE_AI_API_KEY=${GOOGLE_GENERATIVE_AI_API_KEY} \ OLLAMA_API_BASE_URL=${OLLAMA_API_BASE_URL} \ - VITE_LOG_LEVEL=${VITE_LOG_LEVEL} + VITE_LOG_LEVEL=${VITE_LOG_LEVEL} \ + DEFAULT_NUM_CTX=${DEFAULT_NUM_CTX} # Pre-configure wrangler to disable metrics RUN mkdir -p /root/.config/.wrangler && \ @@ -57,6 +59,7 @@ ARG OPEN_ROUTER_API_KEY ARG GOOGLE_GENERATIVE_AI_API_KEY ARG OLLAMA_API_BASE_URL ARG VITE_LOG_LEVEL=debug +ARG DEFAULT_NUM_CTX ENV GROQ_API_KEY=${GROQ_API_KEY} \ HuggingFace_API_KEY=${HuggingFace_API_KEY} \ @@ -65,7 +68,8 @@ ENV GROQ_API_KEY=${GROQ_API_KEY} \ OPEN_ROUTER_API_KEY=${OPEN_ROUTER_API_KEY} \ GOOGLE_GENERATIVE_AI_API_KEY=${GOOGLE_GENERATIVE_AI_API_KEY} \ OLLAMA_API_BASE_URL=${OLLAMA_API_BASE_URL} \ - VITE_LOG_LEVEL=${VITE_LOG_LEVEL} + VITE_LOG_LEVEL=${VITE_LOG_LEVEL} \ + DEFAULT_NUM_CTX=${DEFAULT_NUM_CTX} RUN mkdir -p ${WORKDIR}/run CMD pnpm run dev --host diff --git a/README.md b/README.md index 52889233..f952e0af 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,12 @@ [![Bolt.new: AI-Powered Full-Stack Web Development in the Browser](./public/social_preview_index.jpg)](https://bolt.new) -# Bolt.new Fork by Cole Medin +# Bolt.new Fork by Cole Medin - oTToDev -This fork of Bolt.new allows you to choose the LLM that you use for each prompt! Currently, you can use OpenAI, Anthropic, Ollama, OpenRouter, Gemini, or Groq models - and it is easily extended to use any other model supported by the Vercel AI SDK! See the instructions below for running this locally and extending it to include more models. +This fork of Bolt.new (oTToDev) allows you to choose the LLM that you use for each prompt! Currently, you can use OpenAI, Anthropic, Ollama, OpenRouter, Gemini, LMStudio, Mistral, xAI, HuggingFace, DeepSeek, or Groq models - and it is easily extended to use any other model supported by the Vercel AI SDK! See the instructions below for running this locally and extending it to include more models. + +Join the community for oTToDev! + +https://thinktank.ottomator.ai # Requested Additions to this Fork - Feel Free to Contribute!! @@ -20,21 +24,24 @@ This fork of Bolt.new allows you to choose the LLM that you use for each prompt! - ✅ Publish projects directly to GitHub (@goncaloalves) - ✅ Ability to enter API keys in the UI (@ali00209) - ✅ xAI Grok Beta Integration (@milutinke) +- ✅ LM Studio Integration (@karrot0) +- ✅ HuggingFace Integration (@ahsan3219) +- ✅ Bolt terminal to see the output of LLM run commands (@thecodacus) +- ✅ Streaming of code output (@thecodacus) +- ✅ Ability to revert code to earlier version (@wonderwhy-er) - ⬜ **HIGH PRIORITY** - Prevent Bolt from rewriting files as often (file locking and diffs) - ⬜ **HIGH PRIORITY** - Better prompting for smaller LLMs (code window sometimes doesn't start) -- ⬜ **HIGH PRIORITY** Load local projects into the app +- ⬜ **HIGH PRIORITY** - Load local projects into the app - ⬜ **HIGH PRIORITY** - Attach images to prompts - ⬜ **HIGH PRIORITY** - Run agents in the backend as opposed to a single model call - ⬜ Mobile friendly -- ⬜ LM Studio Integration - ⬜ Together Integration - ⬜ Azure Open AI API Integration -- ⬜ HuggingFace Integration - ⬜ Perplexity Integration - ⬜ Vertex AI Integration -- ⬜ Cohere Integration +- ✅ Cohere Integration (@hasanraiyan) +- ✅ Dynamic model max token length (@hasanraiyan) - ⬜ Deploy directly to Vercel/Netlify/other similar platforms -- ⬜ Ability to revert code to earlier version - ⬜ Prompt caching - ⬜ Better prompt enhancing - ⬜ Have LLM plan the project in a MD file for better results/transparency diff --git a/app/components/chat/APIKeyManager.tsx b/app/components/chat/APIKeyManager.tsx index 5b2c85e4..c61e466d 100644 --- a/app/components/chat/APIKeyManager.tsx +++ b/app/components/chat/APIKeyManager.tsx @@ -10,11 +10,7 @@ interface APIKeyManagerProps { labelForGetApiKey?: string; } -export const APIKeyManager: React.FC = ({ - provider, - apiKey, - setApiKey, - }) => { +export const APIKeyManager: React.FC = ({ provider, apiKey, setApiKey }) => { const [isEditing, setIsEditing] = useState(false); const [tempKey, setTempKey] = useState(apiKey); @@ -24,15 +20,29 @@ export const APIKeyManager: React.FC = ({ }; return ( -
- {provider?.name} API Key: +
+
+ {provider?.name} API Key: + {!isEditing && ( +
+ + {apiKey ? '••••••••' : 'Not set (will still work if set in .env file)'} + + setIsEditing(true)} title="Edit API Key"> +
+ +
+ )} +
+ {isEditing ? ( - <> +
setTempKey(e.target.value)} - className="flex-1 p-1 text-sm rounded border border-bolt-elements-borderColor bg-bolt-elements-prompt-background text-bolt-elements-textPrimary focus:outline-none focus:ring-2 focus:ring-bolt-elements-focus" + className="flex-1 px-2 py-1 text-xs lg:text-sm rounded border border-bolt-elements-borderColor bg-bolt-elements-prompt-background text-bolt-elements-textPrimary focus:outline-none focus:ring-2 focus:ring-bolt-elements-focus" />
@@ -40,20 +50,15 @@ export const APIKeyManager: React.FC = ({ setIsEditing(false)} title="Cancel">
- +
) : ( <> - - {apiKey ? '••••••••' : 'Not set (will still work if set in .env file)'} - - setIsEditing(true)} title="Edit API Key"> -
- - - {provider?.getApiKeyLink && window.open(provider?.getApiKeyLink)} title="Edit API Key"> - {provider?.labelForGetApiKey || 'Get API Key'} -
- } + {provider?.getApiKeyLink && ( + window.open(provider?.getApiKeyLink)} title="Edit API Key"> + {provider?.labelForGetApiKey || 'Get API Key'} +
+ + )} )}
diff --git a/app/components/chat/BaseChat.tsx b/app/components/chat/BaseChat.tsx index d3ccb053..adfd1bfb 100644 --- a/app/components/chat/BaseChat.tsx +++ b/app/components/chat/BaseChat.tsx @@ -29,9 +29,9 @@ const EXAMPLE_PROMPTS = [ const providerList = PROVIDER_LIST; -const ModelSelector = ({ model, setModel, provider, setProvider, modelList, providerList }) => { +const ModelSelector = ({ model, setModel, provider, setProvider, modelList, providerList, apiKeys }) => { return ( -
+