UniqAI/bolt.new-any-llm
1
0
Fork 0
mirror of https://github.com/coleam00/bolt.new-any-llm synced 2024-12-27 22:33:03 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Created DEFAULT_NUM_CTX VAR with a deafult of 32768

Browse Source
This commit is contained in:
Aaron Bolton 2024-11-18 20:48:35 +00:00
parent 233d22e080
commit bfaaf86c69
2 changed files with 12 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.env.example
View File

@ -56,3 +56,10 @@ XAI_API_KEY=
# Include this environment variable if you want more logging for debugging locally
VITE_LOG_LEVEL=debug
# Example Context Values for qwen2.5-coder:32b
#
# DEFAULT_NUM_CTX=32768 # Consumes 36GB of VRAM
# DEFAULT_NUM_CTX=24576 # Consumes 32GB of VRAM
# DEFAULT_NUM_CTX=12288 # Consumes 26GB of VRAM
# DEFAULT_NUM_CTX=6144 # Consumes 24GB of VRAM

6
app/lib/.server/llm/model.ts
View File

@ -8,6 +8,10 @@ import { ollama } from 'ollama-ai-provider';
import { createOpenRouter } from "@openrouter/ai-sdk-provider";
import { createMistral } from '@ai-sdk/mistral';
export const DEFAULT_NUM_CTX = process.env.DEFAULT_NUM_CTX ?
parseInt(process.env.DEFAULT_NUM_CTX, 10) :
32768;
export function getAnthropicModel(apiKey: string, model: string) {
const anthropic = createAnthropic({
apiKey,
@ -58,7 +62,7 @@ export function getGroqModel(apiKey: string, model: string) {
export function getOllamaModel(baseURL: string, model: string) {
let Ollama = ollama(model, {
numCtx: 32768,
numCtx: DEFAULT_NUM_CTX,
});
Ollama.config.baseURL = `${baseURL}/api`;