import { BaseProvider } from '~/lib/modules/llm/base-provider'; import type { ModelInfo } from '~/lib/modules/llm/types'; import type { IProviderSetting } from '~/types/model'; import { createOpenAI } from '@ai-sdk/openai'; import type { LanguageModelV1 } from 'ai'; import { logger } from '~/utils/logger'; export default class LMStudioProvider extends BaseProvider { name = 'LMStudio'; getApiKeyLink = 'https://lmstudio.ai/'; labelForGetApiKey = 'Get LMStudio'; icon = 'i-ph:cloud-arrow-down'; config = { baseUrlKey: 'LMSTUDIO_API_BASE_URL', baseUrl: 'http://localhost:1234/', }; staticModels: ModelInfo[] = []; async getDynamicModels( apiKeys?: Record, settings?: IProviderSetting, serverEnv: Record = {}, ): Promise { let { baseUrl } = this.getProviderBaseUrlAndKey({ apiKeys, providerSettings: settings, serverEnv, defaultBaseUrlKey: 'LMSTUDIO_API_BASE_URL', defaultApiTokenKey: '', }); if (!baseUrl) { throw new Error('No baseUrl found for LMStudio provider'); } if (typeof window === 'undefined') { /* * Running in Server * Backend: Check if we're running in Docker */ const isDocker = process.env.RUNNING_IN_DOCKER === 'true'; baseUrl = isDocker ? baseUrl.replace('localhost', 'host.docker.internal') : baseUrl; baseUrl = isDocker ? baseUrl.replace('127.0.0.1', 'host.docker.internal') : baseUrl; } const response = await fetch(`${baseUrl}/v1/models`); const data = (await response.json()) as { data: Array<{ id: string }> }; return data.data.map((model) => ({ name: model.id, label: model.id, provider: this.name, maxTokenAllowed: 8000, })); } getModelInstance: (options: { model: string; serverEnv: Env; apiKeys?: Record; providerSettings?: Record; }) => LanguageModelV1 = (options) => { const { apiKeys, providerSettings, serverEnv, model } = options; let { baseUrl } = this.getProviderBaseUrlAndKey({ apiKeys, providerSettings: providerSettings?.[this.name], serverEnv: serverEnv as any, defaultBaseUrlKey: 'LMSTUDIO_API_BASE_URL', defaultApiTokenKey: '', }); if (!baseUrl) { throw new Error('No baseUrl found for LMStudio provider'); } if (typeof window === 'undefined') { const isDocker = process.env.RUNNING_IN_DOCKER === 'true'; baseUrl = isDocker ? baseUrl.replace('localhost', 'host.docker.internal') : baseUrl; baseUrl = isDocker ? baseUrl.replace('127.0.0.1', 'host.docker.internal') : baseUrl; } logger.debug('LMStudio Base Url used: ', baseUrl); const lmstudio = createOpenAI({ baseUrl: `${baseUrl}/v1`, apiKey: '', }); return lmstudio(model); }; }