wip: direct models

src/lib/apis/index.ts

@@ -1,6 +1,11 @@
 import { WEBUI_API_BASE_URL, WEBUI_BASE_URL } from '$lib/constants';
+import { getOpenAIModelsDirect } from './openai';
 
-export const getModels = async (token: string = '', base: boolean = false) => {
+export const getModels = async (
+	token: string = '',
+	connections: object | null = null,
+	base: boolean = false
+) => {
 	let error = null;
 	const res = await fetch(`${WEBUI_BASE_URL}/api/models${base ? '/base' : ''}`, {
 		method: 'GET',
@@ -25,6 +30,76 @@ export const getModels = async (token: string = '', base: boolean = false) => {
 	}
 
 	let models = res?.data ?? [];
+
+	if (connections && !base) {
+		let localModels = [];
+
+		if (connections) {
+			const OPENAI_API_BASE_URLS = connections.OPENAI_API_BASE_URLS;
+			const OPENAI_API_KEYS = connections.OPENAI_API_KEYS;
+			const OPENAI_API_CONFIGS = connections.OPENAI_API_CONFIGS;
+
+			const requests = [];
+			for (const idx in OPENAI_API_BASE_URLS) {
+				const url = OPENAI_API_BASE_URLS[idx];
+
+				if (idx.toString() in OPENAI_API_CONFIGS) {
+					const apiConfig = OPENAI_API_CONFIGS[idx.toString()] ?? {};
+
+					const enable = apiConfig?.enable ?? true;
+					const modelIds = apiConfig?.model_ids ?? [];
+
+					if (enable) {
+						if (modelIds.length > 0) {
+							const modelList = {
+								object: 'list',
+								data: modelIds.map((modelId) => ({
+									id: modelId,
+									name: modelId,
+									owned_by: 'openai',
+									openai: { id: modelId },
+									urlIdx: idx
+								}))
+							};
+
+							requests.push(() => modelList);
+						} else {
+							requests.push(getOpenAIModelsDirect(url, OPENAI_API_KEYS[idx]));
+						}
+					} else {
+						requests.push(() => {});
+					}
+				}
+			}
+			const responses = await Promise.all(requests);
+
+			for (const idx in responses) {
+				const response = responses[idx];
+				const apiConfig = OPENAI_API_CONFIGS[idx.toString()] ?? {};
+
+				let models = Array.isArray(response) ? response : (response?.data ?? []);
+				models = models.map((model) => ({ ...model, openai: { id: model.id }, urlIdx: idx }));
+
+				const prefixId = apiConfig.prefix_id;
+				if (prefixId) {
+					for (const model of models) {
+						model.id = `${prefixId}.${model.id}`;
+					}
+				}
+
+				localModels = localModels.concat(models);
+			}
+		}
+
+		models = models.concat(
+			localModels.map((model) => ({
+				...model,
+				name: model?.name ?? model?.id,
+				direct: true
+			}))
+		);
+	}
+
 	return models;
 };
 

src/lib/apis/openai/index.ts

@@ -208,6 +208,33 @@ export const updateOpenAIKeys = async (token: string = '', keys: string[]) => {
 	return res.OPENAI_API_KEYS;
 };
 
+export const getOpenAIModelsDirect = async (url: string, key: string) => {
+	let error = null;
+
+	const res = await fetch(`${url}/models`, {
+		method: 'GET',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			...(key && { authorization: `Bearer ${key}` })
+		}
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			error = `OpenAI: ${err?.error?.message ?? 'Network Problem'}`;
+			return [];
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
 export const getOpenAIModels = async (token: string, urlIdx?: number) => {
 	let error = null;
 

src/lib/components/admin/Functions.svelte

@@ -3,7 +3,7 @@
 	import fileSaver from 'file-saver';
 	const { saveAs } = fileSaver;
 
-	import { WEBUI_NAME, config, functions, models } from '$lib/stores';
+	import { WEBUI_NAME, config, functions, models, settings } from '$lib/stores';
 	import { onMount, getContext, tick } from 'svelte';
 
 	import { goto } from '$app/navigation';
@@ -126,7 +126,7 @@
 			toast.success($i18n.t('Function deleted successfully'));
 
 			functions.set(await getFunctions(localStorage.token));
-			models.set(await getModels(localStorage.token));
+			models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 		}
 	};
 
@@ -147,7 +147,7 @@
 			}
 
 			functions.set(await getFunctions(localStorage.token));
-			models.set(await getModels(localStorage.token));
+			models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 		}
 	};
 
@@ -359,7 +359,9 @@
 							bind:state={func.is_active}
 							on:change={async (e) => {
 								toggleFunctionById(localStorage.token, func.id);
-								models.set(await getModels(localStorage.token));
+								models.set(
+									await getModels(localStorage.token, $settings?.directConnections ?? null)
+								);
 							}}
 						/>
 					</Tooltip>
@@ -496,7 +498,7 @@
 	id={selectedFunction?.id ?? null}
 	on:save={async () => {
 		await tick();
-		models.set(await getModels(localStorage.token));
+		models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 	}}
 />
 
@@ -517,7 +519,7 @@
 
 			toast.success($i18n.t('Functions imported successfully'));
 			functions.set(await getFunctions(localStorage.token));
-			models.set(await getModels(localStorage.token));
+			models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 		};
 
 		reader.readAsText(importFiles[0]);

src/lib/components/admin/Settings/Audio.svelte

@@ -10,7 +10,7 @@
 		getModels as _getModels,
 		getVoices as _getVoices
 	} from '$lib/apis/audio';
-	import { config } from '$lib/stores';
+	import { config, settings } from '$lib/stores';
 
 	import SensitiveInput from '$lib/components/common/SensitiveInput.svelte';
 
@@ -51,9 +51,11 @@
 		if (TTS_ENGINE === '') {
 			models = [];
 		} else {
-			const res = await _getModels(localStorage.token).catch((e) => {
-				toast.error(`${e}`);
-			});
+			const res = await _getModels(localStorage.token, $settings?.directConnections ?? null).catch(
+				(e) => {
+					toast.error(`${e}`);
+				}
+			);
 
 			if (res) {
 				console.log(res);

src/lib/components/admin/Settings/Connections.svelte

@@ -9,7 +9,7 @@
 	import { getModels as _getModels } from '$lib/apis';
 	import { getDirectConnectionsConfig, setDirectConnectionsConfig } from '$lib/apis/configs';
 
-	import { models, user } from '$lib/stores';
+	import { models, settings, user } from '$lib/stores';
 
 	import Switch from '$lib/components/common/Switch.svelte';
 	import Spinner from '$lib/components/common/Spinner.svelte';
@@ -23,7 +23,7 @@
 	const i18n = getContext('i18n');
 
 	const getModels = async () => {
-		const models = await _getModels(localStorage.token);
+		const models = await _getModels(localStorage.token, $settings?.directConnections ?? null);
 		return models;
 	};
 

src/lib/components/admin/Settings/Evaluations.svelte

@@ -1,6 +1,6 @@
 <script lang="ts">
 	import { toast } from 'svelte-sonner';
-	import { models, user } from '$lib/stores';
+	import { models, settings, user } from '$lib/stores';
 	import { createEventDispatcher, onMount, getContext, tick } from 'svelte';
 
 	const dispatch = createEventDispatcher();
@@ -27,7 +27,7 @@
 
 		if (config) {
 			toast.success('Settings saved successfully');
-			models.set(await getModels(localStorage.token));
+			models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 		}
 	};
 
@@ -36,7 +36,7 @@
 		config.EVALUATION_ARENA_MODELS = [...config.EVALUATION_ARENA_MODELS];
 
 		await submitHandler();
-		models.set(await getModels(localStorage.token));
+		models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 	};
 
 	const editModelHandler = async (model, modelIdx) => {
@@ -44,7 +44,7 @@
 		config.EVALUATION_ARENA_MODELS = [...config.EVALUATION_ARENA_MODELS];
 
 		await submitHandler();
-		models.set(await getModels(localStorage.token));
+		models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 	};
 
 	const deleteModelHandler = async (modelIdx) => {
@@ -53,7 +53,7 @@
 		);
 
 		await submitHandler();
-		models.set(await getModels(localStorage.token));
+		models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 	};
 
 	onMount(async () => {

src/lib/components/admin/Settings/Models.svelte

@@ -68,7 +68,7 @@
 
 	const init = async () => {
 		workspaceModels = await getBaseModels(localStorage.token);
-		baseModels = await getModels(localStorage.token, true);
+		baseModels = await getModels(localStorage.token, null, true);
 
 		models = baseModels.map((m) => {
 			const workspaceModel = workspaceModels.find((wm) => wm.id === m.id);
@@ -111,7 +111,7 @@
 			}
 		}
 
-		_models.set(await getModels(localStorage.token));
+		_models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 		await init();
 	};
 
@@ -133,7 +133,7 @@
 		}
 
 		// await init();
-		_models.set(await getModels(localStorage.token));
+		_models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 	};
 
 	onMount(async () => {
@@ -330,7 +330,9 @@
 									}
 								}
 
-								await _models.set(await getModels(localStorage.token));
+								await _models.set(
+									await getModels(localStorage.token, $settings?.directConnections ?? null)
+								);
 								init();
 							};
 

src/lib/components/admin/Settings/Models/Manage/ManageOllama.svelte

@@ -3,7 +3,7 @@
 	import { getContext, onMount } from 'svelte';
 	const i18n = getContext('i18n');
 
-	import { WEBUI_NAME, models, MODEL_DOWNLOAD_POOL, user, config } from '$lib/stores';
+	import { WEBUI_NAME, models, MODEL_DOWNLOAD_POOL, user, config, settings } from '$lib/stores';
 	import { splitStream } from '$lib/utils';
 
 	import {
@@ -235,7 +235,7 @@
 					})
 				);
 
-				models.set(await getModels(localStorage.token));
+				models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 			} else {
 				toast.error($i18n.t('Download canceled'));
 			}
@@ -394,7 +394,7 @@
 		modelTransferring = false;
 		uploadProgress = null;
 
-		models.set(await getModels(localStorage.token));
+		models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 	};
 
 	const deleteModelHandler = async () => {
@@ -407,7 +407,7 @@
 		}
 
 		deleteModelTag = '';
-		models.set(await getModels(localStorage.token));
+		models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 	};
 
 	const cancelModelPullHandler = async (model: string) => {
@@ -506,7 +506,7 @@
 			}
 		}
 
-		models.set(await getModels(localStorage.token));
+		models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 
 		createModelLoading = false;
 

src/lib/components/admin/Settings/Pipelines.svelte

@@ -2,7 +2,7 @@
 	import { v4 as uuidv4 } from 'uuid';
 
 	import { toast } from 'svelte-sonner';
-	import { models } from '$lib/stores';
+	import { models, settings } from '$lib/stores';
 	import { getContext, onMount, tick } from 'svelte';
 	import type { Writable } from 'svelte/store';
 	import type { i18n as i18nType } from 'i18next';
@@ -63,7 +63,7 @@
 			if (res) {
 				toast.success($i18n.t('Valves updated successfully'));
 				setPipelines();
-				models.set(await getModels(localStorage.token));
+				models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 				saveHandler();
 			}
 		} else {
@@ -125,7 +125,7 @@
 		if (res) {
 			toast.success($i18n.t('Pipeline downloaded successfully'));
 			setPipelines();
-			models.set(await getModels(localStorage.token));
+			models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 		}
 
 		downloading = false;
@@ -150,7 +150,7 @@
 			if (res) {
 				toast.success($i18n.t('Pipeline downloaded successfully'));
 				setPipelines();
-				models.set(await getModels(localStorage.token));
+				models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 			}
 		} else {
 			toast.error($i18n.t('No file selected'));
@@ -179,7 +179,7 @@
 		if (res) {
 			toast.success($i18n.t('Pipeline deleted successfully'));
 			setPipelines();
-			models.set(await getModels(localStorage.token));
+			models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 		}
 	};
 

src/lib/components/chat/ModelSelector/Selector.svelte

@@ -12,7 +12,14 @@
 
 	import { deleteModel, getOllamaVersion, pullModel } from '$lib/apis/ollama';
 
-	import { user, MODEL_DOWNLOAD_POOL, models, mobile, temporaryChatEnabled } from '$lib/stores';
+	import {
+		user,
+		MODEL_DOWNLOAD_POOL,
+		models,
+		mobile,
+		temporaryChatEnabled,
+		settings
+	} from '$lib/stores';
 	import { toast } from 'svelte-sonner';
 	import { capitalizeFirstLetter, sanitizeResponseContent, splitStream } from '$lib/utils';
 	import { getModels } from '$lib/apis';
@@ -186,7 +193,7 @@
 					})
 				);
 
-				models.set(await getModels(localStorage.token));
+				models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 			} else {
 				toast.error($i18n.t('Download canceled'));
 			}
@@ -358,7 +365,24 @@
 
 								<!-- {JSON.stringify(item.info)} -->
 
-								{#if item.model.owned_by === 'openai'}
+								{#if item.model?.direct}
+									<Tooltip content={`${'Direct'}`}>
+										<div class="translate-y-[1px]">
+											<svg
+												xmlns="http://www.w3.org/2000/svg"
+												viewBox="0 0 16 16"
+												fill="currentColor"
+												class="size-3"
+											>
+												<path
+													fill-rule="evenodd"
+													d="M2 2.75A.75.75 0 0 1 2.75 2C8.963 2 14 7.037 14 13.25a.75.75 0 0 1-1.5 0c0-5.385-4.365-9.75-9.75-9.75A.75.75 0 0 1 2 2.75Zm0 4.5a.75.75 0 0 1 .75-.75 6.75 6.75 0 0 1 6.75 6.75.75.75 0 0 1-1.5 0C8 10.35 5.65 8 2.75 8A.75.75 0 0 1 2 7.25ZM3.5 11a1.5 1.5 0 1 0 0 3 1.5 1.5 0 0 0 0-3Z"
+													clip-rule="evenodd"
+												/>
+											</svg>
+										</div>
+									</Tooltip>
+								{:else if item.model.owned_by === 'openai'}
 									<Tooltip content={`${'External'}`}>
 										<div class="translate-y-[1px]">
 											<svg

src/lib/components/chat/Settings/Connections.svelte

@@ -58,14 +58,6 @@
 		});
 	};
 
-	const submitHandler = async () => {
-		await updateHandler();
-
-		await saveSettings({
-			directConnections: config
-		});
-	};
-
 	onMount(async () => {
 		config = $settings?.directConnections ?? {
 			OPENAI_API_BASE_URLS: [],
@@ -77,7 +69,12 @@
 
 <AddConnectionModal direct bind:show={showConnectionModal} onSubmit={addConnectionHandler} />
 
-<form class="flex flex-col h-full justify-between text-sm" on:submit|preventDefault={submitHandler}>
+<form
+	class="flex flex-col h-full justify-between text-sm"
+	on:submit|preventDefault={() => {
+		updateHandler();
+	}}
+>
 	<div class=" overflow-y-scroll scrollbar-hidden h-full">
 		{#if config !== null}
 			<div class="">

src/lib/components/chat/SettingsModal.svelte

@@ -322,7 +322,7 @@
 	};
 
 	const getModels = async () => {
-		return await _getModels(localStorage.token);
+		return await _getModels(localStorage.token, $settings?.directConnections ?? null);
 	};
 
 	let selectedTab = 'general';
@@ -654,8 +654,8 @@
 					/>
 				{:else if selectedTab === 'connections'}
 					<Connections
-						{saveSettings}
-						on:save={() => {
+						saveSettings={async (updated) => {
+							await saveSettings(updated);
 							toast.success($i18n.t('Settings saved successfully!'));
 						}}
 					/>

src/lib/components/workspace/Models.svelte

@@ -68,7 +68,7 @@
 			toast.success($i18n.t(`Deleted {{name}}`, { name: model.id }));
 		}
 
-		await _models.set(await getModels(localStorage.token));
+		await _models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 		models = await getWorkspaceModels(localStorage.token);
 	};
 
@@ -134,7 +134,7 @@
 			);
 		}
 
-		await _models.set(await getModels(localStorage.token));
+		await _models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 		models = await getWorkspaceModels(localStorage.token);
 	};
 
@@ -371,7 +371,9 @@
 										bind:state={model.is_active}
 										on:change={async (e) => {
 											toggleModelById(localStorage.token, model.id);
-											_models.set(await getModels(localStorage.token));
+											_models.set(
+												await getModels(localStorage.token, $settings?.directConnections ?? null)
+											);
 										}}
 									/>
 								</Tooltip>
@@ -417,7 +419,9 @@
 								}
 							}
 
-							await _models.set(await getModels(localStorage.token));
+							await _models.set(
+								await getModels(localStorage.token, $settings?.directConnections ?? null)
+							);
 							models = await getWorkspaceModels(localStorage.token);
 						};
 

src/routes/(app)/+layout.svelte

@@ -93,7 +93,7 @@
 				settings.set(localStorageSettings);
 			}
 
-			models.set(await getModels(localStorage.token));
+			models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 			banners.set(await getBanners(localStorage.token));
 			tools.set(await getTools(localStorage.token));
 

src/routes/(app)/admin/functions/create/+page.svelte

@@ -3,7 +3,7 @@
 	import { onMount, getContext } from 'svelte';
 	import { goto } from '$app/navigation';
 
-	import { functions, models } from '$lib/stores';
+	import { functions, models, settings } from '$lib/stores';
 	import { createNewFunction, getFunctions } from '$lib/apis/functions';
 	import FunctionEditor from '$lib/components/admin/Functions/FunctionEditor.svelte';
 	import { getModels } from '$lib/apis';
@@ -47,7 +47,7 @@
 		if (res) {
 			toast.success($i18n.t('Function created successfully'));
 			functions.set(await getFunctions(localStorage.token));
-			models.set(await getModels(localStorage.token));
+			models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 
 			await goto('/admin/functions');
 		}

src/routes/(app)/admin/functions/edit/+page.svelte

@@ -4,7 +4,7 @@
 
 	import { goto } from '$app/navigation';
 	import { page } from '$app/stores';
-	import { functions, models } from '$lib/stores';
+	import { functions, models, settings } from '$lib/stores';
 	import { updateFunctionById, getFunctions, getFunctionById } from '$lib/apis/functions';
 
 	import FunctionEditor from '$lib/components/admin/Functions/FunctionEditor.svelte';
@@ -48,7 +48,7 @@
 		if (res) {
 			toast.success($i18n.t('Function updated successfully'));
 			functions.set(await getFunctions(localStorage.token));
-			models.set(await getModels(localStorage.token));
+			models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 		}
 	};
 

src/routes/(app)/workspace/models/+page.svelte

@@ -1,13 +1,13 @@
 <script>
 	import { onMount } from 'svelte';
-	import { models } from '$lib/stores';
+	import { models, settings } from '$lib/stores';
 	import { getModels } from '$lib/apis';
 	import Models from '$lib/components/workspace/Models.svelte';
 
 	onMount(async () => {
 		await Promise.all([
 			(async () => {
-				models.set(await getModels(localStorage.token));
+				models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 			})()
 		]);
 	});

src/routes/(app)/workspace/models/create/+page.svelte

@@ -42,7 +42,7 @@
 			});
 
 			if (res) {
-				await models.set(await getModels(localStorage.token));
+				await models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 				toast.success($i18n.t('Model created successfully!'));
 				await goto('/workspace/models');
 			}

src/routes/(app)/workspace/models/edit/+page.svelte

@@ -34,7 +34,7 @@
 		const res = await updateModelById(localStorage.token, modelInfo.id, modelInfo);
 
 		if (res) {
-			await models.set(await getModels(localStorage.token));
+			await models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 			toast.success($i18n.t('Model updated successfully'));
 			await goto('/workspace/models');
 		}

src/routes/s/[id]/+page.svelte

@@ -61,7 +61,7 @@
 	//////////////////////////
 
 	const loadSharedChat = async () => {
-		await models.set(await getModels(localStorage.token));
+		await models.set(await getModels(localStorage.token, $settings?.directConnections ?? null));
 		await chatId.set($page.params.id);
 		chat = await getChatByShareId(localStorage.token, $chatId).catch(async (error) => {
 			await goto('/');