feat: model capabilities

backend/apps/web/models/models.py

@@ -31,7 +31,6 @@ class ModelParams(BaseModel):
 
 
 # ModelMeta is a model for the data stored in the meta field of the Model table
-# It isn't currently used in the backend, but it's here as a reference
 class ModelMeta(BaseModel):
     profile_image_url: Optional[str] = "/favicon.png"
 
@@ -40,10 +39,7 @@ class ModelMeta(BaseModel):
         User-facing description of the model.
     """
 
-    vision_capable: Optional[bool] = None
+    capabilities: Optional[dict] = None
-    """
-        A flag indicating if the model is capable of vision and thus image inputs
-    """
 
     model_config = ConfigDict(extra="allow")
 

src/lib/components/chat/MessageInput.svelte

@@ -53,7 +53,11 @@
 	export let messages = [];
 
 	let speechRecognition;
-	let visionCapableState = 'all';
+
+	let visionCapableModels = [];
+	$: visionCapableModels = [...(atSelectedModel ? [atSelectedModel] : selectedModels)].filter(
+		(model) => $models.find((m) => m.id === model)?.info?.meta?.capabilities?.vision ?? true
+	);
 
 	$: if (prompt) {
 		if (chatTextAreaElement) {
@@ -62,49 +66,6 @@
 		}
 	}
 
-	// $: {
-	// 	if (atSelectedModel || selectedModels) {
-	// 		visionCapableState = checkModelsAreVisionCapable();
-	// 		if (visionCapableState === 'none') {
-	// 			// Remove all image files
-	// 			const fileCount = files.length;
-	// 			files = files.filter((file) => file.type != 'image');
-	// 			if (files.length < fileCount) {
-	// 				toast.warning($i18n.t('All selected models do not support image input, removed images'));
-	// 			}
-	// 		}
-	// 	}
-	// }
-
-	const checkModelsAreVisionCapable = () => {
-		let modelsToCheck = [];
-		if (atSelectedModel !== undefined) {
-			modelsToCheck = [atSelectedModel.id];
-		} else {
-			modelsToCheck = selectedModels;
-		}
-		if (modelsToCheck.length == 0 || modelsToCheck[0] == '') {
-			return 'all';
-		}
-		let visionCapableCount = 0;
-		for (const modelName of modelsToCheck) {
-			const model = $models.find((m) => m.id === modelName);
-			if (!model) {
-				continue;
-			}
-			if (model.custom_info?.meta.vision_capable ?? true) {
-				visionCapableCount++;
-			}
-		}
-		if (visionCapableCount == modelsToCheck.length) {
-			return 'all';
-		} else if (visionCapableCount == 0) {
-			return 'none';
-		} else {
-			return 'some';
-		}
-	};
-
 	let mediaRecorder;
 	let audioChunks = [];
 	let isRecording = false;
@@ -404,8 +365,8 @@
 					inputFiles.forEach((file) => {
 						console.log(file, file.name.split('.').at(-1));
 						if (['image/gif', 'image/webp', 'image/jpeg', 'image/png'].includes(file['type'])) {
-							if (visionCapableState == 'none') {
+							if (visionCapableModels.length === 0) {
-								toast.error($i18n.t('Selected models do not support image inputs'));
+								toast.error($i18n.t('Selected model(s) do not support image inputs'));
 								return;
 							}
 							let reader = new FileReader();
@@ -600,8 +561,8 @@
 									if (
 										['image/gif', 'image/webp', 'image/jpeg', 'image/png'].includes(file['type'])
 									) {
-										if (visionCapableState === 'none') {
+										if (visionCapableModels.length === 0) {
-											toast.error($i18n.t('Selected models do not support image inputs'));
+											toast.error($i18n.t('Selected model(s) do not support image inputs'));
 											inputFiles = null;
 											filesInputElement.value = '';
 											return;
@@ -645,6 +606,7 @@
 						dir={$settings?.chatDirection ?? 'LTR'}
 						class=" flex flex-col relative w-full rounded-3xl px-1.5 bg-gray-50 dark:bg-gray-850 dark:text-gray-100"
 						on:submit|preventDefault={() => {
+							// check if selectedModels support image input
 							submitPrompt(prompt, user);
 						}}
 					>
@@ -659,16 +621,20 @@
 													alt="input"
 													class=" h-16 w-16 rounded-xl object-cover"
 												/>
-												{#if visionCapableState === 'some'}
+												{#if atSelectedModel ? visionCapableModels.length === 0 : selectedModels.length !== visionCapableModels.length}
 													<Tooltip
-														className=" absolute top-0 left-0"
+														className=" absolute top-1 left-1"
-														content={$i18n.t('A selected model does not support image input')}
+														content={$i18n.t('{{ models }}', {
+															models: [...(atSelectedModel ? [atSelectedModel] : selectedModels)]
+																.filter((id) => !visionCapableModels.includes(id))
+																.join(', ')
+														})}
 													>
 														<svg
 															xmlns="http://www.w3.org/2000/svg"
 															viewBox="0 0 24 24"
 															fill="currentColor"
-															class="w-6 h-6 fill-yellow-300"
+															class="size-4 fill-yellow-300"
 														>
 															<path
 																fill-rule="evenodd"

src/lib/components/chat/Settings/Advanced/AdvancedParams.svelte

@@ -31,7 +31,7 @@
 	}
 </script>
 
-<div class=" space-y-3 text-xs">
+<div class=" space-y-1 text-xs">
 	<div class=" py-0.5 w-full justify-between">
 		<div class="flex w-full justify-between">
 			<div class=" self-center text-xs font-medium">{$i18n.t('Seed')}</div>

src/lib/components/common/Checkbox.svelte

@@ -29,6 +29,7 @@
 			dispatch('change', _state);
 		}
 	}}
+	type="button"
 >
 	<div class="top-0 left-0 absolute w-full flex justify-center">
 		{#if _state === 'checked'}

src/lib/components/workspace/Models.svelte

@@ -142,7 +142,7 @@
 				<div class=" flex-1 self-center">
 					<div class=" font-bold line-clamp-1">{model.name}</div>
 					<div class=" text-sm overflow-hidden text-ellipsis line-clamp-1">
-						{model?.info?.meta?.description ?? model.id}
+						{!!model?.info?.meta?.description ? model?.info?.meta?.description : model.id}
 					</div>
 				</div>
 			</a>

src/routes/(app)/workspace/models/create/+page.svelte

@@ -9,6 +9,7 @@
 	import { getModels } from '$lib/apis';
 
 	import AdvancedParams from '$lib/components/chat/Settings/Advanced/AdvancedParams.svelte';
+	import Checkbox from '$lib/components/common/Checkbox.svelte';
 
 	const i18n = getContext('i18n');
 
@@ -48,6 +49,10 @@
 
 	let params = {};
 
+	let capabilities = {
+		vision: false
+	};
+
 	$: if (name) {
 		id = name.replace(/\s+/g, '-').toLowerCase();
 	}
@@ -57,6 +62,7 @@
 
 		info.id = id;
 		info.name = name;
+		info.meta.capabilities = capabilities;
 
 		if ($models.find((m) => m.id === info.id)) {
 			toast.error(
@@ -298,14 +304,13 @@
 		</div>
 
 		<div class="my-2">
-			<div class=" text-sm font-semibold mb-2">{$i18n.t('Description')}*</div>
+			<div class=" text-sm font-semibold mb-2">{$i18n.t('Description')}</div>
 
 			<div>
 				<input
 					class="px-3 py-1.5 text-sm w-full bg-transparent border dark:border-gray-600 outline-none rounded-lg"
 					placeholder={$i18n.t('Add a short description about what this model does')}
 					bind:value={info.meta.description}
-					required
 				/>
 			</div>
 		</div>
@@ -329,7 +334,7 @@
 				</div>
 
 				<div class="flex w-full justify-between">
-					<div class=" self-center text-sm font-semibold">
+					<div class=" self-center text-xs font-semibold">
 						{$i18n.t('Advanced Params')}
 					</div>
 
@@ -422,6 +427,28 @@
 			</div>
 		</div>
 
+		<div class="my-2">
+			<div class="flex w-full justify-between">
+				<div class=" self-center text-sm font-semibold">{$i18n.t('Capabilities')}</div>
+			</div>
+			<div class="flex flex-col">
+				{#each Object.keys(capabilities) as capability}
+					<div class=" flex items-center gap-2">
+						<Checkbox
+							state={capabilities[capability] ? 'checked' : 'unchecked'}
+							on:change={(e) => {
+								capabilities[capability] = e.detail === 'checked';
+							}}
+						/>
+
+						<div class=" py-1.5 text-sm w-full capitalize">
+							{$i18n.t(capability)}
+						</div>
+					</div>
+				{/each}
+			</div>
+		</div>
+
 		<div class="my-2 text-gray-500">
 			<div class="flex w-full justify-between mb-2">
 				<div class=" self-center text-sm font-semibold">{$i18n.t('JSON Preview')}</div>

src/routes/(app)/workspace/models/edit/+page.svelte

@@ -12,6 +12,7 @@
 
 	import AdvancedParams from '$lib/components/chat/Settings/Advanced/AdvancedParams.svelte';
 	import { getModels } from '$lib/apis';
+	import Checkbox from '$lib/components/common/Checkbox.svelte';
 
 	const i18n = getContext('i18n');
 
@@ -53,11 +54,16 @@
 
 	let params = {};
 
+	let capabilities = {
+		vision: true
+	};
+
 	const updateHandler = async () => {
 		loading = true;
 
 		info.id = id;
 		info.name = name;
+		info.meta.capabilities = capabilities;
 
 		const res = await updateModelById(localStorage.token, info.id, info);
 
@@ -98,6 +104,10 @@
 					info.base_model_id = `${info.base_model_id}:latest`;
 				}
 
+				if (model?.info?.meta?.capabilities) {
+					capabilities = { ...capabilities, ...model?.info?.meta?.capabilities };
+				}
+
 				console.log(model);
 			} else {
 				goto('/workspace/models');
@@ -291,14 +301,13 @@
 			{/if}
 
 			<div class="my-2">
-				<div class=" text-sm font-semibold mb-2">{$i18n.t('Description')}*</div>
+				<div class=" text-sm font-semibold mb-2">{$i18n.t('Description')}</div>
 
 				<div>
 					<input
 						class="px-3 py-1.5 text-sm w-full bg-transparent border dark:border-gray-600 outline-none rounded-lg"
 						placeholder={$i18n.t('Add a short description about what this model does')}
 						bind:value={info.meta.description}
-						required
 					/>
 				</div>
 			</div>
@@ -324,7 +333,7 @@
 					</div>
 
 					<div class="flex w-full justify-between">
-						<div class=" self-center text-sm font-semibold">
+						<div class=" self-center text-xs font-semibold">
 							{$i18n.t('Advanced Params')}
 						</div>
 
@@ -417,6 +426,28 @@
 				</div>
 			</div>
 
+			<div class="my-2">
+				<div class="flex w-full justify-between">
+					<div class=" self-center text-sm font-semibold">{$i18n.t('Capabilities')}</div>
+				</div>
+				<div class="flex flex-col">
+					{#each Object.keys(capabilities) as capability}
+						<div class=" flex items-center gap-2">
+							<Checkbox
+								state={capabilities[capability] ? 'checked' : 'unchecked'}
+								on:change={(e) => {
+									capabilities[capability] = e.detail === 'checked';
+								}}
+							/>
+
+							<div class=" py-1.5 text-sm w-full capitalize">
+								{$i18n.t(capability)}
+							</div>
+						</div>
+					{/each}
+				</div>
+			</div>
+
 			<div class="my-2 text-gray-500">
 				<div class="flex w-full justify-between mb-2">
 					<div class=" self-center text-sm font-semibold">{$i18n.t('JSON Preview')}</div>