From ecd3b4ebd41563632bd6a4793a52425a2df32fe6 Mon Sep 17 00:00:00 2001
From: Timothy Jaeryang Baek <tim@openwebui.com>
Date: Mon, 23 Dec 2024 14:43:58 -0700
Subject: [PATCH] enh: channel file upload

---
 .../7826ab40b532_update_file_table.py         |  26 ++
 backend/open_webui/models/files.py            |   5 +
 src/lib/components/channel/Channel.svelte     |  13 +-
 .../components/channel/MessageInput.svelte    | 356 +++++++++++++++++-
 .../channel/MessageInput/InputMenu.svelte     |  77 ++++
 .../channel/Messages/Message.svelte           |  23 ++
 src/lib/components/common/Image.svelte        |   2 +-
 7 files changed, 477 insertions(+), 25 deletions(-)
 create mode 100644 backend/open_webui/migrations/versions/7826ab40b532_update_file_table.py
 create mode 100644 src/lib/components/channel/MessageInput/InputMenu.svelte

diff --git a/backend/open_webui/migrations/versions/7826ab40b532_update_file_table.py b/backend/open_webui/migrations/versions/7826ab40b532_update_file_table.py
new file mode 100644
index 000000000..c8afe9d51
--- /dev/null
+++ b/backend/open_webui/migrations/versions/7826ab40b532_update_file_table.py
@@ -0,0 +1,26 @@
+"""Update file table
+
+Revision ID: 7826ab40b532
+Revises: 57c599a3cb57
+Create Date: 2024-12-23 03:00:00.000000
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+revision = "7826ab40b532"
+down_revision = "57c599a3cb57"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "file",
+        sa.Column("access_control", sa.JSON(), nullable=True),
+    )
+
+
+def downgrade():
+    op.drop_column("file", "access_control")
diff --git a/backend/open_webui/models/files.py b/backend/open_webui/models/files.py
index 4050b0140..91dea5444 100644
--- a/backend/open_webui/models/files.py
+++ b/backend/open_webui/models/files.py
@@ -27,6 +27,8 @@ class File(Base):
     data = Column(JSON, nullable=True)
     meta = Column(JSON, nullable=True)
 
+    access_control = Column(JSON, nullable=True)
+
     created_at = Column(BigInteger)
     updated_at = Column(BigInteger)
 
@@ -44,6 +46,8 @@ class FileModel(BaseModel):
     data: Optional[dict] = None
     meta: Optional[dict] = None
 
+    access_control: Optional[dict] = None
+
     created_at: Optional[int]  # timestamp in epoch
     updated_at: Optional[int]  # timestamp in epoch
 
@@ -90,6 +94,7 @@ class FileForm(BaseModel):
     path: str
     data: dict = {}
     meta: dict = {}
+    access_control: Optional[dict] = None
 
 
 class FilesTable:
diff --git a/src/lib/components/channel/Channel.svelte b/src/lib/components/channel/Channel.svelte
index da12e3d2d..51c813765 100644
--- a/src/lib/components/channel/Channel.svelte
+++ b/src/lib/components/channel/Channel.svelte
@@ -80,15 +80,17 @@
 		}
 	};
 
-	const submitHandler = async ({ content }) => {
+	const submitHandler = async ({ content, data }) => {
 		if (!content) {
 			return;
 		}
 
-		const res = await sendMessage(localStorage.token, id, { content: content }).catch((error) => {
-			toast.error(error);
-			return null;
-		});
+		const res = await sendMessage(localStorage.token, id, { content: content, data: data }).catch(
+			(error) => {
+				toast.error(error);
+				return null;
+			}
+		);
 
 		if (res) {
 			messagesContainerElement.scrollTop = messagesContainerElement.scrollHeight;
@@ -108,6 +110,7 @@
 	class="h-screen max-h-[100dvh] {$showSidebar
 		? 'md:max-w-[calc(100%-260px)]'
 		: ''} w-full max-w-full flex flex-col"
+	id="channel-container"
 >
 	<Navbar {channel} />
 
diff --git a/src/lib/components/channel/MessageInput.svelte b/src/lib/components/channel/MessageInput.svelte
index 89e32f3ab..e16480908 100644
--- a/src/lib/components/channel/MessageInput.svelte
+++ b/src/lib/components/channel/MessageInput.svelte
@@ -1,43 +1,282 @@
 <script lang="ts">
 	import { toast } from 'svelte-sonner';
-	import { tick, getContext } from 'svelte';
+	import { v4 as uuidv4 } from 'uuid';
+
+	import { tick, getContext, onMount, onDestroy } from 'svelte';
 
 	const i18n = getContext('i18n');
 
-	import { mobile, settings } from '$lib/stores';
+	import { config, mobile, settings } from '$lib/stores';
+	import { blobToFile } from '$lib/utils';
 
 	import Tooltip from '../common/Tooltip.svelte';
 	import RichTextInput from '../common/RichTextInput.svelte';
 	import VoiceRecording from '../chat/MessageInput/VoiceRecording.svelte';
+	import InputMenu from './MessageInput/InputMenu.svelte';
+	import { uploadFile } from '$lib/apis/files';
+	import { WEBUI_API_BASE_URL } from '$lib/constants';
+	import FileItem from '../common/FileItem.svelte';
+	import Image from '../common/Image.svelte';
+	import { transcribeAudio } from '$lib/apis/audio';
+	import FilesOverlay from '../chat/MessageInput/FilesOverlay.svelte';
 
 	export let placeholder = $i18n.t('Send a Message');
 	export let transparentBackground = false;
 
-	let recording = false;
+	let draggedOver = false;
 
+	let recording = false;
 	let content = '';
+	let files = [];
+
+	let filesInputElement;
+	let inputFiles;
 
 	export let onSubmit: Function;
 	export let scrollEnd = true;
 	export let scrollToBottom: Function;
 
-	let submitHandler = async () => {
+	const screenCaptureHandler = async () => {
+		try {
+			// Request screen media
+			const mediaStream = await navigator.mediaDevices.getDisplayMedia({
+				video: { cursor: 'never' },
+				audio: false
+			});
+			// Once the user selects a screen, temporarily create a video element
+			const video = document.createElement('video');
+			video.srcObject = mediaStream;
+			// Ensure the video loads without affecting user experience or tab switching
+			await video.play();
+			// Set up the canvas to match the video dimensions
+			const canvas = document.createElement('canvas');
+			canvas.width = video.videoWidth;
+			canvas.height = video.videoHeight;
+			// Grab a single frame from the video stream using the canvas
+			const context = canvas.getContext('2d');
+			context.drawImage(video, 0, 0, canvas.width, canvas.height);
+			// Stop all video tracks (stop screen sharing) after capturing the image
+			mediaStream.getTracks().forEach((track) => track.stop());
+
+			// bring back focus to this current tab, so that the user can see the screen capture
+			window.focus();
+
+			// Convert the canvas to a Base64 image URL
+			const imageUrl = canvas.toDataURL('image/png');
+			// Add the captured image to the files array to render it
+			files = [...files, { type: 'image', url: imageUrl }];
+			// Clean memory: Clear video srcObject
+			video.srcObject = null;
+		} catch (error) {
+			// Handle any errors (e.g., user cancels screen sharing)
+			console.error('Error capturing screen:', error);
+		}
+	};
+
+	const inputFilesHandler = async (inputFiles) => {
+		inputFiles.forEach((file) => {
+			console.log('Processing file:', {
+				name: file.name,
+				type: file.type,
+				size: file.size,
+				extension: file.name.split('.').at(-1)
+			});
+
+			if (
+				($config?.file?.max_size ?? null) !== null &&
+				file.size > ($config?.file?.max_size ?? 0) * 1024 * 1024
+			) {
+				console.log('File exceeds max size limit:', {
+					fileSize: file.size,
+					maxSize: ($config?.file?.max_size ?? 0) * 1024 * 1024
+				});
+				toast.error(
+					$i18n.t(`File size should not exceed {{maxSize}} MB.`, {
+						maxSize: $config?.file?.max_size
+					})
+				);
+				return;
+			}
+
+			if (['image/gif', 'image/webp', 'image/jpeg', 'image/png'].includes(file['type'])) {
+				let reader = new FileReader();
+				reader.onload = (event) => {
+					files = [
+						...files,
+						{
+							type: 'image',
+							url: `${event.target.result}`
+						}
+					];
+				};
+				reader.readAsDataURL(file);
+			} else {
+				uploadFileHandler(file);
+			}
+		});
+	};
+
+	const uploadFileHandler = async (file) => {
+		const tempItemId = uuidv4();
+		const fileItem = {
+			type: 'file',
+			file: '',
+			id: null,
+			url: '',
+			name: file.name,
+			collection_name: '',
+			status: 'uploading',
+			size: file.size,
+			error: '',
+			itemId: tempItemId
+		};
+
+		if (fileItem.size == 0) {
+			toast.error($i18n.t('You cannot upload an empty file.'));
+			return null;
+		}
+
+		files = [...files, fileItem];
+		// Check if the file is an audio file and transcribe/convert it to text file
+		if (['audio/mpeg', 'audio/wav', 'audio/ogg', 'audio/x-m4a'].includes(file['type'])) {
+			const res = await transcribeAudio(localStorage.token, file).catch((error) => {
+				toast.error(error);
+				return null;
+			});
+
+			if (res) {
+				console.log(res);
+				const blob = new Blob([res.text], { type: 'text/plain' });
+				file = blobToFile(blob, `${file.name}.txt`);
+
+				fileItem.name = file.name;
+				fileItem.size = file.size;
+			}
+		}
+
+		try {
+			// During the file upload, file content is automatically extracted.
+			const uploadedFile = await uploadFile(localStorage.token, file);
+
+			if (uploadedFile) {
+				console.log('File upload completed:', {
+					id: uploadedFile.id,
+					name: fileItem.name,
+					collection: uploadedFile?.meta?.collection_name
+				});
+
+				if (uploadedFile.error) {
+					console.warn('File upload warning:', uploadedFile.error);
+					toast.warning(uploadedFile.error);
+				}
+
+				fileItem.status = 'uploaded';
+				fileItem.file = uploadedFile;
+				fileItem.id = uploadedFile.id;
+				fileItem.collection_name =
+					uploadedFile?.meta?.collection_name || uploadedFile?.collection_name;
+				fileItem.url = `${WEBUI_API_BASE_URL}/files/${uploadedFile.id}`;
+
+				files = files;
+			} else {
+				files = files.filter((item) => item?.itemId !== tempItemId);
+			}
+		} catch (e) {
+			toast.error(e);
+			files = files.filter((item) => item?.itemId !== tempItemId);
+		}
+	};
+
+	const handleKeyDown = (event: KeyboardEvent) => {
+		if (event.key === 'Escape') {
+			console.log('Escape');
+			draggedOver = false;
+		}
+	};
+
+	const onDragOver = (e) => {
+		e.preventDefault();
+
+		// Check if a file is being draggedOver.
+		if (e.dataTransfer?.types?.includes('Files')) {
+			draggedOver = true;
+		} else {
+			draggedOver = false;
+		}
+	};
+
+	const onDragLeave = () => {
+		draggedOver = false;
+	};
+
+	const onDrop = async (e) => {
+		e.preventDefault();
+		console.log(e);
+
+		if (e.dataTransfer?.files) {
+			const inputFiles = Array.from(e.dataTransfer?.files);
+			if (inputFiles && inputFiles.length > 0) {
+				console.log(inputFiles);
+				inputFilesHandler(inputFiles);
+			}
+		}
+
+		draggedOver = false;
+	};
+
+	const submitHandler = async () => {
 		if (content === '') {
 			return;
 		}
 
 		onSubmit({
-			content
+			content,
+			data: {
+				files: files
+			}
 		});
 
 		content = '';
+		files = [];
+
 		await tick();
 
 		const chatInputElement = document.getElementById('chat-input');
 		chatInputElement?.focus();
 	};
+
+	onMount(async () => {
+		window.setTimeout(() => {
+			const chatInput = document.getElementById('chat-input');
+			chatInput?.focus();
+		}, 0);
+
+		window.addEventListener('keydown', handleKeyDown);
+		await tick();
+
+		const dropzoneElement = document.getElementById('channel-container');
+
+		dropzoneElement?.addEventListener('dragover', onDragOver);
+		dropzoneElement?.addEventListener('drop', onDrop);
+		dropzoneElement?.addEventListener('dragleave', onDragLeave);
+	});
+
+	onDestroy(() => {
+		console.log('destroy');
+		window.removeEventListener('keydown', handleKeyDown);
+
+		const dropzoneElement = document.getElementById('channel-container');
+
+		if (dropzoneElement) {
+			dropzoneElement?.removeEventListener('dragover', onDragOver);
+			dropzoneElement?.removeEventListener('drop', onDrop);
+			dropzoneElement?.removeEventListener('dragleave', onDragLeave);
+		}
+	});
 </script>
 
+<FilesOverlay show={draggedOver} />
+
 <div class=" mx-auto inset-x-0 bg-transparent flex justify-center">
 	<div class="flex flex-col px-3 max-w-6xl w-full">
 		<div class="relative">
@@ -69,6 +308,22 @@
 	</div>
 </div>
 
+<input
+	bind:this={filesInputElement}
+	bind:files={inputFiles}
+	type="file"
+	hidden
+	multiple
+	on:change={async () => {
+		if (inputFiles && inputFiles.length > 0) {
+			inputFilesHandler(Array.from(inputFiles));
+		} else {
+			toast.error($i18n.t(`File not found.`));
+		}
+
+		filesInputElement.value = '';
+	}}
+/>
 <div class="{transparentBackground ? 'bg-transparent' : 'bg-white dark:bg-gray-900'} ">
 	<div class="max-w-6xl px-2.5 mx-auto inset-x-0">
 		<div class="">
@@ -101,24 +356,87 @@
 						class="flex-1 flex flex-col relative w-full rounded-3xl px-1 bg-gray-50 dark:bg-gray-400/5 dark:text-gray-100"
 						dir={$settings?.chatDirection ?? 'LTR'}
 					>
+						{#if files.length > 0}
+							<div class="mx-1 mt-2.5 mb-1 flex flex-wrap gap-2">
+								{#each files as file, fileIdx}
+									{#if file.type === 'image'}
+										<div class=" relative group">
+											<div class="relative">
+												<Image
+													src={file.url}
+													alt="input"
+													imageClassName=" h-16 w-16 rounded-xl object-cover"
+												/>
+											</div>
+											<div class=" absolute -top-1 -right-1">
+												<button
+													class=" bg-gray-400 text-white border border-white rounded-full group-hover:visible invisible transition"
+													type="button"
+													on:click={() => {
+														files.splice(fileIdx, 1);
+														files = files;
+													}}
+												>
+													<svg
+														xmlns="http://www.w3.org/2000/svg"
+														viewBox="0 0 20 20"
+														fill="currentColor"
+														class="w-4 h-4"
+													>
+														<path
+															d="M6.28 5.22a.75.75 0 00-1.06 1.06L8.94 10l-3.72 3.72a.75.75 0 101.06 1.06L10 11.06l3.72 3.72a.75.75 0 101.06-1.06L11.06 10l3.72-3.72a.75.75 0 00-1.06-1.06L10 8.94 6.28 5.22z"
+														/>
+													</svg>
+												</button>
+											</div>
+										</div>
+									{:else}
+										<FileItem
+											item={file}
+											name={file.name}
+											type={file.type}
+											size={file?.size}
+											loading={file.status === 'uploading'}
+											dismissible={true}
+											edit={true}
+											on:dismiss={() => {
+												files.splice(fileIdx, 1);
+												files = files;
+											}}
+											on:click={() => {
+												console.log(file);
+											}}
+										/>
+									{/if}
+								{/each}
+							</div>
+						{/if}
+
 						<div class=" flex">
 							<div class="ml-1 self-end mb-1.5 flex space-x-1">
-								<button
-									class="bg-transparent hover:bg-white/80 text-gray-800 dark:text-white dark:hover:bg-gray-800 transition rounded-full p-2 outline-none focus:outline-none"
-									type="button"
-									aria-label="More"
+								<InputMenu
+									{screenCaptureHandler}
+									uploadFilesHandler={() => {
+										filesInputElement.click();
+									}}
 								>
-									<svg
-										xmlns="http://www.w3.org/2000/svg"
-										viewBox="0 0 20 20"
-										fill="currentColor"
-										class="size-5"
+									<button
+										class="bg-transparent hover:bg-white/80 text-gray-800 dark:text-white dark:hover:bg-gray-800 transition rounded-full p-2 outline-none focus:outline-none"
+										type="button"
+										aria-label="More"
 									>
-										<path
-											d="M10.75 4.75a.75.75 0 0 0-1.5 0v4.5h-4.5a.75.75 0 0 0 0 1.5h4.5v4.5a.75.75 0 0 0 1.5 0v-4.5h4.5a.75.75 0 0 0 0-1.5h-4.5v-4.5Z"
-										/>
-									</svg>
-								</button>
+										<svg
+											xmlns="http://www.w3.org/2000/svg"
+											viewBox="0 0 20 20"
+											fill="currentColor"
+											class="size-5"
+										>
+											<path
+												d="M10.75 4.75a.75.75 0 0 0-1.5 0v4.5h-4.5a.75.75 0 0 0 0 1.5h4.5v4.5a.75.75 0 0 0 1.5 0v-4.5h4.5a.75.75 0 0 0 0-1.5h-4.5v-4.5Z"
+											/>
+										</svg>
+									</button>
+								</InputMenu>
 							</div>
 
 							{#if $settings?.richTextInput ?? true}
diff --git a/src/lib/components/channel/MessageInput/InputMenu.svelte b/src/lib/components/channel/MessageInput/InputMenu.svelte
new file mode 100644
index 000000000..2363b75ec
--- /dev/null
+++ b/src/lib/components/channel/MessageInput/InputMenu.svelte
@@ -0,0 +1,77 @@
+<script lang="ts">
+	import { DropdownMenu } from 'bits-ui';
+	import { flyAndScale } from '$lib/utils/transitions';
+	import { getContext, onMount, tick } from 'svelte';
+
+	import { config, user, tools as _tools, mobile } from '$lib/stores';
+	import { getTools } from '$lib/apis/tools';
+
+	import Dropdown from '$lib/components/common/Dropdown.svelte';
+	import Tooltip from '$lib/components/common/Tooltip.svelte';
+	import DocumentArrowUpSolid from '$lib/components/icons/DocumentArrowUpSolid.svelte';
+	import Switch from '$lib/components/common/Switch.svelte';
+	import GlobeAltSolid from '$lib/components/icons/GlobeAltSolid.svelte';
+	import WrenchSolid from '$lib/components/icons/WrenchSolid.svelte';
+	import CameraSolid from '$lib/components/icons/CameraSolid.svelte';
+
+	const i18n = getContext('i18n');
+
+	export let screenCaptureHandler: Function;
+	export let uploadFilesHandler: Function;
+
+	export let onClose: Function = () => {};
+
+	let show = false;
+
+	$: if (show) {
+		init();
+	}
+
+	const init = async () => {};
+</script>
+
+<Dropdown
+	bind:show
+	on:change={(e) => {
+		if (e.detail === false) {
+			onClose();
+		}
+	}}
+>
+	<Tooltip content={$i18n.t('More')}>
+		<slot />
+	</Tooltip>
+
+	<div slot="content">
+		<DropdownMenu.Content
+			class="w-full max-w-[200px] rounded-xl px-1 py-1  border-gray-300/30 dark:border-gray-700/50 z-50 bg-white dark:bg-gray-850 dark:text-white shadow"
+			sideOffset={15}
+			alignOffset={-8}
+			side="top"
+			align="start"
+			transition={flyAndScale}
+		>
+			{#if !$mobile}
+				<DropdownMenu.Item
+					class="flex gap-2 items-center px-3 py-2 text-sm  font-medium cursor-pointer hover:bg-gray-50 dark:hover:bg-gray-800  rounded-xl"
+					on:click={() => {
+						screenCaptureHandler();
+					}}
+				>
+					<CameraSolid />
+					<div class=" line-clamp-1">{$i18n.t('Capture')}</div>
+				</DropdownMenu.Item>
+			{/if}
+
+			<DropdownMenu.Item
+				class="flex gap-2 items-center px-3 py-2 text-sm font-medium cursor-pointer hover:bg-gray-50 dark:hover:bg-gray-800 rounded-xl"
+				on:click={() => {
+					uploadFilesHandler();
+				}}
+			>
+				<DocumentArrowUpSolid />
+				<div class="line-clamp-1">{$i18n.t('Upload Files')}</div>
+			</DropdownMenu.Item>
+		</DropdownMenu.Content>
+	</div>
+</Dropdown>
diff --git a/src/lib/components/channel/Messages/Message.svelte b/src/lib/components/channel/Messages/Message.svelte
index 8b468ab7b..2182e0a30 100644
--- a/src/lib/components/channel/Messages/Message.svelte
+++ b/src/lib/components/channel/Messages/Message.svelte
@@ -23,6 +23,8 @@
 	import Pencil from '$lib/components/icons/Pencil.svelte';
 	import Tooltip from '$lib/components/common/Tooltip.svelte';
 	import Textarea from '$lib/components/common/Textarea.svelte';
+	import Image from '$lib/components/common/Image.svelte';
+	import FileItem from '$lib/components/common/FileItem.svelte';
 
 	export let message;
 	export let showUserProfile = true;
@@ -142,6 +144,27 @@
 					</Name>
 				{/if}
 
+				{#if (message?.data?.files ?? []).length > 0}
+					<div class="my-2.5 w-full flex overflow-x-auto gap-2 flex-wrap">
+						{#each message?.data?.files as file}
+							<div>
+								{#if file.type === 'image'}
+									<Image src={file.url} alt={file.name} imageClassName=" max-h-96 rounded-lg" />
+								{:else}
+									<FileItem
+										item={file}
+										url={file.url}
+										name={file.name}
+										type={file.type}
+										size={file?.size}
+										colorClassName="bg-white dark:bg-gray-850 "
+									/>
+								{/if}
+							</div>
+						{/each}
+					</div>
+				{/if}
+
 				{#if edit}
 					<div class="py-2">
 						<Textarea
diff --git a/src/lib/components/common/Image.svelte b/src/lib/components/common/Image.svelte
index e3c723564..1c9231b8f 100644
--- a/src/lib/components/common/Image.svelte
+++ b/src/lib/components/common/Image.svelte
@@ -5,7 +5,7 @@
 	export let src = '';
 	export let alt = '';
 
-	export let className = ' w-full';
+	export let className = ' w-full outline-none focus:outline-none';
 	export let imageClassName = 'rounded-lg';
 
 	let _src = '';