Merge branch 'dev' of https://github.com/open-webui/open-webui into remove-ollama

.github/ISSUE_TEMPLATE/bug_report.md

@@ -8,36 +8,41 @@ assignees: ''
 
 # Bug Report
 
-## Description
+## Installation Method
 
-**Bug Summary:**
-[Provide a brief but clear summary of the bug]
-
-**Steps to Reproduce:**
-[Outline the steps to reproduce the bug. Be as detailed as possible.]
-
-**Expected Behavior:**
-[Describe what you expected to happen.]
-
-**Actual Behavior:**
-[Describe what actually happened.]
+[Describe the method you used to install the project, e.g., git clone, Docker, pip, etc.]
 
 ## Environment
 
-- **Open WebUI Version:** [e.g., 0.1.120]
-- **Ollama (if applicable):** [e.g., 0.1.30, 0.1.32-rc1]
+- **Open WebUI Version:** [e.g., v0.3.11]
+- **Ollama (if applicable):** [e.g., v0.2.0, v0.1.32-rc1]
 
 - **Operating System:** [e.g., Windows 10, macOS Big Sur, Ubuntu 20.04]
 - **Browser (if applicable):** [e.g., Chrome 100.0, Firefox 98.0]
 
-## Reproduction Details
-
 **Confirmation:**
 
 - [ ] I have read and followed all the instructions provided in the README.md.
 - [ ] I am on the latest version of both Open WebUI and Ollama.
 - [ ] I have included the browser console logs.
 - [ ] I have included the Docker container logs.
+- [ ] I have provided the exact steps to reproduce the bug in the "Steps to Reproduce" section below.
+
+## Expected Behavior:
+[Describe what you expected to happen.]
+
+## Actual Behavior:
+[Describe what actually happened.]
+
+## Description
+
+**Bug Summary:**
+[Provide a brief but clear summary of the bug]
+
+## Reproduction Details
+
+**Steps to Reproduce:**
+[Outline the steps to reproduce the bug. Be as detailed as possible.]
 
 ## Logs and Screenshots
 
@@ -47,13 +52,9 @@ assignees: ''
 **Docker Container Logs:**
 [Include relevant Docker container logs, if applicable]
 
-**Screenshots (if applicable):**
+**Screenshots/Screen Recordings (if applicable):**
 [Attach any relevant screenshots to help illustrate the issue]
 
-## Installation Method
-
-[Describe the method you used to install the project, e.g., manual installation, Docker, package manager, etc.]
-
 ## Additional Information
 
 [Include any additional details that may help in understanding and reproducing the issue. This could include specific configurations, error messages, or anything else relevant to the bug.]

docs/SECURITY.md

@@ -11,10 +11,25 @@ Our primary goal is to ensure the protection and confidentiality of sensitive da
 
 ## Reporting a Vulnerability
 
-If you discover a security issue within our system, please notify us immediately via a pull request or contact us on discord.
+We appreciate the community's interest in identifying potential vulnerabilities. However, effective immediately, we will **not** accept low-effort vulnerability reports. To ensure that submissions are constructive and actionable, please adhere to the following guidelines:
+
+1. **No Vague Reports**: Submissions such as "I found a vulnerability" without any details will be treated as spam and will not be accepted.
+
+2. **In-Depth Understanding Required**: Reports must reflect a clear understanding of the codebase and provide specific details about the vulnerability, including the affected components and potential impacts.
+
+3. **Proof of Concept (PoC) is Mandatory**: Each submission must include a well-documented proof of concept (PoC) that demonstrates the vulnerability. If confidentiality is a concern, reporters are encouraged to create a private fork of the repository and share access with the maintainers. Reports lacking valid evidence will be disregarded.
+
+4. **Required Patch Submission**: Along with the PoC, reporters must provide a patch or actionable steps to remediate the identified vulnerability. This helps us evaluate and implement fixes rapidly.
+
+5. **Streamlined Merging Process**: When vulnerability reports meet the above criteria, we can consider them for immediate merging, similar to regular pull requests. Well-structured and thorough submissions will expedite the process of enhancing our security.
+
+Submissions that do not meet these criteria will be closed, and repeat offenders may face a ban from future submissions. We aim to create a respectful and constructive reporting environment, where high-quality submissions foster better security for everyone.
 
 ## Product Security
 
-We regularly audit our internal processes and system's architecture for vulnerabilities using a combination of automated and manual testing techniques.
+We regularly audit our internal processes and system architecture for vulnerabilities using a combination of automated and manual testing techniques. We are also planning to implement SAST and SCA scans in our project soon.
 
-We are planning on implementing SAST and SCA scans in our project soon.
+For immediate concerns or detailed reports that meet our guidelines, please create an issue in our [issue tracker](/open-webui/open-webui/issues) or contact us on [Discord](https://discord.gg/5rJgQTnV4s).
+
+---
+_Last updated on **2024-08-06**._

src/lib/components/chat/Messages/Markdown.svelte

@@ -1,132 +0,0 @@
-<script lang="ts">
-	import { onMount } from 'svelte';
-	import { marked } from 'marked';
-	import type { Token } from 'marked';
-
-	import {
-		replaceTokens,
-		revertSanitizedResponseContent,
-		sanitizeResponseContent,
-		unescapeHtml
-	} from '$lib/utils';
-	import CodeBlock from '$lib/components/chat/Messages/CodeBlock.svelte';
-	import MarkdownInlineTokens from '$lib/components/chat/Messages/MarkdownInlineTokens.svelte';
-	import { user } from '$lib/stores';
-
-	export let id: string;
-	export let tokens: Token[];
-	export let model = null;
-	export let content = '';
-	export let top = true;
-
-	$: tokens = marked.lexer(
-		replaceTokens(sanitizeResponseContent(content), model?.name, $user?.name)
-	);
-
-	const headerComponent = (depth: number) => {
-		return 'h' + depth;
-	};
-
-	onMount(() => {
-		console.log('MarkdownTokens', id, tokens, top);
-	});
-</script>
-
-{#each tokens as token, tokenIdx}
-	{#if token.type === 'hr'}
-		<hr />
-	{:else if token.type === 'heading'}
-		<svelte:element this={headerComponent(token.depth)}>
-			<MarkdownInlineTokens id={`${id}-${tokenIdx}-h`} tokens={token.tokens} />
-		</svelte:element>
-	{:else if token.type === 'code'}
-		<CodeBlock
-			{id}
-			lang={token?.lang ?? ''}
-			code={revertSanitizedResponseContent(token?.text ?? '')}
-		/>
-	{:else if token.type === 'table'}
-		<table>
-			<thead>
-				<tr>
-					{#each token.header as header, headerIdx}
-						<th style={token.align[headerIdx] ? '' : `text-align: ${token.align[headerIdx]}`}>
-							<MarkdownInlineTokens
-								id={`${id}-${tokenIdx}-header-${headerIdx}`}
-								tokens={header.tokens}
-							/>
-						</th>
-					{/each}
-				</tr>
-			</thead>
-			<tbody>
-				{#each token.rows as row, rowIdx}
-					<tr>
-						{#each row ?? [] as cell, cellIdx}
-							<td style={token.align[cellIdx] ? '' : `text-align: ${token.align[cellIdx]}`}>
-								<MarkdownInlineTokens
-									id={`${id}-${tokenIdx}-row-${rowIdx}-${cellIdx}`}
-									tokens={cell.tokens}
-								/>
-							</td>
-						{/each}
-					</tr>
-				{/each}
-			</tbody>
-		</table>
-	{:else if token.type === 'blockquote'}
-		<blockquote>
-			<svelte:self id={`${id}-${tokenIdx}`} tokens={token.tokens} />
-		</blockquote>
-	{:else if token.type === 'list'}
-		{#if token.ordered}
-			<ol start={token.start || 1}>
-				{#each token.items as item, itemIdx}
-					<li>
-						<svelte:self
-							id={`${id}-${tokenIdx}-${itemIdx}`}
-							tokens={item.tokens}
-							top={token.loose}
-						/>
-					</li>
-				{/each}
-			</ol>
-		{:else}
-			<ul>
-				{#each token.items as item, itemIdx}
-					<li>
-						<svelte:self
-							id={`${id}-${tokenIdx}-${itemIdx}`}
-							tokens={item.tokens}
-							top={token.loose}
-						/>
-					</li>
-				{/each}
-			</ul>
-		{/if}
-	{:else if token.type === 'html'}
-		{@html token.text}
-	{:else if token.type === 'paragraph'}
-		<p>
-			<MarkdownInlineTokens id={`${id}-${tokenIdx}-p`} tokens={token.tokens ?? []} />
-		</p>
-	{:else if token.type === 'text'}
-		{#if top}
-			<p>
-				{#if token.tokens}
-					<MarkdownInlineTokens id={`${id}-${tokenIdx}-t`} tokens={token.tokens} />
-				{:else}
-					{unescapeHtml(token.text)}
-				{/if}
-			</p>
-		{:else if token.tokens}
-			<MarkdownInlineTokens id={`${id}-${tokenIdx}-p`} tokens={token.tokens ?? []} />
-		{:else}
-			{unescapeHtml(token.text)}
-		{/if}
-	{:else if token.type === 'space'}
-		{''}
-	{:else}
-		{console.log('Unknown token', token)}
-	{/if}
-{/each}

src/lib/components/chat/Messages/MarkdownInlineTokens.svelte

@@ -1,16 +1,10 @@
 <script lang="ts">
 	import type { Token } from 'marked';
 	import { unescapeHtml } from '$lib/utils';
-	import { onMount } from 'svelte';
-	import { revertSanitizedResponseContent } from '$lib/utils/index.js';
 	import Image from '$lib/components/common/Image.svelte';
 
 	export let id: string;
 	export let tokens: Token[];
-
-	onMount(() => {
-		console.log('MarkdownInlineTokens', id, tokens, top);
-	});
 </script>
 
 {#each tokens as token}

src/lib/components/chat/Messages/MarkdownTokens.svelte

@@ -0,0 +1,222 @@
+<script lang="ts">
+	import { marked } from 'marked';
+	import type { Token } from 'marked';
+	import { revertSanitizedResponseContent, unescapeHtml } from '$lib/utils';
+
+	import { onMount } from 'svelte';
+
+	import Image from '$lib/components/common/Image.svelte';
+	import CodeBlock from '$lib/components/chat/Messages/CodeBlock.svelte';
+
+	import MarkdownInlineTokens from '$lib/components/chat/Messages/MarkdownInlineTokens.svelte';
+
+	export let id: string;
+	export let tokens: Token[];
+	export let top = true;
+
+	let containerElement;
+
+	const headerComponent = (depth: number) => {
+		return 'h' + depth;
+	};
+
+	const renderer = new marked.Renderer();
+	// For code blocks with simple backticks
+	renderer.codespan = (code) => {
+		return `<code class="codespan">${code.replaceAll('&amp;', '&')}</code>`;
+	};
+
+	let codes = [];
+	renderer.code = (code, lang) => {
+		codes.push({
+			code: code,
+			lang: lang
+		});
+		codes = codes;
+		const codeId = `${id}-${codes.length}`;
+
+		const interval = setInterval(() => {
+			const codeElement = document.getElementById(`code-${codeId}`);
+			if (codeElement) {
+				clearInterval(interval);
+				// If the code is already loaded, don't load it again
+				if (codeElement.innerHTML) {
+					return;
+				}
+
+				new CodeBlock({
+					target: codeElement,
+					props: {
+						id: `${id}-${codes.length}`,
+						lang: lang,
+						code: revertSanitizedResponseContent(code)
+					},
+					hydrate: true,
+					$$inline: true
+				});
+			}
+		}, 10);
+
+		return `<div id="code-${id}-${codes.length}"></div>`;
+	};
+
+	let images = [];
+	renderer.image = (href, title, text) => {
+		images.push({
+			href: href,
+			title: title,
+			text: text
+		});
+		images = images;
+
+		const imageId = `${id}-${images.length}`;
+		const interval = setInterval(() => {
+			const imageElement = document.getElementById(`image-${imageId}`);
+			if (imageElement) {
+				clearInterval(interval);
+
+				// If the image is already loaded, don't load it again
+				if (imageElement.innerHTML) {
+					return;
+				}
+
+				console.log('image', href, text);
+				new Image({
+					target: imageElement,
+					props: {
+						src: href,
+						alt: text
+					},
+					$$inline: true
+				});
+			}
+		}, 10);
+
+		return `<div id="image-${id}-${images.length}"></div>`;
+	};
+
+	// Open all links in a new tab/window (from https://github.com/markedjs/marked/issues/655#issuecomment-383226346)
+	const origLinkRenderer = renderer.link;
+	renderer.link = (href, title, text) => {
+		const html = origLinkRenderer.call(renderer, href, title, text);
+		return html.replace(/^<a /, '<a target="_blank" rel="nofollow" ');
+	};
+
+	const { extensions, ...defaults } = marked.getDefaults() as marked.MarkedOptions & {
+		// eslint-disable-next-line @typescript-eslint/no-explicit-any
+		extensions: any;
+	};
+
+	$: if (tokens) {
+		images = [];
+		codes = [];
+	}
+</script>
+
+<div bind:this={containerElement} class="flex flex-col">
+	{#each tokens as token, tokenIdx (`${id}-${tokenIdx}`)}
+		{#if token.type === 'code'}
+			{#if token.lang === 'mermaid'}
+				<pre class="mermaid">{revertSanitizedResponseContent(token.text)}</pre>
+			{:else}
+				<CodeBlock
+					id={`${id}-${tokenIdx}`}
+					lang={token?.lang ?? ''}
+					code={revertSanitizedResponseContent(token?.text ?? '')}
+				/>
+			{/if}
+			<!-- {:else if token.type === 'heading'}
+		<svelte:element this={headerComponent(token.depth)}>
+			<MarkdownInlineTokens id={`${id}-${tokenIdx}-h`} tokens={token.tokens} />
+		</svelte:element>
+	{:else if token.type === 'hr'}
+		<hr />
+	{:else if token.type === 'blockquote'}
+		<blockquote>
+			<svelte:self id={`${id}-${tokenIdx}`} tokens={token.tokens} />
+		</blockquote>
+	{:else if token.type === 'html'}
+		{@html token.text}
+	{:else if token.type === 'paragraph'}
+		<p>
+			<MarkdownInlineTokens id={`${id}-${tokenIdx}-p`} tokens={token.tokens ?? []} />
+		</p>
+	{:else if token.type === 'list'}
+		{#if token.ordered}
+			<ol start={token.start || 1}>
+				{#each token.items as item, itemIdx}
+					<li>
+						<svelte:self
+							id={`${id}-${tokenIdx}-${itemIdx}`}
+							tokens={item.tokens}
+							top={token.loose}
+						/>
+					</li>
+				{/each}
+			</ol>
+		{:else}
+			<ul>
+				{#each token.items as item, itemIdx}
+					<li>
+						<svelte:self
+							id={`${id}-${tokenIdx}-${itemIdx}`}
+							tokens={item.tokens}
+							top={token.loose}
+						/>
+					</li>
+				{/each}
+			</ul>
+		{/if}
+	{:else if token.type === 'table'}
+		<table>
+			<thead>
+				<tr>
+					{#each token.header as header, headerIdx}
+						<th style={token.align[headerIdx] ? '' : `text-align: ${token.align[headerIdx]}`}>
+							<MarkdownInlineTokens
+								id={`${id}-${tokenIdx}-header-${headerIdx}`}
+								tokens={header.tokens}
+							/>
+						</th>
+					{/each}
+				</tr>
+			</thead>
+			<tbody>
+				{#each token.rows as row, rowIdx}
+					<tr>
+						{#each row ?? [] as cell, cellIdx}
+							<td style={token.align[cellIdx] ? '' : `text-align: ${token.align[cellIdx]}`}>
+								<MarkdownInlineTokens
+									id={`${id}-${tokenIdx}-row-${rowIdx}-${cellIdx}`}
+									tokens={cell.tokens}
+								/>
+							</td>
+						{/each}
+					</tr>
+				{/each}
+			</tbody>
+		</table>
+	{:else if token.type === 'text'} -->
+			<!-- {#if top}
+			<p>
+				{#if token.tokens}
+					<MarkdownInlineTokens id={`${id}-${tokenIdx}-t`} tokens={token.tokens} />
+				{:else}
+					{unescapeHtml(token.text)}
+				{/if}
+			</p>
+		{:else if token.tokens}
+			<MarkdownInlineTokens id={`${id}-${tokenIdx}-p`} tokens={token.tokens ?? []} />
+		{:else}
+			{unescapeHtml(token.text)}
+		{/if} -->
+		{:else}
+			{@html marked.parse(token.raw, {
+				...defaults,
+				gfm: true,
+				breaks: true,
+				renderer
+			})}
+		{/if}
+	{/each}
+</div>

src/lib/components/chat/Messages/ResponseMessage.svelte

@@ -1,6 +1,7 @@
 <script lang="ts">
 	import { toast } from 'svelte-sonner';
 	import dayjs from 'dayjs';
+	import { marked } from 'marked';
 	import tippy from 'tippy.js';
 	import auto_render from 'katex/dist/contrib/auto-render.mjs';
 	import 'katex/dist/katex.min.css';
@@ -37,7 +38,7 @@
 	import Spinner from '$lib/components/common/Spinner.svelte';
 	import WebSearchResults from './ResponseMessage/WebSearchResults.svelte';
 	import Sparkles from '$lib/components/icons/Sparkles.svelte';
-	import Markdown from './Markdown.svelte';
+	import MarkdownTokens from './MarkdownTokens.svelte';
 
 	export let message;
 	export let siblings;
@@ -76,6 +77,17 @@
 
 	let selectedCitation = null;
 
+	let tokens;
+
+	$: (async () => {
+		if (message?.content) {
+			tokens = marked.lexer(
+				replaceTokens(sanitizeResponseContent(message?.content), model?.name, $user?.name)
+			);
+			// console.log(message?.content, tokens);
+		}
+	})();
+
 	$: if (message) {
 		renderStyling();
 	}
@@ -408,7 +420,7 @@
 				{/if}
 
 				<div
-					class="prose chat-{message.role} w-full max-w-full dark:prose-invert prose-p:my-0 prose-img:my-1 prose-headings:my-1.5 prose-ol:my-1 prose-ul:my-1 whitespace-pre-line"
+					class="prose chat-{message.role} w-full max-w-full dark:prose-invert prose-p:my-0 prose-img:my-1 prose-headings:my-1 prose-pre:my-0 prose-table:my-0 prose-blockquote:my-0 prose-ul:-my-2 prose-ol:-my-2 prose-li:-my-3 whitespace-pre-line"
 				>
 					<div>
 						{#if (message?.statusHistory ?? [...(message?.status ? [message?.status] : [])]).length > 0}
@@ -488,15 +500,14 @@
 								</div>
 							</div>
 						{:else}
-							<div class="w-full">
+							<div class="w-full flex flex-col">
 								{#if message.content === '' && !message.error}
 									<Skeleton />
 								{:else if message.content && message.error !== true}
 									<!-- always show message contents even if there's an error -->
 									<!-- unless message.error === true which is legacy error handling, where the error message is stored in message.content -->
-
 									{#key message.id}
-										<Markdown id={message.id} {model} content={message.content} />
+										<MarkdownTokens id={message.id} {tokens} />
 									{/key}
 								{/if}
 

src/lib/components/common/Image.svelte

@@ -13,18 +13,13 @@
 	let showImagePreview = false;
 </script>
 
-<div class={className}>
-	<!-- svelte-ignore a11y-click-events-have-key-events -->
-	<!-- svelte-ignore a11y-no-noninteractive-element-interactions -->
-	<img
-		on:click={() => {
-			showImagePreview = true;
-		}}
-		src={_src}
-		{alt}
-		class=" rounded-lg cursor-pointer"
-		draggable="false"
-		data-cy="image"
-	/>
-</div>
+<button
+	class={className}
+	on:click={() => {
+		showImagePreview = true;
+	}}
+>
+	<img src={_src} {alt} class=" rounded-lg cursor-pointer" draggable="false" data-cy="image" />
+</button>
+
 <ImagePreview bind:show={showImagePreview} src={_src} {alt} />