From 9ca198c776700b8379b278bea9b078e288689c6c Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Mon, 12 May 2025 21:14:57 +0200 Subject: [PATCH 1/4] Markdownfix (#1) * Fix/feat: rich text input * fix * fix * fix * fix * fix * fix * Update PromptEditor.svelte * Update RichTextInput.svelte * Update RichTextInput.svelte * fix * Update RichTextInput.svelte * Update RichTextInput.svelte * Update RichTextInput.svelte * Update RichTextInput.svelte * Update RichTextInput.svelte * Update RichTextInput.svelte * Update RichTextInput.svelte * Update RichTextInput.svelte * Update RichTextInput.svelte * Update RichTextInput.svelte * Update RichTextInput.svelte * Update RichTextInput.svelte * Update RichTextInput.svelte * Update RichTextInput.svelte * Update Prompts.svelte * revert * Update RichTextInput.svelte * Update PromptEditor.svelte * test * Update Prompts.svelte * Update Prompts.svelte * Update Prompts.svelte * Update Prompts.svelte * fix? * Update RichTextInput.svelte * revert * Update MessageInput.svelte * Update Prompts.svelte * Update Prompts.svelte * Update Prompts.svelte * Update Prompts.svelte * Update Prompts.svelte * Update RichTextInput.svelte * Update RichTextInput.svelte * Update PromptEditor.svelte * Update PromptEditor.svelte --------- Co-authored-by: Tim Jaeryang Baek --- src/lib/components/chat/MessageInput.svelte | 1 + .../chat/MessageInput/Commands/Prompts.svelte | 62 ++++++++++++++----- .../components/common/RichTextInput.svelte | 50 ++++++++++----- .../workspace/Prompts/PromptEditor.svelte | 1 + 4 files changed, 80 insertions(+), 34 deletions(-) diff --git a/src/lib/components/chat/MessageInput.svelte b/src/lib/components/chat/MessageInput.svelte index 9e5593ea9..ec5b60878 100644 --- a/src/lib/components/chat/MessageInput.svelte +++ b/src/lib/components/chat/MessageInput.svelte @@ -611,6 +611,7 @@ bind:this={chatInputElement} bind:value={prompt} id="chat-input" + preserveBreaks={true} messageInput={true} shiftEnter={!($settings?.ctrlEnterToSend ?? false) && (!$mobile || diff --git a/src/lib/components/chat/MessageInput/Commands/Prompts.svelte b/src/lib/components/chat/MessageInput/Commands/Prompts.svelte index da88acc45..b0424572f 100644 --- a/src/lib/components/chat/MessageInput/Commands/Prompts.svelte +++ b/src/lib/components/chat/MessageInput/Commands/Prompts.svelte @@ -22,9 +22,18 @@ let selectedPromptIdx = 0; let filteredPrompts = []; - $: filteredPrompts = $prompts - .filter((p) => p.command.toLowerCase().includes(command.toLowerCase())) - .sort((a, b) => a.title.localeCompare(b.title)); + $: { + if (command && command.length > 1) { + const commandName = command.substring(1).toLowerCase(); + const cleanedCommandName = commandName.replace(/<\/?p>/gi, '').trim(); + + filteredPrompts = $prompts + .filter((p) => p.command.toLowerCase().includes(cleanedCommandName)) + .sort((a, b) => a.title.localeCompare(b.title)); + } else { + filteredPrompts = []; + } + } $: if (command) { selectedPromptIdx = 0; @@ -120,23 +129,42 @@ text = text.replaceAll('{{CURRENT_WEEKDAY}}', weekday); } - const lines = prompt.split('\n'); - const lastLine = lines.pop(); - - const lastLineWords = lastLine.split(' '); - const lastWord = lastLineWords.pop(); - if ($settings?.richTextInput ?? true) { - lastLineWords.push( - `${text.replace(//g, '>').replaceAll('\n', '
')}` - ); + const allPromptLines = prompt.split('\n'); + const lastLineWithTrigger = allPromptLines.pop() || ''; // Line where command was typed + const wordsInLastLine = lastLineWithTrigger.split(' '); + wordsInLastLine.pop(); // Remove the command trigger itself (e.g., /mycommand) + + let fullPromptPrefix = ''; + if (allPromptLines.length > 0) { + fullPromptPrefix = allPromptLines.join('\n'); + } + if (wordsInLastLine.length > 0) { + if (fullPromptPrefix.length > 0) { + fullPromptPrefix += '\n'; + } + fullPromptPrefix += wordsInLastLine.join(' '); + } + fullPromptPrefix = fullPromptPrefix.trimEnd(); + + if (text && text.trim().length > 0) { // If 'text' (the command's content) is not empty + if (fullPromptPrefix.length > 0) { + prompt = fullPromptPrefix + '\n\n' + text; + } else { + prompt = text; + } + } else { + prompt = fullPromptPrefix; + } - lines.push(lastLineWords.join(' ')); - prompt = lines.join('
'); } else { - lastLineWords.push(text); - lines.push(lastLineWords.join(' ')); - prompt = lines.join('\n'); + const currentInputLines = prompt.split('\n'); + const lastCurrentInputLine = currentInputLines.pop() || ''; + const lastCurrentInputLineWords = lastCurrentInputLine.split(' '); + lastCurrentInputLineWords.pop(); + lastCurrentInputLineWords.push(command.content); + currentInputLines.push(lastCurrentInputLineWords.join(' ')); + prompt = currentInputLines.join('\n'); } const chatInputContainerElement = document.getElementById('chat-input-container'); diff --git a/src/lib/components/common/RichTextInput.svelte b/src/lib/components/common/RichTextInput.svelte index b7bc03e89..84dfbad5a 100644 --- a/src/lib/components/common/RichTextInput.svelte +++ b/src/lib/components/common/RichTextInput.svelte @@ -156,25 +156,34 @@ } if (!raw) { - async function tryParse(value, attempts = 3, interval = 100) { + async function tryParse(value, attempts = 3, interval = 100, useMarkdownBreaks = false) { try { + // Normalize all types of newlines to \n + const normalizedTextValue = (value || '').replace(/\r\n|\r/g, '\n'); // Try parsing the value - return marked.parse(value.replaceAll(`\n
`, `
`), { - breaks: false + return marked.parse(normalizedTextValue, { + breaks: useMarkdownBreaks, + gfm: true // Ensure GFM is active for paragraph handling }); } catch (error) { // If no attempts remain, fallback to plain text if (attempts <= 1) { - return value; + return (value || '').replace(/\r\n|\r/g, '\n'); } // Wait for the interval, then retry await new Promise((resolve) => setTimeout(resolve, interval)); - return tryParse(value, attempts - 1, interval); // Recursive call + return tryParse(value, attempts - 1, interval, useMarkdownBreaks); } } // Usage example - content = await tryParse(value); + content = await tryParse(value, 3, 100, preserveBreaks); + + if (preserveBreaks && content) { + // Ensure truly empty paragraphs generated by marked get an   + // so they are visually represented and can be saved back correctly. + content = content.replace(/

(?: |)?<\/p>/gi, '

​

'); + } } } else { if (html && !content) { @@ -233,9 +242,7 @@ if (!raw) { let newValue = turndownService .turndown( - editor - .getHTML() - .replace(/

<\/p>/g, '
') + html .replace(/ {2,}/g, (m) => m.replace(/ /g, '\u00a0')) ) .replace(/\u00a0/g, ' '); @@ -419,14 +426,23 @@ ) .replace(/\u00a0/g, ' ') ) { - preserveBreaks - ? editor.commands.setContent(value) - : editor.commands.setContent( - marked.parse(value.replaceAll(`\n
`, `
`), { - breaks: false - }) - ); // Update editor content - + if (preserveBreaks) { + const normalizedValue = (value || '').replace(/\r\n|\r/g, '\n'); + // Parse Markdown with breaks:true to convert \n to
for Tiptap + let htmlToSet = marked.parse(normalizedValue, { breaks: true, gfm: true }); + if (htmlToSet) { + // htmlToSet = htmlToSet.replace(/

<\/p>/g, '

 

'); // Old way + htmlToSet = htmlToSet.replace(/

(?: |)?<\/p>/gi, '

​

'); + } + editor.commands.setContent( + marked.parse(normalizedValue, { breaks: true, gfm: true }) + ); + } else { + const normalizedValue = (value || '').replace(/\r\n|\r/g, '\n'); + editor.commands.setContent( + marked.parse(normalizedValue, { breaks: false, gfm: true }) + ); + } selectTemplate(); } } diff --git a/src/lib/components/workspace/Prompts/PromptEditor.svelte b/src/lib/components/workspace/Prompts/PromptEditor.svelte index 624bb551e..f4a7e9771 100644 --- a/src/lib/components/workspace/Prompts/PromptEditor.svelte +++ b/src/lib/components/workspace/Prompts/PromptEditor.svelte @@ -155,6 +155,7 @@ bind:value={content} rows={6} required + preserveBreaks={true} /> From f4cae6fb49603d52c4e387ff5df60c346931f36a Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Fri, 23 May 2025 08:33:21 +0200 Subject: [PATCH 2/4] Update Prompts.svelte --- src/lib/components/chat/MessageInput/Commands/Prompts.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/components/chat/MessageInput/Commands/Prompts.svelte b/src/lib/components/chat/MessageInput/Commands/Prompts.svelte index 7a7251be3..9d49cee25 100644 --- a/src/lib/components/chat/MessageInput/Commands/Prompts.svelte +++ b/src/lib/components/chat/MessageInput/Commands/Prompts.svelte @@ -23,7 +23,7 @@ let filteredPrompts = []; $: { - if (command && command.length > 1) { + if (command && command.length > 0) { const commandName = command.substring(1).toLowerCase(); const cleanedCommandName = commandName.replace(/<\/?p>/gi, '').trim(); From 17e100661cd2acbab9fe56cb7e56313739e25673 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Tue, 10 Jun 2025 12:40:28 +0200 Subject: [PATCH 3/4] Update youtube.py --- .../open_webui/retrieval/loaders/youtube.py | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/backend/open_webui/retrieval/loaders/youtube.py b/backend/open_webui/retrieval/loaders/youtube.py index be5e53358..f5ff3a638 100644 --- a/backend/open_webui/retrieval/loaders/youtube.py +++ b/backend/open_webui/retrieval/loaders/youtube.py @@ -40,11 +40,13 @@ def _parse_video_id(url: str) -> Optional[str]: video_id = ids if isinstance(ids, str) else ids[0] else: return None + elif parsed_url.netloc == "youtu.be": + video_id = parsed_url.path.lstrip("/").split("?")[0] else: path = parsed_url.path.lstrip("/") - video_id = path.split("/")[-1] + video_id = path.split("/")[-1].split("?")[0] - if len(video_id) != 11: # Video IDs are 11 characters long + if len(video_id) != 11: return None return video_id @@ -109,19 +111,19 @@ class YoutubeLoader: # Try each language in order of priority for lang in self.language: try: - transcript = transcript_list.find_transcript([lang]) - if transcript.is_generated: - log.debug(f"Found generated transcript for language '{lang}'") - try: - transcript = transcript_list.find_manually_created_transcript( - [lang] - ) - log.debug(f"Found manual transcript for language '{lang}'") - except NoTranscriptFound: - log.debug( - f"No manual transcript found for language '{lang}', using generated" - ) - pass + try: + transcript = transcript_list.find_manually_created_transcript([lang]) + log.debug(f"Found manual transcript for language '{lang}'") + except NoTranscriptFound: + transcript = transcript_list.find_generated_transcript([lang]) + log.debug(f"Found auto-generated transcript for language '{lang}'") + + log.debug(f"Found transcript for language '{lang}'") + try: + transcript_pieces: List[Dict[str, Any]] = transcript.fetch() + except ParseError: + log.debug(f"Empty or invalid transcript for language '{lang}'") + continue log.debug(f"Found transcript for language '{lang}'") try: From 3216e8f9048f24ec37147eeabe7be9c55eb47538 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Tue, 10 Jun 2025 12:46:21 +0200 Subject: [PATCH 4/4] Update youtube.py --- .../open_webui/retrieval/loaders/youtube.py | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/backend/open_webui/retrieval/loaders/youtube.py b/backend/open_webui/retrieval/loaders/youtube.py index f5ff3a638..be5e53358 100644 --- a/backend/open_webui/retrieval/loaders/youtube.py +++ b/backend/open_webui/retrieval/loaders/youtube.py @@ -40,13 +40,11 @@ def _parse_video_id(url: str) -> Optional[str]: video_id = ids if isinstance(ids, str) else ids[0] else: return None - elif parsed_url.netloc == "youtu.be": - video_id = parsed_url.path.lstrip("/").split("?")[0] else: path = parsed_url.path.lstrip("/") - video_id = path.split("/")[-1].split("?")[0] + video_id = path.split("/")[-1] - if len(video_id) != 11: + if len(video_id) != 11: # Video IDs are 11 characters long return None return video_id @@ -111,19 +109,19 @@ class YoutubeLoader: # Try each language in order of priority for lang in self.language: try: - try: - transcript = transcript_list.find_manually_created_transcript([lang]) - log.debug(f"Found manual transcript for language '{lang}'") - except NoTranscriptFound: - transcript = transcript_list.find_generated_transcript([lang]) - log.debug(f"Found auto-generated transcript for language '{lang}'") - - log.debug(f"Found transcript for language '{lang}'") - try: - transcript_pieces: List[Dict[str, Any]] = transcript.fetch() - except ParseError: - log.debug(f"Empty or invalid transcript for language '{lang}'") - continue + transcript = transcript_list.find_transcript([lang]) + if transcript.is_generated: + log.debug(f"Found generated transcript for language '{lang}'") + try: + transcript = transcript_list.find_manually_created_transcript( + [lang] + ) + log.debug(f"Found manual transcript for language '{lang}'") + except NoTranscriptFound: + log.debug( + f"No manual transcript found for language '{lang}', using generated" + ) + pass log.debug(f"Found transcript for language '{lang}'") try: