Fix dev upstream merge conflicts

src/lib/utils/index.ts

@@ -8,7 +8,11 @@ import { TTS_RESPONSE_SPLIT } from '$lib/types';
 // Helper functions
 //////////////////////////
 
-export const replaceTokens = (content, char, user) => {
+function escapeRegExp(string: string): string {
+	return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+export const replaceTokens = (content, sourceIds, char, user) => {
 	const charToken = /{{char}}/gi;
 	const userToken = /{{user}}/gi;
 	const videoIdToken = /{{VIDEO_FILE_ID_([a-f0-9-]+)}}/gi; // Regex to capture the video ID
@@ -36,6 +40,20 @@ export const replaceTokens = (content, char, user) => {
 		return `<iframe src="${htmlUrl}" width="100%" frameborder="0" onload="this.style.height=(this.contentWindow.document.body.scrollHeight+20)+'px';"></iframe>`;
 	});
 
+	// Remove sourceIds from the content and replace them with <source_id>...</source_id>
+	if (Array.isArray(sourceIds)) {
+		sourceIds.forEach((sourceId) => {
+			// Escape special characters in the sourceId
+			const escapedSourceId = escapeRegExp(sourceId);
+
+			// Create a token based on the exact `[sourceId]` string
+			const sourceToken = `\\[${escapedSourceId}\\]`; // Escape special characters for RegExp
+			const sourceRegex = new RegExp(sourceToken, 'g'); // Match all occurrences of [sourceId]
+
+			content = content.replace(sourceRegex, `<source_id data="${sourceId}" />`);
+		});
+	}
+
 	return content;
 };
 
@@ -534,7 +552,31 @@ export const removeEmojis = (str: string) => {
 };
 
 export const removeFormattings = (str: string) => {
-	return str.replace(/(\*)(.*?)\1/g, '').replace(/(```)(.*?)\1/gs, '');
+	return str
+        // Block elements (remove completely)
+        .replace(/(```[\s\S]*?```)/g, '')                    // Code blocks
+        .replace(/^\|.*\|$/gm, '')                          // Tables
+	// Inline elements (preserve content)
+        .replace(/(?:\*\*|__)(.*?)(?:\*\*|__)/g, '$1')      // Bold
+        .replace(/(?:[*_])(.*?)(?:[*_])/g, '$1')            // Italic
+        .replace(/~~(.*?)~~/g, '$1')                        // Strikethrough
+        .replace(/`([^`]+)`/g, '$1')                        // Inline code
+        
+        // Links and images
+        .replace(/!?\[([^\]]*)\](?:\([^)]+\)|\[[^\]]*\])/g, '$1') // Links & images
+        .replace(/^\[[^\]]+\]:\s*.*$/gm, '')               // Reference definitions
+        
+        // Block formatting
+        .replace(/^#{1,6}\s+/gm, '')                       // Headers
+        .replace(/^\s*[-*+]\s+/gm, '')                     // Lists
+        .replace(/^\s*(?:\d+\.)\s+/gm, '')                 // Numbered lists
+        .replace(/^\s*>[> ]*/gm, '')                       // Blockquotes
+        .replace(/^\s*:\s+/gm, '')                         // Definition lists
+        
+        // Cleanup
+        .replace(/\[\^[^\]]*\]/g, '')                      // Footnotes
+        .replace(/[-*_~]/g, '')                            // Remaining markers
+        .replace(/\n{2,}/g, '\n')                          // Multiple newlines
 };
 
 export const cleanText = (content: string) => {