UniqAI/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui synced 2025-03-25 06:58:04 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

refac: latex

Browse Source
This commit is contained in:
Timothy J. Baek 2024-08-14 16:07:39 +02:00
parent 0ec1f9e331
commit 6aefc79807
3 changed files with 120 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/lib/components/chat/Messages/MarkdownInlineTokens.svelte
View File

@ -37,10 +37,7 @@
</del> </del>
{:else if token.type === 'inlineKatex'} {:else if token.type === 'inlineKatex'}
{#if token.text} {#if token.text}
<KatexRenderer <KatexRenderer content={revertSanitizedResponseContent(token.text)} displayMode={false} />
content={revertSanitizedResponseContent(token.text)}
displayMode={token?.displayMode ?? false}
/>
{/if} {/if}
{:else if token.type === 'text'} {:else if token.type === 'text'}
{token.raw} {token.raw}

7
src/lib/components/chat/Messages/MarkdownTokens.svelte
View File

@ -116,6 +116,13 @@
displayMode={token?.displayMode ?? false} displayMode={token?.displayMode ?? false}
/> />
{/if} {/if}
{:else if token.type === 'blockKatex'}
{#if token.text}
<KatexRenderer
content={revertSanitizedResponseContent(token.text)}
displayMode={token?.displayMode ?? false}
/>
{/if}
{:else if token.type === 'space'} {:else if token.type === 'space'}
{''} {''}
{:else} {:else}

178
src/lib/utils/katex-extension.ts
View File

@ -1,83 +1,129 @@
import katex from 'katex'; import katex from 'katex';
const inlineRule = const DELIMITER_LIST = [
/^(\${1,2})(?!\$)((?:\\.|[^\\\n])*?(?:\\.|[^\\\n\$]))\1(?=[\s?!\.,:？！。，：]|$)/; { left: '$$', right: '$$', display: false },
const inlineRuleNonStandard = /^(\${1,2})(?!\$)((?:\\.|[^\\\n])*?(?:\\.|[^\\\n\$]))\1/; // Non-standard, even if there are no spaces before and after $ or $$, try to parse { left: '$', right: '$', display: false },
{ left: '\\pu{', right: '}', display: false },
{ left: '\\ce{', right: '}', display: false },
{ left: '\\(', right: '\\)', display: false },
{ left: '( ', right: ' )', display: false },
{ left: '\\[', right: '\\]', display: true },
{ left: '[', right: ']', display: true }
]
const blockRule = /^(\${1,2})\n((?:\\[^]|[^\\])+?)\n\1(?:\n|$)/; // const DELIMITER_LIST = [
// { left: '$$', right: '$$', display: false },
// { left: '$', right: '$', display: false },
// ];
export default function (options = {}) { // const inlineRule = /^(\${1,2})(?!\$)((?:\\.|[^\\\n])*?(?:\\.|[^\\\n\$]))\1(?=[\s?!\.,:？！。，：]|$)/;
return { // const blockRule = /^(\${1,2})\n((?:\\[^]|[^\\])+?)\n\1(?:\n|$)/;
extensions: [
inlineKatex(options, createRenderer(options, false)), let inlinePatterns = [];
blockKatex(options, createRenderer(options, true)) let blockPatterns = [];
]
}; function escapeRegex(string) {
return string.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&');
}
function generateRegexRules(delimiters) {
delimiters.forEach(delimiter => {
const { left, right } = delimiter;
// Ensure regex-safe delimiters
const escapedLeft = escapeRegex(left);
const escapedRight = escapeRegex(right);
// Inline pattern - Capture group $1, token content, followed by end delimiter and normal punctuation marks.
// Example: $text$
inlinePatterns.push(`${escapedLeft}((?:\\\\.|[^\\\\\\n])*?(?:\\\\.|[^\\\\\\n${escapedRight}]))${escapedRight}`);
// Block pattern - Starts and ends with the delimiter on new lines. Example:
// $$\ncontent here\n$$
blockPatterns.push(`${escapedLeft}\n((?:\\\\[^]|[^\\\\])+?)\n${escapedRight}`);
});
const inlineRule = new RegExp(`^(${inlinePatterns.join('|')})(?=[\\s?!.,:？！。，：]|$)`, 'u');
const blockRule = new RegExp(`^(${blockPatterns.join('|')})(?:\n|$)`, 'u');
return { inlineRule, blockRule };
}
const { inlineRule, blockRule } = generateRegexRules(DELIMITER_LIST);
export default function(options = {}) {
return {
extensions: [
inlineKatex(options, createRenderer(options, false)),
blockKatex(options, createRenderer(options, true)),
],
};
} }
function createRenderer(options, newlineAfter) { function createRenderer(options, newlineAfter) {
return (token) => return (token) => katex.renderToString(token.text, { ...options, displayMode: token.displayMode }) + (newlineAfter ? '\n' : '');
katex.renderToString(token.text, { ...options, displayMode: token.displayMode }) +
(newlineAfter ? '\n' : '');
} }
function inlineKatex(options, renderer) { function inlineKatex(options, renderer) {
const nonStandard = options && options.nonStandard; const ruleReg = inlineRule;
const ruleReg = nonStandard ? inlineRuleNonStandard : inlineRule; return {
return { name: 'inlineKatex',
name: 'inlineKatex', level: 'inline',
level: 'inline', start(src) {
start(src) { let index;
let index; let indexSrc = src;
let indexSrc = src;
while (indexSrc) { while (indexSrc) {
index = indexSrc.indexOf('$'); index = indexSrc.indexOf('$');
if (index === -1) { if (index === -1) {
return; return;
} }
const f = nonStandard ? index > -1 : index === 0 || indexSrc.charAt(index - 1) === ' '; const f = index === 0 || indexSrc.charAt(index - 1) === ' ';
if (f) { if (f) {
const possibleKatex = indexSrc.substring(index); const possibleKatex = indexSrc.substring(index);
if (possibleKatex.match(ruleReg)) { if (possibleKatex.match(ruleReg)) {
return index; return index;
} }
} }
indexSrc = indexSrc.substring(index + 1).replace(/^\$+/, ''); indexSrc = indexSrc.substring(index + 1).replace(/^\$+/, '');
} }
}, },
tokenizer(src, tokens) { tokenizer(src, tokens) {
const match = src.match(ruleReg); const match = src.match(ruleReg);
if (match) {
return { if (match) {
type: 'inlineKatex', console.log(match)
raw: match[0], const text = match.slice(2).filter((item) => item).find((item) => item.trim());
text: match[2].trim(),
displayMode: match[1].length === 2 return {
}; type: 'inlineKatex',
} raw: match[0],
}, text: text,
renderer };
}; }
},
renderer,
};
} }
function blockKatex(options, renderer) { function blockKatex(options, renderer) {
return { return {
name: 'blockKatex', name: 'blockKatex',
level: 'block', level: 'block',
tokenizer(src, tokens) { tokenizer(src, tokens) {
const match = src.match(blockRule); const match = src.match(blockRule);
if (match) { if (match) {
return { return {
type: 'blockKatex', type: 'blockKatex',
raw: match[0], raw: match[0],
text: match[2].trim(), text: match[0],
displayMode: match[1].length === 2 };
}; }
} },
}, renderer,
renderer };
};
} }