mirror of https://github.com/open-webui/open-webui
Timothy Jaeryang Baek
GitHub
commit
ba20c71963
|
|
@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
|
|||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [0.3.27] - 2024-09-24
|
||||
|
||||
### Fixed
|
||||
|
||||
- **🔄 Periodic Cleanup Error Resolved**: Fixed a critical RuntimeError related to the 'periodic_usage_pool_cleanup' coroutine, ensuring smooth and efficient performance post-pip install, correcting a persisting issue from version 0.3.26.
|
||||
- **📊 Enhanced LaTeX Rendering**: Improved rendering for LaTeX content, enhancing clarity and visual presentation in documents and mathematical models.
|
||||
|
||||
## [0.3.26] - 2024-09-24
|
||||
|
||||
### Fixed
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ def serve(
|
|||
)
|
||||
os.environ["USE_CUDA_DOCKER"] = "false"
|
||||
os.environ["LD_LIBRARY_PATH"] = ":".join(LD_LIBRARY_PATH)
|
||||
|
||||
import open_webui.main # we need set environment variables before importing main
|
||||
|
||||
uvicorn.run(open_webui.main.app, host=host, port=port, forwarded_allow_ips="*")
|
||||
|
|
|
|||
|
|
@ -93,7 +93,6 @@ async def periodic_usage_pool_cleanup():
|
|||
app = socketio.ASGIApp(
|
||||
sio,
|
||||
socketio_path="/ws/socket.io",
|
||||
on_startup=asyncio.create_task(periodic_usage_pool_cleanup()),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@ import shutil
|
|||
import sys
|
||||
import time
|
||||
import uuid
|
||||
import asyncio
|
||||
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Optional
|
||||
|
||||
|
|
@ -31,7 +33,7 @@ from open_webui.apps.openai.main import (
|
|||
from open_webui.apps.openai.main import get_all_models as get_openai_models
|
||||
from open_webui.apps.rag.main import app as rag_app
|
||||
from open_webui.apps.rag.utils import get_rag_context, rag_template
|
||||
from open_webui.apps.socket.main import app as socket_app
|
||||
from open_webui.apps.socket.main import app as socket_app, periodic_usage_pool_cleanup
|
||||
from open_webui.apps.socket.main import get_event_call, get_event_emitter
|
||||
from open_webui.apps.webui.internal.db import Session
|
||||
from open_webui.apps.webui.main import app as webui_app
|
||||
|
|
@ -184,6 +186,8 @@ https://github.com/open-webui/open-webui
|
|||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
run_migrations()
|
||||
|
||||
asyncio.create_task(periodic_usage_pool_cleanup())
|
||||
yield
|
||||
|
||||
|
||||
|
|
@ -851,7 +855,6 @@ async def inspect_websocket(request: Request, call_next):
|
|||
|
||||
|
||||
app.mount("/ws", socket_app)
|
||||
|
||||
app.mount("/ollama", ollama_app)
|
||||
app.mount("/openai", openai_app)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"name": "open-webui",
|
||||
"version": "0.3.26",
|
||||
"version": "0.3.27",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "open-webui",
|
||||
"version": "0.3.26",
|
||||
"version": "0.3.27",
|
||||
"dependencies": {
|
||||
"@codemirror/lang-javascript": "^6.2.2",
|
||||
"@codemirror/lang-python": "^6.1.6",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "open-webui",
|
||||
"version": "0.3.26",
|
||||
"version": "0.3.27",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "npm run pyodide:fetch && vite dev --host",
|
||||
|
|
|
|||
|
|
@ -8,23 +8,6 @@ import { TTS_RESPONSE_SPLIT } from '$lib/types';
|
|||
// Helper functions
|
||||
//////////////////////////
|
||||
|
||||
const convertLatexToSingleLine = (content) => {
|
||||
// Patterns to match multiline LaTeX blocks
|
||||
const patterns = [
|
||||
/(\$\$\s[\s\S]*?\s\$\$)/g, // Match $$ ... $$
|
||||
/(\\\[[\s\S]*?\\\])/g, // Match \[ ... \]
|
||||
/(\\begin\{[a-z]+\}[\s\S]*?\\end\{[a-z]+\})/g // Match \begin{...} ... \end{...}
|
||||
];
|
||||
|
||||
patterns.forEach((pattern) => {
|
||||
content = content.replace(pattern, (match) => {
|
||||
return match.replace(/\s*\n\s*/g, ' ').trim();
|
||||
});
|
||||
});
|
||||
|
||||
return content;
|
||||
};
|
||||
|
||||
export const replaceTokens = (content, char, user) => {
|
||||
const charToken = /{{char}}/gi;
|
||||
const userToken = /{{user}}/gi;
|
||||
|
|
@ -68,7 +51,6 @@ export const sanitizeResponseContent = (content: string) => {
|
|||
};
|
||||
|
||||
export const processResponseContent = (content: string) => {
|
||||
content = convertLatexToSingleLine(content);
|
||||
return content.trim();
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,14 +1,14 @@
|
|||
import katex from 'katex';
|
||||
|
||||
const DELIMITER_LIST = [
|
||||
{ left: '$$', right: '$$', display: false },
|
||||
{ left: '$$\n', right: '\n$$', display: true },
|
||||
{ left: '$$', right: '$$', display: false }, // This should be on top to prevent conflict with $ delimiter
|
||||
{ left: '$', right: '$', display: false },
|
||||
{ left: '\\pu{', right: '}', display: false },
|
||||
{ left: '\\ce{', right: '}', display: false },
|
||||
{ left: '\\(', right: '\\)', display: false },
|
||||
{ left: '( ', right: ' )', display: false },
|
||||
{ left: '\\[', right: '\\]', display: true },
|
||||
{ left: '[ ', right: ' ]', display: true }
|
||||
{ left: '\\[\n', right: '\n\\]', display: true },
|
||||
{ left: '\\[', right: '\\]', display: false }
|
||||
];
|
||||
|
||||
// const DELIMITER_LIST = [
|
||||
|
|
@ -28,24 +28,20 @@ function escapeRegex(string) {
|
|||
|
||||
function generateRegexRules(delimiters) {
|
||||
delimiters.forEach((delimiter) => {
|
||||
const { left, right } = delimiter;
|
||||
const { left, right, display } = delimiter;
|
||||
// Ensure regex-safe delimiters
|
||||
const escapedLeft = escapeRegex(left);
|
||||
const escapedRight = escapeRegex(right);
|
||||
|
||||
// Inline pattern - Capture group $1, token content, followed by end delimiter and normal punctuation marks.
|
||||
// Example: $text$
|
||||
inlinePatterns.push(
|
||||
`${escapedLeft}((?:\\\\.|[^\\\\\\n])*?(?:\\\\.|[^\\\\\\n${escapedRight}]))${escapedRight}`
|
||||
);
|
||||
|
||||
// Block pattern - Starts and ends with the delimiter on new lines. Example:
|
||||
// $$\ncontent here\n$$
|
||||
blockPatterns.push(`${escapedLeft}\n((?:\\\\[^]|[^\\\\])+?)\n${escapedRight}`);
|
||||
if (!display) {
|
||||
inlinePatterns.push(`${escapedLeft}((?:\\\\[^]|[^\\\\])+?)${escapedRight}`);
|
||||
} else {
|
||||
blockPatterns.push(`${escapedLeft}((?:\\\\[^]|[^\\\\])+?)${escapedRight}`);
|
||||
}
|
||||
});
|
||||
|
||||
const inlineRule = new RegExp(`^(${inlinePatterns.join('|')})(?=[\\s?!.,:？！。，：]|$)`, 'u');
|
||||
const blockRule = new RegExp(`^(${blockPatterns.join('|')})(?:\n|$)`, 'u');
|
||||
const blockRule = new RegExp(`^(${blockPatterns.join('|')})(?=[\\s?!.,:？！。，：]|$)`, 'u');
|
||||
|
||||
return { inlineRule, blockRule };
|
||||
}
|
||||
|
|
@ -55,84 +51,97 @@ const { inlineRule, blockRule } = generateRegexRules(DELIMITER_LIST);
|
|||
export default function (options = {}) {
|
||||
return {
|
||||
extensions: [
|
||||
inlineKatex(options, createRenderer(options, false)),
|
||||
blockKatex(options, createRenderer(options, true))
|
||||
blockKatex(options), // This should be on top to prevent conflict with inline delimiters.
|
||||
inlineKatex(options)
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
function createRenderer(options, newlineAfter) {
|
||||
return (token) =>
|
||||
katex.renderToString(token.text, { ...options, displayMode: token.displayMode }) +
|
||||
(newlineAfter ? '\n' : '');
|
||||
function katexStart(src, displayMode: boolean) {
|
||||
let ruleReg = displayMode ? blockRule : inlineRule;
|
||||
|
||||
let indexSrc = src;
|
||||
|
||||
while (indexSrc) {
|
||||
let index = -1;
|
||||
let startIndex = -1;
|
||||
let startDelimiter = '';
|
||||
let endDelimiter = '';
|
||||
for (let delimiter of DELIMITER_LIST) {
|
||||
if (delimiter.display !== displayMode) {
|
||||
continue;
|
||||
}
|
||||
|
||||
startIndex = indexSrc.indexOf(delimiter.left);
|
||||
if (startIndex === -1) {
|
||||
continue;
|
||||
}
|
||||
|
||||
index = startIndex;
|
||||
startDelimiter = delimiter.left;
|
||||
endDelimiter = delimiter.right;
|
||||
}
|
||||
|
||||
if (index === -1) {
|
||||
return;
|
||||
}
|
||||
|
||||
const f = index === 0 || indexSrc.charAt(index - 1) === ' ';
|
||||
if (f) {
|
||||
const possibleKatex = indexSrc.substring(index);
|
||||
|
||||
if (possibleKatex.match(ruleReg)) {
|
||||
return index;
|
||||
}
|
||||
}
|
||||
|
||||
indexSrc = indexSrc.substring(index + startDelimiter.length).replace(endDelimiter, '');
|
||||
}
|
||||
}
|
||||
|
||||
function inlineKatex(options, renderer) {
|
||||
const ruleReg = inlineRule;
|
||||
function katexTokenizer(src, tokens, displayMode: boolean) {
|
||||
let ruleReg = displayMode ? blockRule : inlineRule;
|
||||
let type = displayMode ? 'blockKatex' : 'inlineKatex';
|
||||
|
||||
const match = src.match(ruleReg);
|
||||
|
||||
if (match) {
|
||||
const text = match
|
||||
.slice(2)
|
||||
.filter((item) => item)
|
||||
.find((item) => item.trim());
|
||||
|
||||
return {
|
||||
type,
|
||||
raw: match[0],
|
||||
text: text,
|
||||
displayMode
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function inlineKatex(options) {
|
||||
return {
|
||||
name: 'inlineKatex',
|
||||
level: 'inline',
|
||||
start(src) {
|
||||
let index;
|
||||
let indexSrc = src;
|
||||
|
||||
while (indexSrc) {
|
||||
index = indexSrc.indexOf('$');
|
||||
if (index === -1) {
|
||||
return;
|
||||
}
|
||||
const f = index === 0 || indexSrc.charAt(index - 1) === ' ';
|
||||
if (f) {
|
||||
const possibleKatex = indexSrc.substring(index);
|
||||
|
||||
if (possibleKatex.match(ruleReg)) {
|
||||
return index;
|
||||
}
|
||||
}
|
||||
|
||||
indexSrc = indexSrc.substring(index + 1).replace(/^\$+/, '');
|
||||
}
|
||||
return katexStart(src, false);
|
||||
},
|
||||
tokenizer(src, tokens) {
|
||||
const match = src.match(ruleReg);
|
||||
|
||||
if (match) {
|
||||
const text = match
|
||||
.slice(2)
|
||||
.filter((item) => item)
|
||||
.find((item) => item.trim());
|
||||
|
||||
return {
|
||||
type: 'inlineKatex',
|
||||
raw: match[0],
|
||||
text: text
|
||||
};
|
||||
}
|
||||
},
|
||||
renderer
|
||||
return katexTokenizer(src, tokens, false);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function blockKatex(options, renderer) {
|
||||
function blockKatex(options) {
|
||||
return {
|
||||
name: 'blockKatex',
|
||||
level: 'block',
|
||||
tokenizer(src, tokens) {
|
||||
const match = src.match(blockRule);
|
||||
|
||||
if (match) {
|
||||
const text = match
|
||||
.slice(2)
|
||||
.filter((item) => item)
|
||||
.find((item) => item.trim());
|
||||
|
||||
return {
|
||||
type: 'blockKatex',
|
||||
raw: match[0],
|
||||
text: text
|
||||
};
|
||||
}
|
||||
start(src) {
|
||||
return katexStart(src, true);
|
||||
},
|
||||
renderer
|
||||
tokenizer(src, tokens) {
|
||||
return katexTokenizer(src, tokens, true);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue