OpenWebUI/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui synced 2025-06-26 18:26:48 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: merge with dev

Browse Source
This commit is contained in:
Fabio Polito
2025-03-07 00:22:47 +00:00
parent 17dbb3ec57 1173459eee
commit e3eef58310
63 changed files with 226 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
backend/open_webui/config.py
View File

@@ -593,7 +593,10 @@ for file_path in (FRONTEND_BUILD_DIR / "static").glob("**/*"):
(FRONTEND_BUILD_DIR / "static")
)
target_path.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(file_path, target_path)
try:
shutil.copyfile(file_path, target_path)
except Exception as e:
logging.error(f"An error occurred: {e}")
frontend_favicon = FRONTEND_BUILD_DIR / "static" / "favicon.png"
@@ -1377,6 +1380,11 @@ Responses from models: {{responses}}"""
# Code Interpreter
####################################
ENABLE_CODE_EXECUTION = PersistentConfig(
"ENABLE_CODE_EXECUTION",
"code_execution.enable",
os.environ.get("ENABLE_CODE_EXECUTION", "True").lower() == "true",
)
CODE_EXECUTION_ENGINE = PersistentConfig(
"CODE_EXECUTION_ENGINE",
@@ -1553,7 +1561,9 @@ ELASTICSEARCH_USERNAME = os.environ.get("ELASTICSEARCH_USERNAME", None)
ELASTICSEARCH_PASSWORD = os.environ.get("ELASTICSEARCH_PASSWORD", None)
ELASTICSEARCH_CLOUD_ID = os.environ.get("ELASTICSEARCH_CLOUD_ID", None)
SSL_ASSERT_FINGERPRINT = os.environ.get("SSL_ASSERT_FINGERPRINT", None)
ELASTICSEARCH_INDEX_PREFIX = os.environ.get("ELASTICSEARCH_INDEX_PREFIX", "open_webui_collections")
ELASTICSEARCH_INDEX_PREFIX = os.environ.get(
"ELASTICSEARCH_INDEX_PREFIX", "open_webui_collections"
)
# Pgvector
PGVECTOR_DB_URL = os.environ.get("PGVECTOR_DB_URL", DATABASE_URL)
if VECTOR_DB == "pgvector" and not PGVECTOR_DB_URL.startswith("postgres"):

3
backend/open_webui/main.py
View File

@@ -105,6 +105,7 @@ from open_webui.config import (
# Direct Connections
ENABLE_DIRECT_CONNECTIONS,
# Code Execution
ENABLE_CODE_EXECUTION,
CODE_EXECUTION_ENGINE,
CODE_EXECUTION_JUPYTER_URL,
CODE_EXECUTION_JUPYTER_AUTH,
@@ -662,6 +663,7 @@ app.state.EMBEDDING_FUNCTION = get_embedding_function(
#
########################################
app.state.config.ENABLE_CODE_EXECUTION = ENABLE_CODE_EXECUTION
app.state.config.CODE_EXECUTION_ENGINE = CODE_EXECUTION_ENGINE
app.state.config.CODE_EXECUTION_JUPYTER_URL = CODE_EXECUTION_JUPYTER_URL
app.state.config.CODE_EXECUTION_JUPYTER_AUTH = CODE_EXECUTION_JUPYTER_AUTH
@@ -1175,6 +1177,7 @@ async def get_app_config(request: Request):
"enable_direct_connections": app.state.config.ENABLE_DIRECT_CONNECTIONS,
"enable_channels": app.state.config.ENABLE_CHANNELS,
"enable_web_search": app.state.config.ENABLE_RAG_WEB_SEARCH,
"enable_code_execution": app.state.config.ENABLE_CODE_EXECUTION,
"enable_code_interpreter": app.state.config.ENABLE_CODE_INTERPRETER,
"enable_image_generation": app.state.config.ENABLE_IMAGE_GENERATION,
"enable_autocomplete_generation": app.state.config.ENABLE_AUTOCOMPLETE_GENERATION,

183
backend/open_webui/retrieval/vector/dbs/elasticsearch.py
View File

@@ -1,30 +1,28 @@
from elasticsearch import Elasticsearch, BadRequestError
from typing import Optional
import ssl
from elasticsearch.helpers import bulk,scan
from elasticsearch.helpers import bulk, scan
from open_webui.retrieval.vector.main import VectorItem, SearchResult, GetResult
from open_webui.config import (
ELASTICSEARCH_URL,
ELASTICSEARCH_CA_CERTS,
ELASTICSEARCH_CA_CERTS,
ELASTICSEARCH_API_KEY,
ELASTICSEARCH_USERNAME,
ELASTICSEARCH_PASSWORD,
ELASTICSEARCH_PASSWORD,
ELASTICSEARCH_CLOUD_ID,
ELASTICSEARCH_INDEX_PREFIX,
SSL_ASSERT_FINGERPRINT,
)
class ElasticsearchClient:
"""
Important:
in order to reduce the number of indexes and since the embedding vector length is fixed, we avoid creating
an index for each file but store it as a text field, while seperating to different index
in order to reduce the number of indexes and since the embedding vector length is fixed, we avoid creating
an index for each file but store it as a text field, while seperating to different index
baesd on the embedding length.
"""
def __init__(self):
self.index_prefix = ELASTICSEARCH_INDEX_PREFIX
self.client = Elasticsearch(
@@ -32,15 +30,19 @@ class ElasticsearchClient:
ca_certs=ELASTICSEARCH_CA_CERTS,
api_key=ELASTICSEARCH_API_KEY,
cloud_id=ELASTICSEARCH_CLOUD_ID,
basic_auth=(ELASTICSEARCH_USERNAME,ELASTICSEARCH_PASSWORD) if ELASTICSEARCH_USERNAME and ELASTICSEARCH_PASSWORD else None,
ssl_assert_fingerprint=SSL_ASSERT_FINGERPRINT
basic_auth=(
(ELASTICSEARCH_USERNAME, ELASTICSEARCH_PASSWORD)
if ELASTICSEARCH_USERNAME and ELASTICSEARCH_PASSWORD
else None
),
ssl_assert_fingerprint=SSL_ASSERT_FINGERPRINT,
)
#Status: works
def _get_index_name(self,dimension:int)->str:
# Status: works
def _get_index_name(self, dimension: int) -> str:
return f"{self.index_prefix}_d{str(dimension)}"
#Status: works
# Status: works
def _scan_result_to_get_result(self, result) -> GetResult:
if not result:
return None
@@ -55,7 +57,7 @@ class ElasticsearchClient:
return GetResult(ids=[ids], documents=[documents], metadatas=[metadatas])
#Status: works
# Status: works
def _result_to_get_result(self, result) -> GetResult:
if not result["hits"]["hits"]:
return None
@@ -70,7 +72,7 @@ class ElasticsearchClient:
return GetResult(ids=[ids], documents=[documents], metadatas=[metadatas])
#Status: works
# Status: works
def _result_to_search_result(self, result) -> SearchResult:
ids = []
distances = []
@@ -84,19 +86,21 @@ class ElasticsearchClient:
metadatas.append(hit["_source"].get("metadata"))
return SearchResult(
ids=[ids], distances=[distances], documents=[documents], metadatas=[metadatas]
ids=[ids],
distances=[distances],
documents=[documents],
metadatas=[metadatas],
)
#Status: works
# Status: works
def _create_index(self, dimension: int):
body = {
"mappings": {
"dynamic_templates": [
{
"strings": {
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
"strings": {
"match_mapping_type": "string",
"mapping": {"type": "keyword"},
}
}
],
@@ -111,68 +115,52 @@ class ElasticsearchClient:
},
"text": {"type": "text"},
"metadata": {"type": "object"},
}
},
}
}
self.client.indices.create(index=self._get_index_name(dimension), body=body)
#Status: works
# Status: works
def _create_batches(self, items: list[VectorItem], batch_size=100):
for i in range(0, len(items), batch_size):
yield items[i : min(i + batch_size,len(items))]
yield items[i : min(i + batch_size, len(items))]
#Status: works
def has_collection(self,collection_name) -> bool:
# Status: works
def has_collection(self, collection_name) -> bool:
query_body = {"query": {"bool": {"filter": []}}}
query_body["query"]["bool"]["filter"].append({"term": {"collection": collection_name}})
query_body["query"]["bool"]["filter"].append(
{"term": {"collection": collection_name}}
)
try:
result = self.client.count(
index=f"{self.index_prefix}*",
body=query_body
)
return result.body["count"]>0
result = self.client.count(index=f"{self.index_prefix}*", body=query_body)
return result.body["count"] > 0
except Exception as e:
return None
def delete_collection(self, collection_name: str):
query = {
"query": {
"term": {"collection": collection_name}
}
}
query = {"query": {"term": {"collection": collection_name}}}
self.client.delete_by_query(index=f"{self.index_prefix}*", body=query)
#Status: works
# Status: works
def search(
self, collection_name: str, vectors: list[list[float]], limit: int
) -> Optional[SearchResult]:
query = {
"size": limit,
"_source": [
"text",
"metadata"
],
"_source": ["text", "metadata"],
"query": {
"script_score": {
"query": {
"bool": {
"filter": [
{
"term": {
"collection": collection_name
}
}
]
}
"bool": {"filter": [{"term": {"collection": collection_name}}]}
},
"script": {
"source": "cosineSimilarity(params.vector, 'vector') + 1.0",
"params": {
"vector": vectors[0]
}, # Assuming single query vector
}, # Assuming single query vector
},
}
},
@@ -183,7 +171,8 @@ class ElasticsearchClient:
)
return self._result_to_search_result(result)
#Status: only tested halfwat
# Status: only tested halfwat
def query(
self, collection_name: str, filter: dict, limit: Optional[int] = None
) -> Optional[GetResult]:
@@ -197,7 +186,9 @@ class ElasticsearchClient:
for field, value in filter.items():
query_body["query"]["bool"]["filter"].append({"term": {field: value}})
query_body["query"]["bool"]["filter"].append({"term": {"collection": collection_name}})
query_body["query"]["bool"]["filter"].append(
{"term": {"collection": collection_name}}
)
size = limit if limit else 10
try:
@@ -206,59 +197,53 @@ class ElasticsearchClient:
body=query_body,
size=size,
)
return self._result_to_get_result(result)
except Exception as e:
return None
#Status: works
def _has_index(self,dimension:int):
return self.client.indices.exists(index=self._get_index_name(dimension=dimension))
# Status: works
def _has_index(self, dimension: int):
return self.client.indices.exists(
index=self._get_index_name(dimension=dimension)
)
def get_or_create_index(self, dimension: int):
if not self._has_index(dimension=dimension):
self._create_index(dimension=dimension)
#Status: works
# Status: works
def get(self, collection_name: str) -> Optional[GetResult]:
# Get all the items in the collection.
query = {
"query": {
"bool": {
"filter": [
{
"term": {
"collection": collection_name
}
}
]
}
}, "_source": ["text", "metadata"]}
"query": {"bool": {"filter": [{"term": {"collection": collection_name}}]}},
"_source": ["text", "metadata"],
}
results = list(scan(self.client, index=f"{self.index_prefix}*", query=query))
return self._scan_result_to_get_result(results)
#Status: works
# Status: works
def insert(self, collection_name: str, items: list[VectorItem]):
if not self._has_index(dimension=len(items[0]["vector"])):
self._create_index(dimension=len(items[0]["vector"]))
for batch in self._create_batches(items):
actions = [
{
"_index":self._get_index_name(dimension=len(items[0]["vector"])),
"_id": item["id"],
"_source": {
"collection": collection_name,
"vector": item["vector"],
"text": item["text"],
"metadata": item["metadata"],
},
}
{
"_index": self._get_index_name(dimension=len(items[0]["vector"])),
"_id": item["id"],
"_source": {
"collection": collection_name,
"vector": item["vector"],
"text": item["text"],
"metadata": item["metadata"],
},
}
for item in batch
]
bulk(self.client,actions)
bulk(self.client, actions)
# Upsert documents using the update API with doc_as_upsert=True.
def upsert(self, collection_name: str, items: list[VectorItem]):
@@ -280,8 +265,7 @@ class ElasticsearchClient:
}
for item in batch
]
bulk(self.client,actions)
bulk(self.client, actions)
# Delete specific documents from a collection by filtering on both collection and document IDs.
def delete(
@@ -292,21 +276,16 @@ class ElasticsearchClient:
):
query = {
"query": {
"bool": {
"filter": [
{"term": {"collection": collection_name}}
]
}
}
"query": {"bool": {"filter": [{"term": {"collection": collection_name}}]}}
}
#logic based on chromaDB
# logic based on chromaDB
if ids:
query["query"]["bool"]["filter"].append({"terms": {"_id": ids}})
elif filter:
for field, value in filter.items():
query["query"]["bool"]["filter"].append({"term": {f"metadata.{field}": value}})
query["query"]["bool"]["filter"].append(
{"term": {f"metadata.{field}": value}}
)
self.client.delete_by_query(index=f"{self.index_prefix}*", body=query)

5
backend/open_webui/routers/configs.py
View File

@@ -70,6 +70,7 @@ async def set_direct_connections_config(
# CodeInterpreterConfig
############################
class CodeInterpreterConfigForm(BaseModel):
ENABLE_CODE_EXECUTION: bool
CODE_EXECUTION_ENGINE: str
CODE_EXECUTION_JUPYTER_URL: Optional[str]
CODE_EXECUTION_JUPYTER_AUTH: Optional[str]
@@ -89,6 +90,7 @@ class CodeInterpreterConfigForm(BaseModel):
@router.get("/code_execution", response_model=CodeInterpreterConfigForm)
async def get_code_execution_config(request: Request, user=Depends(get_admin_user)):
return {
"ENABLE_CODE_EXECUTION": request.app.state.config.ENABLE_CODE_EXECUTION,
"CODE_EXECUTION_ENGINE": request.app.state.config.CODE_EXECUTION_ENGINE,
"CODE_EXECUTION_JUPYTER_URL": request.app.state.config.CODE_EXECUTION_JUPYTER_URL,
"CODE_EXECUTION_JUPYTER_AUTH": request.app.state.config.CODE_EXECUTION_JUPYTER_AUTH,
@@ -111,6 +113,8 @@ async def set_code_execution_config(
request: Request, form_data: CodeInterpreterConfigForm, user=Depends(get_admin_user)
):
request.app.state.config.ENABLE_CODE_EXECUTION = form_data.ENABLE_CODE_EXECUTION
request.app.state.config.CODE_EXECUTION_ENGINE = form_data.CODE_EXECUTION_ENGINE
request.app.state.config.CODE_EXECUTION_JUPYTER_URL = (
form_data.CODE_EXECUTION_JUPYTER_URL
@@ -153,6 +157,7 @@ async def set_code_execution_config(
)
return {
"ENABLE_CODE_EXECUTION": request.app.state.config.ENABLE_CODE_EXECUTION,
"CODE_EXECUTION_ENGINE": request.app.state.config.CODE_EXECUTION_ENGINE,
"CODE_EXECUTION_JUPYTER_URL": request.app.state.config.CODE_EXECUTION_JUPYTER_URL,
"CODE_EXECUTION_JUPYTER_AUTH": request.app.state.config.CODE_EXECUTION_JUPYTER_AUTH,

40
backend/open_webui/static/site.webmanifest
View File

@@ -1,21 +1,21 @@
{
"name": "Open WebUI",
"short_name": "WebUI",
"icons": [
{
"src": "/static/web-app-manifest-192x192.png",
"sizes": "192x192",
"type": "image/png",
"purpose": "maskable"
},
{
"src": "/static/web-app-manifest-512x512.png",
"sizes": "512x512",
"type": "image/png",
"purpose": "maskable"
}
],
"theme_color": "#ffffff",
"background_color": "#ffffff",
"display": "standalone"
}
"name": "Open WebUI",
"short_name": "WebUI",
"icons": [
{
"src": "/static/web-app-manifest-192x192.png",
"sizes": "192x192",
"type": "image/png",
"purpose": "maskable"
},
{
"src": "/static/web-app-manifest-512x512.png",
"sizes": "512x512",
"type": "image/png",
"purpose": "maskable"
}
],
"theme_color": "#ffffff",
"background_color": "#ffffff",
"display": "standalone"
}

2
backend/open_webui/utils/auth.py
View File

@@ -72,7 +72,7 @@ def get_license_data(app, key):
if key:
try:
res = requests.post(
"https://api.openwebui.com/api/v1/license",
"https://api.openwebui.com/api/v1/license/",
json={"key": key, "version": "1"},
timeout=5,
)