UniqAI/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui synced 2025-01-19 09:16:44 +00:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

updated to use credentials json, tested with local built image and upload to gcs works

Browse Source
This commit is contained in:
kahghi 2025-01-17 21:53:41 +08:00
parent 8efc543f49
commit 1764de41f3
5 changed files with 47 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
backend/open_webui/config.py
View File

@ -582,15 +582,15 @@ if CUSTOM_NAME:
#################################### ####################################
STORAGE_PROVIDER = os.environ.get("STORAGE_PROVIDER", "local") # defaults to local, s3 STORAGE_PROVIDER = os.environ.get("STORAGE_PROVIDER", "local") # defaults to local, s3
print(f"storage provider: {STORAGE_PROVIDER}")
S3_ACCESS_KEY_ID = os.environ.get("S3_ACCESS_KEY_ID", None) S3_ACCESS_KEY_ID = os.environ.get("S3_ACCESS_KEY_ID", None)
S3_SECRET_ACCESS_KEY = os.environ.get("S3_SECRET_ACCESS_KEY", None) S3_SECRET_ACCESS_KEY = os.environ.get("S3_SECRET_ACCESS_KEY", None)
S3_REGION_NAME = os.environ.get("S3_REGION_NAME", None) S3_REGION_NAME = os.environ.get("S3_REGION_NAME", None)
S3_BUCKET_NAME = os.environ.get("S3_BUCKET_NAME", None) S3_BUCKET_NAME = os.environ.get("S3_BUCKET_NAME", None)
S3_ENDPOINT_URL = os.environ.get("S3_ENDPOINT_URL", None) S3_ENDPOINT_URL = os.environ.get("S3_ENDPOINT_URL", None)
GCS_PROJECT_ID = os.environ.get("GCS_PROJECT_ID", None) GCS_BUCKET_NAME = os.environ.get("GCS_BUCKET_NAME", None)
GCS_BUCKET_NAME = os.environ.get("GCS_BUCKET_NAME", FileNotFoundError) GOOGLE_APPLICATION_CREDENTIALS_JSON = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS_JSON", None)
#################################### ####################################
# File Upload DIR # File Upload DIR

18
backend/open_webui/storage/provider.py
View File

@ -1,7 +1,9 @@
import os import os
import shutil import shutil
import json
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from typing import BinaryIO, Tuple from typing import BinaryIO, Tuple
from io import BytesIO
import boto3 import boto3
from botocore.exceptions import ClientError from botocore.exceptions import ClientError
@ -11,8 +13,8 @@ from open_webui.config import (
S3_ENDPOINT_URL, S3_ENDPOINT_URL,
S3_REGION_NAME, S3_REGION_NAME,
S3_SECRET_ACCESS_KEY, S3_SECRET_ACCESS_KEY,
GCS_PROJECT_ID,
GCS_BUCKET_NAME, GCS_BUCKET_NAME,
GOOGLE_APPLICATION_CREDENTIALS_JSON,
STORAGE_PROVIDER, STORAGE_PROVIDER,
UPLOAD_DIR, UPLOAD_DIR,
) )
@ -143,7 +145,7 @@ class S3StorageProvider(StorageProvider):
class GCSStorageProvider(StorageProvider): class GCSStorageProvider(StorageProvider):
def __init__(self): def __init__(self):
self.gcs_client = storage.Client(project=GCS_PROJECT_ID) self.gcs_client = storage.Client.from_service_account_info(info=json.loads(GOOGLE_APPLICATION_CREDENTIALS_JSON))
self.bucket_name = self.gcs_client.bucket(GCS_BUCKET_NAME) self.bucket_name = self.gcs_client.bucket(GCS_BUCKET_NAME)
def upload_file(self, file: BinaryIO, filename: str): def upload_file(self, file: BinaryIO, filename: str):
@ -154,26 +156,28 @@ class GCSStorageProvider(StorageProvider):
blob = self.bucket_name.blob(filename) blob = self.bucket_name.blob(filename)
# Upload the file to the bucket # Upload the file to the bucket
blob.upload_from_file(BytesIO(contents)) blob.upload_from_file(BytesIO(contents))
print("file successfully uploaded") return contents, _
except GoogleCloudError as e: except GoogleCloudError as e:
raise RuntimeError(f"Error uploading file to GCS: {e}") raise RuntimeError(f"Error uploading file to GCS: {e}")
def get_file(self, file_path:str) -> str: def get_file(self, file_path:str) -> str:
"""Handles downloading of the file from GCS storage.""" """Handles downloading of the file from GCS storage."""
try: try:
local_file_path = f"{UPLOAD_DIR}/{file_path}" local_file_path=file_path.removeprefix(UPLOAD_DIR + "/")
# Get the blob (object in the bucket) # Get the blob (object in the bucket)
blob = self.bucket_name.blob(file_path) blob = self.bucket_name.blob(local_file_path)
# Download the file to a local destination # Download the file to a local destination
blob.download_to_filename(local_file_path) blob.download_to_filename(file_path)
return file_path
except NotFound as e: except NotFound as e:
raise RuntimeError(f"Error downloading file from GCS: {e}") raise RuntimeError(f"Error downloading file from GCS: {e}")
def delete_file(self, file_path:str) -> None: def delete_file(self, file_path:str) -> None:
"""Handles deletion of the file from GCS storage.""" """Handles deletion of the file from GCS storage."""
try: try:
local_file_path = file_path.removeprefix(UPLOAD_DIR + "/")
# Get the blob (object in the bucket) # Get the blob (object in the bucket)
blob = self.bucket_name.blob(file_path) blob = self.bucket_name.blob(local_file_path)
# Delete the file # Delete the file
blob.delete() blob.delete()

1
backend/requirements.txt
View File

@ -102,6 +102,7 @@ pytest~=8.3.2
pytest-docker~=3.1.1 pytest-docker~=3.1.1
googleapis-common-protos==1.63.2 googleapis-common-protos==1.63.2
google-cloud-storage==2.19.0
## LDAP ## LDAP
ldap3==2.9.1 ldap3==2.9.1

20
pyproject.toml
View File

@ -10,16 +10,20 @@ dependencies = [
"uvicorn[standard]==0.30.6", "uvicorn[standard]==0.30.6",
"pydantic==2.9.2", "pydantic==2.9.2",
"python-multipart==0.0.18", "python-multipart==0.0.18",
"Flask==3.1.0", "Flask==3.1.0",
"Flask-Cors==5.0.0", "Flask-Cors==5.0.0",
"python-socketio==5.11.3", "python-socketio==5.11.3",
"python-jose==3.3.0", "python-jose==3.3.0",
"passlib[bcrypt]==1.7.4", "passlib[bcrypt]==1.7.4",
"requests==2.32.3", "requests==2.32.3",
"aiohttp==3.11.8", "aiohttp==3.11.8",
"async-timeout", "async-timeout",
"aiocache", "aiocache",
"aiofiles", "aiofiles",
"sqlalchemy==2.0.32", "sqlalchemy==2.0.32",
"alembic==1.14.0", "alembic==1.14.0",
"peewee==3.17.8", "peewee==3.17.8",
@ -28,26 +32,33 @@ dependencies = [
"pgvector==0.3.5", "pgvector==0.3.5",
"PyMySQL==1.1.1", "PyMySQL==1.1.1",
"bcrypt==4.2.0", "bcrypt==4.2.0",
"pymongo", "pymongo",
"redis", "redis",
"boto3==1.35.53", "boto3==1.35.53",
"argon2-cffi==23.1.0", "argon2-cffi==23.1.0",
"APScheduler==3.10.4", "APScheduler==3.10.4",
"openai", "openai",
"anthropic", "anthropic",
"google-generativeai==0.7.2", "google-generativeai==0.7.2",
"tiktoken", "tiktoken",
"langchain==0.3.7", "langchain==0.3.7",
"langchain-community==0.3.7", "langchain-community==0.3.7",
"fake-useragent==1.5.1", "fake-useragent==1.5.1",
"chromadb==0.6.2", "chromadb==0.6.2",
"pymilvus==2.5.0", "pymilvus==2.5.0",
"qdrant-client~=1.12.0", "qdrant-client~=1.12.0",
"opensearch-py==2.7.1", "opensearch-py==2.7.1",
"transformers", "transformers",
"sentence-transformers==3.3.1", "sentence-transformers==3.3.1",
"colbert-ai==0.2.21", "colbert-ai==0.2.21",
"einops==0.8.0", "einops==0.8.0",
"ftfy==6.2.3", "ftfy==6.2.3",
"pypdf==4.3.1", "pypdf==4.3.1",
"fpdf2==2.8.2", "fpdf2==2.8.2",
@ -66,26 +77,33 @@ dependencies = [
"psutil", "psutil",
"sentencepiece", "sentencepiece",
"soundfile==0.12.1", "soundfile==0.12.1",
"opencv-python-headless==4.10.0.84", "opencv-python-headless==4.10.0.84",
"rapidocr-onnxruntime==1.3.24", "rapidocr-onnxruntime==1.3.24",
"rank-bm25==0.2.2", "rank-bm25==0.2.2",
"faster-whisper==1.0.3", "faster-whisper==1.0.3",
"PyJWT[crypto]==2.10.1", "PyJWT[crypto]==2.10.1",
"authlib==1.3.2", "authlib==1.3.2",
"black==24.8.0", "black==24.8.0",
"langfuse==2.44.0", "langfuse==2.44.0",
"youtube-transcript-api==0.6.3", "youtube-transcript-api==0.6.3",
"pytube==15.0.0", "pytube==15.0.0",
"extract_msg", "extract_msg",
"pydub", "pydub",
"duckduckgo-search~=6.3.5", "duckduckgo-search~=6.3.5",
"docker~=7.1.0", "docker~=7.1.0",
"pytest~=8.3.2", "pytest~=8.3.2",
"pytest-docker~=3.1.1", "pytest-docker~=3.1.1",
"moto[s3]>=5.0.26", "moto[s3]>=5.0.26",
"googleapis-common-protos==1.63.2", "googleapis-common-protos==1.63.2",
"ldap3==2.9.1", "ldap3==2.9.1",
"google-cloud-storage>=2.19.0", "google-cloud-storage==2.19.0",
] ]
readme = "README.md" readme = "README.md"
requires-python = ">= 3.11, < 3.13.0a1" requires-python = ">= 3.11, < 3.13.0a1"

20
uv.lock
View File

@ -19,8 +19,10 @@ resolution-markers = [
"python_full_version < '3.12' and platform_system == 'Darwin'", "python_full_version < '3.12' and platform_system == 'Darwin'",
"python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_system == 'Darwin'", "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_system == 'Darwin'",
"python_full_version < '3.12' and platform_system == 'Darwin'", "python_full_version < '3.12' and platform_system == 'Darwin'",
"python_full_version < '3.12.4' and platform_system == 'Darwin'", "python_full_version < '3.12' and platform_system == 'Darwin'",
"python_full_version >= '3.12.4' and platform_system == 'Darwin'", "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_system == 'Darwin'",
"python_full_version < '3.13' and platform_system == 'Darwin'",
"python_full_version >= '3.13' and platform_system == 'Darwin'",
"python_full_version >= '3.13' and platform_system == 'Darwin'", "python_full_version >= '3.13' and platform_system == 'Darwin'",
"python_full_version >= '3.13' and platform_system == 'Darwin'", "python_full_version >= '3.13' and platform_system == 'Darwin'",
"python_full_version >= '3.13' and platform_system == 'Darwin'", "python_full_version >= '3.13' and platform_system == 'Darwin'",
@ -41,8 +43,10 @@ resolution-markers = [
"python_full_version < '3.12' and platform_machine == 'aarch64' and platform_system == 'Linux'", "python_full_version < '3.12' and platform_machine == 'aarch64' and platform_system == 'Linux'",
"python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine == 'aarch64' and platform_system == 'Linux'", "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine == 'aarch64' and platform_system == 'Linux'",
"python_full_version < '3.12' and platform_machine == 'aarch64' and platform_system == 'Linux'", "python_full_version < '3.12' and platform_machine == 'aarch64' and platform_system == 'Linux'",
"python_full_version < '3.12.4' and platform_machine == 'aarch64' and platform_system == 'Linux'", "python_full_version < '3.12' and platform_machine == 'aarch64' and platform_system == 'Linux'",
"python_full_version >= '3.12.4' and platform_machine == 'aarch64' and platform_system == 'Linux'", "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine == 'aarch64' and platform_system == 'Linux'",
"python_full_version < '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux'",
"python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux'",
"python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux'", "python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux'",
"python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux'", "python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux'",
"python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux'", "python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux'",
@ -63,8 +67,10 @@ resolution-markers = [
"(python_full_version < '3.12' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version < '3.12' and platform_system != 'Darwin' and platform_system != 'Linux')", "(python_full_version < '3.12' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version < '3.12' and platform_system != 'Darwin' and platform_system != 'Linux')",
"(python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_system != 'Darwin' and platform_system != 'Linux')", "(python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_system != 'Darwin' and platform_system != 'Linux')",
"(python_full_version < '3.12' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version < '3.12' and platform_system != 'Darwin' and platform_system != 'Linux')", "(python_full_version < '3.12' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version < '3.12' and platform_system != 'Darwin' and platform_system != 'Linux')",
"(python_full_version < '3.12.4' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version < '3.12.4' and platform_system != 'Darwin' and platform_system != 'Linux')", "(python_full_version < '3.12' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version < '3.12' and platform_system != 'Darwin' and platform_system != 'Linux')",
"(python_full_version >= '3.12.4' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version >= '3.12.4' and platform_system != 'Darwin' and platform_system != 'Linux')", "(python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_system != 'Darwin' and platform_system != 'Linux')",
"(python_full_version < '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version < '3.13' and platform_system != 'Darwin' and platform_system != 'Linux')",
"(python_full_version >= '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version >= '3.13' and platform_system != 'Darwin' and platform_system != 'Linux')",
"(python_full_version >= '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version >= '3.13' and platform_system != 'Darwin' and platform_system != 'Linux')", "(python_full_version >= '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version >= '3.13' and platform_system != 'Darwin' and platform_system != 'Linux')",
"(python_full_version >= '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version >= '3.13' and platform_system != 'Darwin' and platform_system != 'Linux')", "(python_full_version >= '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version >= '3.13' and platform_system != 'Darwin' and platform_system != 'Linux')",
"(python_full_version >= '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version >= '3.13' and platform_system != 'Darwin' and platform_system != 'Linux')", "(python_full_version >= '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version >= '3.13' and platform_system != 'Darwin' and platform_system != 'Linux')",
@ -2847,7 +2853,7 @@ requires-dist = [
{ name = "flask-cors", specifier = "==5.0.0" }, { name = "flask-cors", specifier = "==5.0.0" },
{ name = "fpdf2", specifier = "==2.8.2" }, { name = "fpdf2", specifier = "==2.8.2" },
{ name = "ftfy", specifier = "==6.2.3" }, { name = "ftfy", specifier = "==6.2.3" },
{ name = "google-cloud-storage", specifier = ">=2.19.0" }, { name = "google-cloud-storage", specifier = "==2.19.0" },
{ name = "google-generativeai", specifier = "==0.7.2" }, { name = "google-generativeai", specifier = "==0.7.2" },
{ name = "googleapis-common-protos", specifier = "==1.63.2" }, { name = "googleapis-common-protos", specifier = "==1.63.2" },
{ name = "langchain", specifier = "==0.3.7" }, { name = "langchain", specifier = "==0.3.7" },