diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index dc8035bf5..306686195 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -582,15 +582,15 @@ if CUSTOM_NAME: #################################### STORAGE_PROVIDER = os.environ.get("STORAGE_PROVIDER", "local") # defaults to local, s3 -print(f"storage provider: {STORAGE_PROVIDER}") + S3_ACCESS_KEY_ID = os.environ.get("S3_ACCESS_KEY_ID", None) S3_SECRET_ACCESS_KEY = os.environ.get("S3_SECRET_ACCESS_KEY", None) S3_REGION_NAME = os.environ.get("S3_REGION_NAME", None) S3_BUCKET_NAME = os.environ.get("S3_BUCKET_NAME", None) S3_ENDPOINT_URL = os.environ.get("S3_ENDPOINT_URL", None) -GCS_PROJECT_ID = os.environ.get("GCS_PROJECT_ID", None) -GCS_BUCKET_NAME = os.environ.get("GCS_BUCKET_NAME", FileNotFoundError) +GCS_BUCKET_NAME = os.environ.get("GCS_BUCKET_NAME", None) +GOOGLE_APPLICATION_CREDENTIALS_JSON = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS_JSON", None) #################################### # File Upload DIR diff --git a/backend/open_webui/storage/provider.py b/backend/open_webui/storage/provider.py index af90519d3..84cfefe2a 100644 --- a/backend/open_webui/storage/provider.py +++ b/backend/open_webui/storage/provider.py @@ -1,7 +1,9 @@ import os import shutil +import json from abc import ABC, abstractmethod from typing import BinaryIO, Tuple +from io import BytesIO import boto3 from botocore.exceptions import ClientError @@ -11,8 +13,8 @@ from open_webui.config import ( S3_ENDPOINT_URL, S3_REGION_NAME, S3_SECRET_ACCESS_KEY, - GCS_PROJECT_ID, GCS_BUCKET_NAME, + GOOGLE_APPLICATION_CREDENTIALS_JSON, STORAGE_PROVIDER, UPLOAD_DIR, ) @@ -143,7 +145,7 @@ class S3StorageProvider(StorageProvider): class GCSStorageProvider(StorageProvider): def __init__(self): - self.gcs_client = storage.Client(project=GCS_PROJECT_ID) + self.gcs_client = storage.Client.from_service_account_info(info=json.loads(GOOGLE_APPLICATION_CREDENTIALS_JSON)) self.bucket_name = self.gcs_client.bucket(GCS_BUCKET_NAME) def upload_file(self, file: BinaryIO, filename: str): @@ -154,26 +156,28 @@ class GCSStorageProvider(StorageProvider): blob = self.bucket_name.blob(filename) # Upload the file to the bucket blob.upload_from_file(BytesIO(contents)) - print("file successfully uploaded") + return contents, _ except GoogleCloudError as e: raise RuntimeError(f"Error uploading file to GCS: {e}") def get_file(self, file_path:str) -> str: """Handles downloading of the file from GCS storage.""" try: - local_file_path = f"{UPLOAD_DIR}/{file_path}" + local_file_path=file_path.removeprefix(UPLOAD_DIR + "/") # Get the blob (object in the bucket) - blob = self.bucket_name.blob(file_path) + blob = self.bucket_name.blob(local_file_path) # Download the file to a local destination - blob.download_to_filename(local_file_path) + blob.download_to_filename(file_path) + return file_path except NotFound as e: raise RuntimeError(f"Error downloading file from GCS: {e}") def delete_file(self, file_path:str) -> None: """Handles deletion of the file from GCS storage.""" try: + local_file_path = file_path.removeprefix(UPLOAD_DIR + "/") # Get the blob (object in the bucket) - blob = self.bucket_name.blob(file_path) + blob = self.bucket_name.blob(local_file_path) # Delete the file blob.delete() diff --git a/backend/requirements.txt b/backend/requirements.txt index fcd53ea74..de62eef6b 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -102,6 +102,7 @@ pytest~=8.3.2 pytest-docker~=3.1.1 googleapis-common-protos==1.63.2 +google-cloud-storage==2.19.0 ## LDAP ldap3==2.9.1 diff --git a/pyproject.toml b/pyproject.toml index e79a0253c..d9eeeca90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,16 +10,20 @@ dependencies = [ "uvicorn[standard]==0.30.6", "pydantic==2.9.2", "python-multipart==0.0.18", + "Flask==3.1.0", "Flask-Cors==5.0.0", + "python-socketio==5.11.3", "python-jose==3.3.0", "passlib[bcrypt]==1.7.4", + "requests==2.32.3", "aiohttp==3.11.8", "async-timeout", "aiocache", "aiofiles", + "sqlalchemy==2.0.32", "alembic==1.14.0", "peewee==3.17.8", @@ -28,26 +32,33 @@ dependencies = [ "pgvector==0.3.5", "PyMySQL==1.1.1", "bcrypt==4.2.0", + "pymongo", "redis", "boto3==1.35.53", + "argon2-cffi==23.1.0", "APScheduler==3.10.4", + "openai", "anthropic", "google-generativeai==0.7.2", "tiktoken", + "langchain==0.3.7", "langchain-community==0.3.7", + "fake-useragent==1.5.1", "chromadb==0.6.2", "pymilvus==2.5.0", "qdrant-client~=1.12.0", "opensearch-py==2.7.1", + "transformers", "sentence-transformers==3.3.1", "colbert-ai==0.2.21", "einops==0.8.0", + "ftfy==6.2.3", "pypdf==4.3.1", "fpdf2==2.8.2", @@ -66,26 +77,33 @@ dependencies = [ "psutil", "sentencepiece", "soundfile==0.12.1", + "opencv-python-headless==4.10.0.84", "rapidocr-onnxruntime==1.3.24", "rank-bm25==0.2.2", + "faster-whisper==1.0.3", + "PyJWT[crypto]==2.10.1", "authlib==1.3.2", + "black==24.8.0", "langfuse==2.44.0", "youtube-transcript-api==0.6.3", "pytube==15.0.0", + "extract_msg", "pydub", "duckduckgo-search~=6.3.5", + "docker~=7.1.0", "pytest~=8.3.2", "pytest-docker~=3.1.1", "moto[s3]>=5.0.26", + "googleapis-common-protos==1.63.2", "ldap3==2.9.1", - "google-cloud-storage>=2.19.0", + "google-cloud-storage==2.19.0", ] readme = "README.md" requires-python = ">= 3.11, < 3.13.0a1" diff --git a/uv.lock b/uv.lock index 58ad3d922..bb70fbe0c 100644 --- a/uv.lock +++ b/uv.lock @@ -19,8 +19,10 @@ resolution-markers = [ "python_full_version < '3.12' and platform_system == 'Darwin'", "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_system == 'Darwin'", "python_full_version < '3.12' and platform_system == 'Darwin'", - "python_full_version < '3.12.4' and platform_system == 'Darwin'", - "python_full_version >= '3.12.4' and platform_system == 'Darwin'", + "python_full_version < '3.12' and platform_system == 'Darwin'", + "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_system == 'Darwin'", + "python_full_version < '3.13' and platform_system == 'Darwin'", + "python_full_version >= '3.13' and platform_system == 'Darwin'", "python_full_version >= '3.13' and platform_system == 'Darwin'", "python_full_version >= '3.13' and platform_system == 'Darwin'", "python_full_version >= '3.13' and platform_system == 'Darwin'", @@ -41,8 +43,10 @@ resolution-markers = [ "python_full_version < '3.12' and platform_machine == 'aarch64' and platform_system == 'Linux'", "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine == 'aarch64' and platform_system == 'Linux'", "python_full_version < '3.12' and platform_machine == 'aarch64' and platform_system == 'Linux'", - "python_full_version < '3.12.4' and platform_machine == 'aarch64' and platform_system == 'Linux'", - "python_full_version >= '3.12.4' and platform_machine == 'aarch64' and platform_system == 'Linux'", + "python_full_version < '3.12' and platform_machine == 'aarch64' and platform_system == 'Linux'", + "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine == 'aarch64' and platform_system == 'Linux'", + "python_full_version < '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux'", + "python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux'", "python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux'", "python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux'", "python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux'", @@ -63,8 +67,10 @@ resolution-markers = [ "(python_full_version < '3.12' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version < '3.12' and platform_system != 'Darwin' and platform_system != 'Linux')", "(python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_system != 'Darwin' and platform_system != 'Linux')", "(python_full_version < '3.12' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version < '3.12' and platform_system != 'Darwin' and platform_system != 'Linux')", - "(python_full_version < '3.12.4' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version < '3.12.4' and platform_system != 'Darwin' and platform_system != 'Linux')", - "(python_full_version >= '3.12.4' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version >= '3.12.4' and platform_system != 'Darwin' and platform_system != 'Linux')", + "(python_full_version < '3.12' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version < '3.12' and platform_system != 'Darwin' and platform_system != 'Linux')", + "(python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_system != 'Darwin' and platform_system != 'Linux')", + "(python_full_version < '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version < '3.13' and platform_system != 'Darwin' and platform_system != 'Linux')", + "(python_full_version >= '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version >= '3.13' and platform_system != 'Darwin' and platform_system != 'Linux')", "(python_full_version >= '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version >= '3.13' and platform_system != 'Darwin' and platform_system != 'Linux')", "(python_full_version >= '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version >= '3.13' and platform_system != 'Darwin' and platform_system != 'Linux')", "(python_full_version >= '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version >= '3.13' and platform_system != 'Darwin' and platform_system != 'Linux')", @@ -2847,7 +2853,7 @@ requires-dist = [ { name = "flask-cors", specifier = "==5.0.0" }, { name = "fpdf2", specifier = "==2.8.2" }, { name = "ftfy", specifier = "==6.2.3" }, - { name = "google-cloud-storage", specifier = ">=2.19.0" }, + { name = "google-cloud-storage", specifier = "==2.19.0" }, { name = "google-generativeai", specifier = "==0.7.2" }, { name = "googleapis-common-protos", specifier = "==1.63.2" }, { name = "langchain", specifier = "==0.3.7" },