mirror of
https://github.com/open-webui/open-webui
synced 2025-01-19 01:06:45 +00:00
updated to use credentials json, tested with local built image and upload to gcs works
This commit is contained in:
kahghi
parent
8efc543f49
commit
1764de41f3
@ -582,15 +582,15 @@ if CUSTOM_NAME:
|
||||
####################################
|
||||
|
||||
STORAGE_PROVIDER = os.environ.get("STORAGE_PROVIDER", "local") # defaults to local, s3
|
||||
print(f"storage provider: {STORAGE_PROVIDER}")
|
||||
|
||||
S3_ACCESS_KEY_ID = os.environ.get("S3_ACCESS_KEY_ID", None)
|
||||
S3_SECRET_ACCESS_KEY = os.environ.get("S3_SECRET_ACCESS_KEY", None)
|
||||
S3_REGION_NAME = os.environ.get("S3_REGION_NAME", None)
|
||||
S3_BUCKET_NAME = os.environ.get("S3_BUCKET_NAME", None)
|
||||
S3_ENDPOINT_URL = os.environ.get("S3_ENDPOINT_URL", None)
|
||||
|
||||
GCS_PROJECT_ID = os.environ.get("GCS_PROJECT_ID", None)
|
||||
GCS_BUCKET_NAME = os.environ.get("GCS_BUCKET_NAME", FileNotFoundError)
|
||||
GCS_BUCKET_NAME = os.environ.get("GCS_BUCKET_NAME", None)
|
||||
GOOGLE_APPLICATION_CREDENTIALS_JSON = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS_JSON", None)
|
||||
|
||||
####################################
|
||||
# File Upload DIR
|
||||
|
||||
@ -1,7 +1,9 @@
|
||||
import os
|
||||
import shutil
|
||||
import json
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import BinaryIO, Tuple
|
||||
from io import BytesIO
|
||||
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError
|
||||
@ -11,8 +13,8 @@ from open_webui.config import (
|
||||
S3_ENDPOINT_URL,
|
||||
S3_REGION_NAME,
|
||||
S3_SECRET_ACCESS_KEY,
|
||||
GCS_PROJECT_ID,
|
||||
GCS_BUCKET_NAME,
|
||||
GOOGLE_APPLICATION_CREDENTIALS_JSON,
|
||||
STORAGE_PROVIDER,
|
||||
UPLOAD_DIR,
|
||||
)
|
||||
@ -143,7 +145,7 @@ class S3StorageProvider(StorageProvider):
|
||||
|
||||
class GCSStorageProvider(StorageProvider):
|
||||
def __init__(self):
|
||||
self.gcs_client = storage.Client(project=GCS_PROJECT_ID)
|
||||
self.gcs_client = storage.Client.from_service_account_info(info=json.loads(GOOGLE_APPLICATION_CREDENTIALS_JSON))
|
||||
self.bucket_name = self.gcs_client.bucket(GCS_BUCKET_NAME)
|
||||
|
||||
def upload_file(self, file: BinaryIO, filename: str):
|
||||
@ -154,26 +156,28 @@ class GCSStorageProvider(StorageProvider):
|
||||
blob = self.bucket_name.blob(filename)
|
||||
# Upload the file to the bucket
|
||||
blob.upload_from_file(BytesIO(contents))
|
||||
print("file successfully uploaded")
|
||||
return contents, _
|
||||
except GoogleCloudError as e:
|
||||
raise RuntimeError(f"Error uploading file to GCS: {e}")
|
||||
|
||||
def get_file(self, file_path:str) -> str:
|
||||
"""Handles downloading of the file from GCS storage."""
|
||||
try:
|
||||
local_file_path = f"{UPLOAD_DIR}/{file_path}"
|
||||
local_file_path=file_path.removeprefix(UPLOAD_DIR + "/")
|
||||
# Get the blob (object in the bucket)
|
||||
blob = self.bucket_name.blob(file_path)
|
||||
blob = self.bucket_name.blob(local_file_path)
|
||||
# Download the file to a local destination
|
||||
blob.download_to_filename(local_file_path)
|
||||
blob.download_to_filename(file_path)
|
||||
return file_path
|
||||
except NotFound as e:
|
||||
raise RuntimeError(f"Error downloading file from GCS: {e}")
|
||||
|
||||
def delete_file(self, file_path:str) -> None:
|
||||
"""Handles deletion of the file from GCS storage."""
|
||||
try:
|
||||
local_file_path = file_path.removeprefix(UPLOAD_DIR + "/")
|
||||
# Get the blob (object in the bucket)
|
||||
blob = self.bucket_name.blob(file_path)
|
||||
blob = self.bucket_name.blob(local_file_path)
|
||||
|
||||
# Delete the file
|
||||
blob.delete()
|
||||
|
||||
@ -102,6 +102,7 @@ pytest~=8.3.2
|
||||
pytest-docker~=3.1.1
|
||||
|
||||
googleapis-common-protos==1.63.2
|
||||
google-cloud-storage==2.19.0
|
||||
|
||||
## LDAP
|
||||
ldap3==2.9.1
|
||||
|
||||
@ -10,16 +10,20 @@ dependencies = [
|
||||
"uvicorn[standard]==0.30.6",
|
||||
"pydantic==2.9.2",
|
||||
"python-multipart==0.0.18",
|
||||
|
||||
"Flask==3.1.0",
|
||||
"Flask-Cors==5.0.0",
|
||||
|
||||
"python-socketio==5.11.3",
|
||||
"python-jose==3.3.0",
|
||||
"passlib[bcrypt]==1.7.4",
|
||||
|
||||
"requests==2.32.3",
|
||||
"aiohttp==3.11.8",
|
||||
"async-timeout",
|
||||
"aiocache",
|
||||
"aiofiles",
|
||||
|
||||
"sqlalchemy==2.0.32",
|
||||
"alembic==1.14.0",
|
||||
"peewee==3.17.8",
|
||||
@ -28,26 +32,33 @@ dependencies = [
|
||||
"pgvector==0.3.5",
|
||||
"PyMySQL==1.1.1",
|
||||
"bcrypt==4.2.0",
|
||||
|
||||
"pymongo",
|
||||
"redis",
|
||||
"boto3==1.35.53",
|
||||
|
||||
"argon2-cffi==23.1.0",
|
||||
"APScheduler==3.10.4",
|
||||
|
||||
"openai",
|
||||
"anthropic",
|
||||
"google-generativeai==0.7.2",
|
||||
"tiktoken",
|
||||
|
||||
"langchain==0.3.7",
|
||||
"langchain-community==0.3.7",
|
||||
|
||||
"fake-useragent==1.5.1",
|
||||
"chromadb==0.6.2",
|
||||
"pymilvus==2.5.0",
|
||||
"qdrant-client~=1.12.0",
|
||||
"opensearch-py==2.7.1",
|
||||
|
||||
"transformers",
|
||||
"sentence-transformers==3.3.1",
|
||||
"colbert-ai==0.2.21",
|
||||
"einops==0.8.0",
|
||||
|
||||
"ftfy==6.2.3",
|
||||
"pypdf==4.3.1",
|
||||
"fpdf2==2.8.2",
|
||||
@ -66,26 +77,33 @@ dependencies = [
|
||||
"psutil",
|
||||
"sentencepiece",
|
||||
"soundfile==0.12.1",
|
||||
|
||||
"opencv-python-headless==4.10.0.84",
|
||||
"rapidocr-onnxruntime==1.3.24",
|
||||
"rank-bm25==0.2.2",
|
||||
|
||||
"faster-whisper==1.0.3",
|
||||
|
||||
"PyJWT[crypto]==2.10.1",
|
||||
"authlib==1.3.2",
|
||||
|
||||
"black==24.8.0",
|
||||
"langfuse==2.44.0",
|
||||
"youtube-transcript-api==0.6.3",
|
||||
"pytube==15.0.0",
|
||||
|
||||
"extract_msg",
|
||||
"pydub",
|
||||
"duckduckgo-search~=6.3.5",
|
||||
|
||||
"docker~=7.1.0",
|
||||
"pytest~=8.3.2",
|
||||
"pytest-docker~=3.1.1",
|
||||
"moto[s3]>=5.0.26",
|
||||
|
||||
"googleapis-common-protos==1.63.2",
|
||||
"ldap3==2.9.1",
|
||||
"google-cloud-storage>=2.19.0",
|
||||
"google-cloud-storage==2.19.0",
|
||||
]
|
||||
readme = "README.md"
|
||||
requires-python = ">= 3.11, < 3.13.0a1"
|
||||
|
||||
20
uv.lock
20
uv.lock
@ -19,8 +19,10 @@ resolution-markers = [
|
||||
"python_full_version < '3.12' and platform_system == 'Darwin'",
|
||||
"python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_system == 'Darwin'",
|
||||
"python_full_version < '3.12' and platform_system == 'Darwin'",
|
||||
"python_full_version < '3.12.4' and platform_system == 'Darwin'",
|
||||
"python_full_version >= '3.12.4' and platform_system == 'Darwin'",
|
||||
"python_full_version < '3.12' and platform_system == 'Darwin'",
|
||||
"python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_system == 'Darwin'",
|
||||
"python_full_version < '3.13' and platform_system == 'Darwin'",
|
||||
"python_full_version >= '3.13' and platform_system == 'Darwin'",
|
||||
"python_full_version >= '3.13' and platform_system == 'Darwin'",
|
||||
"python_full_version >= '3.13' and platform_system == 'Darwin'",
|
||||
"python_full_version >= '3.13' and platform_system == 'Darwin'",
|
||||
@ -41,8 +43,10 @@ resolution-markers = [
|
||||
"python_full_version < '3.12' and platform_machine == 'aarch64' and platform_system == 'Linux'",
|
||||
"python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine == 'aarch64' and platform_system == 'Linux'",
|
||||
"python_full_version < '3.12' and platform_machine == 'aarch64' and platform_system == 'Linux'",
|
||||
"python_full_version < '3.12.4' and platform_machine == 'aarch64' and platform_system == 'Linux'",
|
||||
"python_full_version >= '3.12.4' and platform_machine == 'aarch64' and platform_system == 'Linux'",
|
||||
"python_full_version < '3.12' and platform_machine == 'aarch64' and platform_system == 'Linux'",
|
||||
"python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine == 'aarch64' and platform_system == 'Linux'",
|
||||
"python_full_version < '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux'",
|
||||
"python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux'",
|
||||
"python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux'",
|
||||
"python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux'",
|
||||
"python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux'",
|
||||
@ -63,8 +67,10 @@ resolution-markers = [
|
||||
"(python_full_version < '3.12' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version < '3.12' and platform_system != 'Darwin' and platform_system != 'Linux')",
|
||||
"(python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_system != 'Darwin' and platform_system != 'Linux')",
|
||||
"(python_full_version < '3.12' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version < '3.12' and platform_system != 'Darwin' and platform_system != 'Linux')",
|
||||
"(python_full_version < '3.12.4' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version < '3.12.4' and platform_system != 'Darwin' and platform_system != 'Linux')",
|
||||
"(python_full_version >= '3.12.4' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version >= '3.12.4' and platform_system != 'Darwin' and platform_system != 'Linux')",
|
||||
"(python_full_version < '3.12' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version < '3.12' and platform_system != 'Darwin' and platform_system != 'Linux')",
|
||||
"(python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_system != 'Darwin' and platform_system != 'Linux')",
|
||||
"(python_full_version < '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version < '3.13' and platform_system != 'Darwin' and platform_system != 'Linux')",
|
||||
"(python_full_version >= '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version >= '3.13' and platform_system != 'Darwin' and platform_system != 'Linux')",
|
||||
"(python_full_version >= '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version >= '3.13' and platform_system != 'Darwin' and platform_system != 'Linux')",
|
||||
"(python_full_version >= '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version >= '3.13' and platform_system != 'Darwin' and platform_system != 'Linux')",
|
||||
"(python_full_version >= '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version >= '3.13' and platform_system != 'Darwin' and platform_system != 'Linux')",
|
||||
@ -2847,7 +2853,7 @@ requires-dist = [
|
||||
{ name = "flask-cors", specifier = "==5.0.0" },
|
||||
{ name = "fpdf2", specifier = "==2.8.2" },
|
||||
{ name = "ftfy", specifier = "==6.2.3" },
|
||||
{ name = "google-cloud-storage", specifier = ">=2.19.0" },
|
||||
{ name = "google-cloud-storage", specifier = "==2.19.0" },
|
||||
{ name = "google-generativeai", specifier = "==0.7.2" },
|
||||
{ name = "googleapis-common-protos", specifier = "==1.63.2" },
|
||||
{ name = "langchain", specifier = "==0.3.7" },
|
||||
|
||||
Loading…
Reference in New Issue
Block a user