From fd3c24af4e1ea80a45a4b26271cb8d08fb12a42a Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 13:25:31 -0500 Subject: [PATCH 01/23] Add AzureStorageProvider --- backend/open_webui/storage/provider.py | 76 +++++++++++++++++++ .../test/apps/webui/storage/test_provider.py | 4 + backend/requirements.txt | 4 + 3 files changed, 84 insertions(+) diff --git a/backend/open_webui/storage/provider.py b/backend/open_webui/storage/provider.py index b03cf0a7e..43f4a6922 100644 --- a/backend/open_webui/storage/provider.py +++ b/backend/open_webui/storage/provider.py @@ -15,12 +15,19 @@ from open_webui.config import ( S3_SECRET_ACCESS_KEY, GCS_BUCKET_NAME, GOOGLE_APPLICATION_CREDENTIALS_JSON, + AZURE_STORAGE_ENDPOINT, + AZURE_STORAGE_CONTAINER_NAME, + AZURE_STORAGE_KEY, STORAGE_PROVIDER, UPLOAD_DIR, ) from google.cloud import storage from google.cloud.exceptions import GoogleCloudError, NotFound from open_webui.constants import ERROR_MESSAGES +from azure.identity import DefaultAzureCredential +from azure.storage.blob import BlobServiceClient +from azure.core.exceptions import ResourceNotFoundError + class StorageProvider(ABC): @@ -221,6 +228,73 @@ class GCSStorageProvider(StorageProvider): LocalStorageProvider.delete_all_files() + +class AzureStorageProvider(StorageProvider): + def __init__(self): + self.endpoint = AZURE_STORAGE_ENDPOINT + self.container_name = AZURE_STORAGE_CONTAINER_NAME + storage_key = AZURE_STORAGE_KEY + + if storage_key: + # Configure using the Azure Storage Account Endpoint and Key + self.blob_service_client = BlobServiceClient( + account_url=self.endpoint, credential=storage_key + ) + else: + # Configure using the Azure Storage Account Endpoint and DefaultAzureCredential + # If the key is not configured, then the DefaultAzureCredential will be used to support Managed Identity authentication + self.blob_service_client = BlobServiceClient( + account_url=self.endpoint, credential=DefaultAzureCredential() + ) + self.container_client = self.blob_service_client.get_container_client(self.container_name) + + def upload_file(self, file: BinaryIO, filename: str) -> Tuple[bytes, str]: + """Handles uploading of the file to Azure Blob Storage.""" + contents, file_path = LocalStorageProvider.upload_file(file, filename) + try: + blob_client = self.container_client.get_blob_client(filename) + blob_client.upload_blob(contents, overwrite=True) + return contents, f"{self.endpoint}/{self.container_name}/{filename}" + except Exception as e: + raise RuntimeError(f"Error uploading file to Azure Blob Storage: {e}") + + def get_file(self, file_path: str) -> str: + """Handles downloading of the file from Azure Blob Storage.""" + try: + filename = file_path.split("/")[-1] + local_file_path = f"{UPLOAD_DIR}/{filename}" + blob_client = self.container_client.get_blob_client(filename) + with open(local_file_path, "wb") as download_file: + download_file.write(blob_client.download_blob().readall()) + return local_file_path + except ResourceNotFoundError as e: + raise RuntimeError(f"Error downloading file from Azure Blob Storage: {e}") + + def delete_file(self, file_path: str) -> None: + """Handles deletion of the file from Azure Blob Storage.""" + try: + filename = file_path.split("/")[-1] + blob_client = self.container_client.get_blob_client(filename) + blob_client.delete_blob() + except ResourceNotFoundError as e: + raise RuntimeError(f"Error deleting file from Azure Blob Storage: {e}") + + # Always delete from local storage + LocalStorageProvider.delete_file(file_path) + + def delete_all_files(self) -> None: + """Handles deletion of all files from Azure Blob Storage.""" + try: + blobs = self.container_client.list_blobs() + for blob in blobs: + self.container_client.delete_blob(blob.name) + except Exception as e: + raise RuntimeError(f"Error deleting all files from Azure Blob Storage: {e}") + + # Always delete from local storage + LocalStorageProvider.delete_all_files() + + def get_storage_provider(storage_provider: str): if storage_provider == "local": Storage = LocalStorageProvider() @@ -228,6 +302,8 @@ def get_storage_provider(storage_provider: str): Storage = S3StorageProvider() elif storage_provider == "gcs": Storage = GCSStorageProvider() + elif storage_provider == "azure": + Storage = AzureStorageProvider() else: raise RuntimeError(f"Unsupported storage provider: {storage_provider}") return Storage diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 863106e75..4c3112526 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -22,6 +22,7 @@ def test_imports(): provider.LocalStorageProvider provider.S3StorageProvider provider.GCSStorageProvider + provider.AzureStorageProvider provider.Storage @@ -32,6 +33,8 @@ def test_get_storage_provider(): assert isinstance(Storage, provider.S3StorageProvider) Storage = provider.get_storage_provider("gcs") assert isinstance(Storage, provider.GCSStorageProvider) + Storage = provider.get_storage_provider("azure") + assert isinstance(Storage, provider.AzureStorageProvider) with pytest.raises(RuntimeError): provider.get_storage_provider("invalid") @@ -48,6 +51,7 @@ def test_class_instantiation(): provider.LocalStorageProvider() provider.S3StorageProvider() provider.GCSStorageProvider() + provider.AzureStorageProvider() class TestLocalStorageProvider: diff --git a/backend/requirements.txt b/backend/requirements.txt index 9b859b84a..f8e5f6684 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -103,5 +103,9 @@ pytest-docker~=3.1.1 googleapis-common-protos==1.63.2 google-cloud-storage==2.19.0 +azure-identity==1.20.0 +azure-storage-blob==12.24.1 + + ## LDAP ldap3==2.9.1 From e4febfa0974c6433b97f5e483430a33bb88e9941 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 13:25:49 -0500 Subject: [PATCH 02/23] Add AzureStorageProvider config options --- backend/open_webui/config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index adfdcfec8..6bc2b7636 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -668,6 +668,10 @@ GOOGLE_APPLICATION_CREDENTIALS_JSON = os.environ.get( "GOOGLE_APPLICATION_CREDENTIALS_JSON", None ) +AZURE_STORAGE_ENDPOINT = os.environ.get("AZURE_STORAGE_ENDPOINT", None) +AZURE_STORAGE_CONTAINER_NAME = os.environ.get("AZURE_STORAGE_CONTAINER_NAME", "open-webui") +AZURE_STORAGE_KEY = os.environ.get("AZURE_STORAGE_KEY", None) + #################################### # File Upload DIR #################################### From aee57107bcafac2e7a363c9a9b7b046398c00c55 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 13:27:37 -0500 Subject: [PATCH 03/23] Update config.py --- backend/open_webui/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index 6bc2b7636..176511948 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -669,7 +669,7 @@ GOOGLE_APPLICATION_CREDENTIALS_JSON = os.environ.get( ) AZURE_STORAGE_ENDPOINT = os.environ.get("AZURE_STORAGE_ENDPOINT", None) -AZURE_STORAGE_CONTAINER_NAME = os.environ.get("AZURE_STORAGE_CONTAINER_NAME", "open-webui") +AZURE_STORAGE_CONTAINER_NAME = os.environ.get("AZURE_STORAGE_CONTAINER_NAME", None) AZURE_STORAGE_KEY = os.environ.get("AZURE_STORAGE_KEY", None) #################################### From cc4598c41baea60dbaee2b26eab77d0e489bc876 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 13:39:27 -0500 Subject: [PATCH 04/23] Update build-release.yml --- .github/workflows/build-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index 443d90419..ac2e28130 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -4,6 +4,7 @@ on: push: branches: - main # or whatever branch you want to use + - azure-storage jobs: release: From 2f4f4c23764b03230cd626e3ba47b4ba93067450 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 14:09:00 -0500 Subject: [PATCH 05/23] add tests --- .github/workflows/integration-test.disabled | 1 + .../test/apps/webui/storage/test_provider.py | 100 +++++++++++++++++- 2 files changed, 100 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.disabled b/.github/workflows/integration-test.disabled index b248df4b5..946735c5b 100644 --- a/.github/workflows/integration-test.disabled +++ b/.github/workflows/integration-test.disabled @@ -5,6 +5,7 @@ on: branches: - main - dev + - azure-storage pull_request: branches: - main diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 4c3112526..5cfb1b6a8 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -3,10 +3,11 @@ import os import boto3 import pytest from botocore.exceptions import ClientError -from moto import mock_aws +from moto import mock_aws, mock_azure from open_webui.storage import provider from gcp_storage_emulator.server import create_server from google.cloud import storage +from azure.storage.blob import BlobServiceClient def mock_upload_dir(monkeypatch, tmp_path): @@ -276,3 +277,100 @@ class TestGCSStorageProvider: assert not (upload_dir / self.filename_extra).exists() assert self.Storage.bucket.get_blob(self.filename) == None assert self.Storage.bucket.get_blob(self.filename_extra) == None + + +class TestAzureStorageProvider: + def __init__(self): + self.Storage = provider.AzureStorageProvider() + self.Storage.container_name = "my-container" + self.file_content = b"test content" + self.filename = "test.txt" + self.filename_extra = "test_exyta.txt" + self.file_bytesio_empty = io.BytesIO() + super().__init__() + + @pytest.fixture(scope="class") + def setup(self, monkeypatch): + connection_string = "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtl6rE4rWlgEoMF1rA==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" + self.Storage.blob_service_client = BlobServiceClient.from_connection_string(connection_string) + self.Storage.container_client = self.Storage.blob_service_client.get_container_client(self.Storage.container_name) + monkeypatch.setattr(self.Storage, "blob_service_client", self.Storage.blob_service_client) + monkeypatch.setattr(self.Storage, "container_client", self.Storage.container_client) + yield + self.Storage.container_client.delete_container() + + def test_upload_file(self, monkeypatch, tmp_path, setup): + upload_dir = mock_upload_dir(monkeypatch, tmp_path) + # Azure checks + with pytest.raises(Exception): + self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) + self.Storage.create_container() + contents, azure_file_path = self.Storage.upload_file( + io.BytesIO(self.file_content), self.filename + ) + blob = self.Storage.blob_service_client.get_blob_client( + container=self.Storage.container_name, blob=self.filename + ) + assert self.file_content == blob.download_blob().readall() + # local checks + assert (upload_dir / self.filename).exists() + assert (upload_dir / self.filename).read_bytes() == self.file_content + assert contents == self.file_content + assert azure_file_path == "azure://" + self.Storage.container_name + "/" + self.filename + with pytest.raises(ValueError): + self.Storage.upload_file(self.file_bytesio_empty, self.filename) + + def test_get_file(self, monkeypatch, tmp_path, setup): + upload_dir = mock_upload_dir(monkeypatch, tmp_path) + self.Storage.create_container() + contents, azure_file_path = self.Storage.upload_file( + io.BytesIO(self.file_content), self.filename + ) + file_path = self.Storage.get_file(azure_file_path) + assert file_path == str(upload_dir / self.filename) + assert (upload_dir / self.filename).exists() + + def test_delete_file(self, monkeypatch, tmp_path, setup): + upload_dir = mock_upload_dir(monkeypatch, tmp_path) + self.Storage.create_container() + contents, azure_file_path = self.Storage.upload_file( + io.BytesIO(self.file_content), self.filename + ) + assert (upload_dir / self.filename).exists() + self.Storage.delete_file(azure_file_path) + assert not (upload_dir / self.filename).exists() + blob = self.Storage.blob_service_client.get_blob_client( + container=self.Storage.container_name, blob=self.filename + ) + with pytest.raises(Exception): + blob.download_blob().readall() + + def test_delete_all_files(self, monkeypatch, tmp_path, setup): + upload_dir = mock_upload_dir(monkeypatch, tmp_path) + self.Storage.create_container() + self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) + blob = self.Storage.blob_service_client.get_blob_client( + container=self.Storage.container_name, blob=self.filename + ) + assert self.file_content == blob.download_blob().readall() + assert (upload_dir / self.filename).exists() + self.Storage.upload_file(io.BytesIO(self.file_content), self.filename_extra) + blob = self.Storage.blob_service_client.get_blob_client( + container=self.Storage.container_name, blob=self.filename_extra + ) + assert self.file_content == blob.download_blob().readall() + assert (upload_dir / self.filename_extra).exists() + + self.Storage.delete_all_files() + assert not (upload_dir / self.filename).exists() + assert not (upload_dir / self.filename_extra).exists() + blob = self.Storage.blob_service_client.get_blob_client( + container=self.Storage.container_name, blob=self.filename + ) + with pytest.raises(Exception): + blob.download_blob().readall() + blob = self.Storage.blob_service_client.get_blob_client( + container=self.Storage.container_name, blob=self.filename_extra + ) + with pytest.raises(Exception): + blob.download_blob().readall() \ No newline at end of file From cb0666ac216ab5794e4ef795e367929c4dbec91a Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 14:13:54 -0500 Subject: [PATCH 06/23] update --- .github/workflows/build-release.yml | 1 - .github/workflows/integration-test.disabled | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index ac2e28130..443d90419 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -4,7 +4,6 @@ on: push: branches: - main # or whatever branch you want to use - - azure-storage jobs: release: diff --git a/.github/workflows/integration-test.disabled b/.github/workflows/integration-test.disabled index 946735c5b..b248df4b5 100644 --- a/.github/workflows/integration-test.disabled +++ b/.github/workflows/integration-test.disabled @@ -5,7 +5,6 @@ on: branches: - main - dev - - azure-storage pull_request: branches: - main From b86f8df29f6f2101787350990401b51b0341db22 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 14:37:10 -0500 Subject: [PATCH 07/23] Update test_provider.py --- .../test/apps/webui/storage/test_provider.py | 144 +++++++++++------- 1 file changed, 91 insertions(+), 53 deletions(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 5cfb1b6a8..e434523f8 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -7,7 +7,8 @@ from moto import mock_aws, mock_azure from open_webui.storage import provider from gcp_storage_emulator.server import create_server from google.cloud import storage -from azure.storage.blob import BlobServiceClient +from azure.storage.blob import BlobServiceClient, BlobContainerClient, BlobClient +from unittest.mock import MagicMock def mock_upload_dir(monkeypatch, tmp_path): @@ -279,98 +280,135 @@ class TestGCSStorageProvider: assert self.Storage.bucket.get_blob(self.filename_extra) == None + class TestAzureStorageProvider: def __init__(self): self.Storage = provider.AzureStorageProvider() + self.Storage.endpoint = "https://myaccount.blob.core.windows.net" self.Storage.container_name = "my-container" self.file_content = b"test content" self.filename = "test.txt" - self.filename_extra = "test_exyta.txt" + self.filename_extra = "test_extra.txt" self.file_bytesio_empty = io.BytesIO() super().__init__() - @pytest.fixture(scope="class") + @pytest.fixture def setup(self, monkeypatch): - connection_string = "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtl6rE4rWlgEoMF1rA==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" - self.Storage.blob_service_client = BlobServiceClient.from_connection_string(connection_string) - self.Storage.container_client = self.Storage.blob_service_client.get_container_client(self.Storage.container_name) - monkeypatch.setattr(self.Storage, "blob_service_client", self.Storage.blob_service_client) - monkeypatch.setattr(self.Storage, "container_client", self.Storage.container_client) + """Mock BlobServiceClient and BlobContainerClient for local testing""" + # Create mock Blob Service Client + mock_blob_service_client = MagicMock() + mock_container_client = MagicMock() + mock_blob_client = MagicMock() + + # Set up return values + mock_blob_service_client.get_container_client.return_value = mock_container_client + mock_container_client.get_blob_client.return_value = mock_blob_client + + # Mock `from_connection_string` and `BlobServiceClient` constructor + monkeypatch.setattr("azure.storage.blob.BlobServiceClient", lambda *_: mock_blob_service_client) + + # Apply to instance variables + self.Storage.blob_service_client = mock_blob_service_client + self.Storage.container_client = mock_container_client + yield - self.Storage.container_client.delete_container() def test_upload_file(self, monkeypatch, tmp_path, setup): + """Test uploading a file to mocked Azure Storage.""" upload_dir = mock_upload_dir(monkeypatch, tmp_path) - # Azure checks + + # Simulate an error when container does not exist + self.Storage.container_client.get_blob_client.side_effect = Exception("Container does not exist") + with pytest.raises(Exception): self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) + + # Reset side effect and create container + self.Storage.container_client.get_blob_client.side_effect = None + self.Storage.create_container() contents, azure_file_path = self.Storage.upload_file( io.BytesIO(self.file_content), self.filename ) - blob = self.Storage.blob_service_client.get_blob_client( - container=self.Storage.container_name, blob=self.filename - ) - assert self.file_content == blob.download_blob().readall() - # local checks + + # Assertions + self.Storage.container_client.get_blob_client.assert_called_with(self.filename) + self.Storage.container_client.get_blob_client().upload_blob.assert_called_once_with(self.file_content, overwrite=True) + + assert contents == self.file_content + assert azure_file_path == f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}" assert (upload_dir / self.filename).exists() assert (upload_dir / self.filename).read_bytes() == self.file_content - assert contents == self.file_content - assert azure_file_path == "azure://" + self.Storage.container_name + "/" + self.filename + with pytest.raises(ValueError): self.Storage.upload_file(self.file_bytesio_empty, self.filename) def test_get_file(self, monkeypatch, tmp_path, setup): + """Test retrieving a file from mocked Azure Storage.""" upload_dir = mock_upload_dir(monkeypatch, tmp_path) self.Storage.create_container() - contents, azure_file_path = self.Storage.upload_file( - io.BytesIO(self.file_content), self.filename - ) - file_path = self.Storage.get_file(azure_file_path) + + # Mock upload behavior + self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) + + # Mock blob download behavior + self.Storage.container_client.get_blob_client().download_blob().readall.return_value = self.file_content + + file_path = self.Storage.get_file(f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}") + assert file_path == str(upload_dir / self.filename) assert (upload_dir / self.filename).exists() + assert (upload_dir / self.filename).read_bytes() == self.file_content def test_delete_file(self, monkeypatch, tmp_path, setup): + """Test deleting a file from mocked Azure Storage.""" upload_dir = mock_upload_dir(monkeypatch, tmp_path) self.Storage.create_container() - contents, azure_file_path = self.Storage.upload_file( - io.BytesIO(self.file_content), self.filename - ) - assert (upload_dir / self.filename).exists() - self.Storage.delete_file(azure_file_path) + + # Mock upload + self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) + + # Mock deletion + self.Storage.container_client.get_blob_client().delete_blob.return_value = None + + self.Storage.delete_file(f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}") + + # Assertions + self.Storage.container_client.get_blob_client().delete_blob.assert_called_once() assert not (upload_dir / self.filename).exists() - blob = self.Storage.blob_service_client.get_blob_client( - container=self.Storage.container_name, blob=self.filename - ) - with pytest.raises(Exception): - blob.download_blob().readall() def test_delete_all_files(self, monkeypatch, tmp_path, setup): + """Test deleting all files from mocked Azure Storage.""" upload_dir = mock_upload_dir(monkeypatch, tmp_path) self.Storage.create_container() - self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) - blob = self.Storage.blob_service_client.get_blob_client( - container=self.Storage.container_name, blob=self.filename - ) - assert self.file_content == blob.download_blob().readall() - assert (upload_dir / self.filename).exists() - self.Storage.upload_file(io.BytesIO(self.file_content), self.filename_extra) - blob = self.Storage.blob_service_client.get_blob_client( - container=self.Storage.container_name, blob=self.filename_extra - ) - assert self.file_content == blob.download_blob().readall() - assert (upload_dir / self.filename_extra).exists() + # Mock file uploads + self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) + self.Storage.upload_file(io.BytesIO(self.file_content), self.filename_extra) + + # Mock listing and deletion behavior + self.Storage.container_client.list_blobs.return_value = [ + {"name": self.filename}, + {"name": self.filename_extra}, + ] + self.Storage.container_client.get_blob_client().delete_blob.return_value = None + + # Call delete all files self.Storage.delete_all_files() + + # Assertions + self.Storage.container_client.list_blobs.assert_called_once() + self.Storage.container_client.get_blob_client().delete_blob.assert_any_call() assert not (upload_dir / self.filename).exists() assert not (upload_dir / self.filename_extra).exists() - blob = self.Storage.blob_service_client.get_blob_client( - container=self.Storage.container_name, blob=self.filename - ) - with pytest.raises(Exception): - blob.download_blob().readall() - blob = self.Storage.blob_service_client.get_blob_client( - container=self.Storage.container_name, blob=self.filename_extra - ) - with pytest.raises(Exception): - blob.download_blob().readall() \ No newline at end of file + + def test_get_file_not_found(self, monkeypatch, setup): + """Test handling when a requested file does not exist.""" + self.Storage.create_container() + + # Mock behavior to raise an error for missing files + self.Storage.container_client.get_blob_client().download_blob.side_effect = Exception("Blob not found") + + with pytest.raises(Exception, match="Blob not found"): + self.Storage.get_file(f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}") + From 4a9a88b683e0da71dd71a96fc1fb19564f9a053b Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 14:41:42 -0500 Subject: [PATCH 08/23] Update test_provider.py --- backend/open_webui/test/apps/webui/storage/test_provider.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index e434523f8..6770add7b 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -305,6 +305,7 @@ class TestAzureStorageProvider: mock_container_client.get_blob_client.return_value = mock_blob_client # Mock `from_connection_string` and `BlobServiceClient` constructor + monkeypatch.setattr(provider, "BlobServiceClient", lambda *_: mock_blob_service_client) monkeypatch.setattr("azure.storage.blob.BlobServiceClient", lambda *_: mock_blob_service_client) # Apply to instance variables From f674e28263574f33a920c79d282389f0364825d2 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 14:45:13 -0500 Subject: [PATCH 09/23] Update test_provider.py --- backend/open_webui/test/apps/webui/storage/test_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 6770add7b..2a8655027 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -292,7 +292,7 @@ class TestAzureStorageProvider: self.file_bytesio_empty = io.BytesIO() super().__init__() - @pytest.fixture + @pytest.fixture(scope="class") def setup(self, monkeypatch): """Mock BlobServiceClient and BlobContainerClient for local testing""" # Create mock Blob Service Client From 9a2e81f5f01045b424a5b7b30118e2ebbbabc45e Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 14:49:03 -0500 Subject: [PATCH 10/23] Update test_provider.py --- .../test/apps/webui/storage/test_provider.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 2a8655027..2ed2c7b0f 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -283,13 +283,6 @@ class TestGCSStorageProvider: class TestAzureStorageProvider: def __init__(self): - self.Storage = provider.AzureStorageProvider() - self.Storage.endpoint = "https://myaccount.blob.core.windows.net" - self.Storage.container_name = "my-container" - self.file_content = b"test content" - self.filename = "test.txt" - self.filename_extra = "test_extra.txt" - self.file_bytesio_empty = io.BytesIO() super().__init__() @pytest.fixture(scope="class") @@ -305,12 +298,21 @@ class TestAzureStorageProvider: mock_container_client.get_blob_client.return_value = mock_blob_client # Mock `from_connection_string` and `BlobServiceClient` constructor - monkeypatch.setattr(provider, "BlobServiceClient", lambda *_: mock_blob_service_client) monkeypatch.setattr("azure.storage.blob.BlobServiceClient", lambda *_: mock_blob_service_client) + + + self.Storage = provider.AzureStorageProvider() + self.Storage.endpoint = "https://myaccount.blob.core.windows.net" + self.Storage.container_name = "my-container" + self.file_content = b"test content" + self.filename = "test.txt" + self.filename_extra = "test_extra.txt" + self.file_bytesio_empty = io.BytesIO() + # Apply to instance variables - self.Storage.blob_service_client = mock_blob_service_client - self.Storage.container_client = mock_container_client + #self.Storage.blob_service_client = mock_blob_service_client + #self.Storage.container_client = mock_container_client yield From ff5f0c3e3965a5ef44decba7382052548e442a4b Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 14:51:17 -0500 Subject: [PATCH 11/23] Update test_provider.py --- backend/open_webui/test/apps/webui/storage/test_provider.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 2ed2c7b0f..23d020c13 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -311,8 +311,8 @@ class TestAzureStorageProvider: self.file_bytesio_empty = io.BytesIO() # Apply to instance variables - #self.Storage.blob_service_client = mock_blob_service_client - #self.Storage.container_client = mock_container_client + self.Storage.blob_service_client = mock_blob_service_client + self.Storage.container_client = mock_container_client yield From 7d1ec2042905513fbd944f8407f6b1bb3ecd8e64 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 14:57:51 -0500 Subject: [PATCH 12/23] Update test_provider.py --- backend/open_webui/test/apps/webui/storage/test_provider.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 23d020c13..2bc3ccb64 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -298,9 +298,9 @@ class TestAzureStorageProvider: mock_container_client.get_blob_client.return_value = mock_blob_client # Mock `from_connection_string` and `BlobServiceClient` constructor - monkeypatch.setattr("azure.storage.blob.BlobServiceClient", lambda *_: mock_blob_service_client) - - + monkeypatch.setattr(azure.storage.blob.BlobServiceClient, lambda *_: mock_blob_service_client) + monkeypatch.setattr(azure.storage.blob.BlobContainerClient, lambda *_: mock_container_client) + monkeypatch.setattr(azure.storage.blob.BlobClient, lambda *_: mock_blob_client) self.Storage = provider.AzureStorageProvider() self.Storage.endpoint = "https://myaccount.blob.core.windows.net" From 56060db29de3287fdf7df124bce2e44f8fef743e Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 15:01:26 -0500 Subject: [PATCH 13/23] Update test_provider.py --- .../test/apps/webui/storage/test_provider.py | 94 ++++++++----------- 1 file changed, 41 insertions(+), 53 deletions(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 2bc3ccb64..08338375e 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -282,26 +282,8 @@ class TestGCSStorageProvider: class TestAzureStorageProvider: - def __init__(self): - super().__init__() - - @pytest.fixture(scope="class") - def setup(self, monkeypatch): - """Mock BlobServiceClient and BlobContainerClient for local testing""" - # Create mock Blob Service Client - mock_blob_service_client = MagicMock() - mock_container_client = MagicMock() - mock_blob_client = MagicMock() - - # Set up return values - mock_blob_service_client.get_container_client.return_value = mock_container_client - mock_container_client.get_blob_client.return_value = mock_blob_client - - # Mock `from_connection_string` and `BlobServiceClient` constructor - monkeypatch.setattr(azure.storage.blob.BlobServiceClient, lambda *_: mock_blob_service_client) - monkeypatch.setattr(azure.storage.blob.BlobContainerClient, lambda *_: mock_container_client) - monkeypatch.setattr(azure.storage.blob.BlobClient, lambda *_: mock_blob_client) - + @pytest.fixture(autouse=True) + def setup_storage(self, monkeypatch): self.Storage = provider.AzureStorageProvider() self.Storage.endpoint = "https://myaccount.blob.core.windows.net" self.Storage.container_name = "my-container" @@ -310,34 +292,48 @@ class TestAzureStorageProvider: self.filename_extra = "test_extra.txt" self.file_bytesio_empty = io.BytesIO() - # Apply to instance variables + # Create mock Blob Service Client and related clients + mock_blob_service_client = MagicMock() + mock_container_client = MagicMock() + mock_blob_client = MagicMock() + + # Set up return values for the mock + mock_blob_service_client.get_container_client.return_value = mock_container_client + mock_container_client.get_blob_client.return_value = mock_blob_client + + # Monkeypatch the Azure classes to return our mocks + monkeypatch.setattr( + azure.storage.blob, "BlobServiceClient", lambda *args, **kwargs: mock_blob_service_client + ) + monkeypatch.setattr( + azure.storage.blob, "BlobContainerClient", lambda *args, **kwargs: mock_container_client + ) + monkeypatch.setattr( + azure.storage.blob, "BlobClient", lambda *args, **kwargs: mock_blob_client + ) + + # Apply mocks to the Storage instance self.Storage.blob_service_client = mock_blob_service_client self.Storage.container_client = mock_container_client - yield - - def test_upload_file(self, monkeypatch, tmp_path, setup): - """Test uploading a file to mocked Azure Storage.""" + def test_upload_file(self, monkeypatch, tmp_path): upload_dir = mock_upload_dir(monkeypatch, tmp_path) - + # Simulate an error when container does not exist self.Storage.container_client.get_blob_client.side_effect = Exception("Container does not exist") - with pytest.raises(Exception): self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) # Reset side effect and create container self.Storage.container_client.get_blob_client.side_effect = None - self.Storage.create_container() - contents, azure_file_path = self.Storage.upload_file( - io.BytesIO(self.file_content), self.filename - ) + contents, azure_file_path = self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) # Assertions self.Storage.container_client.get_blob_client.assert_called_with(self.filename) - self.Storage.container_client.get_blob_client().upload_blob.assert_called_once_with(self.file_content, overwrite=True) - + self.Storage.container_client.get_blob_client().upload_blob.assert_called_once_with( + self.file_content, overwrite=True + ) assert contents == self.file_content assert azure_file_path == f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}" assert (upload_dir / self.filename).exists() @@ -346,42 +342,38 @@ class TestAzureStorageProvider: with pytest.raises(ValueError): self.Storage.upload_file(self.file_bytesio_empty, self.filename) - def test_get_file(self, monkeypatch, tmp_path, setup): - """Test retrieving a file from mocked Azure Storage.""" + def test_get_file(self, monkeypatch, tmp_path): upload_dir = mock_upload_dir(monkeypatch, tmp_path) self.Storage.create_container() # Mock upload behavior self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) - # Mock blob download behavior self.Storage.container_client.get_blob_client().download_blob().readall.return_value = self.file_content - file_path = self.Storage.get_file(f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}") + file_url = f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}" + file_path = self.Storage.get_file(file_url) assert file_path == str(upload_dir / self.filename) assert (upload_dir / self.filename).exists() assert (upload_dir / self.filename).read_bytes() == self.file_content - def test_delete_file(self, monkeypatch, tmp_path, setup): - """Test deleting a file from mocked Azure Storage.""" + def test_delete_file(self, monkeypatch, tmp_path): upload_dir = mock_upload_dir(monkeypatch, tmp_path) self.Storage.create_container() - # Mock upload + # Mock file upload self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) - # Mock deletion self.Storage.container_client.get_blob_client().delete_blob.return_value = None - self.Storage.delete_file(f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}") + file_url = f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}" + self.Storage.delete_file(file_url) - # Assertions self.Storage.container_client.get_blob_client().delete_blob.assert_called_once() assert not (upload_dir / self.filename).exists() - def test_delete_all_files(self, monkeypatch, tmp_path, setup): - """Test deleting all files from mocked Azure Storage.""" + def test_delete_all_files(self, monkeypatch, tmp_path): upload_dir = mock_upload_dir(monkeypatch, tmp_path) self.Storage.create_container() @@ -396,22 +388,18 @@ class TestAzureStorageProvider: ] self.Storage.container_client.get_blob_client().delete_blob.return_value = None - # Call delete all files self.Storage.delete_all_files() - # Assertions self.Storage.container_client.list_blobs.assert_called_once() self.Storage.container_client.get_blob_client().delete_blob.assert_any_call() assert not (upload_dir / self.filename).exists() assert not (upload_dir / self.filename_extra).exists() - def test_get_file_not_found(self, monkeypatch, setup): - """Test handling when a requested file does not exist.""" + def test_get_file_not_found(self, monkeypatch): self.Storage.create_container() - # Mock behavior to raise an error for missing files + file_url = f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}" + # Mock behavior to raise an error for missing blobs self.Storage.container_client.get_blob_client().download_blob.side_effect = Exception("Blob not found") - with pytest.raises(Exception, match="Blob not found"): - self.Storage.get_file(f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}") - + self.Storage.get_file(file_url) \ No newline at end of file From 2c328cc7c9b9b928ed660864208265bad8034538 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 15:04:49 -0500 Subject: [PATCH 14/23] Update test_provider.py --- .../test/apps/webui/storage/test_provider.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 08338375e..75a105bc8 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -284,14 +284,6 @@ class TestGCSStorageProvider: class TestAzureStorageProvider: @pytest.fixture(autouse=True) def setup_storage(self, monkeypatch): - self.Storage = provider.AzureStorageProvider() - self.Storage.endpoint = "https://myaccount.blob.core.windows.net" - self.Storage.container_name = "my-container" - self.file_content = b"test content" - self.filename = "test.txt" - self.filename_extra = "test_extra.txt" - self.file_bytesio_empty = io.BytesIO() - # Create mock Blob Service Client and related clients mock_blob_service_client = MagicMock() mock_container_client = MagicMock() @@ -312,6 +304,15 @@ class TestAzureStorageProvider: azure.storage.blob, "BlobClient", lambda *args, **kwargs: mock_blob_client ) + + self.Storage = provider.AzureStorageProvider() + self.Storage.endpoint = "https://myaccount.blob.core.windows.net" + self.Storage.container_name = "my-container" + self.file_content = b"test content" + self.filename = "test.txt" + self.filename_extra = "test_extra.txt" + self.file_bytesio_empty = io.BytesIO() + # Apply mocks to the Storage instance self.Storage.blob_service_client = mock_blob_service_client self.Storage.container_client = mock_container_client @@ -402,4 +403,4 @@ class TestAzureStorageProvider: # Mock behavior to raise an error for missing blobs self.Storage.container_client.get_blob_client().download_blob.side_effect = Exception("Blob not found") with pytest.raises(Exception, match="Blob not found"): - self.Storage.get_file(file_url) \ No newline at end of file + self.Storage.get_file(file_url) From 9c8c837ab96dbed4b1e6eb3d0135471486179e65 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 15:12:35 -0500 Subject: [PATCH 15/23] Update test_provider.py --- .../test/apps/webui/storage/test_provider.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 75a105bc8..3bb71735c 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -328,7 +328,9 @@ class TestAzureStorageProvider: # Reset side effect and create container self.Storage.container_client.get_blob_client.side_effect = None self.Storage.create_container() - contents, azure_file_path = self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) + contents, azure_file_path = self.Storage.upload_file( + io.BytesIO(self.file_content), self.filename + ) # Assertions self.Storage.container_client.get_blob_client.assert_called_with(self.filename) @@ -336,7 +338,10 @@ class TestAzureStorageProvider: self.file_content, overwrite=True ) assert contents == self.file_content - assert azure_file_path == f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}" + assert ( + azure_file_path + == f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}" + ) assert (upload_dir / self.filename).exists() assert (upload_dir / self.filename).read_bytes() == self.file_content @@ -350,7 +355,9 @@ class TestAzureStorageProvider: # Mock upload behavior self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) # Mock blob download behavior - self.Storage.container_client.get_blob_client().download_blob().readall.return_value = self.file_content + self.Storage.container_client.get_blob_client().download_blob().readall.return_value = ( + self.file_content + ) file_url = f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}" file_path = self.Storage.get_file(file_url) @@ -401,6 +408,8 @@ class TestAzureStorageProvider: file_url = f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}" # Mock behavior to raise an error for missing blobs - self.Storage.container_client.get_blob_client().download_blob.side_effect = Exception("Blob not found") + self.Storage.container_client.get_blob_client().download_blob.side_effect = ( + Exception("Blob not found") + ) with pytest.raises(Exception, match="Blob not found"): self.Storage.get_file(file_url) From 55bd7a1c6588ec04b76869c4bcbd96183e5059c4 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 15:13:30 -0500 Subject: [PATCH 16/23] Update test_provider.py --- .../test/apps/webui/storage/test_provider.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 3bb71735c..77bbcc5e2 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -290,8 +290,12 @@ class TestAzureStorageProvider: mock_blob_client = MagicMock() # Set up return values for the mock - mock_blob_service_client.get_container_client.return_value = mock_container_client - mock_container_client.get_blob_client.return_value = mock_blob_client + mock_blob_service_client.get_container_client.return_value = ( + mock_container_client + ) + mock_container_client.get_blob_client.return_value = ( + mock_blob_client + ) # Monkeypatch the Azure classes to return our mocks monkeypatch.setattr( @@ -321,7 +325,9 @@ class TestAzureStorageProvider: upload_dir = mock_upload_dir(monkeypatch, tmp_path) # Simulate an error when container does not exist - self.Storage.container_client.get_blob_client.side_effect = Exception("Container does not exist") + self.Storage.container_client.get_blob_client.side_effect = Exception( + "Container does not exist" + ) with pytest.raises(Exception): self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) From 4c352ff9747280f56d03156f4532566a2fa05dd8 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 15:14:46 -0500 Subject: [PATCH 17/23] Update test_provider.py --- .../test/apps/webui/storage/test_provider.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 77bbcc5e2..c92dd176b 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -293,19 +293,23 @@ class TestAzureStorageProvider: mock_blob_service_client.get_container_client.return_value = ( mock_container_client ) - mock_container_client.get_blob_client.return_value = ( - mock_blob_client - ) + mock_container_client.get_blob_client.return_value = mock_blob_client # Monkeypatch the Azure classes to return our mocks monkeypatch.setattr( - azure.storage.blob, "BlobServiceClient", lambda *args, **kwargs: mock_blob_service_client + azure.storage.blob, + "BlobServiceClient", + lambda *args, **kwargs: mock_blob_service_client ) monkeypatch.setattr( - azure.storage.blob, "BlobContainerClient", lambda *args, **kwargs: mock_container_client + azure.storage.blob, + "BlobContainerClient", + lambda *args, **kwargs: mock_container_client ) monkeypatch.setattr( - azure.storage.blob, "BlobClient", lambda *args, **kwargs: mock_blob_client + azure.storage.blob, + "BlobClient", + lambda *args, **kwargs: mock_blob_client ) From a29f83c4e77452416deb28cf6a09632ea0a8606f Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 15:17:49 -0500 Subject: [PATCH 18/23] updates to formatting --- backend/open_webui/storage/provider.py | 7 ++++--- .../open_webui/test/apps/webui/storage/test_provider.py | 8 +++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/backend/open_webui/storage/provider.py b/backend/open_webui/storage/provider.py index 43f4a6922..ae119e39b 100644 --- a/backend/open_webui/storage/provider.py +++ b/backend/open_webui/storage/provider.py @@ -228,7 +228,6 @@ class GCSStorageProvider(StorageProvider): LocalStorageProvider.delete_all_files() - class AzureStorageProvider(StorageProvider): def __init__(self): self.endpoint = AZURE_STORAGE_ENDPOINT @@ -246,7 +245,9 @@ class AzureStorageProvider(StorageProvider): self.blob_service_client = BlobServiceClient( account_url=self.endpoint, credential=DefaultAzureCredential() ) - self.container_client = self.blob_service_client.get_container_client(self.container_name) + self.container_client = self.blob_service_client.get_container_client( + self.container_name + ) def upload_file(self, file: BinaryIO, filename: str) -> Tuple[bytes, str]: """Handles uploading of the file to Azure Blob Storage.""" @@ -293,7 +294,7 @@ class AzureStorageProvider(StorageProvider): # Always delete from local storage LocalStorageProvider.delete_all_files() - + def get_storage_provider(storage_provider: str): if storage_provider == "local": diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index c92dd176b..0d5d81f96 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -299,17 +299,15 @@ class TestAzureStorageProvider: monkeypatch.setattr( azure.storage.blob, "BlobServiceClient", - lambda *args, **kwargs: mock_blob_service_client + lambda *args, **kwargs: mock_blob_service_client, ) monkeypatch.setattr( azure.storage.blob, "BlobContainerClient", - lambda *args, **kwargs: mock_container_client + lambda *args, **kwargs: mock_container_client, ) monkeypatch.setattr( - azure.storage.blob, - "BlobClient", - lambda *args, **kwargs: mock_blob_client + azure.storage.blob, "BlobClient", lambda *args, **kwargs: mock_blob_client ) From 7404494772bb9fee85dfafa2e91e5ee1d8818e6d Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 15:19:35 -0500 Subject: [PATCH 19/23] formatting --- backend/open_webui/storage/provider.py | 1 - backend/open_webui/test/apps/webui/storage/test_provider.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/backend/open_webui/storage/provider.py b/backend/open_webui/storage/provider.py index ae119e39b..160a45153 100644 --- a/backend/open_webui/storage/provider.py +++ b/backend/open_webui/storage/provider.py @@ -29,7 +29,6 @@ from azure.storage.blob import BlobServiceClient from azure.core.exceptions import ResourceNotFoundError - class StorageProvider(ABC): @abstractmethod def get_file(self, file_path: str) -> str: diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 0d5d81f96..e9b34aa18 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -280,7 +280,6 @@ class TestGCSStorageProvider: assert self.Storage.bucket.get_blob(self.filename_extra) == None - class TestAzureStorageProvider: @pytest.fixture(autouse=True) def setup_storage(self, monkeypatch): @@ -310,7 +309,6 @@ class TestAzureStorageProvider: azure.storage.blob, "BlobClient", lambda *args, **kwargs: mock_blob_client ) - self.Storage = provider.AzureStorageProvider() self.Storage.endpoint = "https://myaccount.blob.core.windows.net" self.Storage.container_name = "my-container" From 7b5f82ffc783cf3c906954bec1fe18a35a6eb5a2 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 15:26:04 -0500 Subject: [PATCH 20/23] Update test_provider.py --- backend/open_webui/test/apps/webui/storage/test_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index e9b34aa18..c9c63a008 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -3,7 +3,7 @@ import os import boto3 import pytest from botocore.exceptions import ClientError -from moto import mock_aws, mock_azure +from moto import mock_aws from open_webui.storage import provider from gcp_storage_emulator.server import create_server from google.cloud import storage From 888ae008679b16d3730ac972a6f0f3102d1f20fe Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 15:39:47 -0500 Subject: [PATCH 21/23] Update test_provider.py --- backend/open_webui/test/apps/webui/storage/test_provider.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index c9c63a008..08e8acb3e 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -7,7 +7,7 @@ from moto import mock_aws from open_webui.storage import provider from gcp_storage_emulator.server import create_server from google.cloud import storage -from azure.storage.blob import BlobServiceClient, BlobContainerClient, BlobClient +from azure.storage.blob import BlobServiceClient, ContainerClient, BlobClient from unittest.mock import MagicMock @@ -302,7 +302,7 @@ class TestAzureStorageProvider: ) monkeypatch.setattr( azure.storage.blob, - "BlobContainerClient", + "ContainerClient", lambda *args, **kwargs: mock_container_client, ) monkeypatch.setattr( @@ -354,6 +354,8 @@ class TestAzureStorageProvider: with pytest.raises(ValueError): self.Storage.upload_file(self.file_bytesio_empty, self.filename) + assert (true == false).equals(true) + def test_get_file(self, monkeypatch, tmp_path): upload_dir = mock_upload_dir(monkeypatch, tmp_path) self.Storage.create_container() From 9864185b57a0810155e5cbb9ecf3f9fe281dc8b2 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 15:49:44 -0500 Subject: [PATCH 22/23] Update test_provider.py --- backend/open_webui/test/apps/webui/storage/test_provider.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 08e8acb3e..accbb05d1 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -281,7 +281,10 @@ class TestGCSStorageProvider: class TestAzureStorageProvider: - @pytest.fixture(autouse=True) + def __init__(self): + super().__init__() + + @pytest.fixture(scope="class") def setup_storage(self, monkeypatch): # Create mock Blob Service Client and related clients mock_blob_service_client = MagicMock() @@ -354,7 +357,6 @@ class TestAzureStorageProvider: with pytest.raises(ValueError): self.Storage.upload_file(self.file_bytesio_empty, self.filename) - assert (true == false).equals(true) def test_get_file(self, monkeypatch, tmp_path): upload_dir = mock_upload_dir(monkeypatch, tmp_path) From a232f1f34ee5d931fc1d803b526bcdae8b0b9396 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 15:53:54 -0500 Subject: [PATCH 23/23] Update test_provider.py --- backend/open_webui/test/apps/webui/storage/test_provider.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index accbb05d1..a5ef13504 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -357,7 +357,6 @@ class TestAzureStorageProvider: with pytest.raises(ValueError): self.Storage.upload_file(self.file_bytesio_empty, self.filename) - def test_get_file(self, monkeypatch, tmp_path): upload_dir = mock_upload_dir(monkeypatch, tmp_path) self.Storage.create_container()