ClearML/clearml-server
1
0
Fork 0
mirror of https://github.com/clearml/clearml-server synced 2025-04-06 05:55:11 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Use an auth token in async_urls_delete when contacting the fileserver

Browse Source
This commit is contained in:
allegroai 2024-06-20 18:00:19 +03:00
parent f1c876089b
commit f7d36bea4f
2 changed files with 12 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
apiserver/config/default/services/async_urls_delete.conf
View File

@ -9,4 +9,5 @@ fileserver {
# Can be in the form <schema>://host:port/path or /path # Can be in the form <schema>://host:port/path or /path
url_prefixes: ["https://files.community-master.hosted.allegro.ai/"] url_prefixes: ["https://files.community-master.hosted.allegro.ai/"]
timeout_sec: 300 timeout_sec: 300
token_expiration_sec: 600
} }

18
apiserver/jobs/async_urls_delete.py
View File

@ -19,7 +19,9 @@ from google.cloud import storage as google_storage
from mongoengine import Q from mongoengine import Q
from mypy_boto3_s3.service_resource import Bucket as AWSBucket from mypy_boto3_s3.service_resource import Bucket as AWSBucket
from apiserver.bll.auth import AuthBLL
from apiserver.bll.storage import StorageBLL from apiserver.bll.storage import StorageBLL
from apiserver.config.info import get_default_company
from apiserver.config_repo import config from apiserver.config_repo import config
from apiserver.database import db from apiserver.database import db
from apiserver.database.model.url_to_delete import UrlToDelete, StorageType, DeletionStatus from apiserver.database.model.url_to_delete import UrlToDelete, StorageType, DeletionStatus
@ -200,6 +202,8 @@ class FileserverStorage(Storage):
res_data = res.json() res_data = res.json()
return list(res_data.get("deleted", {})), res_data.get("errors", {}) return list(res_data.get("deleted", {})), res_data.get("errors", {})
token_expiration_sec = conf.get("fileserver.token_expiration_sec", 600)
def __init__(self, company: str, fileserver_host: str = None): def __init__(self, company: str, fileserver_host: str = None):
fileserver_host = fileserver_host or config.get("hosts.fileserver", None) fileserver_host = fileserver_host or config.get("hosts.fileserver", None)
self.host = fileserver_host.rstrip("/") self.host = fileserver_host.rstrip("/")
@ -220,13 +224,6 @@ class FileserverStorage(Storage):
self.company = company self.company = company
# @classmethod
# def validate_fileserver_access(cls, fileserver_host: str):
# res = requests.get(
# url=fileserver_host
# )
# res.raise_for_status()
@property @property
def name(self) -> str: def name(self) -> str:
return "Fileserver" return "Fileserver"
@ -260,7 +257,13 @@ class FileserverStorage(Storage):
def get_client(self, base: str, urls: Sequence[UrlToDelete]) -> Client: def get_client(self, base: str, urls: Sequence[UrlToDelete]) -> Client:
host = base host = base
token = AuthBLL.get_token_for_user(
user_id="__apiserver__",
company_id=get_default_company(),
expiration_sec=self.token_expiration_sec,
).token
session = requests.session() session = requests.session()
session.headers.update({"Authorization": "Bearer {}".format(token)})
res = session.get(url=host, timeout=self.Client.timeout) res = session.get(url=host, timeout=self.Client.timeout)
res.raise_for_status() res.raise_for_status()
@ -285,6 +288,7 @@ class AzureStorage(Storage):
): ):
raise ValueError("No path found following container name") raise ValueError("No path found following container name")
# noinspection PyTypeChecker
return os.path.join(*parsed.path.segments[1:]) return os.path.join(*parsed.path.segments[1:])
@staticmethod @staticmethod