ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-02-07 21:33:25 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix filename too long bug (trains-server #54)

Browse Source
This commit is contained in:
allegroai 2020-07-30 14:58:48 +03:00
parent d8671beb89
commit 2ec5726812
1 changed files with 15 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
trains/backend_interface/metrics/events.py
View File

@ -1,4 +1,5 @@
import abc import abc
import hashlib
import time import time
from multiprocessing import Lock from multiprocessing import Lock
@ -309,13 +310,26 @@ class UploadEvent(MetricsEventAdapter):
) )
def get_target_full_upload_uri(self, storage_uri, storage_key_prefix=None, quote_uri=True): def get_target_full_upload_uri(self, storage_uri, storage_key_prefix=None, quote_uri=True):
def limit_path_folder_length(folder_path):
if not folder_path or len(folder_path) <= 250:
return folder_path
parts = folder_path.split('.')
if len(parts) > 1:
prefix = hashlib.md5(str('.'.join(parts[:-1])).encode('utf-8')).hexdigest()
new_path = '{}.{}'.format(prefix, parts[-1])
if len(new_path) <= 250:
return new_path
return hashlib.md5(str(folder_path).encode('utf-8')).hexdigest()
e_storage_uri = self._upload_uri or storage_uri e_storage_uri = self._upload_uri or storage_uri
# if we have an entry (with or without a stream), we'll generate the URL and store it in the event # if we have an entry (with or without a stream), we'll generate the URL and store it in the event
filename = self._upload_filename filename = self._upload_filename
if self._override_storage_key_prefix or not storage_key_prefix: if self._override_storage_key_prefix or not storage_key_prefix:
storage_key_prefix = self._override_storage_key_prefix storage_key_prefix = self._override_storage_key_prefix
key = '/'.join(x for x in (storage_key_prefix, self._replace_slash(self.metric), key = '/'.join(x for x in (storage_key_prefix, self._replace_slash(self.metric),
self._replace_slash(self.variant), self._replace_slash(filename)) if x) self._replace_slash(self.variant), self._replace_slash(filename)
) if x)
key = '/'.join(limit_path_folder_length(x) for x in key.split('/'))
url = '/'.join(x.strip('/') for x in (e_storage_uri, key)) url = '/'.join(x.strip('/') for x in (e_storage_uri, key))
# make sure we preserve local path root # make sure we preserve local path root
if e_storage_uri.startswith('/'): if e_storage_uri.startswith('/'):