ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-03 18:52:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Release lock after dataset cache is downloaded (#708)

Browse Source
This commit is contained in:
Giacomo Ferroni 2022-07-13 16:33:55 +02:00 committed by GitHub
parent e2208fc4c5
commit 397dcfacda
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 17 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
clearml/datasets/dataset.py
View File

@ -2150,6 +2150,11 @@ class Dataset(object):
local_folder.mkdir(parents=True, exist_ok=True) local_folder.mkdir(parents=True, exist_ok=True)
return local_folder return local_folder
def _release_lock_ds_target_folder(self, target_folder):
# type: () -> None
cache = CacheManager.get_cache_manager(cache_context=self.__cache_context)
cache.unlock_cache_folder(target_folder)
def _get_data_artifact_names(self): def _get_data_artifact_names(self):
# type: () -> List[str] # type: () -> List[str]
data_artifact_entries = [ data_artifact_entries = [
@ -2212,6 +2217,7 @@ class Dataset(object):
if target_base_folder and next(target_base_folder.iterdir(), None): if target_base_folder and next(target_base_folder.iterdir(), None):
if self._verify_dataset_folder(target_base_folder, part, chunk_selection): if self._verify_dataset_folder(target_base_folder, part, chunk_selection):
target_base_folder.touch() target_base_folder.touch()
self._release_lock_ds_target_folder(target_base_folder)
return target_base_folder.as_posix() return target_base_folder.as_posix()
else: else:
LoggerRoot.get_base_logger().info('Dataset needs refreshing, fetching all parent datasets') LoggerRoot.get_base_logger().info('Dataset needs refreshing, fetching all parent datasets')
@ -2240,6 +2246,7 @@ class Dataset(object):
# if we have no dependencies, we can just return now # if we have no dependencies, we can just return now
if not dependencies_by_order: if not dependencies_by_order:
self._release_lock_ds_target_folder(target_base_folder)
return target_base_folder.absolute().as_posix() return target_base_folder.absolute().as_posix()
# extract parent datasets # extract parent datasets
@ -2257,6 +2264,7 @@ class Dataset(object):
chunk_selection=chunk_selection, use_soft_links=use_soft_links, chunk_selection=chunk_selection, use_soft_links=use_soft_links,
raise_on_error=raise_on_error, force=True) raise_on_error=raise_on_error, force=True)
self._release_lock_ds_target_folder(target_base_folder)
return target_base_folder.absolute().as_posix() return target_base_folder.absolute().as_posix()
def _get_dependencies_by_order(self, include_unused=False, include_current=True): def _get_dependencies_by_order(self, include_unused=False, include_current=True):

29
clearml/storage/cache.py
View File

@ -308,26 +308,15 @@ class CacheManager(object):
atexit.register(self._lock_file_cleanup_callback) atexit.register(self._lock_file_cleanup_callback)
lock = self._folder_locks.get(local_path.as_posix()) lock = self._folder_locks.get(local_path.as_posix())
i = 0 # try to create a lock. If it exists, wait for the lock to be released
# try to create a lock if we do not already have one (if we do, we assume it is locked) lock_path = local_path.parent / "{}{:03d}.{}{}".format(
while not lock: CacheManager._lockfile_prefix,
lock_path = local_path.parent / "{}{:03d}.{}{}".format( 0,
CacheManager._lockfile_prefix, local_path.name,
i, CacheManager._lockfile_suffix,
local_path.name, )
CacheManager._lockfile_suffix, lock = FileLock(filename=lock_path)
) lock.acquire(timeout=300, check_interval=1.0)
lock = FileLock(filename=lock_path)
# try to lock folder (if we failed to create lock, try nex number)
try:
lock.acquire(timeout=0)
break
except LockException:
# failed locking, maybe someone else already locked it.
del lock
lock = None
i += 1
# store lock # store lock
self._folder_locks[local_path.as_posix()] = lock self._folder_locks[local_path.as_posix()] = lock