diff --git a/clearml/datasets/dataset.py b/clearml/datasets/dataset.py index aa606f0d..331ecf8a 100644 --- a/clearml/datasets/dataset.py +++ b/clearml/datasets/dataset.py @@ -2150,6 +2150,11 @@ class Dataset(object): local_folder.mkdir(parents=True, exist_ok=True) return local_folder + def _release_lock_ds_target_folder(self, target_folder): + # type: () -> None + cache = CacheManager.get_cache_manager(cache_context=self.__cache_context) + cache.unlock_cache_folder(target_folder) + def _get_data_artifact_names(self): # type: () -> List[str] data_artifact_entries = [ @@ -2212,6 +2217,7 @@ class Dataset(object): if target_base_folder and next(target_base_folder.iterdir(), None): if self._verify_dataset_folder(target_base_folder, part, chunk_selection): target_base_folder.touch() + self._release_lock_ds_target_folder(target_base_folder) return target_base_folder.as_posix() else: LoggerRoot.get_base_logger().info('Dataset needs refreshing, fetching all parent datasets') @@ -2240,6 +2246,7 @@ class Dataset(object): # if we have no dependencies, we can just return now if not dependencies_by_order: + self._release_lock_ds_target_folder(target_base_folder) return target_base_folder.absolute().as_posix() # extract parent datasets @@ -2257,6 +2264,7 @@ class Dataset(object): chunk_selection=chunk_selection, use_soft_links=use_soft_links, raise_on_error=raise_on_error, force=True) + self._release_lock_ds_target_folder(target_base_folder) return target_base_folder.absolute().as_posix() def _get_dependencies_by_order(self, include_unused=False, include_current=True): diff --git a/clearml/storage/cache.py b/clearml/storage/cache.py index 5af8053e..ac8b130c 100644 --- a/clearml/storage/cache.py +++ b/clearml/storage/cache.py @@ -308,26 +308,15 @@ class CacheManager(object): atexit.register(self._lock_file_cleanup_callback) lock = self._folder_locks.get(local_path.as_posix()) - i = 0 - # try to create a lock if we do not already have one (if we do, we assume it is locked) - while not lock: - lock_path = local_path.parent / "{}{:03d}.{}{}".format( - CacheManager._lockfile_prefix, - i, - local_path.name, - CacheManager._lockfile_suffix, - ) - lock = FileLock(filename=lock_path) - - # try to lock folder (if we failed to create lock, try nex number) - try: - lock.acquire(timeout=0) - break - except LockException: - # failed locking, maybe someone else already locked it. - del lock - lock = None - i += 1 + # try to create a lock. If it exists, wait for the lock to be released + lock_path = local_path.parent / "{}{:03d}.{}{}".format( + CacheManager._lockfile_prefix, + 0, + local_path.name, + CacheManager._lockfile_suffix, + ) + lock = FileLock(filename=lock_path) + lock.acquire(timeout=300, check_interval=1.0) # store lock self._folder_locks[local_path.as_posix()] = lock