ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-02-07 13:23:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add loading bar to SHA2 generation (#396)

Browse Source
This commit is contained in:
Eren Balatkan 2021-07-06 15:18:29 +03:00 committed by GitHub
parent 75d0577570
commit 6b9297660e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
clearml/datasets/dataset.py
View File

@ -983,7 +983,12 @@ class Dataset(object):
for f in files if f.is_file()]
self._task.get_logger().report_text('Generating SHA2 hash for {} files'.format(len(file_entries)))
pool = ThreadPool(cpu_count() * 2)
pool.map(self._calc_file_hash, file_entries)
try:
import tqdm
for _ in tqdm.tqdm(pool.imap_unordered(self._calc_file_hash, file_entries), total=len(file_entries)):
pass
except ImportError:
pool.map(self._calc_file_hash, file_entries)
pool.close()
self._task.get_logger().report_text('Hash generation completed')