ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-04 02:57:24 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add "sdk.storage.log.report_upload_chunk_size_mb" and "sdk.storage.log.report_download_chunk_size_mb", control upload/download log reporting (issue #424)

Browse Source
This commit is contained in:
allegroai 2021-08-20 00:34:20 +03:00
parent 14108c4f23
commit 5295708e18
1 changed files with 8 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
clearml/storage/callbacks.py
View File

@ -2,6 +2,7 @@ import logging
import os import os
from time import time from time import time
from typing import Optional, AnyStr, IO from typing import Optional, AnyStr, IO
from ..config import config
class ProgressReport(object): class ProgressReport(object):
@ -33,7 +34,9 @@ class ProgressReport(object):
class UploadProgressReport(ProgressReport): class UploadProgressReport(ProgressReport):
def __init__(self, filename, verbose, total_size, log, report_chunk_size_mb=5): def __init__(self, filename, verbose, total_size, log, report_chunk_size_mb=0):
if not report_chunk_size_mb:
report_chunk_size_mb = int(config.get('storage.log.report_upload_chunk_size_mb', 0) or 5)
super(UploadProgressReport, self).__init__(verbose, total_size, log, report_chunk_size_mb) super(UploadProgressReport, self).__init__(verbose, total_size, log, report_chunk_size_mb)
self._filename = filename self._filename = filename
@ -71,7 +74,10 @@ class UploadProgressReport(ProgressReport):
class DownloadProgressReport(ProgressReport): class DownloadProgressReport(ProgressReport):
def __init__(self, total_size, verbose, remote_path, log, report_chunk_size_mb=5): def __init__(self, total_size, verbose, remote_path, log, report_chunk_size_mb=0):
if not report_chunk_size_mb:
report_chunk_size_mb = int(config.get('storage.log.report_download_chunk_size_mb', 0) or 5)
super(DownloadProgressReport, self).__init__(verbose, total_size, log, report_chunk_size_mb) super(DownloadProgressReport, self).__init__(verbose, total_size, log, report_chunk_size_mb)
self._remote_path = remote_path self._remote_path = remote_path