ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-03 18:52:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add HTTP download timeout control (change default connection timeout to 30sec)

Browse Source
This commit is contained in:
allegroai 2021-12-22 12:53:37 +02:00
parent d647a33470
commit 69846d200a
1 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
clearml/storage/helper.py
View File

@ -999,7 +999,8 @@ class StorageHelper(object):
class _HttpDriver(_Driver): class _HttpDriver(_Driver):
""" LibCloud http/https adapter (simple, enough for now) """ """ LibCloud http/https adapter (simple, enough for now) """
timeout = (5.0, 30.) timeout_connection = deferred_config('http.timeout.connection', 30)
timeout_total = deferred_config('http.timeout.total', 30)
min_kbps_speed = 50 min_kbps_speed = 50
schemes = ('http', 'https') schemes = ('http', 'https')
@ -1042,7 +1043,7 @@ class _HttpDriver(_Driver):
url_path = object_name[len(url) + 1:] url_path = object_name[len(url) + 1:]
full_url = container.name + url full_url = container.name + url
# when sending data in post, there is no connection timeout, just an entire upload timeout # when sending data in post, there is no connection timeout, just an entire upload timeout
timeout = self.timeout[-1] timeout = int(self.timeout_total)
stream_size = 0 stream_size = 0
if hasattr(iterator, 'tell') and hasattr(iterator, 'seek'): if hasattr(iterator, 'tell') and hasattr(iterator, 'seek'):
pos = iterator.tell() pos = iterator.tell()
@ -1091,7 +1092,9 @@ class _HttpDriver(_Driver):
container = self._containers[obj.container_name] container = self._containers[obj.container_name]
# set stream flag before we send the request # set stream flag before we send the request
container.session.stream = obj.is_stream container.session.stream = obj.is_stream
res = container.session.get(obj.url, timeout=self.timeout, headers=container.get_headers(obj.url)) res = container.session.get(
obj.url, timeout=(int(self.timeout_connection), int(self.timeout_total)),
headers=container.get_headers(obj.url))
if res.status_code != requests.codes.ok: if res.status_code != requests.codes.ok:
raise ValueError('Failed getting object %s (%d): %s' % (obj.object_name, res.status_code, res.text)) raise ValueError('Failed getting object %s (%d): %s' % (obj.object_name, res.status_code, res.text))
return res return res