From 8ab982cbc90f9a907a30989b2cf157cd21e5f96f Mon Sep 17 00:00:00 2001
From: allegroai <>
Date: Sat, 23 Nov 2019 01:26:18 +0200
Subject: [PATCH] Fix support for uploading large files over slow http
 connection

---
 trains/storage/helper.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/trains/storage/helper.py b/trains/storage/helper.py
index 63cf10d2..f952cdfb 100644
--- a/trains/storage/helper.py
+++ b/trains/storage/helper.py
@@ -947,6 +947,7 @@ class _HttpDriver(_Driver):
     """ LibCloud http/https adapter (simple, enough for now) """
 
     timeout = (5.0, 30.)
+    min_kbps_speed = 50
 
     class _Container(object):
         _default_backend_session = None
@@ -980,7 +981,16 @@ class _HttpDriver(_Driver):
         url = object_name[:object_name.index('/')]
         url_path = object_name[len(url)+1:]
         full_url = container.name+url
-        res = container.session.post(full_url, files={url_path: iterator}, timeout=self.timeout,
+        # when sending data in post, there is no connection timeout, just an entire upload timeout
+        timeout = self.timeout[-1]
+        if hasattr(iterator, 'tell') and hasattr(iterator, 'seek'):
+            pos = iterator.tell()
+            iterator.seek(0, 2)
+            stream_size = iterator.tell() - pos
+            iterator.seek(pos, 0)
+            timeout = max(timeout, (stream_size / 1024) / float(self.min_kbps_speed))
+
+        res = container.session.post(full_url, files={url_path: iterator}, timeout=timeout,
                                      headers=container.get_headers(full_url))
         if res.status_code != requests.codes.ok:
             raise ValueError('Failed uploading object %s (%d): %s' % (object_name, res.status_code, res.text))