Add separate api/web/file server configuration (backward support included).

OS environment override with:  TRAINS_API_HOST / TRAINS_WEB_HOST / TRAINS_FILES_HOST

docs/trains.conf

@@ -1,7 +1,13 @@
 # TRAINS SDK configuration file
 api {
-    # Notice: 'host' is the api server (default port 8008), not the web server.
+    # web_server on port 8080
-    host: http://localhost:8008
+    web_server: "http://localhost:8080"
+
+    # Notice: 'api_server' is the api server (default port 8008), not the web server.
+    api_server: "http://localhost:8008"
+
+    # file server onport 8081
+    files_server: "http://localhost:8081"
 
     # Credentials are generated in the webapp, http://localhost:8080/admin
     credentials {"access_key": "EGRTCO8JMSIGI6S39GTP43NFWXDQOW", "secret_key": "x!XTov_G-#vspE*Y(h$Anm&DIc5Ou-F)jsl$PdOyj5wG1&E!Z8"}

trains/backend_api/config/default/api.conf

@@ -1,7 +1,11 @@
 {
     version: 1.5
-    # default https://demoapi.trainsai.io host
+    # default api_server: https://demoapi.trainsai.io
-    host: ""
+    api_server: ""
+    # default web_server: https://demoapp.trainsai.io
+    web_server: ""
+    # default files_server: https://demofiles.trainsai.io
+    files_server: ""
 
     # verify host ssl certificate, set to False only if you have a very good reason
     verify_certificate: True

trains/backend_api/session/defs.py

@@ -2,6 +2,8 @@ from ...backend_config import EnvEntry
 
 
 ENV_HOST = EnvEntry("TRAINS_API_HOST", "ALG_API_HOST")
+ENV_WEB_HOST = EnvEntry("TRAINS_WEB_HOST", "ALG_WEB_HOST")
+ENV_FILES_HOST = EnvEntry("TRAINS_FILES_HOST", "ALG_FILES_HOST")
 ENV_ACCESS_KEY = EnvEntry("TRAINS_API_ACCESS_KEY", "ALG_API_ACCESS_KEY")
 ENV_SECRET_KEY = EnvEntry("TRAINS_API_SECRET_KEY", "ALG_API_SECRET_KEY")
 ENV_VERBOSE = EnvEntry("TRAINS_API_VERBOSE", "ALG_API_VERBOSE", type=bool, default=False)

trains/backend_api/session/session.py

@@ -2,6 +2,7 @@ import json as json_lib
 import sys
 import types
 from socket import gethostname
+from six.moves.urllib.parse import urlparse, urlunparse
 
 import jwt
 import requests
@@ -10,11 +11,11 @@ from pyhocon import ConfigTree
 from requests.auth import HTTPBasicAuth
 
 from .callresult import CallResult
-from .defs import ENV_VERBOSE, ENV_HOST, ENV_ACCESS_KEY, ENV_SECRET_KEY
+from .defs import ENV_VERBOSE, ENV_HOST, ENV_ACCESS_KEY, ENV_SECRET_KEY, ENV_WEB_HOST, ENV_FILES_HOST
 from .request import Request, BatchRequest
 from .token_manager import TokenManager
 from ..config import load
-from ..utils import get_http_session_with_retry
+from ..utils import get_http_session_with_retry, urllib_log_warning_setup
 from ..version import __version__
 
 
@@ -32,11 +33,13 @@ class Session(TokenManager):
 
     _async_status_code = 202
     _session_requests = 0
-    _session_initial_timeout = (1.0, 10)
+    _session_initial_timeout = (3.0, 10.)
-    _session_timeout = (5.0, None)
+    _session_timeout = (5.0, 300.)
 
     api_version = '2.1'
     default_host = "https://demoapi.trainsai.io"
+    default_web = "https://demoapp.trainsai.io"
+    default_files = "https://demofiles.trainsai.io"
     default_key = "EGRTCO8JMSIGI6S39GTP43NFWXDQOW"
     default_secret = "x!XTov_G-#vspE*Y(h$Anm&DIc5Ou-F)jsl$PdOyj5wG1&E!Z8"
 
@@ -97,7 +100,7 @@ class Session(TokenManager):
         self._logger = logger
 
         self.__access_key = api_key or ENV_ACCESS_KEY.get(
-            default=(self.config.get("api.credentials.access_key") or self.default_key)
+            default=(self.config.get("api.credentials.access_key", None) or self.default_key)
         )
         if not self.access_key:
             raise ValueError(
@@ -105,7 +108,7 @@ class Session(TokenManager):
             )
 
         self.__secret_key = secret_key or ENV_SECRET_KEY.get(
-            default=(self.config.get("api.credentials.secret_key") or self.default_secret)
+            default=(self.config.get("api.credentials.secret_key", None) or self.default_secret)
         )
         if not self.secret_key:
             raise ValueError(
@@ -125,7 +128,7 @@ class Session(TokenManager):
 
         self.__worker = worker or gethostname()
 
-        self.__max_req_size = self.config.get("api.http.max_req_size")
+        self.__max_req_size = self.config.get("api.http.max_req_size", None)
         if not self.__max_req_size:
             raise ValueError("missing max request size")
 
@@ -140,6 +143,11 @@ class Session(TokenManager):
         except (jwt.DecodeError, ValueError):
             pass
 
+        # now setup the session reporting, so one consecutive retries will show warning
+        # we do that here, so if we have problems authenticating, we see them immediately
+        # notice: this is across the board warning omission
+        urllib_log_warning_setup(total_retries=http_retries_config.get('total', 0), display_warning_after=3)
+
     def _send_request(
         self,
         service,
@@ -394,7 +402,65 @@ class Session(TokenManager):
         if not config:
             from ...config import config_obj
             config = config_obj
-        return ENV_HOST.get(default=(config.get("api.host") or cls.default_host))
+        return ENV_HOST.get(default=(config.get("api.api_server", None) or
+                                     config.get("api.host", None) or cls.default_host))
+
+    @classmethod
+    def get_app_server_host(cls, config=None):
+        if not config:
+            from ...config import config_obj
+            config = config_obj
+
+        # get from config/environment
+        web_host = ENV_WEB_HOST.get(default=config.get("api.web_server", None))
+        if web_host:
+            return web_host
+
+        # return default
+        host = cls.get_api_server_host(config)
+        if host == cls.default_host:
+            return cls.default_web
+
+        # compose ourselves
+        if '://demoapi.' in host:
+            return host.replace('://demoapi.', '://demoapp.', 1)
+        if '://api.' in host:
+            return host.replace('://api.', '://app.', 1)
+
+        parsed = urlparse(host)
+        if parsed.port == 8008:
+            return host.replace(':8008', ':8080', 1)
+
+        raise ValueError('Could not detect TRAINS web application server')
+
+    @classmethod
+    def get_files_server_host(cls, config=None):
+        if not config:
+            from ...config import config_obj
+            config = config_obj
+        # get from config/environment
+        files_host = ENV_FILES_HOST.get(default=(config.get("api.files_server", None)))
+        if files_host:
+            return files_host
+
+        # return default
+        host = cls.get_api_server_host(config)
+        if host == cls.default_host:
+            return cls.default_files
+
+        # compose ourselves
+        app_host = cls.get_app_server_host(config)
+        parsed = urlparse(app_host)
+        if parsed.port:
+            parsed = parsed._replace(netloc=parsed.netloc.replace(':%d' % parsed.port, ':8081', 1))
+        elif parsed.netloc.startswith('demoapp.'):
+            parsed = parsed._replace(netloc=parsed.netloc.replace('demoapp.', 'demofiles.', 1))
+        elif parsed.netloc.startswith('app.'):
+            parsed = parsed._replace(netloc=parsed.netloc.replace('app.', 'files.', 1))
+        else:
+            parsed = parsed._replace(netloc=parsed.netloc + ':8081')
+
+        return urlunparse(parsed)
 
     def _do_refresh_token(self, old_token, exp=None):
         """ TokenManager abstract method implementation.

trains/backend_api/utils.py

@@ -27,6 +27,29 @@ def get_config():
     return config_obj
 
 
+def urllib_log_warning_setup(total_retries=10, display_warning_after=5):
+    class RetryFilter(logging.Filter):
+        last_instance = None
+
+        def __init__(self, total, warning_after=5):
+            super(RetryFilter, self).__init__()
+            self.total = total
+            self.display_warning_after = warning_after
+            self.last_instance = self
+
+        def filter(self, record):
+            if record.args and len(record.args) > 0 and isinstance(record.args[0], Retry):
+                retry_left = self.total - record.args[0].total
+                return retry_left >= self.display_warning_after
+
+            return True
+
+    urllib3_log = logging.getLogger('urllib3.connectionpool')
+    if urllib3_log:
+        urllib3_log.removeFilter(RetryFilter.last_instance)
+        urllib3_log.addFilter(RetryFilter(total_retries, display_warning_after))
+
+
 class TLSv1HTTPAdapter(HTTPAdapter):
     def init_poolmanager(self, connections, maxsize, block=False, **pool_kwargs):
         self.poolmanager = PoolManager(num_pools=connections,

trains/config/default/__main__.py

@@ -11,19 +11,20 @@ from trains.config import config_obj
 
 
 description = """
-Please create new credentials using the web app: {}/admin
+Please create new credentials using the web app: {}/profile
 In the Admin page, press "Create new credentials", then press "Copy to clipboard"
 
 Paste credentials here: """
 
 try:
-    def_host = ENV_HOST.get(default=config_obj.get("api.host"))
+    def_host = ENV_HOST.get(default=config_obj.get("api.web_server")) or 'http://localhost:8080'
 except Exception:
     def_host = 'http://localhost:8080'
 
 host_description = """
 Editing configuration file: {CONFIG_FILE}
-Enter the url of the trains-server's api service, for example: http://localhost:8008 : """.format(
+Enter the url of the trains-server's Web service, for example: {HOST}
+""".format(
     CONFIG_FILE=LOCAL_CONFIG_FILES[0],
     HOST=def_host,
 )
@@ -37,64 +38,60 @@ def main():
         print('Leaving setup, feel free to edit the configuration file.')
         return
 
-    print(host_description, end='')
+    print(host_description)
-    parsed_host = None
+    web_host = input_url('Web Application Host', '')
-    while not parsed_host:
+    parsed_host = verify_url(web_host)
-        parse_input = input()
-        if not parse_input:
-            parse_input = def_host
-        # noinspection PyBroadException
-        try:
-            if not parse_input.startswith('http://') and not parse_input.startswith('https://'):
-                parse_input = 'http://'+parse_input
-            parsed_host = urlparse(parse_input)
-            if parsed_host.scheme not in ('http', 'https'):
-                parsed_host = None
-        except Exception:
-            parsed_host = None
-            print('Could not parse url {}\nEnter your trains-server host: '.format(parse_input), end='')
 
-    if parsed_host.port == 8080:
+    if parsed_host.port == 8008:
-        # this is a docker 8080 is the web address, we need the api address, it is 8008
+        print('Port 8008 is the api port. Replacing 8080 with 8008 for Web application')
-        print('Port 8080 is the web port, we need the api port. Replacing 8080 with 8008')
+        api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
+        web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8008', ':8080', 1) + parsed_host.path
+        files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8008', ':8081', 1) + parsed_host.path
+    elif parsed_host.port == 8080:
         api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8080', ':8008', 1) + parsed_host.path
         web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
+        files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8080', ':8081', 1) + parsed_host.path
     elif parsed_host.netloc.startswith('demoapp.'):
-        print('{} is the web server, we need the api server. Replacing \'demoapp.\' with \'demoapi.\''.format(
-            parsed_host.netloc))
         # this is our demo server
         api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('demoapp.', 'demoapi.', 1) + parsed_host.path
         web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
+        files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('demoapp.', 'demofiles.', 1) + parsed_host.path
     elif parsed_host.netloc.startswith('app.'):
-        print('{} is the web server, we need the api server. Replacing \'app.\' with \'api.\''.format(
-            parsed_host.netloc))
         # this is our application server
         api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('app.', 'api.', 1) + parsed_host.path
         web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
-    elif parsed_host.port == 8008:
+        files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('app.', 'files.', 1) + parsed_host.path
-        api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
-        web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8008', ':8080', 1) + parsed_host.path
     elif parsed_host.netloc.startswith('demoapi.'):
+        print('{} is the api server, we need the web server. Replacing \'demoapi.\' with \'demoapp.\''.format(
+            parsed_host.netloc))
         api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
         web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('demoapi.', 'demoapp.', 1) + parsed_host.path
+        files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('demoapi.', 'demofiles.', 1) + parsed_host.path
     elif parsed_host.netloc.startswith('api.'):
+        print('{} is the api server, we need the web server. Replacing \'api.\' with \'app.\''.format(
+            parsed_host.netloc))
         api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
         web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('api.', 'app.', 1) + parsed_host.path
+        files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('api.', 'files.', 1) + parsed_host.path
     else:
-        api_host = None
+        api_host = ''
-        web_host = None
+        web_host = ''
+        files_host = ''
         if not parsed_host.port:
             print('Host port not detected, do you wish to use the default 8008 port n/[y]? ', end='')
             replace_port = input().lower()
             if not replace_port or replace_port == 'y' or replace_port == 'yes':
                 api_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8008' + parsed_host.path
                 web_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8080' + parsed_host.path
+                files_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8081' + parsed_host.path
         if not api_host:
             api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
-        if not web_host:
-            web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
 
-    print('Host configured to: {}'.format(api_host))
+    api_host = input_url('API Host', api_host)
+    files_host = input_url('File Store Host', files_host)
+
+    print('\nTRAINS Hosts configuration:\nAPI: {}\nWeb App: {}\nFile Store: {}\n'.format(
+        api_host, web_host, files_host))
 
     print(description.format(web_host), end='')
     parse_input = input()
@@ -133,11 +130,14 @@ def main():
             header = '# TRAINS SDK configuration file\n' \
                      'api {\n' \
                      '    # Notice: \'host\' is the api server (default port 8008), not the web server.\n' \
-                     '    host: %s\n' \
+                     '    api_server: %s\n' \
-                     '    # Credentials are generated in the webapp, %s/admin\n' \
+                     '    web_server: %s\n' \
+                     '    files_server: %s\n' \
+                     '    # Credentials are generated in the webapp, %s/profile\n' \
                      '    credentials {"access_key": "%s", "secret_key": "%s"}\n' \
                      '}\n' \
-                     'sdk ' % (api_host, web_host, credentials['access_key'], credentials['secret_key'])
+                     'sdk ' % (api_host, web_host, files_host,
+                               web_host, credentials['access_key'], credentials['secret_key'])
             f.write(header)
             f.write(default_sdk)
     except Exception:
@@ -148,5 +148,30 @@ def main():
     print('TRAINS setup completed successfully.')
 
 
+def input_url(host_type, host=None):
+    while True:
+        print('{} configured to: [{}] '.format(host_type, host), end='')
+        parse_input = input()
+        if host and (not parse_input or parse_input.lower() == 'yes' or parse_input.lower() == 'y'):
+            break
+        if parse_input and verify_url(parse_input):
+            host = parse_input
+            break
+    return host
+
+
+def verify_url(parse_input):
+    try:
+        if not parse_input.startswith('http://') and not parse_input.startswith('https://'):
+            parse_input = 'http://' + parse_input
+        parsed_host = urlparse(parse_input)
+        if parsed_host.scheme not in ('http', 'https'):
+            parsed_host = None
+    except Exception:
+        parsed_host = None
+        print('Could not parse url {}\nEnter your trains-server host: '.format(parse_input), end='')
+    return parsed_host
+
+
 if __name__ == '__main__':
     main()