diff --git a/docs/trains.conf b/docs/trains.conf index a27a3948..7e12205e 100644 --- a/docs/trains.conf +++ b/docs/trains.conf @@ -1,7 +1,13 @@ # TRAINS SDK configuration file api { - # Notice: 'host' is the api server (default port 8008), not the web server. - host: http://localhost:8008 + # web_server on port 8080 + web_server: "http://localhost:8080" + + # Notice: 'api_server' is the api server (default port 8008), not the web server. + api_server: "http://localhost:8008" + + # file server onport 8081 + files_server: "http://localhost:8081" # Credentials are generated in the webapp, http://localhost:8080/admin credentials {"access_key": "EGRTCO8JMSIGI6S39GTP43NFWXDQOW", "secret_key": "x!XTov_G-#vspE*Y(h$Anm&DIc5Ou-F)jsl$PdOyj5wG1&E!Z8"} diff --git a/trains/backend_api/config/default/api.conf b/trains/backend_api/config/default/api.conf index cbac9189..de96becc 100644 --- a/trains/backend_api/config/default/api.conf +++ b/trains/backend_api/config/default/api.conf @@ -1,7 +1,11 @@ { version: 1.5 - # default https://demoapi.trainsai.io host - host: "" + # default api_server: https://demoapi.trainsai.io + api_server: "" + # default web_server: https://demoapp.trainsai.io + web_server: "" + # default files_server: https://demofiles.trainsai.io + files_server: "" # verify host ssl certificate, set to False only if you have a very good reason verify_certificate: True diff --git a/trains/backend_api/session/defs.py b/trains/backend_api/session/defs.py index 73fc6603..5ea6a97d 100644 --- a/trains/backend_api/session/defs.py +++ b/trains/backend_api/session/defs.py @@ -2,6 +2,8 @@ from ...backend_config import EnvEntry ENV_HOST = EnvEntry("TRAINS_API_HOST", "ALG_API_HOST") +ENV_WEB_HOST = EnvEntry("TRAINS_WEB_HOST", "ALG_WEB_HOST") +ENV_FILES_HOST = EnvEntry("TRAINS_FILES_HOST", "ALG_FILES_HOST") ENV_ACCESS_KEY = EnvEntry("TRAINS_API_ACCESS_KEY", "ALG_API_ACCESS_KEY") ENV_SECRET_KEY = EnvEntry("TRAINS_API_SECRET_KEY", "ALG_API_SECRET_KEY") ENV_VERBOSE = EnvEntry("TRAINS_API_VERBOSE", "ALG_API_VERBOSE", type=bool, default=False) diff --git a/trains/backend_api/session/session.py b/trains/backend_api/session/session.py index da71098e..bdb71aa0 100644 --- a/trains/backend_api/session/session.py +++ b/trains/backend_api/session/session.py @@ -2,6 +2,7 @@ import json as json_lib import sys import types from socket import gethostname +from six.moves.urllib.parse import urlparse, urlunparse import jwt import requests @@ -10,11 +11,11 @@ from pyhocon import ConfigTree from requests.auth import HTTPBasicAuth from .callresult import CallResult -from .defs import ENV_VERBOSE, ENV_HOST, ENV_ACCESS_KEY, ENV_SECRET_KEY +from .defs import ENV_VERBOSE, ENV_HOST, ENV_ACCESS_KEY, ENV_SECRET_KEY, ENV_WEB_HOST, ENV_FILES_HOST from .request import Request, BatchRequest from .token_manager import TokenManager from ..config import load -from ..utils import get_http_session_with_retry +from ..utils import get_http_session_with_retry, urllib_log_warning_setup from ..version import __version__ @@ -32,11 +33,13 @@ class Session(TokenManager): _async_status_code = 202 _session_requests = 0 - _session_initial_timeout = (1.0, 10) - _session_timeout = (5.0, None) + _session_initial_timeout = (3.0, 10.) + _session_timeout = (5.0, 300.) api_version = '2.1' default_host = "https://demoapi.trainsai.io" + default_web = "https://demoapp.trainsai.io" + default_files = "https://demofiles.trainsai.io" default_key = "EGRTCO8JMSIGI6S39GTP43NFWXDQOW" default_secret = "x!XTov_G-#vspE*Y(h$Anm&DIc5Ou-F)jsl$PdOyj5wG1&E!Z8" @@ -97,7 +100,7 @@ class Session(TokenManager): self._logger = logger self.__access_key = api_key or ENV_ACCESS_KEY.get( - default=(self.config.get("api.credentials.access_key") or self.default_key) + default=(self.config.get("api.credentials.access_key", None) or self.default_key) ) if not self.access_key: raise ValueError( @@ -105,7 +108,7 @@ class Session(TokenManager): ) self.__secret_key = secret_key or ENV_SECRET_KEY.get( - default=(self.config.get("api.credentials.secret_key") or self.default_secret) + default=(self.config.get("api.credentials.secret_key", None) or self.default_secret) ) if not self.secret_key: raise ValueError( @@ -125,7 +128,7 @@ class Session(TokenManager): self.__worker = worker or gethostname() - self.__max_req_size = self.config.get("api.http.max_req_size") + self.__max_req_size = self.config.get("api.http.max_req_size", None) if not self.__max_req_size: raise ValueError("missing max request size") @@ -140,6 +143,11 @@ class Session(TokenManager): except (jwt.DecodeError, ValueError): pass + # now setup the session reporting, so one consecutive retries will show warning + # we do that here, so if we have problems authenticating, we see them immediately + # notice: this is across the board warning omission + urllib_log_warning_setup(total_retries=http_retries_config.get('total', 0), display_warning_after=3) + def _send_request( self, service, @@ -394,7 +402,65 @@ class Session(TokenManager): if not config: from ...config import config_obj config = config_obj - return ENV_HOST.get(default=(config.get("api.host") or cls.default_host)) + return ENV_HOST.get(default=(config.get("api.api_server", None) or + config.get("api.host", None) or cls.default_host)) + + @classmethod + def get_app_server_host(cls, config=None): + if not config: + from ...config import config_obj + config = config_obj + + # get from config/environment + web_host = ENV_WEB_HOST.get(default=config.get("api.web_server", None)) + if web_host: + return web_host + + # return default + host = cls.get_api_server_host(config) + if host == cls.default_host: + return cls.default_web + + # compose ourselves + if '://demoapi.' in host: + return host.replace('://demoapi.', '://demoapp.', 1) + if '://api.' in host: + return host.replace('://api.', '://app.', 1) + + parsed = urlparse(host) + if parsed.port == 8008: + return host.replace(':8008', ':8080', 1) + + raise ValueError('Could not detect TRAINS web application server') + + @classmethod + def get_files_server_host(cls, config=None): + if not config: + from ...config import config_obj + config = config_obj + # get from config/environment + files_host = ENV_FILES_HOST.get(default=(config.get("api.files_server", None))) + if files_host: + return files_host + + # return default + host = cls.get_api_server_host(config) + if host == cls.default_host: + return cls.default_files + + # compose ourselves + app_host = cls.get_app_server_host(config) + parsed = urlparse(app_host) + if parsed.port: + parsed = parsed._replace(netloc=parsed.netloc.replace(':%d' % parsed.port, ':8081', 1)) + elif parsed.netloc.startswith('demoapp.'): + parsed = parsed._replace(netloc=parsed.netloc.replace('demoapp.', 'demofiles.', 1)) + elif parsed.netloc.startswith('app.'): + parsed = parsed._replace(netloc=parsed.netloc.replace('app.', 'files.', 1)) + else: + parsed = parsed._replace(netloc=parsed.netloc + ':8081') + + return urlunparse(parsed) def _do_refresh_token(self, old_token, exp=None): """ TokenManager abstract method implementation. diff --git a/trains/backend_api/utils.py b/trains/backend_api/utils.py index da25e129..7bf47c07 100644 --- a/trains/backend_api/utils.py +++ b/trains/backend_api/utils.py @@ -27,6 +27,29 @@ def get_config(): return config_obj +def urllib_log_warning_setup(total_retries=10, display_warning_after=5): + class RetryFilter(logging.Filter): + last_instance = None + + def __init__(self, total, warning_after=5): + super(RetryFilter, self).__init__() + self.total = total + self.display_warning_after = warning_after + self.last_instance = self + + def filter(self, record): + if record.args and len(record.args) > 0 and isinstance(record.args[0], Retry): + retry_left = self.total - record.args[0].total + return retry_left >= self.display_warning_after + + return True + + urllib3_log = logging.getLogger('urllib3.connectionpool') + if urllib3_log: + urllib3_log.removeFilter(RetryFilter.last_instance) + urllib3_log.addFilter(RetryFilter(total_retries, display_warning_after)) + + class TLSv1HTTPAdapter(HTTPAdapter): def init_poolmanager(self, connections, maxsize, block=False, **pool_kwargs): self.poolmanager = PoolManager(num_pools=connections, diff --git a/trains/config/default/__main__.py b/trains/config/default/__main__.py index aa450008..4cdd52c0 100644 --- a/trains/config/default/__main__.py +++ b/trains/config/default/__main__.py @@ -11,19 +11,20 @@ from trains.config import config_obj description = """ -Please create new credentials using the web app: {}/admin +Please create new credentials using the web app: {}/profile In the Admin page, press "Create new credentials", then press "Copy to clipboard" Paste credentials here: """ try: - def_host = ENV_HOST.get(default=config_obj.get("api.host")) + def_host = ENV_HOST.get(default=config_obj.get("api.web_server")) or 'http://localhost:8080' except Exception: def_host = 'http://localhost:8080' host_description = """ Editing configuration file: {CONFIG_FILE} -Enter the url of the trains-server's api service, for example: http://localhost:8008 : """.format( +Enter the url of the trains-server's Web service, for example: {HOST} +""".format( CONFIG_FILE=LOCAL_CONFIG_FILES[0], HOST=def_host, ) @@ -37,64 +38,60 @@ def main(): print('Leaving setup, feel free to edit the configuration file.') return - print(host_description, end='') - parsed_host = None - while not parsed_host: - parse_input = input() - if not parse_input: - parse_input = def_host - # noinspection PyBroadException - try: - if not parse_input.startswith('http://') and not parse_input.startswith('https://'): - parse_input = 'http://'+parse_input - parsed_host = urlparse(parse_input) - if parsed_host.scheme not in ('http', 'https'): - parsed_host = None - except Exception: - parsed_host = None - print('Could not parse url {}\nEnter your trains-server host: '.format(parse_input), end='') + print(host_description) + web_host = input_url('Web Application Host', '') + parsed_host = verify_url(web_host) - if parsed_host.port == 8080: - # this is a docker 8080 is the web address, we need the api address, it is 8008 - print('Port 8080 is the web port, we need the api port. Replacing 8080 with 8008') + if parsed_host.port == 8008: + print('Port 8008 is the api port. Replacing 8080 with 8008 for Web application') + api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path + web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8008', ':8080', 1) + parsed_host.path + files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8008', ':8081', 1) + parsed_host.path + elif parsed_host.port == 8080: api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8080', ':8008', 1) + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path + files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8080', ':8081', 1) + parsed_host.path elif parsed_host.netloc.startswith('demoapp.'): - print('{} is the web server, we need the api server. Replacing \'demoapp.\' with \'demoapi.\''.format( - parsed_host.netloc)) # this is our demo server api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('demoapp.', 'demoapi.', 1) + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path + files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('demoapp.', 'demofiles.', 1) + parsed_host.path elif parsed_host.netloc.startswith('app.'): - print('{} is the web server, we need the api server. Replacing \'app.\' with \'api.\''.format( - parsed_host.netloc)) # this is our application server api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('app.', 'api.', 1) + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path - elif parsed_host.port == 8008: - api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path - web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8008', ':8080', 1) + parsed_host.path + files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('app.', 'files.', 1) + parsed_host.path elif parsed_host.netloc.startswith('demoapi.'): + print('{} is the api server, we need the web server. Replacing \'demoapi.\' with \'demoapp.\''.format( + parsed_host.netloc)) api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('demoapi.', 'demoapp.', 1) + parsed_host.path + files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('demoapi.', 'demofiles.', 1) + parsed_host.path elif parsed_host.netloc.startswith('api.'): + print('{} is the api server, we need the web server. Replacing \'api.\' with \'app.\''.format( + parsed_host.netloc)) api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('api.', 'app.', 1) + parsed_host.path + files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('api.', 'files.', 1) + parsed_host.path else: - api_host = None - web_host = None + api_host = '' + web_host = '' + files_host = '' if not parsed_host.port: print('Host port not detected, do you wish to use the default 8008 port n/[y]? ', end='') replace_port = input().lower() if not replace_port or replace_port == 'y' or replace_port == 'yes': api_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8008' + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8080' + parsed_host.path + files_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8081' + parsed_host.path if not api_host: api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path - if not web_host: - web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path - print('Host configured to: {}'.format(api_host)) + api_host = input_url('API Host', api_host) + files_host = input_url('File Store Host', files_host) + + print('\nTRAINS Hosts configuration:\nAPI: {}\nWeb App: {}\nFile Store: {}\n'.format( + api_host, web_host, files_host)) print(description.format(web_host), end='') parse_input = input() @@ -133,11 +130,14 @@ def main(): header = '# TRAINS SDK configuration file\n' \ 'api {\n' \ ' # Notice: \'host\' is the api server (default port 8008), not the web server.\n' \ - ' host: %s\n' \ - ' # Credentials are generated in the webapp, %s/admin\n' \ + ' api_server: %s\n' \ + ' web_server: %s\n' \ + ' files_server: %s\n' \ + ' # Credentials are generated in the webapp, %s/profile\n' \ ' credentials {"access_key": "%s", "secret_key": "%s"}\n' \ '}\n' \ - 'sdk ' % (api_host, web_host, credentials['access_key'], credentials['secret_key']) + 'sdk ' % (api_host, web_host, files_host, + web_host, credentials['access_key'], credentials['secret_key']) f.write(header) f.write(default_sdk) except Exception: @@ -148,5 +148,30 @@ def main(): print('TRAINS setup completed successfully.') +def input_url(host_type, host=None): + while True: + print('{} configured to: [{}] '.format(host_type, host), end='') + parse_input = input() + if host and (not parse_input or parse_input.lower() == 'yes' or parse_input.lower() == 'y'): + break + if parse_input and verify_url(parse_input): + host = parse_input + break + return host + + +def verify_url(parse_input): + try: + if not parse_input.startswith('http://') and not parse_input.startswith('https://'): + parse_input = 'http://' + parse_input + parsed_host = urlparse(parse_input) + if parsed_host.scheme not in ('http', 'https'): + parsed_host = None + except Exception: + parsed_host = None + print('Could not parse url {}\nEnter your trains-server host: '.format(parse_input), end='') + return parsed_host + + if __name__ == '__main__': main()