From bd2a0dcc2c7a2491628f2368fb6d1c277c9c59c3 Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Thu, 11 Jun 2020 14:41:10 +0300 Subject: [PATCH] Fix trains-init wizard, allow at most two credentials verification tries --- trains/config/default/__main__.py | 205 ++++++++++++++---------------- 1 file changed, 95 insertions(+), 110 deletions(-) diff --git a/trains/config/default/__main__.py b/trains/config/default/__main__.py index a9b1bec3..64f9c8b3 100644 --- a/trains/config/default/__main__.py +++ b/trains/config/default/__main__.py @@ -14,49 +14,25 @@ from trains.backend_config.defs import LOCAL_CONFIG_FILES, LOCAL_CONFIG_FILE_OVE from trains.config import config_obj from trains.utilities.pyhocon import ConfigFactory, ConfigMissingException -description = """ -Please create new trains credentials through the profile page in your trains web app (e.g. http://localhost:8080/profile) -In the profile page, press "Create new credentials", then press "Copy to clipboard". +description = "\n" \ + "Please create new trains credentials through the profile page in " \ + "your trains web app (e.g. http://localhost:8080/profile)\n" \ + "In the profile page, press \"Create new credentials\", then press \"Copy to clipboard\".\n" \ + "\n" \ + "Paste copied configuration here:\n" -Paste copied configuration here: +host_description = """ +Editing configuration file: {CONFIG_FILE} +Enter the url of the trains-server's Web service, for example: {HOST} """ +# noinspection PyBroadException try: def_host = ENV_HOST.get(default=config_obj.get("api.web_server")) or 'http://localhost:8080' except Exception: def_host = 'http://localhost:8080' -def get_user_input(): - """ - Hack: Supporting two forms of input: - - one line credentials, e.g. - 'credentials {"access_key" = "", "secret_key" = ""}' - - line break with all the info, e.g. - api { - web_server: https://demoapp.trains.allegro.ai - api_server: https://demoapi.trains.allegro.ai - credentials { - "access_key" = "" - "secret_key" = "" - } - } - :return: str The paste data - """ - brackets_counter = 0 # check that the parse field is valid - input_list = [] - while True: - user_input = input() - if not user_input: - return - brackets_counter += user_input.count("{") - user_input.count("}") - input_list.append(user_input) - if user_input.strip() == '}' and brackets_counter == 0: # For break line input - return os.linesep.join(input_list) - elif user_input.strip().startswith("credentials") and len(input_list) == 1: # For one line input - return os.linesep.join(input_list) - - def validate_file(string): if not string: raise argparse.ArgumentTypeError("expected a valid file path") @@ -82,11 +58,11 @@ def main(): print('Configuration file already exists: {}'.format(str(conf_file))) print('Leaving setup, feel free to edit the configuration file.') return - print(description, end='') - parse_input = get_user_input() + sentinel = '' + parse_input = '\n'.join(iter(input, sentinel)) credentials = None - api_host = None + api_server = None web_server = None # noinspection PyBroadException try: @@ -94,11 +70,11 @@ def main(): if parsed: # Take the credentials in raw form or from api section credentials = get_parsed_field(parsed, ["credentials"]) - api_host = get_parsed_field(parsed, ["api_server", "host"]) + api_server = get_parsed_field(parsed, ["api_server", "host"]) web_server = get_parsed_field(parsed, ["web_server"]) except Exception: credentials = credentials or None - api_host = api_host or None + api_server = api_server or None web_server = web_server or None while not credentials or set(credentials) != {"access_key", "secret_key"}: @@ -107,74 +83,25 @@ def main(): print('Detected credentials key=\"{}\" secret=\"{}\"'.format(credentials['access_key'], credentials['secret_key'][0:4] + "***")) + web_input = True + if web_server: + host = input_url('WEB Host', web_server) + elif api_server: + web_input = False + host = input_url('API Host', api_server) + else: + print(host_description.format(CONFIG_FILE=args.file, HOST=def_host,)) + host = input_url('WEB Host', '') - host_description = """ - Editing configuration file: {CONFIG_FILE} - Enter the url of the trains-server's Web service, for example: {HOST} - """.format( - CONFIG_FILE=args.file, - HOST=def_host, - ) + parsed_host = verify_url(host) + api_host, files_host, web_host = parse_host(parsed_host, allow_input=True) - if api_host: + # on of these two we configured + if not web_input: + web_host = input_url('Web Application Host', web_host) + else: api_host = input_url('API Host', api_host) - else: - print(host_description) - api_host = input_url('API Host', '') - parsed_host = verify_url(api_host) - if parsed_host.netloc.startswith('demoapp.'): - # this is our demo server - api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('demoapp.', 'demoapi.', 1) + parsed_host.path - web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path - files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( - 'demoapp.', 'demofiles.', 1) + parsed_host.path - elif parsed_host.netloc.startswith('app.'): - # this is our application server - api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('app.', 'api.', 1) + parsed_host.path - web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path - files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('app.', 'files.', 1) + parsed_host.path - elif parsed_host.netloc.startswith('demoapi.'): - print('{} is the api server, we need the web server. Replacing \'demoapi.\' with \'demoapp.\''.format( - parsed_host.netloc)) - api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path - web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('demoapi.', 'demoapp.', 1) + parsed_host.path - files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( - 'demoapi.', 'demofiles.', 1) + parsed_host.path - elif parsed_host.netloc.startswith('api.'): - print('{} is the api server, we need the web server. Replacing \'api.\' with \'app.\''.format( - parsed_host.netloc)) - api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path - web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('api.', 'app.', 1) + parsed_host.path - files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('api.', 'files.', 1) + parsed_host.path - elif parsed_host.port == 8008: - api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path - web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8008', ':8080', 1) + parsed_host.path - files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8008', ':8081', 1) + parsed_host.path - elif parsed_host.port == 8080: - print('Port 8080 is the web port. Using port 8008 for API Host and 8080 for Web Application Host') - api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8080', ':8008', 1) + parsed_host.path - web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path - files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8080', ':8081', 1) + parsed_host.path - else: - api_host = '' - web_host = '' - files_host = '' - if not parsed_host.port: - print('Host port not detected, do you wish to use the default 8080 port n/[y]? ', end='') - replace_port = input().lower() - if not replace_port or replace_port == 'y' or replace_port == 'yes': - api_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8008' + parsed_host.path - web_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8080' + parsed_host.path - files_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8081' + parsed_host.path - elif not replace_port or replace_port.lower() == 'n' or replace_port.lower() == 'no': - web_host = input_host_port("Web", parsed_host) - api_host = input_host_port("API", parsed_host) - files_host = input_host_port("Files", parsed_host) - if not api_host: - api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path - - web_host = input_url('Web Application Host', web_server if web_server else web_host) files_host = input_url('File Store Host', files_host) print('\nTRAINS Hosts configuration:\nWeb App: {}\nAPI: {}\nFile Store: {}\n'.format( @@ -224,6 +151,63 @@ def main(): print('TRAINS setup completed successfully.') +def parse_host(parsed_host, allow_input=True): + if parsed_host.netloc.startswith('demoapp.'): + # this is our demo server + api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('demoapp.', 'demoapi.', 1) + parsed_host.path + web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path + files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('demoapp.', 'demofiles.', + 1) + parsed_host.path + elif parsed_host.netloc.startswith('app.'): + # this is our application server + api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('app.', 'api.', 1) + parsed_host.path + web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path + files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('app.', 'files.', 1) + parsed_host.path + elif parsed_host.netloc.startswith('demoapi.'): + print('{} is the api server, we need the web server. Replacing \'demoapi.\' with \'demoapp.\''.format( + parsed_host.netloc)) + api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path + web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('demoapi.', 'demoapp.', 1) + parsed_host.path + files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('demoapi.', 'demofiles.', + 1) + parsed_host.path + elif parsed_host.netloc.startswith('api.'): + print('{} is the api server, we need the web server. Replacing \'api.\' with \'app.\''.format( + parsed_host.netloc)) + api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path + web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('api.', 'app.', 1) + parsed_host.path + files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('api.', 'files.', 1) + parsed_host.path + elif parsed_host.port == 8008: + print('Port 8008 is the api port. Replacing 8080 with 8008 for Web application') + api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path + web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8008', ':8080', 1) + parsed_host.path + files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8008', ':8081', 1) + parsed_host.path + elif parsed_host.port == 8080: + api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8080', ':8008', 1) + parsed_host.path + web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path + files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8080', ':8081', 1) + parsed_host.path + elif allow_input: + api_host = '' + web_host = '' + files_host = '' + if not parsed_host.port: + print('Host port not detected, do you wish to use the default 8080 port n/[y]? ', end='') + replace_port = input().lower() + if not replace_port or replace_port == 'y' or replace_port == 'yes': + api_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8008' + parsed_host.path + web_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8080' + parsed_host.path + files_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8081' + parsed_host.path + elif not replace_port or replace_port.lower() == 'n' or replace_port.lower() == 'no': + web_host = input_host_port("Web", parsed_host) + api_host = input_host_port("API", parsed_host) + files_host = input_host_port("Files", parsed_host) + if not api_host: + api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path + else: + raise ValueError("Could not parse host name") + + return api_host, files_host, web_host + + def verify_credentials(api_host, credentials): """check if the credentials are valid""" # noinspection PyBroadException @@ -231,15 +215,15 @@ def verify_credentials(api_host, credentials): print('Verifying credentials ...') if api_host: Session(api_key=credentials['access_key'], secret_key=credentials['secret_key'], host=api_host, - http_retries_config={"total": 1}) + http_retries_config={"total": 2}) print('Credentials verified!') return True else: print("Can't verify credentials") return False - except Exception as ex: - print('Exception: {}\nError: could not verify credentials: key={} secret={}'.format( - str(ex), credentials.get('access_key'), credentials.get('secret_key'))) + except Exception: + print('Error: could not verify credentials: key={} secret={}'.format( + credentials.get('access_key'), credentials.get('secret_key'))) return False @@ -273,7 +257,7 @@ def read_manual_credentials(): def input_url(host_type, host=None): while True: - print('{} configured to: [{}] '.format(host_type, host), end='') + print('{} configured to: {}'.format(host_type, '[{}] '.format(host) if host else ''), end='') parse_input = input() if host and (not parse_input or parse_input.lower() == 'yes' or parse_input.lower() == 'y'): break @@ -287,11 +271,12 @@ def input_url(host_type, host=None): def input_host_port(host_type, parsed_host): print('Enter port for {} host '.format(host_type), end='') replace_port = input().lower() - return (parsed_host.scheme + "://" + parsed_host.netloc + (':{}'.format(replace_port) if replace_port else '') + - parsed_host.path) + return parsed_host.scheme + "://" + parsed_host.netloc + ( + ':{}'.format(replace_port) if replace_port else '') + parsed_host.path def verify_url(parse_input): + # noinspection PyBroadException try: if not parse_input.startswith('http://') and not parse_input.startswith('https://'): # if we have a specific port, use http prefix, otherwise assume https