ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-13 07:08:24 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Black formatting

Browse Source
This commit is contained in:
allegroai 2024-08-02 15:21:31 +03:00
parent 386e1874be
commit 028adb45fb
1 changed files with 135 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

248
clearml/cli/config/__main__.py
View File

@ -15,13 +15,15 @@ from clearml.backend_config.defs import LOCAL_CONFIG_FILES, LOCAL_CONFIG_FILE_OV
from clearml.config import config_obj from clearml.config import config_obj
from clearml.utilities.pyhocon import ConfigFactory, ConfigMissingException from clearml.utilities.pyhocon import ConfigFactory, ConfigMissingException
description = "\n" \ description = (
"Please create new clearml credentials through the settings page in " \ "\n"
"your `clearml-server` web app (e.g. http://localhost:8080//settings/workspace-configuration) \n"\ "Please create new clearml credentials through the settings page in "
"Or create a free account at https://app.clear.ml/settings/workspace-configuration\n\n" \ "your `clearml-server` web app (e.g. http://localhost:8080//settings/workspace-configuration) \n"
"In settings page, press \"Create new credentials\", then press \"Copy to clipboard\".\n" \ "Or create a free account at https://app.clear.ml/settings/workspace-configuration\n\n"
"\n" \ 'In settings page, press "Create new credentials", then press "Copy to clipboard".\n'
"\n"
"Paste copied configuration here:\n" "Paste copied configuration here:\n"
)
host_description = """ host_description = """
Editing configuration file: {CONFIG_FILE} Editing configuration file: {CONFIG_FILE}
@ -30,9 +32,9 @@ Enter the url of the clearml-server's Web service, for example: {HOST}
# noinspection PyBroadException # noinspection PyBroadException
try: try:
def_host = ENV_HOST.get(default=config_obj.get("api.web_server")) or 'http://localhost:8080' def_host = ENV_HOST.get(default=config_obj.get("api.web_server")) or "http://localhost:8080"
except Exception: except Exception:
def_host = 'http://localhost:8080' def_host = "http://localhost:8080"
def validate_file(string): def validate_file(string):
@ -51,35 +53,38 @@ def main():
p = argparse.ArgumentParser(description=__doc__) p = argparse.ArgumentParser(description=__doc__)
p.add_argument( p.add_argument(
"--file", "-F", help="Target configuration file path (default is %(default)s)", "--file",
"-F",
help="Target configuration file path (default is %(default)s)",
default=default_config_file, default=default_config_file,
type=validate_file type=validate_file,
) )
args = p.parse_args() args = p.parse_args()
print('ClearML SDK setup process') print("ClearML SDK setup process")
conf_file = Path(os.path.expanduser(args.file)).absolute() conf_file = Path(os.path.expanduser(args.file)).absolute()
if conf_file.exists() and conf_file.is_file() and conf_file.stat().st_size > 0: if conf_file.exists() and conf_file.is_file() and conf_file.stat().st_size > 0:
print('Configuration file already exists: {}'.format(str(conf_file))) print("Configuration file already exists: {}".format(str(conf_file)))
print('Leaving setup, feel free to edit the configuration file.') print("Leaving setup, feel free to edit the configuration file.")
return return
print(description, end='') print(description, end="")
sentinel = '' sentinel = ""
parse_input = '' parse_input = ""
if os.environ.get("JPY_PARENT_PID"): if os.environ.get("JPY_PARENT_PID"):
# When running from a colab instance and calling clearml-init # When running from a colab instance and calling clearml-init
# colab will squish the api credentials into a single line # colab will squish the api credentials into a single line
# The regex splits this single line based on 2 spaces or more # The regex splits this single line based on 2 spaces or more
import re import re
api_input = input() api_input = input()
parse_input = '\n'.join(re.split(r" {2,}", api_input)) parse_input = "\n".join(re.split(r" {2,}", api_input))
else: else:
for line in iter(input, sentinel): for line in iter(input, sentinel):
parse_input += line+'\n' parse_input += line + "\n"
if line.rstrip() == '}': if line.rstrip() == "}":
break break
credentials = None credentials = None
@ -102,11 +107,14 @@ def main():
files_server = files_server or None files_server = files_server or None
while not credentials or set(credentials) != {"access_key", "secret_key"}: while not credentials or set(credentials) != {"access_key", "secret_key"}:
print('Could not parse credentials, please try entering them manually.') print("Could not parse credentials, please try entering them manually.")
credentials = read_manual_credentials() credentials = read_manual_credentials()
print('Detected credentials key=\"{}\" secret=\"{}\"'.format(credentials['access_key'], print(
credentials['secret_key'][0:4] + "***")) 'Detected credentials key="{}" secret="{}"'.format(
credentials["access_key"], credentials["secret_key"][0:4] + "***"
)
)
web_input = True web_input = True
if web_server: if web_server:
host = web_server host = web_server
@ -114,47 +122,43 @@ def main():
web_input = False web_input = False
host = api_server host = api_server
else: else:
print(host_description.format(CONFIG_FILE=args.file, HOST=def_host,)) print(
host = input_url('WEB Host', '') host_description.format(
CONFIG_FILE=args.file,
HOST=def_host,
)
)
host = input_url("WEB Host", "")
parsed_host = verify_url(host) parsed_host = verify_url(host)
api_host, files_host, web_host = parse_known_host(parsed_host) api_host, files_host, web_host = parse_known_host(parsed_host)
hosts_dict = { hosts_dict = {"API": api_server, "Files": files_server, "Web": web_server}
"API": api_server,
"Files": files_server,
"Web": web_server
}
infered_hosts_dict = { infered_hosts_dict = {"API": api_host, "Files": files_host, "Web": web_host}
"API": api_host,
"Files": files_host,
"Web": web_host
}
for host_type, url in six.iteritems(hosts_dict): for host_type, url in six.iteritems(hosts_dict):
if url is None or not ( if url is None or not (url.startswith("http://") or url.startswith("https://")):
url.startswith('http://') or url.startswith('https://')
):
infered_host_url = infered_hosts_dict[host_type] infered_host_url = infered_hosts_dict[host_type]
if infered_host_url != "": if infered_host_url != "":
hosts_dict[host_type] = infered_host_url hosts_dict[host_type] = infered_host_url
else: else:
hosts_dict[host_type] = input_url(host_type) hosts_dict[host_type] = input_url(host_type)
api_host, files_host, web_host = hosts_dict['API'], hosts_dict['Files'], hosts_dict['Web'] api_host, files_host, web_host = hosts_dict["API"], hosts_dict["Files"], hosts_dict["Web"]
# one of these two we configured # one of these two we configured
if not web_input: if not web_input:
web_host = input_url('Web Application Host', web_host) web_host = input_url("Web Application Host", web_host)
else: else:
if web_input is True and not web_host: if web_input is True and not web_host:
web_host = host web_host = host
print('\nClearML Hosts configuration:\nWeb App: {}\nAPI: {}\nFile Store: {}\n'.format( print(
web_host, api_host, files_host)) "\nClearML Hosts configuration:\nWeb App: {}\nAPI: {}\nFile Store: {}\n".format(web_host, api_host, files_host)
)
if len(set([web_host, api_host, files_host])) != 3: if len({web_host, api_host, files_host}) != 3:
raise ValueError("All three server URLs should be distinct") raise ValueError("All three server URLs should be distinct")
retry = 1 retry = 1
@ -166,88 +170,94 @@ def main():
if retry < max_retries + 1: if retry < max_retries + 1:
credentials = read_manual_credentials() credentials = read_manual_credentials()
else: else:
print('Exiting setup without creating configuration file') print("Exiting setup without creating configuration file")
return return
# noinspection PyBroadException # noinspection PyBroadException
try: try:
default_sdk_conf = Path(__file__).absolute().parents[2] / 'config/default/sdk.conf' default_sdk_conf = Path(__file__).absolute().parents[2] / "config/default/sdk.conf"
with open(str(default_sdk_conf), 'rt') as f: with open(str(default_sdk_conf), "rt") as f:
default_sdk = f.read() default_sdk = f.read()
except Exception: except Exception:
print('Error! Could not read default configuration file') print("Error! Could not read default configuration file")
return return
# noinspection PyBroadException # noinspection PyBroadException
try: try:
with open(str(conf_file), 'wt') as f: with open(str(conf_file), "wt") as f:
header = '# ClearML SDK configuration file\n' \ header = (
'api {\n' \ "# ClearML SDK configuration file\n"
' # Notice: \'host\' is the api server (default port 8008), not the web server.\n' \ "api {\n"
' api_server: %s\n' \ " # Notice: 'host' is the api server (default port 8008), not the web server.\n"
' web_server: %s\n' \ " api_server: %s\n"
' files_server: %s\n' \ " web_server: %s\n"
' # Credentials are generated using the webapp, %s/settings\n' \ " files_server: %s\n"
' # Override with os environment: CLEARML_API_ACCESS_KEY / CLEARML_API_SECRET_KEY\n' \ " # Credentials are generated using the webapp, %s/settings\n"
' credentials {"access_key": "%s", "secret_key": "%s"}\n' \ " # Override with os environment: CLEARML_API_ACCESS_KEY / CLEARML_API_SECRET_KEY\n"
'}\n' \ ' credentials {"access_key": "%s", "secret_key": "%s"}\n'
'sdk ' % (api_host, web_host, files_host, "}\n"
web_host, credentials['access_key'], credentials['secret_key']) "sdk "
% (api_host, web_host, files_host, web_host, credentials["access_key"], credentials["secret_key"])
)
f.write(header) f.write(header)
f.write(default_sdk) f.write(default_sdk)
except Exception: except Exception:
print('Error! Could not write configuration file at: {}'.format(str(conf_file))) print("Error! Could not write configuration file at: {}".format(str(conf_file)))
return return
print('\nNew configuration stored in {}'.format(str(conf_file))) print("\nNew configuration stored in {}".format(str(conf_file)))
print('ClearML setup completed successfully.') print("ClearML setup completed successfully.")
def parse_known_host(parsed_host): def parse_known_host(parsed_host):
if parsed_host.netloc.startswith('demoapp.'): if parsed_host.netloc.startswith("demoapp."):
# this is our demo server # this is our demo server
api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('demoapp.', 'demoapi.', 1) + parsed_host.path api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace("demoapp.", "demoapi.", 1) + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('demoapp.', 'demofiles.', files_host = (
1) + parsed_host.path parsed_host.scheme + "://" + parsed_host.netloc.replace("demoapp.", "demofiles.", 1) + parsed_host.path
elif parsed_host.netloc.startswith('app.'): )
elif parsed_host.netloc.startswith("app."):
# this is our application server # this is our application server
api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('app.', 'api.', 1) + parsed_host.path api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace("app.", "api.", 1) + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('app.', 'files.', 1) + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace("app.", "files.", 1) + parsed_host.path
elif parsed_host.netloc.startswith('demoapi.'): elif parsed_host.netloc.startswith("demoapi."):
print('{} is the api server, we need the web server. Replacing \'demoapi.\' with \'demoapp.\''.format( print(
parsed_host.netloc)) "{} is the api server, we need the web server. Replacing 'demoapi.' with 'demoapp.'".format(
parsed_host.netloc
)
)
api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('demoapi.', 'demoapp.', 1) + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace("demoapi.", "demoapp.", 1) + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('demoapi.', 'demofiles.', files_host = (
1) + parsed_host.path parsed_host.scheme + "://" + parsed_host.netloc.replace("demoapi.", "demofiles.", 1) + parsed_host.path
elif parsed_host.netloc.startswith('api.'): )
print('{} is the api server, we need the web server. Replacing \'api.\' with \'app.\''.format( elif parsed_host.netloc.startswith("api."):
parsed_host.netloc)) print("{} is the api server, we need the web server. Replacing 'api.' with 'app.'".format(parsed_host.netloc))
api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('api.', 'app.', 1) + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace("api.", "app.", 1) + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('api.', 'files.', 1) + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace("api.", "files.", 1) + parsed_host.path
elif parsed_host.port == 8008: elif parsed_host.port == 8008:
print('Port 8008 is the api port. Replacing 8008 with 8080 for Web application') print("Port 8008 is the api port. Replacing 8008 with 8080 for Web application")
api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8008', ':8080', 1) + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(":8008", ":8080", 1) + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8008', ':8081', 1) + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(":8008", ":8081", 1) + parsed_host.path
elif parsed_host.port == 8080: elif parsed_host.port == 8080:
print('Port 8080 is the web port. Replacing 8080 with 8008 for API server') print("Port 8080 is the web port. Replacing 8080 with 8008 for API server")
api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8080', ':8008', 1) + parsed_host.path api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(":8080", ":8008", 1) + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8080', ':8081', 1) + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(":8080", ":8081", 1) + parsed_host.path
elif parsed_host.port is None: elif parsed_host.port is None:
print('Web app hosted on standard port using ' + parsed_host.scheme + ' protocol.') print("Web app hosted on standard port using " + parsed_host.scheme + " protocol.")
print('Assuming files and api ports are unchanged and use the same (' + parsed_host.scheme + ') protocol') print("Assuming files and api ports are unchanged and use the same (" + parsed_host.scheme + ") protocol")
api_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8008' + parsed_host.path api_host = parsed_host.scheme + "://" + parsed_host.netloc + ":8008" + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8081' + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc + ":8081" + parsed_host.path
else: else:
print("Warning! Could not parse host name") print("Warning! Could not parse host name")
api_host = '' api_host = ""
web_host = '' web_host = ""
files_host = '' files_host = ""
return api_host, files_host, web_host return api_host, files_host, web_host
@ -256,18 +266,25 @@ def verify_credentials(api_host, credentials):
"""check if the credentials are valid""" """check if the credentials are valid"""
# noinspection PyBroadException # noinspection PyBroadException
try: try:
print('Verifying credentials ...') print("Verifying credentials ...")
if api_host: if api_host:
Session(api_key=credentials['access_key'], secret_key=credentials['secret_key'], host=api_host, Session(
http_retries_config={"total": 2}) api_key=credentials["access_key"],
print('Credentials verified!') secret_key=credentials["secret_key"],
host=api_host,
http_retries_config={"total": 2},
)
print("Credentials verified!")
return True return True
else: else:
print("Can't verify credentials") print("Can't verify credentials")
return False return False
except Exception: except Exception:
print('Error: could not verify credentials: key={} secret={}'.format( print(
credentials.get('access_key'), credentials.get('secret_key'))) "Error: could not verify credentials: key={} secret={}".format(
credentials.get("access_key"), credentials.get("secret_key")
)
)
return False return False
@ -292,18 +309,18 @@ def get_parsed_field(parsed_config, fields):
def read_manual_credentials(): def read_manual_credentials():
print('Enter user access key: ', end='') print("Enter user access key: ", end="")
access_key = input() access_key = input()
print('Enter user secret: ', end='') print("Enter user secret: ", end="")
secret_key = input() secret_key = input()
return {"access_key": access_key, "secret_key": secret_key} return {"access_key": access_key, "secret_key": secret_key}
def input_url(host_type, host=None): def input_url(host_type, host=None):
while True: while True:
print('{} configured to: {}'.format(host_type, '[{}] '.format(host) if host else ''), end='') print("{} configured to: {}".format(host_type, "[{}] ".format(host) if host else ""), end="")
parse_input = input() parse_input = input()
if host and (not parse_input or parse_input.lower() == 'yes' or parse_input.lower() == 'y'): if host and (not parse_input or parse_input.lower() == "yes" or parse_input.lower() == "y"):
break break
parsed_host = verify_url(parse_input) if parse_input else None parsed_host = verify_url(parse_input) if parse_input else None
if parse_input and parsed_host: if parse_input and parsed_host:
@ -313,29 +330,34 @@ def input_url(host_type, host=None):
def input_host_port(host_type, parsed_host): def input_host_port(host_type, parsed_host):
print('Enter port for {} host '.format(host_type), end='') print("Enter port for {} host ".format(host_type), end="")
replace_port = input().lower() replace_port = input().lower()
return parsed_host.scheme + "://" + parsed_host.netloc + ( return (
':{}'.format(replace_port) if replace_port else '') + parsed_host.path parsed_host.scheme
+ "://"
+ parsed_host.netloc
+ (":{}".format(replace_port) if replace_port else "")
+ parsed_host.path
)
def verify_url(parse_input): def verify_url(parse_input):
# noinspection PyBroadException # noinspection PyBroadException
try: try:
if not parse_input.startswith('http://') and not parse_input.startswith('https://'): if not parse_input.startswith("http://") and not parse_input.startswith("https://"):
# if we have a specific port, use http prefix, otherwise assume https # if we have a specific port, use http prefix, otherwise assume https
if ':' in parse_input: if ":" in parse_input:
parse_input = 'http://' + parse_input parse_input = "http://" + parse_input
else: else:
parse_input = 'https://' + parse_input parse_input = "https://" + parse_input
parsed_host = urlparse(parse_input) parsed_host = urlparse(parse_input)
if parsed_host.scheme not in ('http', 'https'): if parsed_host.scheme not in ("http", "https"):
parsed_host = None parsed_host = None
except Exception: except Exception:
parsed_host = None parsed_host = None
print('Could not parse url {}\nEnter your clearml-server host: '.format(parse_input), end='') print("Could not parse url {}\nEnter your clearml-server host: ".format(parse_input), end="")
return parsed_host return parsed_host
if __name__ == '__main__': if __name__ == "__main__":
main() main()