From 9c14f627b1b0418f2cd8f0d9ff4be3118451225c Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Thu, 18 Nov 2021 10:43:46 +0200 Subject: [PATCH] Change --debug to --verbose and added remote session setup progress print by default Update default vscode_version to 3.12.0 and python_ext_version to 2021.10.1365161279 Add message status reporting, fix k8s support Add support for Task runtime properties --- clearml_session/__main__.py | 244 +++++++++++++++----- clearml_session/interactive_session_task.py | 120 +++++++--- 2 files changed, 271 insertions(+), 93 deletions(-) diff --git a/clearml_session/__main__.py b/clearml_session/__main__.py index 804cfd9..87990bf 100644 --- a/clearml_session/__main__.py +++ b/clearml_session/__main__.py @@ -1,3 +1,4 @@ +import base64 import hashlib import json import logging @@ -20,6 +21,7 @@ import psutil from clearml import Task from clearml.backend_api.session.client import APIClient from clearml.config import config_obj +from clearml.backend_api import Session from .tcp_proxy import TcpProxy from .single_thread_proxy import SingleThreadProxy @@ -136,35 +138,59 @@ def create_base_task(state, project_name=None, task_name=None): task = Task.create(project_name=project_name or 'DevOps', task_name=task_name or 'Interactive Session', task_type=Task.TaskTypes.application) - task_state = task.export_task() + task_script = task.data.script.to_dict() base_script_file = os.path.abspath(os.path.join(__file__, '..', 'tcp_proxy.py')) with open(base_script_file, 'rt') as f: - task_state['script']['diff'] = f.read() + task_script['diff'] = f.read() base_script_file = os.path.abspath(os.path.join(__file__, '..', 'interactive_session_task.py')) with open(base_script_file, 'rt') as f: - task_state['script']['diff'] += '\n\n' + f.read() + task_script['diff'] += '\n\n' + f.read() - task_state['script']['working_dir'] = '.' - task_state['script']['entry_point'] = 'interactive_session.py' - task_state['script']['requirements'] = {'pip': '\n'.join( + task_script['working_dir'] = '.' + task_script['entry_point'] = 'interactive_session.py' + task_script['requirements'] = {'pip': '\n'.join( ["clearml"] + (["jupyter", "jupyterlab", "jupyterlab_git"] if state.get('jupyter_lab') else []) + (['pylint'] if state.get('vscode_server') else []))} - task.update_task(task_state) + section, _, _ = _get_config_section_name() - task.set_parameters({ - "{}/user_base_directory".format(section): "~/", - "{}/ssh_server".format(section): True, - "{}/ssh_password".format(section): "training", - "{}/default_docker".format(section): "nvidia/cuda", - "{}/user_key".format(section): '', - "{}/user_secret".format(section): '', - "properties/external_address": '', - "properties/internal_ssh_port": '', - "properties/jupyter_token": '', - "properties/jupyter_port": '', - }) + + if Session.check_min_api_version('2.13'): + _runtime_prop = dict(task._get_runtime_properties()) + _runtime_prop.update({ + "_user_key": '', + "_user_secret": '', + "_jupyter_token": '', + "_ssh_password": "training", + }) + # noinspection PyProtectedMember + task._set_runtime_properties(_runtime_prop) + task.set_parameters({ + "{}/user_base_directory".format(section): "~/", + "{}/ssh_server".format(section): True, + "{}/default_docker".format(section): "nvidia/cuda", + "properties/external_address": '', + "properties/internal_ssh_port": '', + "properties/jupyter_port": '', + }) + else: + task.set_parameters({ + "{}/user_base_directory".format(section): "~/", + "{}/ssh_server".format(section): True, + "{}/ssh_password".format(section): "training", + "{}/default_docker".format(section): "nvidia/cuda", + "{}/user_key".format(section): '', + "{}/user_secret".format(section): '', + "properties/external_address": '', + "properties/internal_ssh_port": '', + "properties/jupyter_token": '', + "properties/jupyter_port": '', + }) + task.set_system_tags([system_tag]) - task.reset(force=True) + + # only update the data at the end, so reload requests are smaller + # noinspection PyProtectedMember + task._edit(script=task_script) return task @@ -197,18 +223,38 @@ def create_debugging_task(state, debug_task_id): (['pylint'] if state.get('vscode_server') else []) task.update_task(task_state) section, _, _ = _get_config_section_name() - task.set_parameters({ - "{}/user_base_directory".format(section): "~/", - "{}/ssh_server".format(section): True, - "{}/ssh_password".format(section): "training", - "{}/default_docker".format(section): "nvidia/cuda", - "{}/user_key".format(section): '', - "{}/user_secret".format(section): '', - "properties/external_address": '', - "properties/internal_ssh_port": '', - "properties/jupyter_token": '', - "properties/jupyter_port": '', - }) + + if Session.check_min_api_version('2.13'): + _runtime_prop = dict(task._get_runtime_properties()) + _runtime_prop.update({ + "_user_key": '', + "_user_secret": '', + "_jupyter_token": '', + "_ssh_password": "training", + }) + # noinspection PyProtectedMember + task._set_runtime_properties(_runtime_prop) + task.set_parameters({ + "{}/user_base_directory".format(section): "~/", + "{}/ssh_server".format(section): True, + "{}/default_docker".format(section): "nvidia/cuda", + "properties/external_address": '', + "properties/internal_ssh_port": '', + "properties/jupyter_port": '', + }) + else: + task.set_parameters({ + "{}/user_base_directory".format(section): "~/", + "{}/ssh_server".format(section): True, + "{}/ssh_password".format(section): "training", + "{}/default_docker".format(section): "nvidia/cuda", + "{}/user_key".format(section): '', + "{}/user_secret".format(section): '', + "properties/external_address": '', + "properties/internal_ssh_port": '', + "properties/jupyter_token": '', + "properties/jupyter_port": '', + }) task.set_system_tags([system_tag]) task.reset(force=True) return task @@ -265,6 +311,17 @@ def _get_user_id(client): return current_user_id +def _b64_encode_file(file): + # noinspection PyBroadException + try: + import gzip + with open(file, 'rt') as f: + git_credentials = gzip.compress(f.read().encode('utf8')) + return base64.encodebytes(git_credentials).decode('ascii') + except Exception: + return None + + def get_project_id(state): project_id = None project_name = state.get('project') or None @@ -383,8 +440,8 @@ def load_state(state_file): state = json.load(f) except Exception: state = {} - # never reload --debug state - state.pop('debug', None) + # never reload --verbose state + state.pop('verbose', None) return state @@ -402,13 +459,31 @@ def clone_task(state, project_id): task = create_base_task(state, project_name=state.get('project')) new_task = True + print('Configuring new session') + runtime_prop_support = Session.check_min_api_version("2.13") + if runtime_prop_support: + # noinspection PyProtectedMember + runtime_properties = dict(task._get_runtime_properties() or {}) + runtime_properties['_jupyter_token'] = '' + runtime_properties['_ssh_password'] = str(state['password']) + runtime_properties['_user_key'] = str(config_obj.get("api.credentials.access_key")) + runtime_properties['_user_secret'] = (config_obj.get("api.credentials.secret_key")) + # noinspection PyProtectedMember + task._set_runtime_properties(runtime_properties) + task_params = task.get_parameters(backwards_compatibility=False) if 'General/ssh_server' in task_params: section = 'General' init_section = 'init_script' else: section, _, init_section = _get_config_section_name() - task_params['properties/jupyter_token'] = '' + + if not runtime_prop_support: + task_params['properties/jupyter_token'] = '' + task_params['{}/ssh_password'.format(section)] = state['password'] + task_params['{}/user_key'.format(section)] = config_obj.get("api.credentials.access_key") + task_params['{}/user_secret'.format(section)] = config_obj.get("api.credentials.secret_key") + task_params['properties/jupyter_port'] = '' if state.get('remote_gateway') is not None: remote_gateway_parts = str(state.get('remote_gateway')).split(':') @@ -416,9 +491,6 @@ def clone_task(state, project_id): if len(remote_gateway_parts) > 1: task_params['properties/external_ssh_port'] = remote_gateway_parts[1] task_params['{}/ssh_server'.format(section)] = str(True) - task_params['{}/ssh_password'.format(section)] = state['password'] - task_params['{}/user_key'.format(section)] = config_obj.get("api.credentials.access_key") - task_params['{}/user_secret'.format(section)] = config_obj.get("api.credentials.secret_key") task_params["{}/jupyterlab".format(section)] = bool(state.get('jupyter_lab')) task_params["{}/vscode_server".format(section)] = bool(state.get('vscode_server')) task_params["{}/public_ip".format(section)] = bool(state.get('public_ip')) @@ -443,13 +515,30 @@ def clone_task(state, project_id): # store the .git-credentials if state.get('git_credentials'): git_cred_file = os.path.join(os.path.expanduser('~'), '.git-credentials') - if os.path.isfile(git_cred_file): - task.connect_configuration( - configuration=git_cred_file, name='git_credentials', description='git credentials') git_conf_file = os.path.join(os.path.expanduser('~'), '.gitconfig') - if os.path.isfile(git_conf_file): - task.connect_configuration( - configuration=git_conf_file, name='git_config', description='git config') + if not os.path.isfile(git_cred_file): + git_cred_file = None + if not os.path.isfile(git_conf_file): + git_conf_file = None + + if runtime_prop_support: + # noinspection PyProtectedMember + runtime_properties = dict(task._get_runtime_properties() or {}) + if git_cred_file: + runtime_properties['_git_credentials'] = _b64_encode_file(git_cred_file) + if git_conf_file: + runtime_properties['_git_config'] = _b64_encode_file(git_conf_file) + # store back + if git_cred_file or git_conf_file: + # noinspection PyProtectedMember + task._set_runtime_properties(runtime_properties) + else: + if git_cred_file: + task.connect_configuration( + configuration=git_cred_file, name='git_credentials', description='git credentials') + if git_conf_file: + task.connect_configuration( + configuration=git_conf_file, name='git_config', description='git config') if state.get('packages'): requirements = task.data.script.requirements or {} @@ -476,21 +565,28 @@ def wait_for_machine(state, task): # wait until task is running print('Waiting for remote machine allocation [id={}]'.format(task.id)) last_status = None - while last_status != 'in_progress' and last_status in (None, 'created', 'queued', 'unknown',): + last_message = None + stopped_counter = 0 + while last_status != 'in_progress' and last_status in (None, 'created', 'queued', 'unknown', 'stopped'): print('.', end='', flush=True) if last_status is not None: sleep(2.) - status = task.get_status() - if last_status != status: + stopped_counter = (stopped_counter+1) if last_status == 'stopped' else 0 + if stopped_counter > 5: + break + # noinspection PyProtectedMember + status, message = task._get_status() + status = str(status) + if last_status != status or last_message != message: # noinspection PyProtectedMember - last_status = task._get_status()[1] - print('Status [{}]{}'.format(status, ' - {}'.format(last_status) if last_status else '')) + print('Status [{}]{} {}'.format(status, ' - {}'.format(last_status) if last_status else '', message)) last_status = status + last_message = message print('Remote machine allocated') print('Setting remote environment [Task id={}]'.format(task.id)) print('Setup process details: {}'.format(task.get_output_log_web_page())) - print('Waiting for environment setup to complete [usually about 20-30 seconds]') + print('Waiting for environment setup to complete [usually about 20-30 seconds, see last log line/s below]') # monitor progress, until we get the new jupyter, then we know it is working task.reload() @@ -513,24 +609,38 @@ def wait_for_machine(state, task): last_lines = [] period_counter = 0 while any(bool(not task.get_parameter(p)) for p in wait_properties) and task.get_status() == 'in_progress': - lines = task.get_reported_console_output(10) if state.get('debug') else [] + lines = task.get_reported_console_output(10 if state.get('verbose') else 1) if last_lines != lines: # new line if we had '.' counter in the previous run if period_counter: - print('') + if state.get('verbose'): + print('') period_counter = 0 try: index = next(i for i, line in enumerate(lines) if last_lines and line == last_lines[-1]) - print('> ' + ''.join(lines[index+1:]).rstrip().replace('\n', '\n> ')) + print_line = '> ' + ''.join(lines[index+1:]).rstrip().replace('\n', '\n> ') except StopIteration: - print('> ' + ''.join(lines).rstrip().replace('\n', '\n> ')) + print_line = '> ' + ''.join(lines).rstrip().replace('\n', '\n> ') + + if state.get('verbose'): + print(print_line) + else: + print_line = [l for l in print_line.split('\n') if l.rstrip()] + if print_line: + print('\r' + print_line[-1], end='', flush=True) last_lines = lines else: - print('.', end='', flush=True) period_counter += 1 + print(('' if state.get('verbose') else '\r') + '.'*period_counter, end='', flush=True) sleep(3.) task.reload() + + # clear the line + if not state.get('verbose'): + print('\r ', end='', flush=True) + print('\n') + if task.get_status() != 'in_progress': raise ValueError("Remote setup failed (status={}) see details: {}".format( task.get_status(), task.get_output_log_web_page())) @@ -628,14 +738,21 @@ def monitor_ssh_tunnel(state, task): if not all([ssh_port, jupyter_token, jupyter_port, internal_ssh_port, ssh_password, remote_address]): task.reload() task_parameters = task.get_parameters() - section = 'General' if 'General/ssh_server' in task_parameters else default_section + if Session.check_min_api_version("2.13"): + # noinspection PyProtectedMember + runtime_prop = task._get_runtime_properties() + ssh_password = runtime_prop.get('_ssh_password') or state.get('password', '') + jupyter_token = runtime_prop.get('_jupyter_token') + else: + section = 'General' if 'General/ssh_server' in task_parameters else default_section + ssh_password = task_parameters.get('{}/ssh_password'.format(section)) or state.get('password', '') + jupyter_token = task_parameters.get('properties/jupyter_token') + remote_address = \ task_parameters.get('properties/k8s-gateway-address') or \ task_parameters.get('properties/external_address') - ssh_password = task_parameters.get('{}/ssh_password'.format(section)) or state.get('password', '') internal_ssh_port = task_parameters.get('properties/internal_ssh_port') jupyter_port = task_parameters.get('properties/jupyter_port') - jupyter_token = task_parameters.get('properties/jupyter_token') ssh_port = \ task_parameters.get('properties/k8s-pod-port') or \ task_parameters.get('properties/external_ssh_port') or internal_ssh_port @@ -671,7 +788,7 @@ def monitor_ssh_tunnel(state, task): state.get('username') or 'root', remote_address, ssh_port, ssh_password, local_remote_pair_list=local_remote_pair_list, - debug=state.get('debug', False), + debug=state.get('verbose', False), ) if ssh_process and ssh_process.isalive(): @@ -826,8 +943,9 @@ def setup_parser(parser): parser.add_argument('--username', type=str, default=None, help='Advanced: Select ssh username for the interactive session ' '(default: `root` or previously used one)') - parser.add_argument('--debug', action='store_true', default=None, - help='Advanced: If set, print debugging information') + parser.add_argument('--verbose', action='store_true', default=None, + help='Advanced: If set, print verbose progress information, ' + 'e.g. the remote machine setup process log') def get_version(): @@ -862,8 +980,8 @@ def cli(): state_file = os.path.abspath(os.path.expandvars(os.path.expanduser(args.config_file))) state = load_state(state_file) - if args.debug: - state['debug'] = args.debug + if args.verbose: + state['verbose'] = args.verbose client = APIClient() diff --git a/clearml_session/interactive_session_task.py b/clearml_session/interactive_session_task.py index dbfbc1d..5b581d0 100644 --- a/clearml_session/interactive_session_task.py +++ b/clearml_session/interactive_session_task.py @@ -1,3 +1,4 @@ +import base64 import json import os import socket @@ -13,6 +14,7 @@ import psutil from pathlib2 import Path from clearml import Task, StorageManager +from clearml.backend_api import Session # noinspection SpellCheckingInspection @@ -97,7 +99,25 @@ def init_task(param, a_default_ssh_fingerprint): project_name="DevOps", task_name="Allocate Jupyter Notebook Instance", task_type=Task.TaskTypes.service) # Add jupyter server base folder - task.connect(param, name=config_section_name) + if Session.check_min_api_version('2.13'): + param.pop('user_key', None) + param.pop('user_secret', None) + param.pop('ssh_password', None) + task.connect(param, name=config_section_name) + # noinspection PyProtectedMember + runtime_prop = dict(task._get_runtime_properties()) + # remove the user key/secret the moment we have it + param['user_key'] = runtime_prop.pop('_user_key', None) + param['user_secret'] = runtime_prop.pop('_user_secret', None) + # no need to reset, we will need it + param['ssh_password'] = runtime_prop.get('_ssh_password') + # Force removing properties + # noinspection PyProtectedMember + task._edit(runtime=runtime_prop) + task.reload() + else: + task.connect(param, name=config_section_name) + # connect ssh finger print configuration (with fallback if section is missing) old_default_ssh_fingerprint = deepcopy(a_default_ssh_fingerprint) try: @@ -123,16 +143,17 @@ def setup_os_env(param): "_API_SECRET_KEY", "_API_HOST_VERIFY_CERT", "_DOCKER_IMAGE", + "_DOCKER_BASH_SCRIPT", ) # set default docker image, with network configuration if param.get('default_docker', '').strip(): - os.environ["TRAINS_DOCKER_IMAGE"] = param['default_docker'].strip() os.environ["CLEARML_DOCKER_IMAGE"] = param['default_docker'].strip() # setup os environment env = deepcopy(os.environ) for key in os.environ: - if (key.startswith("TRAINS") or key.startswith("CLEARML")) and not any(key.endswith(p) for p in preserve): + # only set CLEARML_ remove any TRAINS_ + if key.startswith("TRAINS") or (key.startswith("CLEARML") and not any(key.endswith(p) for p in preserve)): env.pop(key, None) return env @@ -188,8 +209,7 @@ def monitor_jupyter_server(fd, local_filename, process, task, jupyter_port, host # we could not locate the token, try again if not token: continue - # update the task with the correct links and token - task.set_parameter(name='properties/jupyter_token', value=str(token)) + # we ignore the reported port, because jupyter server will get confused # if we have multiple servers running and will point to the wrong port/server task.set_parameter(name='properties/jupyter_port', value=str(jupyter_port)) @@ -197,8 +217,20 @@ def monitor_jupyter_server(fd, local_filename, process, task, jupyter_port, host 'https' if "https://" in line else 'http', hostnames, jupyter_port, token ) + + # update the task with the correct links and token + if Session.check_min_api_version("2.13"): + # noinspection PyProtectedMember + runtime_prop = task._get_runtime_properties() + runtime_prop['_jupyter_token'] = str(token) + runtime_prop['_jupyter_url'] = str(jupyter_url) + # noinspection PyProtectedMember + task._set_runtime_properties(runtime_prop) + else: + task.set_parameter(name='properties/jupyter_token', value=str(token)) + task.set_parameter(name='properties/jupyter_url', value=jupyter_url) + print('\nJupyter Lab URL: {}\n'.format(jupyter_url)) - task.set_parameter(name='properties/jupyter_url', value=jupyter_url) # cleanup # noinspection PyBroadException @@ -219,8 +251,8 @@ def start_vscode_server(hostname, hostnames, param, task, env): # get vscode version and python extension version # they are extremely flaky, this combination works, most do not. - vscode_version = '3.9.2' - python_ext_version = '2021.3.658691958' + vscode_version = '3.12.0' + python_ext_version = '2021.10.1365161279' if param.get("vscode_version"): vscode_version_parts = param.get("vscode_version").split(':') vscode_version = vscode_version_parts[0] @@ -291,7 +323,7 @@ def start_vscode_server(hostname, hostnames, param, task, env): "--extensions-dir", exts_folder, "--install-extension", "ms-toolsai.jupyter", # "--install-extension", "donjayamanne.python-extension-pack" - ] + ["--install-extension", python_ext] if python_ext else [], + ] + ["--install-extension", "ms-python.python@{}".format(python_ext_version)] if python_ext else [], env=env, stdout=fd, stderr=fd, @@ -343,7 +375,7 @@ def start_jupyter_server(hostname, hostnames, param, task, env): print('no jupyterlab to monitor - going to sleep') while True: sleep(10.) - return + return # noqa # execute jupyter notebook fd, local_filename = mkstemp() @@ -418,15 +450,15 @@ def setup_ssh_server(hostname, hostnames, param, task): "&& " # noqa: W605 "echo 'ClientAliveInterval 10' >> /etc/ssh/sshd_config && " "echo 'ClientAliveCountMax 20' >> /etc/ssh/sshd_config && " - "echo 'AcceptEnv TRAINS_API_ACCESS_KEY TRAINS_API_SECRET_KEY " + "echo 'AcceptEnv CLEARML_API_ACCESS_KEY CLEARML_API_SECRET_KEY " "CLEARML_API_ACCESS_KEY CLEARML_API_SECRET_KEY' >> /etc/ssh/sshd_config && " 'echo "export VISIBLE=now" >> /etc/profile && ' 'echo "export PATH=$PATH" >> /etc/profile && ' 'echo "ldconfig" >> /etc/profile && ' - 'echo "export TRAINS_CONFIG_FILE={trains_config_file}" >> /etc/profile'.format( + 'echo "export CLEARML_CONFIG_FILE={trains_config_file}" >> /etc/profile'.format( password=ssh_password, port=port, - trains_config_file=os.environ.get("CLEARML_CONFIG_FILE") or os.environ.get("TRAINS_CONFIG_FILE"), + trains_config_file=os.environ.get("CLEARML_CONFIG_FILE") or os.environ.get("CLEARML_CONFIG_FILE"), ) ) sshd_path = '/usr/sbin/sshd' @@ -449,7 +481,7 @@ def setup_ssh_server(hostname, hostnames, param, task): "UsePAM yes" + "\n"\ "AuthorizedKeysFile {}".format(os.path.join(ssh_config_path, 'authorized_keys')) + "\n"\ "PidFile {}".format(os.path.join(ssh_config_path, 'sshd.pid')) + "\n"\ - "AcceptEnv TRAINS_API_ACCESS_KEY TRAINS_API_SECRET_KEY "\ + "AcceptEnv CLEARML_API_ACCESS_KEY CLEARML_API_SECRET_KEY "\ "CLEARML_API_ACCESS_KEY CLEARML_API_SECRET_KEY"+"\n" for k in default_ssh_fingerprint: filename = os.path.join(ssh_config_path, '{}'.format(k.replace('__pub', '.pub'))) @@ -511,13 +543,23 @@ def setup_ssh_server(hostname, hostnames, param, task): print("Error: {}\n\n#\n# Error: SSH server could not be launched\n#\n".format(ex)) +def _b64_decode_file(encoded_string): + # noinspection PyBroadException + try: + import gzip + value = gzip.decompress(base64.decodebytes(encoded_string.encode('ascii'))).decode('utf8') + return value + except Exception: + return None + + def setup_user_env(param, task): env = setup_os_env(param) # do not change user bash/profile if os.geteuid() != 0: if param.get("user_key") and param.get("user_secret"): - env['TRAINS_API_ACCESS_KEY'] = param.get("user_key") - env['TRAINS_API_SECRET_KEY'] = param.get("user_secret") + env['CLEARML_API_ACCESS_KEY'] = param.get("user_key") + env['CLEARML_API_SECRET_KEY'] = param.get("user_secret") return env # create symbolic link to the venv @@ -530,20 +572,20 @@ def setup_user_env(param, task): pass # set default user credentials if param.get("user_key") and param.get("user_secret"): - os.system("echo 'export TRAINS_API_ACCESS_KEY=\"{}\"' >> ~/.bashrc".format( + os.system("echo 'export CLEARML_API_ACCESS_KEY=\"{}\"' >> ~/.bashrc".format( param.get("user_key", "").replace('$', '\\$'))) - os.system("echo 'export TRAINS_API_SECRET_KEY=\"{}\"' >> ~/.bashrc".format( + os.system("echo 'export CLEARML_API_SECRET_KEY=\"{}\"' >> ~/.bashrc".format( param.get("user_secret", "").replace('$', '\\$'))) - os.system("echo 'export TRAINS_DOCKER_IMAGE=\"{}\"' >> ~/.bashrc".format( - param.get("default_docker", "").strip() or env.get('TRAINS_DOCKER_IMAGE', ''))) - os.system("echo 'export TRAINS_API_ACCESS_KEY=\"{}\"' >> ~/.profile".format( + os.system("echo 'export CLEARML_DOCKER_IMAGE=\"{}\"' >> ~/.bashrc".format( + param.get("default_docker", "").strip() or env.get('CLEARML_DOCKER_IMAGE', ''))) + os.system("echo 'export CLEARML_API_ACCESS_KEY=\"{}\"' >> ~/.profile".format( param.get("user_key", "").replace('$', '\\$'))) - os.system("echo 'export TRAINS_API_SECRET_KEY=\"{}\"' >> ~/.profile".format( + os.system("echo 'export CLEARML_API_SECRET_KEY=\"{}\"' >> ~/.profile".format( param.get("user_secret", "").replace('$', '\\$'))) - os.system("echo 'export TRAINS_DOCKER_IMAGE=\"{}\"' >> ~/.profile".format( - param.get("default_docker", "").strip() or env.get('TRAINS_DOCKER_IMAGE', ''))) - env['TRAINS_API_ACCESS_KEY'] = param.get("user_key") - env['TRAINS_API_SECRET_KEY'] = param.get("user_secret") + os.system("echo 'export CLEARML_DOCKER_IMAGE=\"{}\"' >> ~/.profile".format( + param.get("default_docker", "").strip() or env.get('CLEARML_DOCKER_IMAGE', ''))) + env['CLEARML_API_ACCESS_KEY'] = param.get("user_key") + env['CLEARML_API_SECRET_KEY'] = param.get("user_secret") # set default folder for user if param.get("user_base_directory"): base_dir = param.get("user_base_directory") @@ -557,8 +599,27 @@ def setup_user_env(param, task): os.system("echo '. {}' >> ~/.profile".format(os.path.join(environment, 'bin', 'activate'))) # check if we need to create .git-credentials - # noinspection PyProtectedMember - git_credentials = task._get_configuration_text('git_credentials') + + runtime_property_support = Session.check_min_api_version("2.13") + if runtime_property_support: + # noinspection PyProtectedMember + runtime_prop = dict(task._get_runtime_properties()) + git_credentials = runtime_prop.pop('_git_credentials', None) + git_config = runtime_prop.pop('_git_config', None) + # force removing properties + # noinspection PyProtectedMember + task._edit(runtime=runtime_prop) + task.reload() + if git_credentials is not None: + git_credentials = _b64_decode_file(git_credentials) + if git_config is not None: + git_config = _b64_decode_file(git_config) + else: + # noinspection PyProtectedMember + git_credentials = task._get_configuration_text('git_credentials') + # noinspection PyProtectedMember + git_config = task._get_configuration_text('git_config') + if git_credentials: git_cred_file = os.path.expanduser('~/.config/git/credentials') # noinspection PyBroadException @@ -568,8 +629,7 @@ def setup_user_env(param, task): f.write(git_credentials) except Exception: print('Could not write {} file'.format(git_cred_file)) - # noinspection PyProtectedMember - git_config = task._get_configuration_text('git_config') + if git_config: git_config_file = os.path.expanduser('~/.config/git/config') # noinspection PyBroadException