From 56d4de04e4e23af632a1410a694d0f73f1d41fd1 Mon Sep 17 00:00:00 2001 From: pollfly <75068813+pollfly@users.noreply.github.com> Date: Tue, 30 May 2023 15:40:43 +0300 Subject: [PATCH 1/5] Edit docstrings (#1024) --- clearml/logger.py | 34 +++++++++++++++++++--------------- clearml/task.py | 34 +++++++++++++++++----------------- 2 files changed, 36 insertions(+), 32 deletions(-) diff --git a/clearml/logger.py b/clearml/logger.py index 884136a3..1c4159f6 100644 --- a/clearml/logger.py +++ b/clearml/logger.py @@ -309,7 +309,7 @@ class Logger(object): extra_data=None, # type: Optional[dict] ): """ - For explicit report, report a table plot. + For explicit reporting, report a table plot. One and only one of the following parameters must be provided. @@ -340,25 +340,29 @@ class Logger(object): See full details on the supported configuration: https://plotly.com/javascript/reference/layout/ For example: - .. code block:: py - logger.report_table( - title='table example', - series='pandas DataFrame', - iteration=0, - table_plot=df, - extra_layout={'height': 600}) + .. code-block:: py + + logger.report_table( + title='table example', + series='pandas DataFrame', + iteration=0, + table_plot=df, + extra_layout={'height': 600} + ) :param extra_data: optional dictionary for data configuration, like column width, passed directly to plotly See full details on the supported configuration: https://plotly.com/javascript/reference/table/ For example: - .. code block:: py - logger.report_table( - title='table example', - series='pandas DataFrame', - iteration=0, - table_plot=df, - extra_data={'columnwidth': [2., 1., 1., 1.]}) + .. code-block:: py + + logger.report_table( + title='table example', + series='pandas DataFrame', + iteration=0, + table_plot=df, + extra_data={'columnwidth': [2., 1., 1., 1.]} + ) """ mutually_exclusive( diff --git a/clearml/task.py b/clearml/task.py index 5e72e3e5..a876b636 100644 --- a/clearml/task.py +++ b/clearml/task.py @@ -1703,30 +1703,30 @@ class Task(_Task): :param total_num_nodes: The total number of nodes to be enqueued, including the master node, which should already be enqueued when running remotely - :param port: Port opened by the master node. If the environment variable `CLEARML_MULTI_NODE_MASTER_DEF_PORT` - is set, the value of this parameter will be set to the one defined in `CLEARML_MULTI_NODE_MASTER_DEF_PORT`. - If `CLEARML_MULTI_NODE_MASTER_DEF_PORT` doesn't exist, but `MASTER_PORT` does, then the value of this - parameter will be set to the one defined in `MASTER_PORT`. If neither environment variables exist, + :param port: Port opened by the master node. If the environment variable ``CLEARML_MULTI_NODE_MASTER_DEF_PORT`` + is set, the value of this parameter will be set to the one defined in ``CLEARML_MULTI_NODE_MASTER_DEF_PORT``. + If ``CLEARML_MULTI_NODE_MASTER_DEF_PORT`` doesn't exist, but ``MASTER_PORT`` does, then the value of this + parameter will be set to the one defined in ``MASTER_PORT``. If neither environment variables exist, the value passed to the parameter will be used - :param queue: The queue to enqueue the nodes to. Can be different than the queue the master + :param queue: The queue to enqueue the nodes to. Can be different from the queue the master node is enqueued to. If None, the nodes will be enqueued to the same queue as the master node :param wait: If True, the master node will wait for the other nodes to start :param addr: The address of the master node's worker. If the environment variable - `CLEARML_MULTI_NODE_MASTER_DEF_ADDR` is set, the value of this parameter will be set to - the one defined in `CLEARML_MULTI_NODE_MASTER_DEF_ADDR`. - If `CLEARML_MULTI_NODE_MASTER_DEF_ADDR` doesn't exist, but `MASTER_ADDR` does, then the value of this - parameter will be set to the one defined in `MASTER_ADDR`. If neither environment variables exist, + ``CLEARML_MULTI_NODE_MASTER_DEF_ADDR`` is set, the value of this parameter will be set to + the one defined in ``CLEARML_MULTI_NODE_MASTER_DEF_ADDR``. + If ``CLEARML_MULTI_NODE_MASTER_DEF_ADDR`` doesn't exist, but ``MASTER_ADDR`` does, then the value of this + parameter will be set to the one defined in ``MASTER_ADDR``. If neither environment variables exist, the value passed to the parameter will be used. If this value is None (default), the private IP of the machine the master node is running on will be used. - :return: A dictionary containing relevant information regarding the multi node run. This dictionary - has the following entries: - - `master_addr` - the address of the machine that the master node is running on - - `master_port` - the open port of the machine that the master node is running on - - `total_num_nodes` - the total number of nodes, including the master - - `queue` - the queue the nodes are enqueued to, excluding the master - - `node_rank` - the rank of the current node (master has rank 0) - - `wait` - if True, the master node will wait for the other nodes to start + :return: A dictionary containing relevant information regarding the multi node run. This dictionary has the following entries: + + - `master_addr` - the address of the machine that the master node is running on + - `master_port` - the open port of the machine that the master node is running on + - `total_num_nodes` - the total number of nodes, including the master + - `queue` - the queue the nodes are enqueued to, excluding the master + - `node_rank` - the rank of the current node (master has rank 0) + - `wait` - if True, the master node will wait for the other nodes to start """ def set_launch_multi_node_runtime_props(task, conf): # noinspection PyProtectedMember From f207e722002b418828f67b629efc569a3d246347 Mon Sep 17 00:00:00 2001 From: Alex Burlacu Date: Thu, 1 Jun 2023 15:12:30 +0300 Subject: [PATCH 2/5] Fix jsonargparse not loading new values from changed config files --- clearml/binding/jsonargs_bind.py | 108 ++++++++++++++++++++++--------- 1 file changed, 78 insertions(+), 30 deletions(-) diff --git a/clearml/binding/jsonargs_bind.py b/clearml/binding/jsonargs_bind.py index 6094de23..9cedebf8 100644 --- a/clearml/binding/jsonargs_bind.py +++ b/clearml/binding/jsonargs_bind.py @@ -4,7 +4,7 @@ import logging try: from jsonargparse import ArgumentParser from jsonargparse.namespace import Namespace - from jsonargparse.util import Path + from jsonargparse.util import Path, change_to_path_dir except ImportError: ArgumentParser = None @@ -30,6 +30,7 @@ class PatchJsonArgParse(object): _command_name = "subcommand" _section_name = "Args" __remote_task_params = {} + __remote_task_params_dict = {} __patched = False @classmethod @@ -54,7 +55,7 @@ class PatchJsonArgParse(object): ) @classmethod - def _update_task_args(cls): + def _update_task_args(cls, parser=None, subcommand=None): if running_remotely() or not cls._current_task or not cls._args: return args = {} @@ -65,7 +66,7 @@ class PatchJsonArgParse(object): if k in cls._args_type: args_type[key_with_section] = cls._args_type[k] continue - if not verify_basic_type(v) and v: + if not verify_basic_type(v, basic_types=(float, int, bool, str, type(None))) and v: # noinspection PyBroadException try: if isinstance(v, Namespace) or (isinstance(v, list) and all(isinstance(sub_v, Namespace) for sub_v in v)): @@ -78,8 +79,24 @@ class PatchJsonArgParse(object): args[key_with_section] = str(v) except Exception: pass + args, args_type = cls.__delete_config_args(parser, args, args_type, subcommand=subcommand) cls._current_task._set_parameters(args, __update=True, __parameters_types=args_type) + @classmethod + def __delete_config_args(cls, parser, args, args_type, subcommand=None): + if not parser: + return args, args_type + paths = PatchJsonArgParse.__get_paths_from_dict(cls._args) + for path in paths: + args_to_delete = PatchJsonArgParse.__get_args_from_path(parser, path, subcommand=subcommand) + for arg_to_delete_key, arg_to_delete_value in args_to_delete.items(): + key_with_section = cls._section_name + cls._args_sep + arg_to_delete_key + if key_with_section in args and args[key_with_section] == arg_to_delete_value: + del args[key_with_section] + if key_with_section in args_type: + del args_type[key_with_section] + return args, args_type + @staticmethod def _adapt_typehints(original_fn, val, *args, **kwargs): if not PatchJsonArgParse._current_task or not running_remotely(): @@ -97,7 +114,7 @@ class PatchJsonArgParse(object): return original_fn(obj, *args, **kwargs) if running_remotely(): try: - PatchJsonArgParse._load_task_params() + PatchJsonArgParse._load_task_params(parser=obj) params = PatchJsonArgParse.__remote_task_params_dict params_namespace = Namespace() for k, v in params.items(): @@ -132,37 +149,68 @@ class PatchJsonArgParse(object): del PatchJsonArgParse._args[subcommand] PatchJsonArgParse._args.update(subcommand_args) PatchJsonArgParse._args = {k: v for k, v in PatchJsonArgParse._args.items()} - PatchJsonArgParse._update_task_args() + PatchJsonArgParse._update_task_args(parser=obj, subcommand=subcommand) except Exception as e: logging.getLogger(__file__).warning("Failed parsing jsonargparse arguments: {}".format(e)) return parsed_args - @staticmethod - def _load_task_params(): - if not PatchJsonArgParse.__remote_task_params: - from clearml import Task + @classmethod + def _load_task_params(cls, parser=None): + if cls.__remote_task_params: + return + from clearml import Task - t = Task.get_task(task_id=get_remote_task_id()) - # noinspection PyProtectedMember - PatchJsonArgParse.__remote_task_params = t._get_task_property("hyperparams") or {} - params_dict = t.get_parameters(backwards_compatibility=False, cast=True) - for key, section_param in PatchJsonArgParse.__remote_task_params[PatchJsonArgParse._section_name].items(): - if section_param.type == PatchJsonArgParse.namespace_type: - params_dict[ - "{}/{}".format(PatchJsonArgParse._section_name, key) - ] = PatchJsonArgParse._get_namespace_from_json(section_param.value) - elif section_param.type == PatchJsonArgParse.path_type: - params_dict[ - "{}/{}".format(PatchJsonArgParse._section_name, key) - ] = PatchJsonArgParse._get_path_from_json(section_param.value) - elif (not section_param.type or section_param.type == "NoneType") and not section_param.value: - params_dict["{}/{}".format(PatchJsonArgParse._section_name, key)] = None - skip = len(PatchJsonArgParse._section_name) + 1 - PatchJsonArgParse.__remote_task_params_dict = { - k[skip:]: v - for k, v in params_dict.items() - if k.startswith(PatchJsonArgParse._section_name + PatchJsonArgParse._args_sep) - } + t = Task.get_task(task_id=get_remote_task_id()) + # noinspection PyProtectedMember + cls.__remote_task_params = t._get_task_property("hyperparams") or {} + params_dict = t.get_parameters(backwards_compatibility=False, cast=True) + for key, section_param in cls.__remote_task_params[cls._section_name].items(): + if section_param.type == cls.namespace_type: + params_dict[ + "{}/{}".format(cls._section_name, key) + ] = cls._get_namespace_from_json(section_param.value) + elif section_param.type == cls.path_type: + params_dict[ + "{}/{}".format(cls._section_name, key) + ] = cls._get_path_from_json(section_param.value) + elif (not section_param.type or section_param.type == "NoneType") and not section_param.value: + params_dict["{}/{}".format(cls._section_name, key)] = None + skip = len(cls._section_name) + 1 + cls.__remote_task_params_dict = { + k[skip:]: v + for k, v in params_dict.items() + if k.startswith(cls._section_name + cls._args_sep) + } + cls.__update_remote_task_params_dict_based_on_paths(parser) + + @classmethod + def __update_remote_task_params_dict_based_on_paths(cls, parser): + paths = PatchJsonArgParse.__get_paths_from_dict(cls.__remote_task_params_dict) + for path in paths: + args = PatchJsonArgParse.__get_args_from_path( + parser, + path, + subcommand=cls.__remote_task_params_dict.get("subcommand") + ) + for subarg_key, subarg_value in args.items(): + if subarg_key not in cls.__remote_task_params_dict: + cls.__remote_task_params_dict[subarg_key] = subarg_value + + @staticmethod + def __get_paths_from_dict(dict_): + paths = [path for path in dict_.values() if isinstance(path, Path)] + for subargs in dict_.values(): + if isinstance(subargs, list) and all(isinstance(path, Path) for path in subargs): + paths.extend(subargs) + return paths + + @staticmethod + def __get_args_from_path(parser, path, subcommand=None): + with change_to_path_dir(path): + parsed_cfg = parser.parse_string(path.get_content(), _skip_check=True, _fail_no_subcommand=False) + if subcommand: + parsed_cfg = {subcommand + PatchJsonArgParse._commands_sep + k: v for k, v in parsed_cfg.items()} + return parsed_cfg @staticmethod def _handle_namespace(value): From 9ec28b47b30ee9e5d5ce4cf7fead91a4017bca87 Mon Sep 17 00:00:00 2001 From: Alex Burlacu Date: Thu, 1 Jun 2023 15:13:03 +0300 Subject: [PATCH 3/5] Bump version to 1.11.1rc0 --- clearml/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clearml/version.py b/clearml/version.py index da77e85c..3f4a9a7b 100644 --- a/clearml/version.py +++ b/clearml/version.py @@ -1 +1 @@ -__version__ = '1.11.0' +__version__ = '1.11.1rc0' From 13c44b0b6aa09298b44d8348e8384a46cda95d7a Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Sun, 4 Jun 2023 10:30:20 +0300 Subject: [PATCH 4/5] Fix lazy evaluated value should be casted to str before passed through to OS calls --- clearml/task.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clearml/task.py b/clearml/task.py index a876b636..c87cb223 100644 --- a/clearml/task.py +++ b/clearml/task.py @@ -591,7 +591,7 @@ class Task(_Task): elif task.get_project_object().default_output_destination: task.output_uri = task.get_project_object().default_output_destination elif cls.__default_output_uri: - task.output_uri = cls.__default_output_uri + task.output_uri = str(cls.__default_output_uri) # store new task ID cls.__update_master_pid_task(task=task) else: @@ -1083,7 +1083,7 @@ class Task(_Task): if value is False: value = None elif value is True: - value = self.__default_output_uri or self._get_default_report_storage_uri() + value = str(self.__default_output_uri or self._get_default_report_storage_uri()) # check if we have the correct packages / configuration if value and value != self.storage_uri: From a343fc4c260e51a54f4e073c09b3853c21cf5385 Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Sun, 4 Jun 2023 10:30:30 +0300 Subject: [PATCH 5/5] Bump version to 1.11.1rc1 --- clearml/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clearml/version.py b/clearml/version.py index 3f4a9a7b..227ae1c5 100644 --- a/clearml/version.py +++ b/clearml/version.py @@ -1 +1 @@ -__version__ = '1.11.1rc0' +__version__ = '1.11.1rc1'