From 3ed63e2154eee7917cca5938af7c59e17248868c Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Tue, 4 Jul 2023 14:37:18 +0300 Subject: [PATCH] Fix docker container backwards compatibility for API <2.13 Fix default docker match rules resolver (used incorrect field "container" instead of "image") Remove "container" (image) match rule option from default docker image resolver --- .../backend_api/config/default/agent.conf | 7 ------ clearml_agent/commands/resolver.py | 22 +++++++++---------- clearml_agent/commands/worker.py | 21 ++++++++++-------- docs/clearml.conf | 2 -- 4 files changed, 23 insertions(+), 29 deletions(-) diff --git a/clearml_agent/backend_api/config/default/agent.conf b/clearml_agent/backend_api/config/default/agent.conf index e3de103..cd2efdb 100644 --- a/clearml_agent/backend_api/config/default/agent.conf +++ b/clearml_agent/backend_api/config/default/agent.conf @@ -388,13 +388,6 @@ # } # }, # { - # "image": "better_container:tag", - # "arguments": "", - # "match": { - # "container": "replace_me_please" - # } - # }, - # { # "image": "another_container:tag", # "arguments": "", # "match": { diff --git a/clearml_agent/commands/resolver.py b/clearml_agent/commands/resolver.py index 61f813b..3ff3e39 100644 --- a/clearml_agent/commands/resolver.py +++ b/clearml_agent/commands/resolver.py @@ -109,15 +109,15 @@ def resolve_default_container(session, task_id, container_config): match.get('script.binary', None), entry)) continue - if match.get('container', None): - # noinspection PyBroadException - try: - if not re.search(match.get('container', None), requested_container.get('image', '')): - continue - except Exception: - print('Failed parsing regular expression \"{}\" in rule: {}'.format( - match.get('container', None), entry)) - continue + # if match.get('image', None): + # # noinspection PyBroadException + # try: + # if not re.search(match.get('image', None), requested_container.get('image', '')): + # continue + # except Exception: + # print('Failed parsing regular expression \"{}\" in rule: {}'.format( + # match.get('image', None), entry)) + # continue matched = True for req_section in ['script.requirements.pip', 'script.requirements.conda']: @@ -156,8 +156,8 @@ def resolve_default_container(session, task_id, container_config): break if matched: - if not container_config.get('container'): - container_config['container'] = entry.get('image', None) + if not container_config.get('image'): + container_config['image'] = entry.get('image', None) if not container_config.get('arguments'): container_config['arguments'] = entry.get('arguments', None) container_config['arguments'] = shlex.split(str(container_config.get('arguments') or '').strip()) diff --git a/clearml_agent/commands/worker.py b/clearml_agent/commands/worker.py index fbbf9cc..0460a05 100644 --- a/clearml_agent/commands/worker.py +++ b/clearml_agent/commands/worker.py @@ -372,16 +372,19 @@ def get_task_container(session, task_id): container = {} else: response = get_task(session, task_id, only_fields=["execution.docker_cmd"]) - task_docker_cmd_parts = shlex.split(str(response.execution.docker_cmd or '').strip()) - try: - container = dict( - container=task_docker_cmd_parts[0], - arguments=task_docker_cmd_parts[1:] if len(task_docker_cmd_parts[0]) > 1 else '' - ) - except (ValueError, TypeError): - container = {} + container = {} + if response.execution: + task_docker_cmd_parts = shlex.split(str(response.execution.docker_cmd or '').strip()) + if task_docker_cmd_parts: + try: + container = dict( + image=task_docker_cmd_parts[0], + arguments=task_docker_cmd_parts[1:] if len(task_docker_cmd_parts[0]) > 1 else '' + ) + except (ValueError, TypeError): + pass - if (not container or not container.get('container')) and session.check_min_api_version("2.13"): + if (not container or not container.get('image')) and session.check_min_api_version("2.13"): container = resolve_default_container(session=session, task_id=task_id, container_config=container) return container diff --git a/docs/clearml.conf b/docs/clearml.conf index 54ab900..5d48e89 100644 --- a/docs/clearml.conf +++ b/docs/clearml.conf @@ -242,8 +242,6 @@ agent { # # no repository matching required # repository: "" # } - # # no container image matching required (allow to replace one requested container with another) - # container: "" # # no repository matching required # project: "" # }