diff --git a/clearml_agent/backend_api/config/default/agent.conf b/clearml_agent/backend_api/config/default/agent.conf index 1486489..037c7d5 100644 --- a/clearml_agent/backend_api/config/default/agent.conf +++ b/clearml_agent/backend_api/config/default/agent.conf @@ -218,6 +218,76 @@ # optional arguments to pass to docker image # arguments: ["--ipc=host", ] + + # Choose the default docker based on the Task properties, + # Notice: Enterprise feature, ignored otherwise + # Examples: 'script.requirements', 'script.binary', 'script.repository', 'script.branch', 'project' + # Notice: Matching is done via regular expression, for example "^searchme$" will match exactly "searchme" string + "match_rules": [ + { + "image": "python:3.6-bullseye", + "arguments": "--ipc=host", + "match": { + "script": { + "binary": "python3.6$", + }, + } + }, + { + "image": "python:3.7-bullseye", + "arguments": "--ipc=host", + "match": { + "script": { + "binary": "python3.7$", + }, + } + }, + { + "image": "python:3.8-bullseye", + "arguments": "--ipc=host", + "match": { + "script": { + "binary": "python3.8$", + }, + } + }, + { + "image": "python:3.9-bullseye", + "arguments": "--ipc=host", + "match": { + "script": { + "binary": "python3.9$", + }, + } + }, + { + "image": "python:3.10-bullseye", + "arguments": "--ipc=host", + "match": { + "script": { + "binary": "python3.10$", + }, + } + }, + { + "image": "python:3.11-bullseye", + "arguments": "--ipc=host", + "match": { + "script": { + "binary": "python3.11$", + }, + } + }, + { + "image": "python:3.12-bullseye", + "arguments": "--ipc=host", + "match": { + "script": { + "binary": "python3.12$", + }, + } + }, + ] } # set the OS environments based on the Task's Environment section before launching the Task process. diff --git a/clearml_agent/commands/resolver.py b/clearml_agent/commands/resolver.py index 3ff3e39..aa7679d 100644 --- a/clearml_agent/commands/resolver.py +++ b/clearml_agent/commands/resolver.py @@ -1,14 +1,16 @@ import json import re import shlex +from copy import copy from clearml_agent.backend_api.session import Request +from clearml_agent.helper.docker_args import DockerArgsSanitizer from clearml_agent.helper.package.requirements import ( RequirementsManager, MarkerRequirement, compare_version_rules, ) -def resolve_default_container(session, task_id, container_config): +def resolve_default_container(session, task_id, container_config, ignore_match_rules=False): container_lookup = session.config.get('agent.default_docker.match_rules', None) if not session.check_min_api_version("2.13") or not container_lookup: return container_config @@ -17,6 +19,12 @@ def resolve_default_container(session, task_id, container_config): try: session.verify_feature_set('advanced') except ValueError: + # ignoring matching rules only supported in higher tiers + return container_config + + if ignore_match_rules: + print("INFO: default docker command line override, ignoring default docker container match rules") + # ignoring matching rules only supported in higher tiers return container_config result = session.send_request( @@ -159,9 +167,10 @@ def resolve_default_container(session, task_id, container_config): if not container_config.get('image'): container_config['image'] = entry.get('image', None) if not container_config.get('arguments'): - container_config['arguments'] = entry.get('arguments', None) - container_config['arguments'] = shlex.split(str(container_config.get('arguments') or '').strip()) - print('Matching default container with rule:\n{}'.format(json.dumps(entry))) + container_config['arguments'] = entry.get('arguments', None) or '' + if isinstance(container_config.get('arguments'), str): + container_config['arguments'] = shlex.split(str(container_config.get('arguments') or '').strip()) + print('INFO: Matching default container with rule:\n{}'.format(json.dumps(entry))) return container_config return container_config diff --git a/clearml_agent/commands/worker.py b/clearml_agent/commands/worker.py index e13c2d5..a207cfd 100644 --- a/clearml_agent/commands/worker.py +++ b/clearml_agent/commands/worker.py @@ -362,7 +362,7 @@ def get_task_fields(session, task_id, fields: list, log=None) -> dict: return {} -def get_task_container(session, task_id): +def get_task_container(session, task_id, ignore_match_rules=False): """ Returns dict with Task docker container setup {container: '', arguments: '', setup_shell_script: ''} """ @@ -398,7 +398,11 @@ def get_task_container(session, task_id): pass if (not container or not container.get('image')) and session.check_min_api_version("2.13"): - container = resolve_default_container(session=session, task_id=task_id, container_config=container) + container = resolve_default_container( + session=session, task_id=task_id, + container_config=container, + ignore_match_rules=ignore_match_rules, + ) return container @@ -732,6 +736,8 @@ class Worker(ServiceCommandSection): self._patch_docker_cmd_func = None self._docker_image = None self._docker_arguments = None + # if True, docker default passed on command line, which means we ignore the default docker match rules + self._docker_default_cmd_override = False PackageManager.set_pip_version(self._session.config.get("agent.package_manager.pip_version", None)) self._extra_docker_arguments = ( ENV_EXTRA_DOCKER_ARGS.get() or self._session.config.get("agent.extra_docker_arguments", None) @@ -943,7 +949,8 @@ class Worker(ServiceCommandSection): if self.docker_image_func: # noinspection PyBroadException try: - task_container = get_task_container(task_session, task_id) + task_container = get_task_container( + task_session, task_id, ignore_match_rules=self._docker_default_cmd_override) except Exception: task_container = {} @@ -2454,7 +2461,8 @@ class Worker(ServiceCommandSection): else: # noinspection PyBroadException try: - task_container = get_task_container(self._session, task_id) + task_container = get_task_container( + self._session, task_id, ignore_match_rules=self._docker_default_cmd_override) if ( task_container.get('image') and not self._session.config.get('agent.disable_task_docker_override', False) @@ -3964,6 +3972,8 @@ class Worker(ServiceCommandSection): if len(docker_arguments) > 1: docker_image = docker_arguments[0] docker_arguments = docker_arguments[1:] + elif docker_args and isinstance(docker_args, list) and len(docker_args) > 1: + docker_arguments = docker_args[1:] else: docker_arguments = self._session.config.get("agent.default_docker.arguments", None) or [] if isinstance(docker_arguments, six.string_types): @@ -3973,9 +3983,16 @@ class Worker(ServiceCommandSection): self._docker_image = docker_image self._docker_arguments = docker_arguments - print("Running in Docker{} mode (v19.03 and above) - using default docker image: {} {}\n".format( - ' *standalone*' if self._standalone_mode else '', self._docker_image, - DockerArgsSanitizer.sanitize_docker_command(self._session, self._docker_arguments) or '')) + if docker_args: + self._docker_default_cmd_override = True + + print("Running in Docker{} mode (v19.03 and above) - using default docker image: {} {} {}\n".format( + ' *standalone*' if self._standalone_mode else '', + self._docker_image, + DockerArgsSanitizer.sanitize_docker_command(self._session, self._docker_arguments) or '', + "\n(default docker commandline override, config matching rules are ignored)" + if self._docker_default_cmd_override else "", + )) temp_config = deepcopy(self._session.config) self.remove_non_backwards_compatible_entries(temp_config)