Add default docker match_rules for enterprise users,

NOTICE: matching_rules are ignored if `--docker container` is passed in command line

clearml_agent/backend_api/config/default/agent.conf

@@ -218,6 +218,76 @@
 
         # optional arguments to pass to docker image
         # arguments: ["--ipc=host", ]
+
+        # Choose the default docker based on the Task properties,
+        # Notice: Enterprise feature, ignored otherwise
+        # Examples: 'script.requirements', 'script.binary', 'script.repository', 'script.branch', 'project'
+        # Notice: Matching is done via regular expression, for example "^searchme$" will match exactly "searchme" string
+        "match_rules": [
+             {
+                 "image": "python:3.6-bullseye",
+                 "arguments": "--ipc=host",
+                 "match": {
+                     "script": {
+                         "binary": "python3.6$",
+                     },
+                 }
+             },
+             {
+                 "image": "python:3.7-bullseye",
+                 "arguments": "--ipc=host",
+                 "match": {
+                     "script": {
+                         "binary": "python3.7$",
+                     },
+                 }
+             },
+             {
+                 "image": "python:3.8-bullseye",
+                 "arguments": "--ipc=host",
+                 "match": {
+                     "script": {
+                         "binary": "python3.8$",
+                     },
+                 }
+             },
+             {
+                 "image": "python:3.9-bullseye",
+                 "arguments": "--ipc=host",
+                 "match": {
+                     "script": {
+                         "binary": "python3.9$",
+                     },
+                 }
+             },
+             {
+                 "image": "python:3.10-bullseye",
+                 "arguments": "--ipc=host",
+                 "match": {
+                     "script": {
+                         "binary": "python3.10$",
+                     },
+                 }
+             },
+             {
+                 "image": "python:3.11-bullseye",
+                 "arguments": "--ipc=host",
+                 "match": {
+                     "script": {
+                         "binary": "python3.11$",
+                     },
+                 }
+             },
+             {
+                 "image": "python:3.12-bullseye",
+                 "arguments": "--ipc=host",
+                 "match": {
+                     "script": {
+                         "binary": "python3.12$",
+                     },
+                 }
+             },
+        ]
     }
 
     # set the OS environments based on the Task's Environment section before launching the Task process.

clearml_agent/commands/resolver.py

@@ -1,14 +1,16 @@
 import json
 import re
 import shlex
+from copy import copy
 
 from clearml_agent.backend_api.session import Request
+from clearml_agent.helper.docker_args import DockerArgsSanitizer
 from clearml_agent.helper.package.requirements import (
     RequirementsManager, MarkerRequirement,
     compare_version_rules, )
 
 
-def resolve_default_container(session, task_id, container_config):
+def resolve_default_container(session, task_id, container_config, ignore_match_rules=False):
     container_lookup = session.config.get('agent.default_docker.match_rules', None)
     if not session.check_min_api_version("2.13") or not container_lookup:
         return container_config
@@ -17,6 +19,12 @@ def resolve_default_container(session, task_id, container_config):
     try:
         session.verify_feature_set('advanced')
     except ValueError:
+        # ignoring matching rules only supported in higher tiers
+        return container_config
+
+    if ignore_match_rules:
+        print("INFO: default docker command line override, ignoring default docker container match rules")
+        # ignoring matching rules only supported in higher tiers
         return container_config
 
     result = session.send_request(
@@ -159,9 +167,10 @@ def resolve_default_container(session, task_id, container_config):
             if not container_config.get('image'):
                 container_config['image'] = entry.get('image', None)
             if not container_config.get('arguments'):
-                container_config['arguments'] = entry.get('arguments', None)
+                container_config['arguments'] = entry.get('arguments', None) or ''
+                if isinstance(container_config.get('arguments'), str):
                     container_config['arguments'] = shlex.split(str(container_config.get('arguments') or '').strip())
-            print('Matching default container with rule:\n{}'.format(json.dumps(entry)))
+            print('INFO: Matching default container with rule:\n{}'.format(json.dumps(entry)))
             return container_config
 
     return container_config

clearml_agent/commands/worker.py

@@ -362,7 +362,7 @@ def get_task_fields(session, task_id, fields: list, log=None) -> dict:
     return {}
 
 
-def get_task_container(session, task_id):
+def get_task_container(session, task_id, ignore_match_rules=False):
     """
     Returns dict with Task docker container setup {container: '', arguments: '', setup_shell_script: ''}
     """
@@ -398,7 +398,11 @@ def get_task_container(session, task_id):
                     pass
 
     if (not container or not container.get('image')) and session.check_min_api_version("2.13"):
-        container = resolve_default_container(session=session, task_id=task_id, container_config=container)
+        container = resolve_default_container(
+            session=session, task_id=task_id,
+            container_config=container,
+            ignore_match_rules=ignore_match_rules,
+        )
 
     return container
 
@@ -732,6 +736,8 @@ class Worker(ServiceCommandSection):
         self._patch_docker_cmd_func = None
         self._docker_image = None
         self._docker_arguments = None
+        # if True, docker default passed on command line, which means we ignore the default docker match rules
+        self._docker_default_cmd_override = False
         PackageManager.set_pip_version(self._session.config.get("agent.package_manager.pip_version", None))
         self._extra_docker_arguments = (
                 ENV_EXTRA_DOCKER_ARGS.get() or self._session.config.get("agent.extra_docker_arguments", None)
@@ -943,7 +949,8 @@ class Worker(ServiceCommandSection):
         if self.docker_image_func:
             # noinspection PyBroadException
             try:
-                task_container = get_task_container(task_session, task_id)
+                task_container = get_task_container(
+                    task_session, task_id, ignore_match_rules=self._docker_default_cmd_override)
             except Exception:
                 task_container = {}
 
@@ -2454,7 +2461,8 @@ class Worker(ServiceCommandSection):
         else:
             # noinspection PyBroadException
             try:
-                task_container = get_task_container(self._session, task_id)
+                task_container = get_task_container(
+                    self._session, task_id, ignore_match_rules=self._docker_default_cmd_override)
                 if (
                     task_container.get('image')
                     and not self._session.config.get('agent.disable_task_docker_override', False)
@@ -3964,6 +3972,8 @@ class Worker(ServiceCommandSection):
         if len(docker_arguments) > 1:
             docker_image = docker_arguments[0]
             docker_arguments = docker_arguments[1:]
+        elif docker_args and isinstance(docker_args, list) and len(docker_args) > 1:
+            docker_arguments = docker_args[1:]
         else:
             docker_arguments = self._session.config.get("agent.default_docker.arguments", None) or []
             if isinstance(docker_arguments, six.string_types):
@@ -3973,9 +3983,16 @@ class Worker(ServiceCommandSection):
         self._docker_image = docker_image
         self._docker_arguments = docker_arguments
 
-        print("Running in Docker{} mode (v19.03 and above) - using default docker image: {} {}\n".format(
+        if docker_args:
-            ' *standalone*' if self._standalone_mode else '', self._docker_image,
+            self._docker_default_cmd_override = True
-            DockerArgsSanitizer.sanitize_docker_command(self._session, self._docker_arguments) or ''))
+
+        print("Running in Docker{} mode (v19.03 and above) - using default docker image: {} {} {}\n".format(
+            ' *standalone*' if self._standalone_mode else '',
+            self._docker_image,
+            DockerArgsSanitizer.sanitize_docker_command(self._session, self._docker_arguments) or '',
+            "\n(default docker commandline override, config matching rules are ignored)"
+            if self._docker_default_cmd_override else "",
+        ))
 
         temp_config = deepcopy(self._session.config)
         self.remove_non_backwards_compatible_entries(temp_config)