Add agent.docker_internal_mounts to control containers internal mounts (non-root containers)

clearml_agent/backend_api/config/default/agent.conf

@@ -198,6 +198,19 @@
         extra_keys: []
     }
 
+    # allow to set internal mount points inside the docker,
+    # especially useful for non-root docker container images.
+    docker_internal_mounts {
+        sdk_cache: "/clearml_agent_cache"
+        apt_cache: "/var/cache/apt/archives"
+        ssh_folder: "/root/.ssh"
+        pip_cache: "/root/.cache/pip"
+        poetry_cache: "/root/.cache/pypoetry"
+        vcs_cache: "/root/.clearml/vcs-cache"
+        venv_build: "/root/.clearml/venvs-builds"
+        pip_download: "/root/.clearml/pip-download-cache"
+    }
+
     # Name docker containers created by the daemon using the following string format (supported from Docker 0.6.5)
     # Allowed variables are task_id, worker_id and rand_string (random lower-case letters string, up to 32 charaters)
     # Note: resulting name must start with an alpha-numeric character and continue with a alpha-numeric characters,

clearml_agent/commands/worker.py

@@ -2959,10 +2959,14 @@ class Worker(ServiceCommandSection):
             '*standalone*' if self._standalone_mode else '', self._docker_image, self._docker_arguments or ''))
 
         temp_config = deepcopy(self._session.config)
-        mounted_cache_dir = self._docker_fixed_user_cache  # '/root/.clearml/cache'
+        mounted_cache_dir = temp_config.get(
-        mounted_pip_dl_dir = '/root/.clearml/pip-download-cache'
+            "agent.docker_internal_mounts.sdk_cache", self._docker_fixed_user_cache)
-        mounted_vcs_cache = '/root/.clearml/vcs-cache'
+        mounted_pip_dl_dir = temp_config.get(
-        mounted_venv_dir = '/root/.clearml/venvs-builds'
+            "agent.docker_internal_mounts.pip_download", '/root/.clearml/pip-download-cache')
+        mounted_vcs_cache = temp_config.get(
+            "agent.docker_internal_mounts.vcs_cache", '/root/.clearml/vcs-cache')
+        mounted_venv_dir = temp_config.get(
+            "agent.docker_internal_mounts.venv_build", '/root/.clearml/venvs-builds')
         temp_config.put("sdk.storage.cache.default_base_dir", mounted_cache_dir)
         temp_config.put("agent.pip_download_cache.path", mounted_pip_dl_dir)
         temp_config.put("agent.vcs_cache.path", mounted_vcs_cache)
@@ -3078,6 +3082,10 @@ class Worker(ServiceCommandSection):
         mounted_pip_dl_dir = temp_config.get("agent.pip_download_cache.path")
         mounted_vcs_cache = temp_config.get("agent.vcs_cache.path")
         mounted_venvs_cache = temp_config.get("agent.venvs_cache.path", "")
+        mount_ssh = temp_config.get("agent.docker_internal_mounts.ssh_folder", None)
+        mount_apt_cache = temp_config.get("agent.docker_internal_mounts.apt_cache", None)
+        mount_pip_cache = temp_config.get("agent.docker_internal_mounts.pip_cache", None)
+        mount_poetry_cache = temp_config.get("agent.docker_internal_mounts.poetry_cache", None)
 
         # Make sure we have created the configuration file for the executor
         if not self.dump_config(self.temp_config_path, config=temp_config, clean_api_credentials=clean_api_credentials):
@@ -3105,6 +3113,10 @@ class Worker(ServiceCommandSection):
             bash_script=bash_script,
             preprocess_bash_script=preprocess_bash_script,
             install_opencv_libs=install_opencv_libs,
+            mount_ssh=mount_ssh,
+            mount_apt_cache=mount_apt_cache,
+            mount_pip_cache=mount_pip_cache,
+            mount_poetry_cache=mount_poetry_cache,
         )
 
         docker_cmd.update(kwargs)
@@ -3156,6 +3168,7 @@ class Worker(ServiceCommandSection):
             auth_token=None,
             worker_tags=None,
             name=None,
+            mount_ssh=None, mount_apt_cache=None, mount_pip_cache=None, mount_poetry_cache=None,
     ):
         docker = 'docker'
 
@@ -3332,13 +3345,18 @@ class Worker(ServiceCommandSection):
                            for line in docker_bash_setup_script.split('\n') if line.strip()) + \
                 ' ; '
 
+        mount_ssh = mount_ssh or '/root/.ssh'
+        mount_apt_cache = mount_apt_cache or '/var/cache/apt/archives'
+        mount_pip_cache = mount_pip_cache or '/root/.cache/pip'
+        mount_poetry_cache = mount_poetry_cache or '/root/.cache/pypoetry'
+
         base_cmd += (
             (['--name', name] if name else []) +
             ['-v', conf_file+':'+DOCKER_ROOT_CONF_FILE] +
-            (['-v', host_ssh_cache+':/root/.ssh'] if host_ssh_cache else []) +
+            (['-v', host_ssh_cache+':'+mount_ssh] if host_ssh_cache else []) +
-            (['-v', host_apt_cache+':/var/cache/apt/archives'] if host_apt_cache else []) +
+            (['-v', host_apt_cache+':'+mount_apt_cache] if host_apt_cache else []) +
-            (['-v', host_pip_cache+':/root/.cache/pip'] if host_pip_cache else []) +
+            (['-v', host_pip_cache+':'+mount_pip_cache] if host_pip_cache else []) +
-            (['-v', host_poetry_cache + ':/root/.cache/pypoetry'] if host_poetry_cache else []) +
+            (['-v', host_poetry_cache + ':'+mount_poetry_cache] if host_poetry_cache else []) +
             (['-v', host_pip_dl+':'+mounted_pip_dl] if host_pip_dl else []) +
             (['-v', host_cache+':'+mounted_cache] if host_cache else []) +
             (['-v', host_vcs_cache+':'+mounted_vcs_cache] if host_vcs_cache else []) +

docs/clearml.conf

@@ -176,10 +176,23 @@ agent {
         extra_keys: []
     }
 
+    # allow to set internal mount points inside the docker,
+    # especially useful for non-root docker container images.
+    # docker_internal_mounts {
+    #     sdk_cache: "/clearml_agent_cache"
+    #     apt_cache: "/var/cache/apt/archives"
+    #     ssh_folder: "/root/.ssh"
+    #     pip_cache: "/root/.cache/pip"
+    #     poetry_cache: "/root/.cache/pypoetry"
+    #     vcs_cache: "/root/.clearml/vcs-cache"
+    #     venv_build: "/root/.clearml/venvs-builds"
+    #     pip_download: "/root/.clearml/pip-download-cache"
+    # }
+
     # Name docker containers created by the daemon using the following string format (supported from Docker 0.6.5)
     # Allowed variables are task_id, worker_id and rand_string (random lower-case letters string, up to 32 charaters)
-    # Note: resulting name must start with an alpha-numeric character and continue with a alpha-numeric characters,
+    # Note: resulting name must start with an alpha-numeric character and
-    #  underscores (_), dots (.) and/or dashes (-)
+    #       continue with a alpha-numeric characters, underscores (_), dots (.) and/or dashes (-)
     # docker_container_name_format: "clearml-id-{task_id}-{rand_string:.8}"
 }