From f693fa165cbd50552b4948d87088848d2ebadb12 Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Thu, 2 Jul 2020 01:33:13 +0300 Subject: [PATCH] Fix .git-credentials and .gitconfig mapping into docker Add agent.docker_init_bash_script allow finer control over docker startup script --- .../backend_api/config/default/agent.conf | 13 ++++- trains_agent/commands/worker.py | 49 ++++++++++++------- 2 files changed, 43 insertions(+), 19 deletions(-) diff --git a/trains_agent/backend_api/config/default/agent.conf b/trains_agent/backend_api/config/default/agent.conf index db16d51..c6de522 100644 --- a/trains_agent/backend_api/config/default/agent.conf +++ b/trains_agent/backend_api/config/default/agent.conf @@ -92,12 +92,23 @@ default_docker: { # default docker image to use when running in docker mode - image: "nvidia/cuda" + image: "nvidia/cuda:10.2-runtime-ubuntu18.04" # optional arguments to pass to docker image # arguments: ["--ipc=host", ] } + # set the initial bash script to execute at the startup of any docker. + # all lines will be executed regardless of their exit code. + # {python_single_digit} is translated to 'python3' or 'python2' according to requested python version + docker_init_bash_script = [ + "echo 'Binary::apt::APT::Keep-Downloaded-Packages \"true\";' > /etc/apt/apt.conf.d/docker-clean", + "chown -R root /root/.cache/pip", + "apt-get update", + "apt-get install -y git libsm6 libxext6 libxrender-dev libglib2.0-0", + "(which {python_single_digit} && {python_single_digit} -m pip --version) || apt-get install -y {python_single_digit}-pip", + ] + # cuda versions used for solving pytorch wheel packages # should be detected automatically. Override with os environment CUDA_VERSION / CUDNN_VERSION # cuda_version: 10.1 diff --git a/trains_agent/commands/worker.py b/trains_agent/commands/worker.py index 92d8303..90429e9 100644 --- a/trains_agent/commands/worker.py +++ b/trains_agent/commands/worker.py @@ -2043,12 +2043,10 @@ class Worker(ServiceCommandSection): pass # check if the .git credentials exist: - host_git_credentials = Path('~/.git-credentials').expanduser() try: - if host_git_credentials.is_file(): - host_git_credentials = host_git_credentials.as_posix() - else: - host_git_credentials = None + host_git_credentials = [ + f.as_posix() for f in [Path('~/.git-credentials').expanduser(), Path('~/.gitconfig').expanduser()] + if f.is_file()] except Exception: host_git_credentials = None @@ -2063,6 +2061,8 @@ class Worker(ServiceCommandSection): cmds = [cmds] extra_shell_script_str = " ; ".join(map(str, cmds)) + " ; " + bash_script = self._session.config.get("agent.docker_init_bash_script", None) + self.temp_config_path = self.temp_config_path or safe_mkstemp( suffix=".cfg", prefix=".trains_agent.", text=True, name_only=True ) @@ -2079,7 +2079,10 @@ class Worker(ServiceCommandSection): host_cache=host_cache, mounted_cache=mounted_cache_dir, host_pip_dl=host_pip_dl, mounted_pip_dl=mounted_pip_dl_dir, host_vcs_cache=host_vcs_cache, mounted_vcs_cache=mounted_vcs_cache, - standalone_mode=self._standalone_mode, force_current_version=self._force_current_version) + standalone_mode=self._standalone_mode, + force_current_version=self._force_current_version, + bash_script=bash_script, + ) return temp_config, partial(docker_cmd_functor, docker_cmd, temp_config) @staticmethod @@ -2092,7 +2095,7 @@ class Worker(ServiceCommandSection): host_pip_dl, mounted_pip_dl, host_vcs_cache, mounted_vcs_cache, standalone_mode=False, extra_docker_arguments=None, extra_shell_script=None, - force_current_version=None, host_git_credentials=None): + force_current_version=None, host_git_credentials=None, bash_script=None): docker = 'docker' base_cmd = [docker, 'run', '-t'] @@ -2193,20 +2196,30 @@ class Worker(ServiceCommandSection): trains_agent_wheel = 'trains-agent{specify_version}'.format(specify_version=specify_version) if not standalone_mode: - update_scheme += \ - "echo 'Binary::apt::APT::Keep-Downloaded-Packages \"true\";' > /etc/apt/apt.conf.d/docker-clean ; " \ - "chown -R root /root/.cache/pip ; " \ - "apt-get update ; " \ - "apt-get install -y git libsm6 libxext6 libxrender-dev libglib2.0-0 {python_single_digit}-pip ; " \ - "{python} -m pip install -U \"pip{pip_version}\" ; " \ - "{python} -m pip install -U {trains_agent_wheel} ; ".format( - python_single_digit=python_version.split('.')[0], - python=python_version, pip_version=PackageManager.get_pip_version(), - trains_agent_wheel=trains_agent_wheel) + if not bash_script: + bash_script = [ + "echo 'Binary::apt::APT::Keep-Downloaded-Packages \"true\";' > /etc/apt/apt.conf.d/docker-clean", + "chown -R root /root/.cache/pip", + "apt-get update", + "apt-get install -y git libsm6 libxext6 libxrender-dev libglib2.0-0 {python_single_digit}-pip", + ] + + docker_bash_script = " ; ".join(bash_script) if not isinstance(bash_script, str) else bash_script + + update_scheme += ( + docker_bash_script + " ; " + + "{python} -m pip install -U \"pip{pip_version}\" ; " + + "{python} -m pip install -U {trains_agent_wheel} ; ").format( + python_single_digit=python_version.split('.')[0], + python=python_version, pip_version=PackageManager.get_pip_version(), + trains_agent_wheel=trains_agent_wheel) + + if host_git_credentials: + for git_credentials in host_git_credentials: + base_cmd += ['-v', '{}:/root/{}'.format(git_credentials, Path(git_credentials).name)] base_cmd += ( ['-v', conf_file+':'+DOCKER_ROOT_CONF_FILE] + - (['-v', host_git_credentials+':/root/.git-credentials'] if host_git_credentials else []) + (['-v', host_ssh_cache+':/root/.ssh'] if host_ssh_cache else []) + (['-v', host_apt_cache+':/var/cache/apt/archives'] if host_apt_cache else []) + (['-v', host_pip_cache+':/root/.cache/pip'] if host_pip_cache else []) +