ClearML/clearml-agent
1
0
Fork 0
mirror of https://github.com/clearml/clearml-agent synced 2025-04-23 07:34:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Use shared git cache between multiple agents on the same machine

Browse Source
This commit is contained in:
allegroai 2021-02-14 13:49:29 +02:00
parent f619969efc
commit 22d5892b12
2 changed files with 44 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
clearml_agent/helper/repo.py
View File

@ -5,6 +5,8 @@ import subprocess
from distutils.spawn import find_executable from distutils.spawn import find_executable
from hashlib import md5 from hashlib import md5
from os import environ from os import environ
from random import random
from threading import Lock
from typing import Text, Sequence, Mapping, Iterable, TypeVar, Callable, Tuple, Optional from typing import Text, Sequence, Mapping, Iterable, TypeVar, Callable, Tuple, Optional
import attr import attr
@ -23,6 +25,7 @@ from clearml_agent.helper.base import (
normalize_path, normalize_path,
create_file_if_not_exists, create_file_if_not_exists,
) )
from clearml_agent.helper.os.locks import FileLock
from clearml_agent.helper.process import DEVNULL, Argv, PathLike, COMMAND_SUCCESS from clearml_agent.helper.process import DEVNULL, Argv, PathLike, COMMAND_SUCCESS
from clearml_agent.session import Session from clearml_agent.session import Session
@ -585,6 +588,9 @@ def clone_repository_cached(session, execution, destination):
:return: repository information :return: repository information
:raises: CommandFailedError if git/hg is not installed :raises: CommandFailedError if git/hg is not installed
""" """
# mock lock
repo_lock = Lock()
repo_lock_timeout_sec = 300
repo_url = execution.repository # type: str repo_url = execution.repository # type: str
parsed_url = furl(repo_url) parsed_url = furl(repo_url)
no_password_url = parsed_url.copy().remove(password=True).url no_password_url = parsed_url.copy().remove(password=True).url
@ -596,12 +602,23 @@ def clone_repository_cached(session, execution, destination):
if standalone_mode: if standalone_mode:
cached_repo_path = clone_folder cached_repo_path = clone_folder
else: else:
cached_repo_path = ( vcs_cache_path = Path(session.config["agent.vcs_cache.path"]).expanduser()
Path(session.config["agent.vcs_cache.path"]).expanduser() repo_hash = md5(ensure_binary(repo_url)).hexdigest()
/ "{}.{}".format(clone_folder_name, md5(ensure_binary(repo_url)).hexdigest()) # create lock
/ clone_folder_name repo_lock = FileLock(filename=(vcs_cache_path / '{}.lock'.format(repo_hash)).as_posix())
) # type: Path # noinspection PyBroadException
try:
repo_lock.acquire(timeout=repo_lock_timeout_sec)
except BaseException:
print('Could not lock cache folder "{}" (timeout {} sec), using temp vcs cache.'.format(
clone_folder_name, repo_lock_timeout_sec))
repo_hash = '{}_{}'.format(repo_hash, str(random()).replace('.', ''))
# use mock lock for the context
repo_lock = Lock()
# select vcs cache folder
cached_repo_path = vcs_cache_path / "{}.{}".format(clone_folder_name, repo_hash) / clone_folder_name
with repo_lock:
vcs = VcsFactory.create( vcs = VcsFactory.create(
session, execution_info=execution, location=cached_repo_path session, execution_info=execution, location=cached_repo_path
) )

2
clearml_agent/session.py
View File

@ -204,7 +204,7 @@ class Session(_Session):
folder_keys = ('agent.venvs_dir', 'agent.vcs_cache.path', folder_keys = ('agent.venvs_dir', 'agent.vcs_cache.path',
'agent.pip_download_cache.path', 'agent.pip_download_cache.path',
'agent.docker_pip_cache', 'agent.docker_apt_cache') 'agent.docker_pip_cache', 'agent.docker_apt_cache')
singleton_folders = ('agent.venvs_dir', 'agent.vcs_cache.path', 'agent.docker_apt_cache') singleton_folders = ('agent.venvs_dir', 'agent.docker_apt_cache')
if ENV_TASK_EXECUTE_AS_USER.get(): if ENV_TASK_EXECUTE_AS_USER.get():
folder_keys = tuple(list(folder_keys) + ['sdk.storage.cache.default_base_dir']) folder_keys = tuple(list(folder_keys) + ['sdk.storage.cache.default_base_dir'])