ClearML/clearml-agent
1
0
Fork 0
mirror of https://github.com/clearml/clearml-agent synced 2025-06-26 18:16:15 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Use shared git cache between multiple agents on the same machine

Browse Source
This commit is contained in:
allegroai 2021-02-14 13:49:29 +02:00
parent f619969efc
commit 22d5892b12
2 changed files with 44 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
clearml_agent/helper/repo.py
View File

@ -5,6 +5,8 @@ import subprocess
from distutils.spawn import find_executable from distutils.spawn import find_executable
from hashlib import md5 from hashlib import md5
from os import environ from os import environ
from random import random
from threading import Lock
from typing import Text, Sequence, Mapping, Iterable, TypeVar, Callable, Tuple, Optional from typing import Text, Sequence, Mapping, Iterable, TypeVar, Callable, Tuple, Optional
import attr import attr
@ -23,6 +25,7 @@ from clearml_agent.helper.base import (
normalize_path, normalize_path,
create_file_if_not_exists, create_file_if_not_exists,
) )
from clearml_agent.helper.os.locks import FileLock
from clearml_agent.helper.process import DEVNULL, Argv, PathLike, COMMAND_SUCCESS from clearml_agent.helper.process import DEVNULL, Argv, PathLike, COMMAND_SUCCESS
from clearml_agent.session import Session from clearml_agent.session import Session
@ -585,6 +588,9 @@ def clone_repository_cached(session, execution, destination):
:return: repository information :return: repository information
:raises: CommandFailedError if git/hg is not installed :raises: CommandFailedError if git/hg is not installed
""" """
# mock lock
repo_lock = Lock()
repo_lock_timeout_sec = 300
repo_url = execution.repository # type: str repo_url = execution.repository # type: str
parsed_url = furl(repo_url) parsed_url = furl(repo_url)
no_password_url = parsed_url.copy().remove(password=True).url no_password_url = parsed_url.copy().remove(password=True).url
@ -596,37 +602,48 @@ def clone_repository_cached(session, execution, destination):
if standalone_mode: if standalone_mode:
cached_repo_path = clone_folder cached_repo_path = clone_folder
else: else:
cached_repo_path = ( vcs_cache_path = Path(session.config["agent.vcs_cache.path"]).expanduser()
Path(session.config["agent.vcs_cache.path"]).expanduser() repo_hash = md5(ensure_binary(repo_url)).hexdigest()
/ "{}.{}".format(clone_folder_name, md5(ensure_binary(repo_url)).hexdigest()) # create lock
/ clone_folder_name repo_lock = FileLock(filename=(vcs_cache_path / '{}.lock'.format(repo_hash)).as_posix())
) # type: Path # noinspection PyBroadException
try:
repo_lock.acquire(timeout=repo_lock_timeout_sec)
except BaseException:
print('Could not lock cache folder "{}" (timeout {} sec), using temp vcs cache.'.format(
clone_folder_name, repo_lock_timeout_sec))
repo_hash = '{}_{}'.format(repo_hash, str(random()).replace('.', ''))
# use mock lock for the context
repo_lock = Lock()
# select vcs cache folder
cached_repo_path = vcs_cache_path / "{}.{}".format(clone_folder_name, repo_hash) / clone_folder_name
vcs = VcsFactory.create( with repo_lock:
session, execution_info=execution, location=cached_repo_path vcs = VcsFactory.create(
) session, execution_info=execution, location=cached_repo_path
if not find_executable(vcs.executable_name): )
raise CommandFailedError(vcs.executable_not_found_error_help()) if not find_executable(vcs.executable_name):
raise CommandFailedError(vcs.executable_not_found_error_help())
if not standalone_mode: if not standalone_mode:
if session.config["agent.vcs_cache.enabled"] and cached_repo_path.exists(): if session.config["agent.vcs_cache.enabled"] and cached_repo_path.exists():
print('Using cached repository in "{}"'.format(cached_repo_path)) print('Using cached repository in "{}"'.format(cached_repo_path))
else: else:
print("cloning: {}".format(no_password_url)) print("cloning: {}".format(no_password_url))
rm_tree(cached_repo_path) rm_tree(cached_repo_path)
# We clone the entire repository, not a specific branch # We clone the entire repository, not a specific branch
vcs.clone() # branch=execution.branch) vcs.clone() # branch=execution.branch)
vcs.pull() vcs.pull()
rm_tree(destination) rm_tree(destination)
shutil.copytree(Text(cached_repo_path), Text(clone_folder)) shutil.copytree(Text(cached_repo_path), Text(clone_folder))
if not clone_folder.is_dir(): if not clone_folder.is_dir():
raise CommandFailedError( raise CommandFailedError(
"copying of repository failed: from {} to {}".format( "copying of repository failed: from {} to {}".format(
cached_repo_path, clone_folder cached_repo_path, clone_folder
)
) )
)
# checkout in the newly copy destination # checkout in the newly copy destination
vcs.location = Text(clone_folder) vcs.location = Text(clone_folder)

2
clearml_agent/session.py
View File

@ -204,7 +204,7 @@ class Session(_Session):
folder_keys = ('agent.venvs_dir', 'agent.vcs_cache.path', folder_keys = ('agent.venvs_dir', 'agent.vcs_cache.path',
'agent.pip_download_cache.path', 'agent.pip_download_cache.path',
'agent.docker_pip_cache', 'agent.docker_apt_cache') 'agent.docker_pip_cache', 'agent.docker_apt_cache')
singleton_folders = ('agent.venvs_dir', 'agent.vcs_cache.path', 'agent.docker_apt_cache') singleton_folders = ('agent.venvs_dir', 'agent.docker_apt_cache')
if ENV_TASK_EXECUTE_AS_USER.get(): if ENV_TASK_EXECUTE_AS_USER.get():
folder_keys = tuple(list(folder_keys) + ['sdk.storage.cache.default_base_dir']) folder_keys = tuple(list(folder_keys) + ['sdk.storage.cache.default_base_dir'])