ClearML/clearml-agent
1
0
Fork 0
mirror of https://github.com/clearml/clearml-agent synced 2025-06-26 18:16:15 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix support for shared cache folder between multiple nodes in the cluster

Browse Source
This commit is contained in:
allegroai 2019-11-23 23:38:36 +02:00
parent bd73be928a
commit 0ed7b2a0c8
2 changed files with 18 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
trains_agent/commands/worker.py
View File

@ -1155,7 +1155,8 @@ class Worker(ServiceCommandSection):
) )
except CommandFailedError: except CommandFailedError:
raise raise
except Exception: except Exception as ex:
print('Repository cloning failed: {}'.format(ex))
task.failed( task.failed(
status_reason="failed cloning repository", status_reason="failed cloning repository",
status_message=self._task_status_change_message, status_message=self._task_status_change_message,

29
trains_agent/helper/repo.py
View File

@ -263,8 +263,9 @@ class VCS(object):
""" """
self._set_ssh_url() self._set_ssh_url()
clone_command = ("clone", self.url_with_auth, self.location) + self.clone_flags clone_command = ("clone", self.url_with_auth, self.location) + self.clone_flags
if branch: # clone all branches regardless of when we want to later checkout
clone_command += ("-b", branch) # if branch:
# clone_command += ("-b", branch)
if self.session.debug_mode: if self.session.debug_mode:
self.call(*clone_command) self.call(*clone_command)
return return
@ -453,13 +454,13 @@ class Git(VCS):
) )
def pull(self): def pull(self):
self.call("fetch", "origin", cwd=self.location) self.call("fetch", "--all", cwd=self.location)
info_commands = dict( info_commands = dict(
url=Argv("git", "remote", "get-url", "origin"), url=Argv(executable_name, "ls-remote", "--get-url", "origin"),
branch=Argv("git", "rev-parse", "--abbrev-ref", "HEAD"), branch=Argv(executable_name, "rev-parse", "--abbrev-ref", "HEAD"),
commit=Argv("git", "rev-parse", "HEAD"), commit=Argv(executable_name, "rev-parse", "HEAD"),
root=Argv("git", "rev-parse", "--show-toplevel"), root=Argv(executable_name, "rev-parse", "--show-toplevel"),
) )
patch_base = ("apply",) patch_base = ("apply",)
@ -493,10 +494,10 @@ class Hg(VCS):
) )
info_commands = dict( info_commands = dict(
url=Argv("hg", "paths", "--verbose"), url=Argv(executable_name, "paths", "--verbose"),
branch=Argv("hg", "--debug", "id", "-b"), branch=Argv(executable_name, "--debug", "id", "-b"),
commit=Argv("hg", "--debug", "id", "-i"), commit=Argv(executable_name, "--debug", "id", "-i"),
root=Argv("hg", "root"), root=Argv(executable_name, "root"),
) )
@ -537,8 +538,6 @@ def clone_repository_cached(session, execution, destination):
vcs.clone() # branch=execution.branch) vcs.clone() # branch=execution.branch)
vcs.pull() vcs.pull()
vcs.checkout()
rm_tree(destination) rm_tree(destination)
shutil.copytree(Text(cached_repo_path), Text(clone_folder)) shutil.copytree(Text(cached_repo_path), Text(clone_folder))
if not clone_folder.is_dir(): if not clone_folder.is_dir():
@ -548,6 +547,10 @@ def clone_repository_cached(session, execution, destination):
) )
) )
# checkout in the newly copy destination
vcs.location = Text(clone_folder)
vcs.checkout()
repo_info = vcs.get_repository_copy_info(clone_folder) repo_info = vcs.get_repository_copy_info(clone_folder)
# make sure we have no user/pass in the returned repository structure # make sure we have no user/pass in the returned repository structure