ClearML/clearml-agent
1
0
Fork 0
mirror of https://github.com/clearml/clearml-agent synced 2025-02-11 23:34:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix clearml-agent build --docker stuck on certain containers

Browse Source
This commit is contained in:
allegroai 2022-12-13 15:24:32 +02:00
parent efa1f71dac
commit f69e16ea9d
2 changed files with 16 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
clearml_agent/commands/worker.py
View File

@ -2103,8 +2103,9 @@ class Worker(ServiceCommandSection):
end_of_build_marker = "build.done=true" end_of_build_marker = "build.done=true"
docker_cmd_suffix = ' build --id {task_id} --install-globally; ' \ docker_cmd_suffix = ' build --id {task_id} --install-globally; ' \
'echo "" >> {conf_file} ; ' \ 'ORG=$(stat -c "%u:%g" {conf_file}) ; chown $(whoami):$(whoami) {conf_file} ; ' \
'echo {end_of_build_marker} >> {conf_file} ; ' \ 'echo "" >> {conf_file} ; echo {end_of_build_marker} >> {conf_file} ; ' \
'chown $ORG {conf_file} ; ' \
'bash'.format( 'bash'.format(
task_id=task_id, task_id=task_id,
end_of_build_marker=end_of_build_marker, end_of_build_marker=end_of_build_marker,
@ -2123,10 +2124,16 @@ class Worker(ServiceCommandSection):
# now we need to wait until the line shows on our configuration file. # now we need to wait until the line shows on our configuration file.
while True: while True:
while temp_config.stat().st_mtime == base_time_stamp: # noinspection PyBroadException
sleep(5.0) try:
with open(temp_config.as_posix()) as f: while temp_config.stat().st_mtime == base_time_stamp:
lines = [l.strip() for l in f.readlines()] sleep(5.0)
with open(temp_config.as_posix()) as f:
lines = [l.strip() for l in f.readlines()]
except Exception as ex:
# print("Failed reading status file [{}], retrying in 2 seconds".format(ex))
sleep(2.0)
if 'build.done=true' in lines: if 'build.done=true' in lines:
break break
base_time_stamp = temp_config.stat().st_mtime base_time_stamp = temp_config.stat().st_mtime
@ -2834,8 +2841,8 @@ class Worker(ServiceCommandSection):
# Todo: add support for poetry caching # Todo: add support for poetry caching
if not self.poetry.enabled: if not self.poetry.enabled:
# add to cache # add to cache
print('Adding venv into cache: {}'.format(add_venv_folder_cache))
if add_venv_folder_cache: if add_venv_folder_cache:
print('Adding venv into cache: {}'.format(add_venv_folder_cache))
self.package_api.add_cached_venv( self.package_api.add_cached_venv(
requirements=[freeze, previous_reqs], requirements=[freeze, previous_reqs],
docker_cmd=execution_info.docker_cmd if execution_info else None, docker_cmd=execution_info.docker_cmd if execution_info else None,

3
clearml_agent/helper/process.py
View File

@ -117,10 +117,11 @@ def terminate_all_child_processes(pid=None, timeout=10., include_parent=True):
def get_docker_id(docker_cmd_contains): def get_docker_id(docker_cmd_contains):
# noinspection PyBroadException
try: try:
containers_running = get_bash_output(cmd='docker ps --no-trunc --format \"{{.ID}}: {{.Command}}\"') containers_running = get_bash_output(cmd='docker ps --no-trunc --format \"{{.ID}}: {{.Command}}\"')
for docker_line in containers_running.split('\n'): for docker_line in containers_running.split('\n'):
parts = docker_line.split(':') parts = docker_line.split(':', 1)
if docker_cmd_contains in parts[-1]: if docker_cmd_contains in parts[-1]:
# we found our docker, return it # we found our docker, return it
return parts[0] return parts[0]