ClearML/clearml-agent
1
0
Fork 0
mirror of https://github.com/clearml/clearml-agent synced 2025-03-03 18:52:22 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add initial Conda support

Browse Source
This commit is contained in:
allegroai 2019-11-15 23:24:47 +02:00
parent 72c26499c0
commit 8b05bb1605
1 changed files with 20 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
trains_agent/commands/worker.py
View File

@ -145,6 +145,11 @@ class LiteralScriptManager(object):
"found task with `script.working_dir` (`%s`) but without `script.repository`, ignoring", "found task with `script.working_dir` (`%s`) but without `script.repository`, ignoring",
execution.working_dir, execution.working_dir,
) )
if not execution.entry_point:
execution.entry_point = 'untitled.py'
else:
# ignore any folders in the entry point we only need the file name
execution.entry_point = execution.entry_point.split(os.path.sep)[-1]
location = None location = None
location = location or (repo_info and repo_info.root) location = location or (repo_info and repo_info.root)
if not location: if not location:
@ -578,10 +583,10 @@ class Worker(ServiceCommandSection):
) )
if is_windows_platform(): if is_windows_platform():
if not self.is_conda: # if not self.is_conda:
self.warning("Worker on Windows without Conda are not supported") # self.warning("Worker on Windows without Conda are not supported")
if self._session.config.agent.venv_update: if self._session.config.agent.venv_update:
self.warning("venv-update is not supported on Windows") # self.warning("venv-update is not supported on Windows")
self.is_venv_update = False self.is_venv_update = False
self._session.print_configuration() self._session.print_configuration()
@ -980,7 +985,9 @@ class Worker(ServiceCommandSection):
requirements_manager=requirements_manager, requirements_manager=requirements_manager,
cached_requirements=requirements, cached_requirements=requirements,
) )
freeze = self.freeze_task_environment(current_task.id) # do not update the task packages if we are using conda,
# it will most likely make the task environment unreproducible
freeze = self.freeze_task_environment(current_task.id if not self.is_conda else None)
script_dir = (directory if isinstance(directory, Path) else Path(directory)).absolute().as_posix() script_dir = (directory if isinstance(directory, Path) else Path(directory)).absolute().as_posix()
# run code # run code
@ -1146,6 +1153,8 @@ class Worker(ServiceCommandSection):
execution=execution, execution=execution,
destination=Path(venv_folder) / WORKING_REPOSITORY_DIR, destination=Path(venv_folder) / WORKING_REPOSITORY_DIR,
) )
except CommandFailedError:
raise
except Exception: except Exception:
task.failed( task.failed(
status_reason="failed cloning repository", status_reason="failed cloning repository",
@ -1303,18 +1312,20 @@ class Worker(ServiceCommandSection):
cached_requirements_failed = False cached_requirements_failed = False
if cached_requirements and ('pip' in cached_requirements or 'conda' in cached_requirements): if cached_requirements and ('pip' in cached_requirements or 'conda' in cached_requirements):
self.log("Found cached requirements, trying to install") self.log("Found task requirements section, trying to install")
try: try:
self.package_api.load_requirements(cached_requirements) self.package_api.load_requirements(cached_requirements)
except Exception as e: except Exception as e:
self.log_traceback(e) self.log_traceback(e)
self.error("Could not install cached requirements, installing requirements from repository") self.error("Could not install task requirements! Trying to install requirements from repository")
cached_requirements_failed = True cached_requirements_failed = True
else: else:
self.log("Cached requirements installation success") self.log("task requirements installation passed")
return return
if not repo_info: if not repo_info:
if cached_requirements_failed:
raise ValueError("Could not install task requirements!")
self.log("no repository to install requirements from") self.log("no repository to install requirements from")
return return
@ -1357,7 +1368,7 @@ class Worker(ServiceCommandSection):
# if we reached here without installing anything, and # if we reached here without installing anything, and
# we failed installing from cached requirements, them this is an error # we failed installing from cached requirements, them this is an error
if cached_requirements_failed and not repo_requirements_installed: if cached_requirements_failed and not repo_requirements_installed:
raise ValueError("Could not install cached requirements or repository requirements") raise ValueError("Failed installing task requirements and repository requirements")
def named_temporary_file(self, *args, **kwargs): def named_temporary_file(self, *args, **kwargs):
kwargs.setdefault("delete", not self._session.debug_mode) kwargs.setdefault("delete", not self._session.debug_mode)
@ -1465,7 +1476,7 @@ class Worker(ServiceCommandSection):
) )
if match: if match:
self.log.debug("Found: {}".format(executable)) self.log.debug("Found: {}".format(executable))
return match.group(1), version, executable return match.group(1), version or '.'.join(match.group(1).split('.')[:2]), executable
raise CommandFailedError( raise CommandFailedError(
"Python executable with version {!r} defined in configuration file, " "Python executable with version {!r} defined in configuration file, "
"key 'agent.default_python', not found in path, tried: {}".format( "key 'agent.default_python', not found in path, tried: {}".format(