ClearML/clearml-agent
1
0
Fork 0
mirror of https://github.com/clearml/clearml-agent synced 2025-06-26 18:16:15 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix running trains-agent from conda environment - conda.sh not found in first conda PATH match

Browse Source
This commit is contained in:
allegroai 2020-12-10 09:53:18 +02:00
parent a41ea52f87
commit 69eb25db1f
3 changed files with 23 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
trains_agent/helper/base.py
View File

@ -210,7 +210,7 @@ def get_python_path(script_dir, entry_point, package_api, is_conda_env=False):
(Path(script_dir) / Path(entry_point)).parent.absolute().as_posix(),
python_path_sep=python_path_sep)
if is_windows_platform():
return python_path.replace('/', '\\') + org_python_path
python_path = python_path.replace('/', '\\')
return python_path if is_conda_env else (python_path + org_python_path)
except Exception:

5
trains_agent/helper/package/base.py
View File

@ -5,7 +5,7 @@ from contextlib import contextmanager
from typing import Text, Iterable, Union
import six
from trains_agent.helper.base import mkstemp, safe_remove_file, join_lines
from trains_agent.helper.base import mkstemp, safe_remove_file, join_lines, select_for_platform
from trains_agent.helper.process import Executable, Argv, PathLike
@ -66,7 +66,8 @@ class PackageManager(object):
pass
def upgrade_pip(self):
result = self._install("pip"+self.get_pip_version(), "--upgrade")
result = self._install(
select_for_platform(windows='"pip{}"', linux='pip{}').format(self.get_pip_version()), "--upgrade")
packages = self.run_with_env(('list',), output=True).splitlines()
# p.split is ('pip', 'x.y.z')
pip = [p.split() for p in packages if len(p.split()) == 2 and p.split()[0] == 'pip']

24
trains_agent/helper/package/conda_api.py
View File

@ -2,7 +2,7 @@ from __future__ import unicode_literals
import json
import re
import shutil
import os
import subprocess
from collections import OrderedDict
from distutils.spawn import find_executable
@ -132,7 +132,7 @@ class CondaAPI(PackageManager):
if self.env_read_only:
print('Conda environment in read-only mode, skipping pip upgrade.')
return ''
return self._install("pip" + self.pip.get_pip_version())
return self._install(select_for_platform(windows='"pip{}"', linux='pip{}').format(self.pip.get_pip_version()))
def create(self):
"""
@ -150,7 +150,7 @@ class CondaAPI(PackageManager):
requirements_manager=self.requirements_manager,
path=self.path,
)
conda_env = Path(self.conda).parent.parent / 'etc' / 'profile.d' / 'conda.sh'
conda_env = self._get_conda_sh()
self.source = self.pip.source = CommandSequence(('source', conda_env.as_posix()), self.source)
self.env_read_only = True
return self
@ -167,7 +167,7 @@ class CondaAPI(PackageManager):
).get_output()
self.source = self.pip.source = ("conda", "activate", self.path.as_posix())
conda_env = Path(self.conda).parent.parent / 'etc' / 'profile.d' / 'conda.sh'
conda_env = self._get_conda_sh()
self.source = self.pip.source = CommandSequence(('source', conda_env.as_posix()), self.source)
# unpack cleanup
print("Fixing prefix in Conda environment {}".format(self.path))
@ -196,7 +196,7 @@ class CondaAPI(PackageManager):
else ("conda", "activate", self.path.as_posix())
)
conda_env = Path(self.conda).parent.parent / 'etc' / 'profile.d' / 'conda.sh'
conda_env = self._get_conda_sh()
if conda_env.is_file() and not is_windows_platform():
self.source = self.pip.source = CommandSequence(('source', conda_env.as_posix()), self.source)
@ -673,6 +673,20 @@ class CondaAPI(PackageManager):
def get_python_command(self, extra=()):
return CommandSequence(self.source, self.pip.get_python_command(extra=extra))
def _get_conda_sh(self):
# type () -> Path
base_conda_env = Path(self.conda).parent.parent / 'etc' / 'profile.d' / 'conda.sh'
if base_conda_env.is_file():
return base_conda_env
for path in os.environ.get('PATH', '').split(select_for_platform(windows=';', linux=':')):
conda = find_executable("conda", path=path)
if not conda:
continue
conda_env = Path(conda).parent.parent / 'etc' / 'profile.d' / 'conda.sh'
if conda_env.is_file():
return conda_env
return base_conda_env
# enable hashing with cmp=False because pdb fails on un-hashable exceptions
exception = attrs(str=True, cmp=False)