mirror of
https://github.com/clearml/clearml-agent
synced 2025-01-31 17:16:51 +00:00
5ed47d2d2c
Add support for FORCE_CLEARML_AGENT_REPO env var to allow installing agent from a repo url when executing a task Implement skip venv installation on execute and allow custom binary Fix services mode limit implementation in docker mode
101 lines
3.3 KiB
Python
101 lines
3.3 KiB
Python
import os
|
|
import sys
|
|
from itertools import chain
|
|
from pathlib import Path
|
|
from typing import Text, Optional
|
|
|
|
from clearml_agent.definitions import PIP_EXTRA_INDICES, PROGRAM_NAME
|
|
from clearml_agent.helper.package.base import PackageManager
|
|
from clearml_agent.helper.process import Argv, DEVNULL
|
|
from clearml_agent.session import Session
|
|
|
|
|
|
class SystemPip(PackageManager):
|
|
|
|
indices_args = None
|
|
|
|
def __init__(self, interpreter=None, session=None):
|
|
# type: (Optional[Text], Optional[Session]) -> ()
|
|
"""
|
|
Program interface to the system pip.
|
|
"""
|
|
super(SystemPip, self).__init__()
|
|
self._bin = Path(interpreter or sys.executable)
|
|
self.session = session
|
|
|
|
@property
|
|
def bin(self):
|
|
return self._bin
|
|
|
|
def create(self):
|
|
pass
|
|
|
|
def remove(self):
|
|
pass
|
|
|
|
def install_from_file(self, path):
|
|
self.run_with_env(('install', '-r', path) + self.install_flags(), cwd=self.cwd)
|
|
|
|
def install_packages(self, *packages):
|
|
self._install(*(packages + self.install_flags()))
|
|
|
|
def _install(self, *args):
|
|
self.run_with_env(('install',) + args, cwd=self.cwd)
|
|
|
|
def uninstall_packages(self, *packages):
|
|
self.run_with_env(('uninstall', '-y') + packages)
|
|
|
|
def download_package(self, package, cache_dir):
|
|
self.run_with_env(
|
|
(
|
|
'download',
|
|
package,
|
|
'--dest', cache_dir,
|
|
'--no-deps',
|
|
) + self.install_flags()
|
|
)
|
|
|
|
def load_requirements(self, requirements):
|
|
requirements = requirements.get('pip') if isinstance(requirements, dict) else requirements
|
|
if not requirements:
|
|
return
|
|
with self.temp_file('cached-reqs', requirements) as path:
|
|
self.install_from_file(path)
|
|
|
|
def uninstall(self, package):
|
|
self.run_with_env(('uninstall', '-y', package))
|
|
|
|
def freeze(self):
|
|
"""
|
|
pip freeze to all install packages except the running program
|
|
:return: Dict contains pip as key and pip's packages to install
|
|
:rtype: Dict[str: List[str]]
|
|
"""
|
|
packages = self.run_with_env(('freeze',), output=True).splitlines()
|
|
packages_without_program = [package for package in packages if PROGRAM_NAME not in package]
|
|
return {'pip': packages_without_program}
|
|
|
|
def run_with_env(self, command, output=False, **kwargs):
|
|
"""
|
|
Run a shell command using environment from a virtualenv script
|
|
:param command: command to run
|
|
:type command: Iterable[Text]
|
|
:param output: return output
|
|
:param kwargs: kwargs for get_output/check_output command
|
|
"""
|
|
command = self._make_command(command)
|
|
# make sure we are not running it with our own PYTHONPATH
|
|
env = dict(**os.environ)
|
|
env.pop('PYTHONPATH', None)
|
|
return (command.get_output if output else command.check_call)(stdin=DEVNULL, env=env, **kwargs)
|
|
|
|
def _make_command(self, command):
|
|
return Argv(self.bin, '-m', 'pip', '--disable-pip-version-check', *command)
|
|
|
|
def install_flags(self):
|
|
if self.indices_args is None:
|
|
self.indices_args = tuple(
|
|
chain.from_iterable(('--extra-index-url', x) for x in PIP_EXTRA_INDICES)
|
|
)
|
|
return self.indices_args
|