diff --git a/clearml_agent/definitions.py b/clearml_agent/definitions.py index 86e2a42..34108a1 100644 --- a/clearml_agent/definitions.py +++ b/clearml_agent/definitions.py @@ -1,6 +1,5 @@ import shlex from datetime import timedelta -from distutils.util import strtobool from enum import IntEnum from os import getenv, environ from typing import Text, Optional, Union, Tuple, Any @@ -9,6 +8,7 @@ import six from pathlib2 import Path from clearml_agent.helper.base import normalize_path +from clearml_agent.helper.environment.converters import strtobool PROGRAM_NAME = "clearml-agent" FROM_FILE_PREFIX_CHARS = "@" @@ -160,6 +160,7 @@ ENV_AGENT_SKIP_PIP_VENV_INSTALL = EnvironmentConfig("CLEARML_AGENT_SKIP_PIP_VENV ENV_AGENT_SKIP_PYTHON_ENV_INSTALL = EnvironmentConfig("CLEARML_AGENT_SKIP_PYTHON_ENV_INSTALL", type=bool) ENV_AGENT_FORCE_CODE_DIR = EnvironmentConfig("CLEARML_AGENT_FORCE_CODE_DIR") ENV_AGENT_FORCE_EXEC_SCRIPT = EnvironmentConfig("CLEARML_AGENT_FORCE_EXEC_SCRIPT") +ENV_AGENT_FORCE_TASK_INIT = EnvironmentConfig("CLEARML_AGENT_FORCE_TASK_INIT", type=bool) ENV_DOCKER_SKIP_GPUS_FLAG = EnvironmentConfig("CLEARML_DOCKER_SKIP_GPUS_FLAG", "TRAINS_DOCKER_SKIP_GPUS_FLAG") ENV_AGENT_GIT_USER = EnvironmentConfig("CLEARML_AGENT_GIT_USER", "TRAINS_AGENT_GIT_USER") ENV_AGENT_GIT_PASS = EnvironmentConfig("CLEARML_AGENT_GIT_PASS", "TRAINS_AGENT_GIT_PASS") diff --git a/clearml_agent/helper/package/poetry_api.py b/clearml_agent/helper/package/poetry_api.py index 45480dd..5a6de53 100644 --- a/clearml_agent/helper/package/poetry_api.py +++ b/clearml_agent/helper/package/poetry_api.py @@ -6,6 +6,7 @@ import sys import os from pathlib2 import Path +from clearml_agent.definitions import ENV_AGENT_FORCE_POETRY from clearml_agent.helper.process import Argv, DEVNULL, check_if_command_exists from clearml_agent.session import Session, POETRY @@ -40,7 +41,7 @@ def prop_guard(prop, log_prop=None): class PoetryConfig: def __init__(self, session, interpreter=None): - # type: (Session, str) -> () + # type: (Session, str) -> None self.session = session self._log = session.get_logger(__name__) self._python = interpreter or sys.executable @@ -52,7 +53,7 @@ class PoetryConfig: @property def enabled(self): - return self.session.config["agent.package_manager.type"] == POETRY + return ENV_AGENT_FORCE_POETRY.get() or self.session.config["agent.package_manager.type"] == POETRY _guard_enabled = prop_guard(enabled, log) @@ -89,30 +90,46 @@ class PoetryConfig: if not self._initialized: if self.session.config.get("agent.package_manager.poetry_version", None) is not None: version = str(self.session.config.get("agent.package_manager.poetry_version")) - print('Upgrading Poetry package {}'.format(version)) - # first upgrade pip if we need to - try: - from clearml_agent.helper.package.pip_api.venv import VirtualenvPip - pip = VirtualenvPip( - session=self.session, python=self._python, - requirements_manager=None, path=None, interpreter=self._python) - pip.upgrade_pip() - except Exception as ex: - self.log.warning("failed upgrading pip: {}".format(ex)) + # get poetry version + version = version.replace(' ', '') + if ('=' in version) or ('~' in version) or ('<' in version) or ('>' in version): + version = version + elif version: + version = "==" + version + # (we are not running it yet) + argv = Argv(self._python, "-m", "pip", "install", "poetry{}".format(version), + "--upgrade", "--disable-pip-version-check") + # this is just for beauty and checks, we already set the verion in the Argv + if not version: + version = "latest" + else: + # mark to install poetry if not already installed (we are not running it yet) + argv = Argv(self._python, "-m", "pip", "install", "poetry", "--disable-pip-version-check") + version = "" + + # first upgrade pip if we need to + try: + from clearml_agent.helper.package.pip_api.venv import VirtualenvPip + pip = VirtualenvPip( + session=self.session, python=self._python, + requirements_manager=None, path=None, interpreter=self._python) + pip.upgrade_pip() + except Exception as ex: + self.log.warning("failed upgrading pip: {}".format(ex)) + + # check if we do not have a specific version and poetry is found skip installation + if not version and check_if_command_exists("poetry"): + print("Notice: Poetry was found, no specific version required, skipping poetry installation") + else: + print('Installing / Upgrading Poetry package to {}'.format(version)) # now install poetry try: - version = version.replace(' ', '') - if ('=' in version) or ('~' in version) or ('<' in version) or ('>' in version): - version = version - elif version: - version = "==" + version - argv = Argv(self._python, "-m", "pip", "install", "poetry{}".format(version), - "--upgrade", "--disable-pip-version-check") print(argv.get_output()) except Exception as ex: - self.log.warning("failed upgrading poetry: {}".format(ex)) + self.log.warning("failed installing poetry: {}".format(ex)) + # now setup poetry self._initialized = True try: self._config("--local", "virtualenvs.in-project", "true", cwd=cwd)