diff --git a/clearml_agent/commands/worker.py b/clearml_agent/commands/worker.py index 79d4f65..83c985c 100644 --- a/clearml_agent/commands/worker.py +++ b/clearml_agent/commands/worker.py @@ -2804,7 +2804,7 @@ class Worker(ServiceCommandSection): if self._session.debug_mode and temp_file: rm_file(temp_file.name) # call post installation callback - requirements_manager.post_install(self._session) + requirements_manager.post_install(self._session, package_manager=package_api) # mark as successful installation repo_requirements_installed = True diff --git a/clearml_agent/helper/package/conda_api.py b/clearml_agent/helper/package/conda_api.py index 2d94f5f..dd2dee7 100644 --- a/clearml_agent/helper/package/conda_api.py +++ b/clearml_agent/helper/package/conda_api.py @@ -424,7 +424,7 @@ class CondaAPI(PackageManager): finally: PackageManager._selected_manager = self - self.requirements_manager.post_install(self.session) + self.requirements_manager.post_install(self.session, package_manager=self) def load_requirements(self, requirements): # if we are in read only mode, do not uninstall anything @@ -642,7 +642,7 @@ class CondaAPI(PackageManager): finally: PackageManager._selected_manager = self - self.requirements_manager.post_install(self.session) + self.requirements_manager.post_install(self.session, package_manager=self) return True def _parse_conda_result_bad_packges(self, result_dict): diff --git a/clearml_agent/helper/package/external_req.py b/clearml_agent/helper/package/external_req.py index 4b572df..6c87d0f 100644 --- a/clearml_agent/helper/package/external_req.py +++ b/clearml_agent/helper/package/external_req.py @@ -46,11 +46,10 @@ class ExternalRequirements(SimpleSubstitution): post_install_req = self.post_install_req self.post_install_req = [] for req in post_install_req: - try: - freeze_base = PackageManager.out_of_scope_freeze() or '' - except: - freeze_base = '' - + if self.is_already_installed(req): + print("No need to reinstall \'{}\' from VCS, " + "the exact same version is already installed".format(req.name)) + continue req_line = self._add_vcs_credentials(req, session) # if we have older pip version we have to make sure we replace back the package name with the diff --git a/clearml_agent/helper/package/pip_api/venv.py b/clearml_agent/helper/package/pip_api/venv.py index 3a41287..018dde4 100644 --- a/clearml_agent/helper/package/pip_api/venv.py +++ b/clearml_agent/helper/package/pip_api/venv.py @@ -39,7 +39,7 @@ class VirtualenvPip(SystemPip, PackageManager): if isinstance(requirements, dict) and requirements.get("pip"): requirements["pip"] = self.requirements_manager.replace(requirements["pip"]) super(VirtualenvPip, self).load_requirements(requirements) - self.requirements_manager.post_install(self.session) + self.requirements_manager.post_install(self.session, package_manager=self) def create_flags(self): """ diff --git a/clearml_agent/helper/package/requirements.py b/clearml_agent/helper/package/requirements.py index b19f8a8..0b50b6a 100644 --- a/clearml_agent/helper/package/requirements.py +++ b/clearml_agent/helper/package/requirements.py @@ -179,7 +179,7 @@ class MarkerRequirement(object): if self.remove_local_file_ref(): # print warning logging.getLogger(__name__).warning( - 'Local file not found [{}], references removed !'.format(line)) + 'Local file not found [{}], references removed'.format(line)) class SimpleVersion: @@ -437,6 +437,7 @@ class RequirementSubstitution(object): self.config = session.config # type: ConfigTree self.suffix = '.post{config[agent.cuda_version]}.dev{config[agent.cudnn_version]}'.format(config=self.config) self.package_manager = self.config['agent.package_manager.type'] + self._is_already_installed_cb = None @abstractmethod def match(self, req): # type: (MarkerRequirement) -> bool @@ -452,6 +453,20 @@ class RequirementSubstitution(object): """ pass + def set_is_already_installed_cb(self, cb): + self._is_already_installed_cb = cb + + def is_already_installed(self, req): + if not self._is_already_installed_cb: + return False + # noinspection PyBroadException + try: + return self._is_already_installed_cb(req) + except BaseException as ex: + # debug could not resolve something + print("Warning: Requirements post install callback exception (check if package installed): {}".format(ex)) + return False + def post_scan_add_req(self): # type: () -> Optional[MarkerRequirement] """ Allows the RequirementSubstitution to add an extra line/requirements after @@ -562,6 +577,7 @@ class RequirementsManager(object): cache_dir=pip_cache_dir.as_posix()) self._base_interpreter = base_interpreter self._cwd = None + self._installed_parsed_packages = set() def register(self, cls): # type: (Type[RequirementSubstitution]) -> None self.handlers.append(cls(self._session)) @@ -619,7 +635,9 @@ class RequirementsManager(object): return join_lines(result) - def post_install(self, session): + def post_install(self, session, package_manager=None): + if package_manager: + self.update_installed_packages_state(package_manager.freeze()) for h in self.handlers: try: h.post_install(session) @@ -641,6 +659,34 @@ class RequirementsManager(object): def get_interpreter(self): return self._base_interpreter + def update_installed_packages_state(self, requirements): + """ + Updates internal Installed Packages objects, so that later we can detect + if we already have a pre-installed package + :param requirements: is the output of a freeze() call, i.e. dict {'pip': "package==version"} + """ + requirements = requirements if not isinstance(requirements, dict) else requirements.get("pip") + self._installed_parsed_packages = self.parse_requirements_section_to_marker_requirements( + requirements=requirements, cwd=self._cwd) + for h in self.handlers: + h.set_is_already_installed_cb(self._callback_is_already_installed) + + def _callback_is_already_installed(self, req): + for p in (self._installed_parsed_packages or []): + if p.name != req.name: + continue + # if this is version control package, only return true of both installed and requests specify commit ID + if req.vcs: + return p.vcs and req.revision and req.revision == p.revision + + if not req.specs and not p.specs: + return True + + # return if this is the same version + return req.specs and p.specs and req.compare_version(p, op="==") + + return False + @staticmethod def get_cuda_version(config): # type: (ConfigTree) -> (Text, Text) # we assume os.environ already updated the config['agent.cuda_version'] & config['agent.cudnn_version']