diff --git a/clearml_agent/commands/worker.py b/clearml_agent/commands/worker.py index df68e59..b1c9efc 100644 --- a/clearml_agent/commands/worker.py +++ b/clearml_agent/commands/worker.py @@ -2446,7 +2446,9 @@ class Worker(ServiceCommandSection): package_api.requirements_manager.update_installed_packages_state(package_api.freeze()) # make sure we run the handlers cached_requirements = \ - {k: package_api.requirements_manager.replace(requirements[k] or '') + {k: package_api.requirements_manager.replace( + requirements=requirements[k] or '', + existing_packages=package_api.requirements_manager.get_installed_packages_state()) for k in requirements} package_api.load_requirements(cached_requirements) # make sure we call the correct freeze @@ -2861,11 +2863,13 @@ class Worker(ServiceCommandSection): requirement_substitutions=[CachedPackageRequirement, OnlyExternalRequirements] ) # manually update the current state, - # for the external git reference chance (in the replace callback) + # for the external git reference chance (in the replaced callback) package_api.requirements_manager.update_installed_packages_state(package_api.freeze()) # make sure we run the handlers cached_requirements = \ - {k: package_api.requirements_manager.replace(requirements[k] or '') + {k: package_api.requirements_manager.replace( + requirements=requirements[k] or '', + existing_packages=package_api.requirements_manager.get_installed_packages_state()) for k in requirements} if str(cached_requirements.get('pip', '')).strip() \ or str(cached_requirements.get('conda', '')).strip(): diff --git a/clearml_agent/helper/base.py b/clearml_agent/helper/base.py index 093c33d..fab4b97 100644 --- a/clearml_agent/helper/base.py +++ b/clearml_agent/helper/base.py @@ -22,7 +22,7 @@ from .._vendor import furl from .._vendor import six from .._vendor.attr import fields_dict from .._vendor.pathlib2 import Path -from .._vendor.six.moves import reduce # noqa +from .._vendor.six.moves import reduce from .._vendor import pyyaml as yaml from clearml_agent.errors import CommandFailedError @@ -372,15 +372,15 @@ def construct_mapping(loader, node): return OrderedDict(loader.construct_pairs(node)) -yaml.SafeDumper.add_representer(OrderedDict, represent_ordered_dict) -yaml.SafeLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping) +# yaml.SafeDumper.add_representer(OrderedDict, represent_ordered_dict) +# yaml.SafeLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping) class AllDumper(yaml.SafeDumper): pass -AllDumper.add_multi_representer(object, lambda dumper, data: dumper.represent_str(str(data))) +# AllDumper.add_multi_representer(object, lambda dumper, data: dumper.represent_str(str(data))) def error(message): diff --git a/clearml_agent/helper/package/requirements.py b/clearml_agent/helper/package/requirements.py index 4486dd0..11d62a3 100644 --- a/clearml_agent/helper/package/requirements.py +++ b/clearml_agent/helper/package/requirements.py @@ -787,6 +787,13 @@ class RequirementsManager(object): for h in self.handlers: h.set_is_already_installed_cb(self._callback_is_already_installed) + def get_installed_packages_state(self): + """ + Return the internal Installed Packages objects, the one used internally so that later we can detect + :return requirements: is the list of requirements objects updated via self.update_installed_packages_state + """ + return self._installed_parsed_packages + def _callback_is_already_installed(self, req): for p in (self._installed_parsed_packages or []): if p.name != req.name: