ClearML/clearml-agent
1
0
Fork 0
mirror of https://github.com/clearml/clearml-agent synced 2025-02-26 05:59:24 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix cached venv tried to reinstall priority packages even through they are preinstalled

Browse Source
This commit is contained in:
clearml 2025-02-24 13:23:00 +02:00
parent a6ae6b2095
commit 546ffff95d
3 changed files with 18 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
clearml_agent/commands/worker.py
View File

@ -2446,7 +2446,9 @@ class Worker(ServiceCommandSection):
package_api.requirements_manager.update_installed_packages_state(package_api.freeze()) package_api.requirements_manager.update_installed_packages_state(package_api.freeze())
# make sure we run the handlers # make sure we run the handlers
cached_requirements = \ cached_requirements = \
{k: package_api.requirements_manager.replace(requirements[k] or '') {k: package_api.requirements_manager.replace(
requirements=requirements[k] or '',
existing_packages=package_api.requirements_manager.get_installed_packages_state())
for k in requirements} for k in requirements}
package_api.load_requirements(cached_requirements) package_api.load_requirements(cached_requirements)
# make sure we call the correct freeze # make sure we call the correct freeze
@ -2861,11 +2863,13 @@ class Worker(ServiceCommandSection):
requirement_substitutions=[CachedPackageRequirement, OnlyExternalRequirements] requirement_substitutions=[CachedPackageRequirement, OnlyExternalRequirements]
) )
# manually update the current state, # manually update the current state,
# for the external git reference chance (in the replace callback) # for the external git reference chance (in the replaced callback)
package_api.requirements_manager.update_installed_packages_state(package_api.freeze()) package_api.requirements_manager.update_installed_packages_state(package_api.freeze())
# make sure we run the handlers # make sure we run the handlers
cached_requirements = \ cached_requirements = \
{k: package_api.requirements_manager.replace(requirements[k] or '') {k: package_api.requirements_manager.replace(
requirements=requirements[k] or '',
existing_packages=package_api.requirements_manager.get_installed_packages_state())
for k in requirements} for k in requirements}
if str(cached_requirements.get('pip', '')).strip() \ if str(cached_requirements.get('pip', '')).strip() \
or str(cached_requirements.get('conda', '')).strip(): or str(cached_requirements.get('conda', '')).strip():

8
clearml_agent/helper/base.py
View File

@ -22,7 +22,7 @@ from .._vendor import furl
from .._vendor import six from .._vendor import six
from .._vendor.attr import fields_dict from .._vendor.attr import fields_dict
from .._vendor.pathlib2 import Path from .._vendor.pathlib2 import Path
from .._vendor.six.moves import reduce # noqa from .._vendor.six.moves import reduce
from .._vendor import pyyaml as yaml from .._vendor import pyyaml as yaml
from clearml_agent.errors import CommandFailedError from clearml_agent.errors import CommandFailedError
@ -372,15 +372,15 @@ def construct_mapping(loader, node):
return OrderedDict(loader.construct_pairs(node)) return OrderedDict(loader.construct_pairs(node))
yaml.SafeDumper.add_representer(OrderedDict, represent_ordered_dict) # yaml.SafeDumper.add_representer(OrderedDict, represent_ordered_dict)
yaml.SafeLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping) # yaml.SafeLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping)
class AllDumper(yaml.SafeDumper): class AllDumper(yaml.SafeDumper):
pass pass
AllDumper.add_multi_representer(object, lambda dumper, data: dumper.represent_str(str(data))) # AllDumper.add_multi_representer(object, lambda dumper, data: dumper.represent_str(str(data)))
def error(message): def error(message):

7
clearml_agent/helper/package/requirements.py
View File

@ -787,6 +787,13 @@ class RequirementsManager(object):
for h in self.handlers: for h in self.handlers:
h.set_is_already_installed_cb(self._callback_is_already_installed) h.set_is_already_installed_cb(self._callback_is_already_installed)
def get_installed_packages_state(self):
"""
Return the internal Installed Packages objects, the one used internally so that later we can detect
:return requirements: is the list of requirements objects updated via self.update_installed_packages_state
"""
return self._installed_parsed_packages
def _callback_is_already_installed(self, req): def _callback_is_already_installed(self, req):
for p in (self._installed_parsed_packages or []): for p in (self._installed_parsed_packages or []):
if p.name != req.name: if p.name != req.name: