Fix conda package manager listed packages with local links (@ file://) should ignore the local package if it does not exist

Fix cuda patch version support in conda

clearml_agent/helper/package/conda_api.py

@@ -449,10 +449,25 @@ class CondaAPI(PackageManager):
         has_torch = False
         has_matplotlib = False
         has_cudatoolkit = False
+        cuda_version_full = 0
+        # noinspection PyBroadException
         try:
             # notice this is an integer version: 112 (means 11.2)
-            cuda_version = int(self.session.config.get('agent.cuda_version', 0))
+            cuda_version = str(self.session.config.get('agent.cuda_version', "")).strip()
-        except:
+            if not cuda_version:
+                cuda_version = 0
+            else:
+                cuda_version_full = str(cuda_version)
+                # if we have patch version we parse it here
+                cuda_version_parts = [int(v) for v in cuda_version.split('.')]
+                if len(cuda_version_parts) > 1 or cuda_version_parts[0] < 60:
+                    cuda_version = 10*cuda_version_parts[0]
+                    if len(cuda_version_parts) > 1:
+                        cuda_version += cuda_version_parts[1]
+                else:
+                    cuda_version = cuda_version_parts[0]
+                    cuda_version_full = "{:.1f}".format(float(cuda_version)/10.)
+        except Exception:
             cuda_version = 0
 
         # notice 'conda' entry with empty string is a valid conda requirements list, it means pip only
@@ -469,6 +484,7 @@ class CondaAPI(PackageManager):
                 continue
 
             m = MarkerRequirement(marker[0])
+            m.validate_local_file_ref()
             # conda does not support version control links
             if m.vcs:
                 pip_requirements.append(m)
@@ -512,7 +528,8 @@ class CondaAPI(PackageManager):
             reqs.append(m)
 
         if not has_cudatoolkit and cuda_version:
-            m = MarkerRequirement(Requirement("cudatoolkit == {}".format(float(cuda_version) / 10.0)))
+            m = MarkerRequirement(Requirement.parse("cudatoolkit == {}".format(cuda_version_full)))
+            has_cudatoolkit = True
             reqs.append(m)
 
         # if we have a conda list, the rest should be installed with pip,
@@ -528,9 +545,9 @@ class CondaAPI(PackageManager):
                     continue
 
                 m = MarkerRequirement(marker[0])
-                # skip over local files (we cannot change the version to a local file)
+                # remove local files reference if it does not exist (leave the package name)
-                if m.local_file:
+                m.validate_local_file_ref()
-                    continue
+
                 m_name = (m.name or '').lower()
                 if m_name in conda_supported_req_names:
                     # this package is in the conda list,

clearml_agent/helper/package/requirements.py

@@ -14,6 +14,7 @@ from pathlib2 import Path
 from pyhocon import ConfigTree
 
 import six
+import logging
 from clearml_agent.definitions import PIP_EXTRA_INDICES
 from clearml_agent.helper.base import warning, is_conda, which, join_lines, is_windows_platform
 from clearml_agent.helper.process import Argv, PathLike
@@ -153,6 +154,31 @@ class MarkerRequirement(object):
         return SimpleVersion.compare_versions(
             version_a=requested_version, op=op, version_b=version, num_parts=num_parts)
 
+    def remove_local_file_ref(self):
+        if not self.local_file or self.vcs or self.editable or self.path:
+            return False
+        parts = re.split(r"@\s*{}".format(self.req.uri), self.req.line)
+        # if we did not find anything do nothing
+        if len(parts) < 2:
+            return False
+        self.req.line = ''.join(parts).strip()
+        self.req.uri = None
+        self.req.local_file = False
+        return True
+
+    def validate_local_file_ref(self):
+        # if local file does not exist, remove the reference to it
+        if self.vcs or self.editable or self.path or not self.local_file or not self.name or \
+                not self.uri or not self.uri.startswith("file://"):
+            return
+        local_path = Path(self.uri[len("file://"):])
+        if not local_path.exists():
+            line = self.line
+            if self.remove_local_file_ref():
+                # print warning
+                logging.getLogger(__name__).warning(
+                    'Local file not found [{}], references removed !'.format(line))
+
 
 class SimpleVersion:
     _sub_versions_pep440 = ['a', 'b', 'rc', '.post', '.dev', '+', ]
@@ -698,12 +724,17 @@ class RequirementsManager(object):
             except Exception as ex:
                 return [Requirement(req_str)]
 
+        def create_req(x):
+            r = MarkerRequirement(x)
+            r.validate_local_file_ref()
+            return r
+
         if not requirements:
             return tuple()
 
         parsed_requirements = tuple(
             map(
-                MarkerRequirement,
+                create_req,
                 [r for line in (requirements.splitlines() if isinstance(requirements, str) else requirements)
                  for r in safe_parse(line)]
             )