Version bump to v1.3.0

Fix --extra-index-url is added for different command line switches

clearml_agent/backend_api/config/default/agent.conf

@@ -83,7 +83,7 @@
         # set the optional priority packages to be installed before the rest of the required packages,
         # In case a package installation fails, the package will be ignored,
         # and the virtual environment process will continue
-        # priority_optional_packages: ["pygobject", ]
+        priority_optional_packages: ["pygobject", ]
 
         # set the post packages to be installed after all the rest of the required packages
         # post_packages: ["horovod", ]

clearml_agent/external/requirements_parser/parser.py

@@ -1,6 +1,9 @@
 import os
+import re
 import warnings
 
+from clearml_agent.definitions import PIP_EXTRA_INDICES
+
 from .requirement import Requirement
 
 
@@ -42,9 +45,14 @@ def parse(reqstr, cwd=None):
                     yield requirement
         elif line.startswith('-f') or line.startswith('--find-links') or \
                 line.startswith('-i') or line.startswith('--index-url') or \
-                line.startswith('--extra-index-url') or \
                 line.startswith('--no-index'):
             warnings.warn('Private repos not supported. Skipping.')
+        elif line.startswith('--extra-index-url'):
+            extra_index = line[len('--extra-index-url'):].strip()
+            extra_index = re.sub(r"\s+#.*$", "", extra_index)  # strip comments
+            if extra_index and extra_index not in PIP_EXTRA_INDICES:
+                PIP_EXTRA_INDICES.append(extra_index)
+                print(f"appended {extra_index} to list of extra pip indices")
             continue
         elif line.startswith('-Z') or line.startswith('--always-unzip'):
             warnings.warn('Unused option --always-unzip. Skipping.')

clearml_agent/helper/package/external_req.py

@@ -95,7 +95,8 @@ class ExternalRequirements(SimpleSubstitution):
                 vcs._set_ssh_url()
                 new_req_line = 'git+{}{}{}'.format(
                     '' if scheme and '://' in vcs.url else scheme,
-                    vcs.url_with_auth, fragment
+                    vcs_url if session.config.get('agent.force_git_ssh_protocol', None) else vcs.url_with_auth,
+                    fragment
                 )
                 if new_req_line != req_line:
                     furl_line = furl(new_req_line)

clearml_agent/helper/package/priority_req.py

@@ -1,3 +1,4 @@
+import re
 from typing import Text
 
 from .base import PackageManager
@@ -11,13 +12,14 @@ class PriorityPackageRequirement(SimpleSubstitution):
 
     def __init__(self, *args, **kwargs):
         super(PriorityPackageRequirement, self).__init__(*args, **kwargs)
+        self._replaced_packages = {}
         # check if we need to replace the packages:
         priority_packages = self.config.get('agent.package_manager.priority_packages', None)
         if priority_packages:
-            self.__class__.name = priority_packages
+            self.__class__.name = [p.lower() for p in priority_packages]
         priority_optional_packages = self.config.get('agent.package_manager.priority_optional_packages', None)
         if priority_optional_packages:
-            self.__class__.optional_package_names = priority_optional_packages
+            self.__class__.optional_package_names = [p.lower() for p in priority_optional_packages]
 
     def match(self, req):
         # match both Cython & cython
@@ -28,7 +30,9 @@ class PriorityPackageRequirement(SimpleSubstitution):
         Replace a requirement
         :raises: ValueError if version is pre-release
         """
-        if req.name in self.optional_package_names:
+        self._replaced_packages[req.name] = req.line
+
+        if req.name.lower() in self.optional_package_names:
             # noinspection PyBroadException
             try:
                 if PackageManager.out_of_scope_install_package(str(req)):
@@ -39,6 +43,41 @@ class PriorityPackageRequirement(SimpleSubstitution):
         PackageManager.out_of_scope_install_package(str(req))
         return Text(req)
 
+    def replace_back(self, list_of_requirements):
+        """
+        :param list_of_requirements: {'pip': ['a==1.0', ]}
+        :return: {'pip': ['a==1.0', ]}
+        """
+        # if we replaced setuptools, it means someone requested it, and since freeze will not contain it,
+        # we need to add it manually
+        if not self._replaced_packages or "setuptools" not in self._replaced_packages:
+            return list_of_requirements
+
+        try:
+            for k, lines in list_of_requirements.items():
+                # k is either pip/conda
+                if k not in ('pip', 'conda'):
+                    continue
+                for i, line in enumerate(lines):
+                    if not line or line.lstrip().startswith('#'):
+                        continue
+                    parts = [p for p in re.split(r'\s|=|\.|<|>|~|!|@|#', line) if p]
+                    if not parts:
+                        continue
+                    # if we found setuptools, do nothing
+                    if parts[0] == "setuptools":
+                        return list_of_requirements
+
+            # if we are here it means we have not found setuptools
+            # we should add it:
+            if "pip" in list_of_requirements:
+                list_of_requirements["pip"] = [self._replaced_packages["setuptools"]] + list_of_requirements["pip"]
+
+        except Exception as ex:  # noqa
+            return list_of_requirements
+
+        return list_of_requirements
+
 
 class PackageCollectorRequirement(SimpleSubstitution):
     """

clearml_agent/helper/package/pytorch.py

@@ -7,7 +7,7 @@ from furl import furl
 import urllib.parse
 from operator import itemgetter
 from html.parser import HTMLParser
-from typing import Text, Optional
+from typing import Text, Optional, Dict
 
 import attr
 import requests
@@ -512,7 +512,7 @@ class PytorchRequirement(SimpleSubstitution):
                 for i, line in enumerate(lines):
                     if not line or line.lstrip().startswith('#'):
                         continue
-                    parts = [p for p in re.split('\s|=|\.|<|>|~|!|@|#', line) if p]
+                    parts = [p for p in re.split(r'\s|=|\.|<|>|~|!|@|#', line) if p]
                     if not parts:
                         continue
                     for req, new_req in self._original_req:

clearml_agent/version.py

@@ -1 +1 @@
-__version__ = '1.2.4rc1'
+__version__ = '1.3.0'

requirements.txt

@@ -8,7 +8,7 @@ psutil>=3.4.2,<5.9.0
 pyhocon>=0.3.38,<0.4.0
 pyparsing>=2.0.3,<2.5.0
 python-dateutil>=2.4.2,<2.9.0
-pyjwt>=1.6.4,<2.1.0
+pyjwt>=2.4.0,<2.5.0
 PyYAML>=3.12,<5.5.0
 requests>=2.20.0,<2.26.0
 six>=1.13.0,<1.16.0

setup.py

@@ -61,6 +61,7 @@ setup(
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
         'License :: OSI Approved :: Apache Software License',
     ],