From c1d91b0d6aa23d07fa1f29c9981bd70130fe616a Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Mon, 13 Jan 2020 12:14:43 +0200 Subject: [PATCH] Use packaging instead of semantic_version --- requirements.txt | 2 +- trains_agent/helper/check_update.py | 6 +-- trains_agent/helper/package/conda_api.py | 6 +-- trains_agent/helper/package/pytorch.py | 43 ++++++++++++++------- trains_agent/helper/package/requirements.py | 4 +- 5 files changed, 37 insertions(+), 24 deletions(-) diff --git a/requirements.txt b/requirements.txt index 42be811..0616d66 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ future>=0.16.0 humanfriendly>=2.1 jsonmodels>=2.2 jsonschema>=2.6.0 +packaging>=16.0 pathlib2>=2.3.0 psutil>=3.4.2 pyhocon>=0.3.38 @@ -15,7 +16,6 @@ PyYAML>=3.12 requests-file>=1.4.2 requests>=2.20.0 requirements_parser>=0.2.0 -semantic_version>=2.6.0 six>=1.11.0 tqdm>=4.19.5 typing>=3.6.4 diff --git a/trains_agent/helper/check_update.py b/trains_agent/helper/check_update.py index efe9af8..41c2148 100644 --- a/trains_agent/helper/check_update.py +++ b/trains_agent/helper/check_update.py @@ -4,7 +4,7 @@ from time import sleep import requests import json from threading import Thread -from semantic_version import Version +from packaging import version as packaging_version from ..version import __version__ __check_update_thread = None @@ -30,8 +30,8 @@ def _check_new_version_available(): return None trains_answer = update_server_releases.get("trains-agent", {}) latest_version = trains_answer.get("version") - cur_version = Version(cur_version) - latest_version = Version(latest_version) + cur_version = packaging_version.parse(cur_version) + latest_version = packaging_version.parse(latest_version or '') if cur_version >= latest_version: return None patch_upgrade = latest_version.major == cur_version.major and latest_version.minor == cur_version.minor diff --git a/trains_agent/helper/package/conda_api.py b/trains_agent/helper/package/conda_api.py index e6d9b70..db81d19 100644 --- a/trains_agent/helper/package/conda_api.py +++ b/trains_agent/helper/package/conda_api.py @@ -14,7 +14,7 @@ import yaml from time import time from attr import attrs, attrib, Factory from pathlib2 import Path -from semantic_version import Version +from packaging import version as packaging_version from requirements import parse from requirements.requirement import Requirement @@ -59,7 +59,7 @@ class CondaAPI(PackageManager): A programmatic interface for controlling conda """ - MINIMUM_VERSION = Version("4.3.30", partial=True) + MINIMUM_VERSION = packaging_version.parse("4.3.30") def __init__(self, session, path, python, requirements_manager): # type: (Session, PathLike, float, RequirementsManager) -> None @@ -93,7 +93,7 @@ class CondaAPI(PackageManager): ) ) self.conda_version = self.get_conda_version(output) - if Version(self.conda_version, partial=True) < self.MINIMUM_VERSION: + if packaging_version.parse(self.conda_version) < self.MINIMUM_VERSION: raise CommandFailedError( "conda version '{}' is smaller than minimum supported conda version '{}'".format( self.conda_version, self.MINIMUM_VERSION diff --git a/trains_agent/helper/package/pytorch.py b/trains_agent/helper/package/pytorch.py index 64934f9..b6b6e1c 100644 --- a/trains_agent/helper/package/pytorch.py +++ b/trains_agent/helper/package/pytorch.py @@ -10,7 +10,8 @@ from typing import Text import attr import requests -from semantic_version import Version, Spec +from packaging import version as packaging_version +from packaging.specifiers import SpecifierSet import six from .requirements import SimpleSubstitution, FatalSpecsResolutionError @@ -155,10 +156,16 @@ class PytorchRequirement(SimpleSubstitution): self.os = os_name or self.get_platform() self.cuda = "cuda{}".format(self.cuda_version).lower() self.python_version_string = str(self.config["agent.default_python"]) - self.python_semantic_version = Version.coerce( - self.python_version_string, partial=True - ) - self.python = "python{}.{}".format(self.python_semantic_version.major, self.python_semantic_version.minor) + self.python_major_minor_str = '.'.join(packaging_version.parse( + self.python_version_string).base_version.split('.')[:2]) + if '.' not in self.python_major_minor_str: + raise PytorchResolutionError( + "invalid python version {!r} defined in configuration file, key 'agent.default_python': " + "must have both major and minor parts of the version (for example: '3.7')".format( + self.python_version_string + ) + ) + self.python = "python{}".format(self.python_major_minor_str) self.exceptions = [ PytorchResolutionError(message) @@ -188,9 +195,7 @@ class PytorchRequirement(SimpleSubstitution): """ Make sure python version has both major and minor versions as required for choosing pytorch wheel """ - if self.is_pip and not ( - self.python_semantic_version.major and self.python_semantic_version.minor - ): + if self.is_pip and not self.python_major_minor_str: raise PytorchResolutionError( "invalid python version {!r} defined in configuration file, key 'agent.default_python': " "must have both major and minor parts of the version (for example: '3.7')".format( @@ -215,8 +220,10 @@ class PytorchRequirement(SimpleSubstitution): links_parser = LinksHTMLParser() links_parser.feed(requests.get(torch_url, timeout=10).text) platform_wheel = "win" if self.get_platform() == "windows" else self.get_platform() - py_ver = "{0.major}{0.minor}".format(self.python_semantic_version) + py_ver = self.python_major_minor_str.replace('.', '') url = None + spec = SpecifierSet(req.format_specs()) + last_v = None # search for our package for l in links_parser.links: parts = l.split('/')[-1].split('-') @@ -225,14 +232,19 @@ class PytorchRequirement(SimpleSubstitution): if parts[0] != req.name: continue # version (ignore +cpu +cu92 etc. + is %2B in the file link) - if parts[1].split('%')[0].split('+')[0] != req.specs[0][1]: + # version ignore .postX suffix (treat as regular version) + try: + v = packaging_version.parse(parts[1].split('%')[0].split('+')[0]) + except Exception: + continue + if v not in spec or (last_v and last_v > v): continue if not parts[2].endswith(py_ver): continue if platform_wheel not in parts[4]: continue url = '/'.join(torch_url.split('/')[:-1] + l.split('/')) - break + last_v = v return url @@ -254,7 +266,8 @@ class PytorchRequirement(SimpleSubstitution): pass # make sure we have a specific version to retrieve - assert req.specs + if not req.specs: + req.specs = [('>', '0')] try: req.specs[0] = (req.specs[0][0], req.specs[0][1].split('+')[0]) @@ -282,7 +295,7 @@ class PytorchRequirement(SimpleSubstitution): if not url: url = PytorchWheel( torch_version=fix_version(version), - python="{0.major}{0.minor}".format(self.python_semantic_version), + python=self.python_major_minor_str.replace('.', ''), os_name=self.os, cuda_version=self.cuda_version, ).make_url() @@ -296,13 +309,13 @@ class PytorchRequirement(SimpleSubstitution): @staticmethod def match_version(req, options): versioned_options = sorted( - ((Version(fix_version(key)), value) for key, value in options.items()), + ((packaging_version.parse(fix_version(key)), value) for key, value in options.items()), key=itemgetter(0), reverse=True, ) req.specs = [(op, fix_version(version)) for op, version in req.specs] if req.specs: - specs = Spec(req.format_specs()) + specs = SpecifierSet(req.format_specs()) else: specs = None try: diff --git a/trains_agent/helper/package/requirements.py b/trains_agent/helper/package/requirements.py index a3439fd..5157c0d 100644 --- a/trains_agent/helper/package/requirements.py +++ b/trains_agent/helper/package/requirements.py @@ -10,7 +10,7 @@ from operator import itemgetter from os import path from typing import Text, List, Type, Optional, Tuple -import semantic_version +from packaging import version as packaging_version from pathlib2 import Path from pyhocon import ConfigTree from requirements import parse @@ -177,7 +177,7 @@ class SimpleSubstitution(RequirementSubstitution): if req.specs: _, version_number = req.specs[0] - assert semantic_version.Version(version_number, partial=True) + assert packaging_version.parse(version_number) else: version_number = self.get_pip_version(self.name)