Reduce required packages

This commit is contained in:
clearml 2025-01-26 23:03:16 +02:00
parent 7e9e3ad08b
commit 28e9280a4f
63 changed files with 130 additions and 143 deletions

View File

@ -1,6 +1,6 @@
from ...backend_config import Config
from ...backend_config.defs import ENV_CONFIG_VERBOSE
from pathlib2 import Path
from ..._vendor.pathlib2 import Path
def load(*additional_module_paths):

View File

@ -1,10 +1,10 @@
import re
from functools import partial
import attr
from attr.converters import optional as optional_converter
from attr.validators import instance_of, optional, and_
from six import string_types
from ..._vendor import attr
from ..._vendor.attr.converters import optional as optional_converter
from ..._vendor.attr.validators import instance_of, optional, and_
from ..._vendor.six import string_types
# noinspection PyTypeChecker
sequence = instance_of((list, tuple))

View File

@ -1,8 +1,8 @@
import itertools
import re
import attr
import six
from ..._vendor import attr
from ..._vendor import six
from clearml_agent.external import pyhocon

View File

@ -4,12 +4,12 @@ auth service
This service provides authentication management and authorization
validation for the entire system.
"""
import six
from ...._vendor import six
import types
from datetime import datetime
import enum
from dateutil.parser import parse as parse_datetime
from ...._vendor.dateutil.parser import parse as parse_datetime
from ....backend_api.session import Request, BatchRequest, Response, DataModel, NonStrictDataModel, CompoundRequest, schema_property, StringEnum

View File

@ -3,12 +3,12 @@ debug service
Debugging utilities
"""
import six
from ...._vendor import six
import types
from datetime import datetime
import enum
from dateutil.parser import parse as parse_datetime
from ...._vendor.dateutil.parser import parse as parse_datetime
from ....backend_api.session import Request, BatchRequest, Response, DataModel, NonStrictDataModel, CompoundRequest, schema_property, StringEnum

View File

@ -3,12 +3,12 @@ events service
Provides an API for running tasks to report events collected by the system.
"""
import six
from ...._vendor import six
import types
from datetime import datetime
import enum
from dateutil.parser import parse as parse_datetime
from ...._vendor.dateutil.parser import parse as parse_datetime
from ....backend_api.session import Request, BatchRequest, Response, DataModel, NonStrictDataModel, CompoundRequest, schema_property, StringEnum

View File

@ -3,12 +3,12 @@ models service
This service provides a management interface for models (results of training tasks) stored in the system.
"""
import six
from ...._vendor import six
import types
from datetime import datetime
import enum
from dateutil.parser import parse as parse_datetime
from ...._vendor.dateutil.parser import parse as parse_datetime
from ....backend_api.session import Request, BatchRequest, Response, DataModel, NonStrictDataModel, CompoundRequest, schema_property, StringEnum

View File

@ -3,12 +3,12 @@ queues service
Provides a management API for queues of tasks waiting to be executed by workers deployed anywhere (see Workers Service).
"""
import six
from ...._vendor import six
import types
from datetime import datetime
import enum
from dateutil.parser import parse as parse_datetime
from ...._vendor.dateutil.parser import parse as parse_datetime
from ....backend_api.session import Request, BatchRequest, Response, DataModel, NonStrictDataModel, CompoundRequest, schema_property, StringEnum

View File

@ -6,8 +6,8 @@ Provides a management API for tasks in the system.
import enum
from datetime import datetime
import six
from dateutil.parser import parse as parse_datetime
from ...._vendor import six
from ...._vendor.dateutil.parser import parse as parse_datetime
from ....backend_api.session import Request, BatchRequest, Response, NonStrictDataModel, schema_property, StringEnum

View File

@ -3,12 +3,12 @@ workers service
Provides an API for worker machines, allowing workers to report status and get tasks for execution
"""
import six
from ...._vendor import six
import types
from datetime import datetime
import enum
from dateutil.parser import parse as parse_datetime
from ...._vendor.dateutil.parser import parse as parse_datetime
from ...session import Request, BatchRequest, Response, DataModel, NonStrictDataModel, CompoundRequest, schema_property, StringEnum

View File

@ -4,12 +4,12 @@ auth service
This service provides authentication management and authorization
validation for the entire system.
"""
import six
from ...._vendor import six
import types
from datetime import datetime
import enum
from dateutil.parser import parse as parse_datetime
from ...._vendor.dateutil.parser import parse as parse_datetime
from ....backend_api.session import Request, BatchRequest, Response, DataModel, NonStrictDataModel, CompoundRequest, schema_property, StringEnum

View File

@ -3,12 +3,12 @@ debug service
Debugging utilities
"""
import six
from ...._vendor import six
import types
from datetime import datetime
import enum
from dateutil.parser import parse as parse_datetime
from ...._vendor.dateutil.parser import parse as parse_datetime
from ....backend_api.session import Request, BatchRequest, Response, DataModel, NonStrictDataModel, CompoundRequest, schema_property, StringEnum

View File

@ -3,12 +3,12 @@ events service
Provides an API for running tasks to report events collected by the system.
"""
import six
from ...._vendor import six
import types
from datetime import datetime
import enum
from dateutil.parser import parse as parse_datetime
from ...._vendor.dateutil.parser import parse as parse_datetime
from ....backend_api.session import Request, BatchRequest, Response, DataModel, NonStrictDataModel, CompoundRequest, schema_property, StringEnum

View File

@ -3,12 +3,12 @@ models service
This service provides a management interface for models (results of training tasks) stored in the system.
"""
import six
from ...._vendor import six
import types
from datetime import datetime
import enum
from dateutil.parser import parse as parse_datetime
from ...._vendor.dateutil.parser import parse as parse_datetime
from ....backend_api.session import Request, BatchRequest, Response, DataModel, NonStrictDataModel, CompoundRequest, schema_property, StringEnum

View File

@ -3,12 +3,12 @@ queues service
Provides a management API for queues of tasks waiting to be executed by workers deployed anywhere (see Workers Service).
"""
import six
from ...._vendor import six
import types
from datetime import datetime
import enum
from dateutil.parser import parse as parse_datetime
from ...._vendor.dateutil.parser import parse as parse_datetime
from ....backend_api.session import Request, BatchRequest, Response, DataModel, NonStrictDataModel, CompoundRequest, schema_property, StringEnum

View File

@ -3,12 +3,12 @@ tasks service
Provides a management API for tasks in the system.
"""
import six
from ...._vendor import six
import types
from datetime import datetime
import enum
from dateutil.parser import parse as parse_datetime
from ...._vendor.dateutil.parser import parse as parse_datetime
from ....backend_api.session import Request, BatchRequest, Response, NonStrictDataModel, schema_property, StringEnum

View File

@ -3,12 +3,12 @@ workers service
Provides an API for worker machines, allowing workers to report status and get tasks for execution
"""
import six
from ...._vendor import six
import types
from datetime import datetime
import enum
from dateutil.parser import parse as parse_datetime
from ...._vendor.dateutil.parser import parse as parse_datetime
from ....backend_api.session import Request, BatchRequest, Response, DataModel, NonStrictDataModel, CompoundRequest, schema_property, StringEnum

View File

@ -12,7 +12,7 @@ from operator import itemgetter
from types import ModuleType
from typing import Dict, Text, Tuple, Type, Any, Sequence
import six
from ...._vendor import six
from ... import services as api_services
from ....backend_api.session import CallResult
from ....backend_api.session import Session, Request as APIRequest

View File

@ -5,10 +5,10 @@ import json
import warnings
from datetime import datetime
import jsonschema
from ..._vendor import jsonschema
from enum import Enum
import six
from ..._vendor import six
def format_date(obj):

View File

@ -4,7 +4,7 @@ from __future__ import absolute_import
from collections import defaultdict
import six
from ...._vendor import six
from . import errors
from .fields import NotSet

View File

@ -2,8 +2,8 @@ import datetime
import re
from weakref import WeakKeyDictionary
import six
from dateutil.parser import parse
from ...._vendor import six
from ...._vendor.dateutil.parser import parse
from .errors import ValidationError
from .collections import ModelCollection

View File

@ -1,4 +1,4 @@
import six
from ...._vendor import six
from . import parsers, errors
from .fields import BaseField

View File

@ -1,6 +1,6 @@
from __future__ import absolute_import
import six
from ...._vendor import six
import re
from collections import namedtuple

View File

@ -1,7 +1,7 @@
"""Predefined validators."""
import re
from six.moves import reduce
from ...._vendor.six.moves import reduce
from .errors import ValidationError
from . import utilities

View File

@ -1,7 +1,7 @@
import abc
import jsonschema
import six
from ..._vendor import jsonschema
from ..._vendor import six
from .apimodel import ApiModel
from .datamodel import DataModel

View File

@ -1,6 +1,6 @@
import requests
import six
from ..._vendor import six
from . import jsonmodels
from .apimodel import ApiModel

View File

@ -8,12 +8,12 @@ from random import SystemRandom
from socket import gethostname
from typing import Optional
import jwt
from ..._vendor import jwt
import requests
import six
from ..._vendor import six
from requests import RequestException
from requests.auth import HTTPBasicAuth
from six.moves.urllib.parse import urlparse, urlunparse
from ..._vendor.six.moves.urllib.parse import urlparse, urlunparse
from clearml_agent.external.pyhocon import ConfigTree, ConfigFactory
from .callresult import CallResult

View File

@ -2,9 +2,9 @@ import sys
from abc import ABCMeta, abstractmethod
from time import time
import jwt
from jwt.algorithms import get_default_algorithms
import six
from ..._vendor import jwt
from ..._vendor.jwt.algorithms import get_default_algorithms
from ..._vendor import six
@six.add_metaclass(ABCMeta)

View File

@ -7,9 +7,9 @@ import sys
from os.path import expanduser
from typing import Any
import six
from pathlib2 import Path
from pyparsing import (
from .._vendor import six
from .._vendor.pathlib2 import Path
from .._vendor.pyparsing import (
ParseFatalException,
ParseException,
RecursiveGrammarException,

View File

@ -1,5 +1,5 @@
from os.path import expanduser
from pathlib2 import Path
from .._vendor.pathlib2 import Path
from ..backend_config.environment import EnvEntry

View File

@ -1,6 +1,6 @@
import logging.config
from pathlib2 import Path
from .._vendor.pathlib2 import Path
def logger(path=None):

View File

@ -12,7 +12,7 @@ from typing import Text
from clearml_agent.helper.console import ListFormatter, print_text
from clearml_agent.helper.dicts import filter_keys
import six
from .._vendor import six
from clearml_agent.backend_api import services
from clearml_agent.errors import APIError, CommandFailedError

View File

@ -2,9 +2,9 @@ from __future__ import print_function
from typing import Dict, Optional
from pathlib2 import Path
from six.moves import input
from six.moves.urllib.parse import urlparse
from .._vendor.pathlib2 import Path
from .._vendor.six.moves import input
from .._vendor.six.moves.urllib.parse import urlparse
from clearml_agent.backend_api.session import Session
from clearml_agent.backend_api.session.defs import ENV_HOST

View File

@ -25,10 +25,10 @@ from tempfile import mkdtemp, NamedTemporaryFile
from time import sleep, time
from typing import Text, Optional, Any, Tuple, List, Dict, Mapping, Union
import attr
import six
from pathlib2 import Path
from six.moves.urllib.parse import quote
from .._vendor import attr
from .._vendor import six
from .._vendor.pathlib2 import Path
from .._vendor.six.moves.urllib.parse import quote
from clearml_agent.external.pyhocon import ConfigTree, ConfigFactory
from clearml_agent.backend_api.services import auth as auth_api

View File

@ -1,4 +1,4 @@
import six
from ._vendor import six
from clearml_agent.external.pyhocon import ConfigTree
from clearml_agent.helper.base import Singleton

View File

@ -4,8 +4,8 @@ from enum import IntEnum
from os import getenv, environ
from typing import Text, Optional, Union, Tuple, Any
import six
from pathlib2 import Path
from ._vendor import six
from ._vendor.pathlib2 import Path
from clearml_agent.helper.base import normalize_path
from clearml_agent.helper.environment.converters import strtobool

View File

@ -1,7 +1,7 @@
from typing import Union, Optional, Text
import requests
import six
from ._vendor import six
from .backend_api import CallResult
from .backend_api.session.client import APIError as ClientAPIError
from .backend_api.session.response import ResponseMeta

View File

@ -6,9 +6,9 @@ import contextlib
import codecs
from datetime import timedelta
from pyparsing import Forward, Keyword, QuotedString, Word, Literal, Suppress, Regex, Optional, SkipTo, ZeroOrMore, \
from ..._vendor.pyparsing import Forward, Keyword, QuotedString, Word, Literal, Suppress, Regex, Optional, SkipTo, ZeroOrMore, \
Group, lineno, col, TokenConverter, replaceWith, alphanums, alphas8bit, ParseSyntaxException, StringEnd
from pyparsing import ParserElement
from ..._vendor.pyparsing import ParserElement
from .config_tree import ConfigTree, ConfigSubstitution, ConfigList, ConfigValues, ConfigUnquotedString, \
ConfigInclude, NoneValue, ConfigQuotedString
from .exceptions import ConfigSubstitutionException, ConfigMissingException, ConfigException
@ -56,7 +56,7 @@ class STR_SUBSTITUTION(object):
def period(period_value, period_unit):
try:
from dateutil.relativedelta import relativedelta as period_impl
from ..._vendor.dateutil.relativedelta import relativedelta as period_impl
except Exception:
from datetime import timedelta as period_impl
@ -219,7 +219,7 @@ class ConfigParser(object):
cls.supported_period_map.update(cls.period_type_map)
try:
from dateutil import relativedelta
from ..._vendor.dateutil import relativedelta
if relativedelta is not None:
cls.supported_period_map.update(cls.optional_period_type_map)

View File

@ -1,6 +1,6 @@
from collections import OrderedDict
from pyparsing import lineno
from pyparsing import col
from ..._vendor.pyparsing import lineno
from ..._vendor.pyparsing import col
try:
basestring
except NameError: # pragma: no cover

View File

@ -16,8 +16,6 @@ from pprint import pformat
from time import sleep, time
from typing import Text, List, Callable, Any, Collection, Optional, Union, Iterable, Dict, Tuple, Set
import yaml
from clearml_agent.commands.events import Events
from clearml_agent.commands.worker import Worker, get_task_container, set_task_container, get_next_task
from clearml_agent.definitions import (
@ -45,6 +43,7 @@ from clearml_agent.glue.definitions import (
ENV_KUBECTL_IGNORE_ERROR,
ENV_DEFAULT_SCHEDULER_QUEUE_TAGS,
)
from .._vendor import pyyaml as yaml
class K8sIntegration(Worker):

View File

@ -17,13 +17,13 @@ from collections import OrderedDict
from functools import total_ordering
from typing import Text, Dict, Any, Optional, AnyStr, IO, Union
import attr
import furl
import six
import yaml
from attr import fields_dict
from pathlib2 import Path
from six.moves import reduce
from .._vendor import attr
from .._vendor import furl
from .._vendor import six
from .._vendor.attr import fields_dict
from .._vendor.pathlib2 import Path
from .._vendor.six.moves import reduce # noqa
from .._vendor import pyyaml as yaml
from clearml_agent.errors import CommandFailedError
from clearml_agent.external import pyhocon

View File

@ -5,10 +5,10 @@ import sys
from collections.abc import Iterable
from typing import List, Dict, Text, Any
from attr import attrs, attrib
from .._vendor.attr import attrs, attrib
import six
from six import binary_type, text_type
from .._vendor import six
from .._vendor.six import binary_type, text_type
from clearml_agent.helper.base import nonstrict_in_place_sort

View File

@ -1,7 +1,7 @@
import base64
from typing import Union, Optional, Any, TypeVar, Callable, Tuple
import six
from ..._vendor import six
try:
from typing import Text
@ -83,4 +83,3 @@ def strtobool(val):
return 0
else:
raise ValueError("invalid truth value %r" % (val,))

View File

@ -1,7 +1,7 @@
import abc
from typing import Optional, Any, Tuple, Callable, Dict
import six
from ..._vendor import six
from .converters import any_to_bool

View File

@ -6,7 +6,7 @@ from time import time
from typing import List, Optional, Sequence
import psutil
from pathlib2 import Path
from ..._vendor.pathlib2 import Path
from .locks import FileLock

View File

@ -6,8 +6,8 @@ from contextlib import contextmanager
from hashlib import md5
from typing import Text, Iterable, Union, Optional, Dict, List
import six
from pathlib2 import Path
from ..._vendor import six
from ..._vendor.pathlib2 import Path
from clearml_agent.definitions import ENV_VENV_CACHE_PATH
from clearml_agent.helper.base import mkstemp, safe_remove_file, join_lines, select_for_platform

View File

@ -9,11 +9,11 @@ from functools import partial
from itertools import chain
from typing import Text, Iterable, Union, Dict, Set, Sequence, Any
import six
import yaml
from ..._vendor import six
from ..._vendor import pyyaml as yaml
from time import time
from attr import attrs, attrib, Factory
from pathlib2 import Path
from ..._vendor.attr import attrs, attrib, Factory
from ..._vendor.pathlib2 import Path
from clearml_agent.external.requirements_parser import parse
from clearml_agent.external.requirements_parser.requirement import Requirement

View File

@ -2,7 +2,7 @@ import re
from collections import OrderedDict
from typing import Text
from pathlib2 import Path
from ..._vendor.pathlib2 import Path
from .base import PackageManager
from .requirements import SimpleSubstitution

View File

@ -1,6 +1,6 @@
from typing import Any
from pathlib2 import Path
from ...._vendor.pathlib2 import Path
from clearml_agent.helper.base import select_for_platform, rm_tree, ExecutionInfo
from clearml_agent.helper.package.base import PackageManager

View File

@ -1,10 +1,10 @@
from copy import deepcopy
from functools import wraps
import attr
from ..._vendor import attr
import sys
import os
from pathlib2 import Path
from ..._vendor.pathlib2 import Path
from clearml_agent.definitions import ENV_AGENT_FORCE_POETRY
from clearml_agent.helper.process import Argv, DEVNULL, check_if_command_exists

View File

@ -3,16 +3,16 @@ from __future__ import unicode_literals
import re
import sys
import platform
from furl import furl
from ..._vendor.furl import furl
import urllib.parse
from operator import itemgetter
from html.parser import HTMLParser
from typing import Text, Optional, Dict
import attr
from ..._vendor import attr
import requests
import six
from ..._vendor import six
from .requirements import (
SimpleSubstitution, FatalSpecsResolutionError, SimpleVersion, MarkerRequirement,
compare_version_rules, )

View File

@ -10,11 +10,11 @@ from operator import itemgetter
from os import path
from typing import Text, List, Type, Optional, Tuple, Dict
from pathlib2 import Path
from ..._vendor.pathlib2 import Path
from clearml_agent.external.pyhocon import ConfigTree
import six
from six.moves.urllib.parse import unquote
from ..._vendor import six
from ..._vendor.six.moves.urllib.parse import unquote # noqa
import logging
from clearml_agent.definitions import PIP_EXTRA_INDICES
from clearml_agent.helper.base import (

View File

@ -1,8 +1,8 @@
from tempfile import mkdtemp
from typing import Text
from furl import furl
from pathlib2 import Path
from ..._vendor.furl import furl
from ..._vendor.pathlib2 import Path
from clearml_agent.config import Config
from .pip_api.system import SystemPip

View File

@ -1,10 +1,10 @@
from copy import deepcopy
from functools import wraps
import attr
from ..._vendor import attr
import sys
import os
from pathlib2 import Path
from ..._vendor.pathlib2 import Path
from clearml_agent.definitions import ENV_AGENT_FORCE_UV
from clearml_agent.helper.base import select_for_platform

View File

@ -1,9 +1,9 @@
from typing import Optional, Text
import requests
from pathlib2 import Path
from ..._vendor.pathlib2 import Path
import six
from ..._vendor import six
from clearml_agent.definitions import CONFIG_DIR
from clearml_agent.helper.process import Argv, DEVNULL
from .pip_api.venv import VirtualenvPip

View File

@ -14,10 +14,10 @@ from time import sleep
from typing import Union, Text, Sequence, Any, TypeVar, Callable
import psutil
from furl import furl
from pathlib2 import Path
from .._vendor.furl import furl
from .._vendor.pathlib2 import Path
import six
from .._vendor import six
from clearml_agent.definitions import PROGRAM_NAME, CONFIG_FILE
from clearml_agent.helper.base import bash_c, is_windows_platform, select_for_platform, chain_map

View File

@ -12,11 +12,11 @@ from random import random
from threading import Lock
from typing import Text, Sequence, Mapping, Iterable, TypeVar, Callable, Tuple, Optional
import attr
from furl import furl
from pathlib2 import Path
from .._vendor import attr
from .._vendor.furl import furl
from .._vendor.pathlib2 import Path
import six
from .._vendor import six
from clearml_agent.definitions import ENV_AGENT_GIT_USER, ENV_AGENT_GIT_PASS, ENV_AGENT_GIT_HOST, ENV_GIT_CLONE_VERBOSE
from clearml_agent.helper.console import ensure_text, ensure_binary

View File

@ -9,9 +9,9 @@ from threading import Thread, Event
from time import time
from typing import Sequence, List, Union, Dict, Optional
import attr
from .._vendor import attr
import psutil
from pathlib2 import Path
from .._vendor.pathlib2 import Path
from clearml_agent.definitions import ENV_WORKER_TAGS, ENV_GPU_FRACTIONS
from clearml_agent.session import Session

View File

@ -9,9 +9,9 @@ from itertools import chain
from types import ModuleType
from typing import Text, Sequence, Union
from pathlib2 import Path
from .._vendor.pathlib2 import Path
import six
from .._vendor import six
try:
from functools import lru_cache

View File

@ -5,8 +5,8 @@ import argparse
from copy import deepcopy
from functools import partial
import six
from pathlib2 import Path
from .._vendor import six
from .._vendor.pathlib2 import Path
from clearml_agent import definitions
from clearml_agent.session import Session

View File

@ -9,8 +9,8 @@ import sys
from copy import deepcopy
from typing import Any, Callable
import attr
from pathlib2 import Path
from ._vendor import attr
from ._vendor.pathlib2 import Path
from clearml_agent.external.pyhocon import ConfigFactory, HOCONConverter, ConfigTree
from clearml_agent.backend_api.session import Session as _Session, Request

View File

@ -1,17 +1,6 @@
attrs>=18.0,<24.0.0
enum34>=0.9,<1.2.0 ; python_version < '3.6'
furl>=2.0.0,<2.2.0
jsonschema>=2.6.0,<5.0.0
pathlib2>=2.3.0,<2.4.0
psutil>=3.4.2,<5.10.0
pyparsing>=2.0.3,<3.2.0
python-dateutil>=2.4.2,<2.9.0
pyjwt>=2.4.0,<2.9.0
PyYAML>=3.12,<6.1
requests>=2.20.0,<=2.31.0
setuptools ; python_version > '3.11'
six>=1.13.0,<1.17.0
typing>=3.6.4,<3.8.0 ; python_version < '3.5'
urllib3>=1.21.1,<2
virtualenv>=16,<21
requests>=2.20.0,<2.33
setuptools ; python_version > '3.11'
pywin32 ; sys_platform == 'win32'

View File

@ -75,7 +75,7 @@ setup(
extras_require={
},
package_data={
'clearml_agent': ['backend_api/config/default/*.conf']
'clearml_agent': ['backend_api/config/default/*.conf', '_vendor/jsonschema/schemas/*.json']
},
include_package_data=True,
# To provide executable scripts, use entry points in preference to the