From 09c5ef99af63837844681513c8c4ae1df68261d7 Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Thu, 29 Feb 2024 14:12:21 +0200 Subject: [PATCH] Fix Python 3.12 support by removing distutil imports --- clearml_agent/commands/worker.py | 5 +-- clearml_agent/helper/base.py | 2 +- .../helper/environment/converters.py | 18 ++++++++- clearml_agent/helper/package/conda_api.py | 3 +- clearml_agent/helper/process.py | 38 ++++++++++++++++++- 5 files changed, 58 insertions(+), 8 deletions(-) diff --git a/clearml_agent/commands/worker.py b/clearml_agent/commands/worker.py index adbcd3c..af0dd8b 100644 --- a/clearml_agent/commands/worker.py +++ b/clearml_agent/commands/worker.py @@ -19,8 +19,6 @@ import traceback from collections import defaultdict from copy import deepcopy, copy from datetime import datetime -from distutils.spawn import find_executable -from distutils.util import strtobool from functools import partial from os.path import basename from tempfile import mkdtemp, NamedTemporaryFile @@ -114,6 +112,7 @@ from clearml_agent.helper.base import ( ) from clearml_agent.helper.check_update import start_check_update_daemon from clearml_agent.helper.console import ensure_text, print_text, decode_binary_lines +from clearml_agent.helper.environment.converters import strtobool from clearml_agent.helper.os.daemonize import daemonize_process from clearml_agent.helper.package.base import PackageManager from clearml_agent.helper.package.conda_api import CondaAPI @@ -140,7 +139,7 @@ from clearml_agent.helper.process import ( commit_docker, terminate_process, check_if_command_exists, - terminate_all_child_processes, + terminate_all_child_processes, find_executable, ) from clearml_agent.helper.repo import clone_repository_cached, RepoInfo, VCS, fix_package_import_diff_patch, \ patch_add_task_init_call diff --git a/clearml_agent/helper/base.py b/clearml_agent/helper/base.py index 9600a2d..bee1cb3 100644 --- a/clearml_agent/helper/base.py +++ b/clearml_agent/helper/base.py @@ -14,7 +14,6 @@ import sys import tempfile from abc import ABCMeta from collections import OrderedDict -from distutils.spawn import find_executable from functools import total_ordering from typing import Text, Dict, Any, Optional, AnyStr, IO, Union @@ -38,6 +37,7 @@ use_powershell = os.getenv("CLEARML_AGENT_USE_POWERSHELL", None) def which(cmd, path=None): + from clearml_agent.helper.process import find_executable result = find_executable(cmd, path) if not result: raise ValueError('command "{}" not found'.format(cmd)) diff --git a/clearml_agent/helper/environment/converters.py b/clearml_agent/helper/environment/converters.py index 943cc69..26e15fe 100644 --- a/clearml_agent/helper/environment/converters.py +++ b/clearml_agent/helper/environment/converters.py @@ -1,5 +1,4 @@ import base64 -from distutils.util import strtobool from typing import Union, Optional, Any, TypeVar, Callable, Tuple import six @@ -68,3 +67,20 @@ def or_(*converters, **kwargs): return value return wrapper + + +def strtobool (val): + """Convert a string representation of truth to true (1) or false (0). + + True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values + are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if + 'val' is anything else. + """ + val = val.lower() + if val in ('y', 'yes', 't', 'true', 'on', '1'): + return 1 + elif val in ('n', 'no', 'f', 'false', 'off', '0'): + return 0 + else: + raise ValueError("invalid truth value %r" % (val,)) + diff --git a/clearml_agent/helper/package/conda_api.py b/clearml_agent/helper/package/conda_api.py index 9c39a03..ffe2894 100644 --- a/clearml_agent/helper/package/conda_api.py +++ b/clearml_agent/helper/package/conda_api.py @@ -5,7 +5,6 @@ import re import os import subprocess from collections import OrderedDict -from distutils.spawn import find_executable from functools import partial from itertools import chain from typing import Text, Iterable, Union, Dict, Set, Sequence, Any @@ -22,7 +21,7 @@ from clearml_agent.errors import CommandFailedError from clearml_agent.helper.base import ( rm_tree, NonStrictAttrs, select_for_platform, is_windows_platform, ExecutionInfo, convert_cuda_version_to_float_single_digit_str, convert_cuda_version_to_int_10_base_str, ) -from clearml_agent.helper.process import Argv, Executable, DEVNULL, CommandSequence, PathLike +from clearml_agent.helper.process import Argv, Executable, DEVNULL, CommandSequence, PathLike, find_executable from clearml_agent.helper.package.requirements import SimpleVersion from clearml_agent.session import Session from .base import PackageManager diff --git a/clearml_agent/helper/process.py b/clearml_agent/helper/process.py index 452f73c..9594450 100644 --- a/clearml_agent/helper/process.py +++ b/clearml_agent/helper/process.py @@ -8,7 +8,6 @@ import subprocess import sys from contextlib import contextmanager from copy import copy -from distutils.spawn import find_executable from itertools import chain, repeat, islice from os.path import devnull from time import sleep @@ -492,3 +491,40 @@ def double_quote(s): # use single quotes, and put single quotes into double quotes # the string $"b is then quoted as "$"""b" return '"' + s.replace('"', '"\'\"\'"') + '"' + + +def find_executable(executable, path=None): + """Tries to find 'executable' in the directories listed in 'path'. + + A string listing directories separated by 'os.pathsep'; defaults to + os.environ['PATH']. Returns the complete filename or None if not found. + """ + _, ext = os.path.splitext(executable) + if (sys.platform == 'win32') and (ext != '.exe'): + executable = executable + '.exe' + + if os.path.isfile(executable): + return executable + + if path is None: + path = os.environ.get('PATH', None) + if path is None: + try: + path = os.confstr("CS_PATH") + except (AttributeError, ValueError): + # os.confstr() or CS_PATH is not available + path = os.defpath + # bpo-35755: Don't use os.defpath if the PATH environment variable is + # set to an empty string + + # PATH='' doesn't match, whereas PATH=':' looks in the current directory + if not path: + return None + + paths = path.split(os.pathsep) + for p in paths: + f = os.path.join(p, executable) + if os.path.isfile(f): + # the file exists, we have a shot at spawn working + return f + return None