Find the correct python version when using a pre-installed python environment

clearml_agent/commands/worker.py

@@ -76,7 +76,9 @@ from clearml_agent.definitions import (
     ENV_EXTRA_DOCKER_LABELS,
     ENV_AGENT_FORCE_CODE_DIR,
     ENV_AGENT_FORCE_EXEC_SCRIPT,
-    ENV_TEMP_STDOUT_FILE_DIR, ENV_AGENT_FORCE_TASK_INIT,
+    ENV_TEMP_STDOUT_FILE_DIR,
+    ENV_AGENT_FORCE_TASK_INIT,
+    ENV_AGENT_DEBUG_GET_NEXT_TASK,
 )
 from clearml_agent.definitions import WORKING_REPOSITORY_DIR, PIP_EXTRA_INDICES
 from clearml_agent.errors import (
@@ -108,7 +110,7 @@ from clearml_agent.helper.base import (
     is_linux_platform,
     rm_file,
     add_python_path,
-    safe_remove_tree,
+    safe_remove_tree, get_python_version,
 )
 from clearml_agent.helper.check_update import start_check_update_daemon
 from clearml_agent.helper.console import ensure_text, print_text, decode_binary_lines
@@ -305,9 +307,12 @@ def get_next_task(session, queue, get_task_info=False):
     """
     Returns dict that contains next task and its additional info (company, user)
     """
+    debug = ENV_AGENT_DEBUG_GET_NEXT_TASK.get()
     request = {'queue': queue}
     if get_task_info:
         request["get_task_info"] = True
+    if debug:
+        print(f"debug> get_next_task: {Request.def_method} payload {request}")
     result = session.send_request(
         service='queues',
         action='get_next_task',
@@ -316,6 +321,8 @@ def get_next_task(session, queue, get_task_info=False):
         method=Request.def_method,
         async_enable=False,
     )
+    if debug:
+        print(f"debug> get_next_task: response {result.status_code} text {result.text}")
     if not result.ok:
         raise APIError(result)
     data = result.json().get('data')
@@ -3567,7 +3574,8 @@ class Worker(ServiceCommandSection):
             if override_interpreter_path:
                 print("Python interpreter {} is set from environment var".format(override_interpreter_path))
                 executable_name = override_interpreter_path
-                executable_version_suffix = self._get_python_version_suffix(executable_name)
+                executable_version_suffix = (get_python_version(executable_name, self.log) or
+                                             self._get_python_version_suffix(executable_name))
             else:
                 try:
                     executable_version, executable_version_suffix, executable_name = \

clearml_agent/helper/base.py

@@ -543,6 +543,36 @@ def convert_cuda_version_to_int_10_base_str(cuda_version):
     return str(int(float(cuda_version)*10))
 
 
+def get_python_version(python_executable, log=None):
+    from clearml_agent.helper.process import Argv
+    try:
+        output = Argv(python_executable, "--version").get_output(
+            stderr=subprocess.STDOUT
+        )
+    except subprocess.CalledProcessError as ex:
+        # Windows returns 9009 code and suggests to install Python from Windows Store
+        if is_windows_platform() and ex.returncode == 9009:
+            if log:
+                log.debug("version not found: {}".format(ex))
+        else:
+            if log:
+                log.warning("error getting %s version: %s", python_executable, ex)
+        return None
+    except FileNotFoundError as ex:
+        if log:
+            log.debug("version not found: {}".format(ex))
+        return None
+
+    match = re.search(r"Python ({}(?:\.\d+)*)".format(r"\d+"), output)
+    if match:
+        if log:
+            log.debug("Found: {}".format(python_executable))
+        # only return major.minor version
+        return ".".join(str(match.group(1)).split(".")[:2])
+
+    return None
+
+
 class NonStrictAttrs(object):
 
     @classmethod

clearml_agent/helper/environment/converters.py

@@ -69,7 +69,7 @@ def or_(*converters, **kwargs):
     return wrapper
 
 
-def strtobool (val):
+def strtobool(val):
     """Convert a string representation of truth to true (1) or false (0).
 
     True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values