Version bump

clearml_agent/commands/worker.py

@@ -686,7 +686,7 @@ class Worker(ServiceCommandSection):
             self._docker_args_filters = []
 
         self._task_ping_interval_sec = max(
-            0, text_to_int(self._session.config.get("agent.task_ping_interval_sec", 120.0))
+            0, text_to_int(self._session.config.get("agent.task_ping_interval_sec", 60.0))
         )
 
     @classmethod
@@ -1784,7 +1784,8 @@ class Worker(ServiceCommandSection):
                 if stderr:
                     stderr.flush()
 
-                if self._task_ping_interval_sec and time() - last_task_ping > self._task_ping_interval_sec:
+                if not stopping and self._task_ping_interval_sec and \
+                        time() - last_task_ping > self._task_ping_interval_sec:
                     # noinspection PyBroadException
                     try:
                         res = (session or self._session).send(tasks_api.PingRequest(task=task_id))
@@ -1793,7 +1794,7 @@ class Worker(ServiceCommandSection):
                     except Exception as ex:
                         self.log.error("Failed sending ping: %s", str(ex))
                     finally:
-                        self._task_ping_interval_sec = time()
+                        last_task_ping = time()
 
                 # get diff from previous poll
                 printed_lines, stdout_pos_count = _print_file(stdout_path, stdout_pos_count)
@@ -2103,8 +2104,9 @@ class Worker(ServiceCommandSection):
 
         end_of_build_marker = "build.done=true"
         docker_cmd_suffix = ' build --id {task_id} --install-globally; ' \
-                            'echo "" >> {conf_file} ; ' \
-                            'echo {end_of_build_marker} >> {conf_file} ; ' \
+                            'ORG=$(stat -c "%u:%g" {conf_file}) ; chown $(whoami):$(whoami) {conf_file} ; ' \
+                            'echo "" >> {conf_file} ; echo {end_of_build_marker} >> {conf_file} ; ' \
+                            'chown $ORG {conf_file} ; ' \
                             'bash'.format(
                                 task_id=task_id,
                                 end_of_build_marker=end_of_build_marker,
@@ -2123,10 +2125,16 @@ class Worker(ServiceCommandSection):
 
         # now we need to wait until the line shows on our configuration file.
         while True:
-            while temp_config.stat().st_mtime == base_time_stamp:
-                sleep(5.0)
-            with open(temp_config.as_posix()) as f:
-                lines = [l.strip() for l in f.readlines()]
+            # noinspection PyBroadException
+            try:
+                while temp_config.stat().st_mtime == base_time_stamp:
+                    sleep(5.0)
+                with open(temp_config.as_posix()) as f:
+                    lines = [l.strip() for l in f.readlines()]
+            except Exception as ex:
+                # print("Failed reading status file [{}], retrying in 2 seconds".format(ex))
+                sleep(2.0)
+
             if 'build.done=true' in lines:
                 break
             base_time_stamp = temp_config.stat().st_mtime
@@ -2155,6 +2163,8 @@ class Worker(ServiceCommandSection):
         print(commit_docker(container_name=target, docker_id=docker_id, apply_change=change))
         shutdown_docker_process(docker_id=docker_id)
 
+        safe_remove_file(temp_config.as_posix())
+
         return
 
     def _get_task_python_version(self, task):
@@ -2834,8 +2844,8 @@ class Worker(ServiceCommandSection):
         # Todo: add support for poetry caching
         if not self.poetry.enabled:
             # add to cache
-            print('Adding venv into cache: {}'.format(add_venv_folder_cache))
             if add_venv_folder_cache:
+                print('Adding venv into cache: {}'.format(add_venv_folder_cache))
                 self.package_api.add_cached_venv(
                     requirements=[freeze, previous_reqs],
                     docker_cmd=execution_info.docker_cmd if execution_info else None,

clearml_agent/helper/package/base.py

@@ -177,7 +177,7 @@ class PackageManager(object):
                 cls._pip_version.append("==" + version)
 
     @classmethod
-    def get_pip_versions(cls, pip="pip", wrap='"'):
+    def get_pip_versions(cls, pip="pip", wrap=''):
         return [
             (wrap + pip + version + wrap)
             for version in cls._pip_version or [pip]

clearml_agent/helper/process.py

@@ -117,10 +117,11 @@ def terminate_all_child_processes(pid=None, timeout=10., include_parent=True):
 
 
 def get_docker_id(docker_cmd_contains):
+    # noinspection PyBroadException
     try:
         containers_running = get_bash_output(cmd='docker ps --no-trunc --format \"{{.ID}}: {{.Command}}\"')
         for docker_line in containers_running.split('\n'):
-            parts = docker_line.split(':')
+            parts = docker_line.split(':', 1)
             if docker_cmd_contains in parts[-1]:
                 # we found our docker, return it
                 return parts[0]

clearml_agent/version.py

@@ -1 +1 @@
-__version__ = '1.5.1rc0'
+__version__ = '1.5.2rc0'

requirements.txt

@@ -9,7 +9,7 @@ python-dateutil>=2.4.2,<2.9.0
 pyjwt>=2.4.0,<2.7.0
 PyYAML>=3.12,<6.1
 requests>=2.20.0,<2.29.0
-six>=1.13.0,<1.16.0
+six>=1.13.0,<1.17.0
 typing>=3.6.4,<3.8.0 ; python_version < '3.5'
 urllib3>=1.21.1,<1.27.0
 virtualenv>=16,<21