Improve server threads shutdown on SIGTERM

server/bll/statistics/resource_monitor.py

@@ -6,6 +6,8 @@ from time import sleep
 import attr
 import psutil
 
+from utilities.threads_manager import ThreadsManager
+
 
 class ResourceMonitor(Thread):
     @attr.s(auto_attribs=True)
@@ -58,7 +60,7 @@ class ResourceMonitor(Thread):
         )
 
     def run(self):
-        while True:
+        while not ThreadsManager.terminating:
             sleep(self.sample_interval_sec)
 
             sample = self._get_sample()

server/bll/statistics/stats_reporter.py

@@ -53,11 +53,8 @@ class StatisticsReporter:
         report_interval = timedelta(
             hours=config.get("apiserver.statistics.report_interval_hours", 24)
         )
-
-        while True:
-
-            sleep(report_interval.total_seconds())
-
+        sleep(report_interval.total_seconds())
+        while not ThreadsManager.terminating:
             try:
                 for company in Company.objects(
                     defaults__stats_option__enabled=True
@@ -68,6 +65,8 @@ class StatisticsReporter:
             except Exception as ex:
                 log.exception(f"Failed collecting stats: {str(ex)}")
 
+            sleep(report_interval.total_seconds())
+
     @classmethod
     @threads.register("sender", daemon=True)
     def start_sender(cls):
@@ -86,7 +85,7 @@ class StatisticsReporter:
 
         WarningFilter.attach()
 
-        while True:
+        while not ThreadsManager.terminating:
             try:
                 report = cls.send_queue.get()
 

server/bll/task/task_bll.py

@@ -569,13 +569,11 @@ class TaskBLL(object):
                 "services.tasks.non_responsive_tasks_watchdog.threshold_sec", 7200
             )
         )
-        while True:
-            sleep(
-                config.get(
-                    "services.tasks.non_responsive_tasks_watchdog.watch_interval_sec",
-                    900,
-                )
-            )
+        watch_interval = config.get(
+            "services.tasks.non_responsive_tasks_watchdog.watch_interval_sec", 900
+        )
+        sleep(watch_interval)
+        while not ThreadsManager.terminating:
             try:
 
                 ref_time = datetime.utcnow() - threshold
@@ -611,6 +609,8 @@ class TaskBLL(object):
             except Exception as ex:
                 log.exception(f"Failed stopping tasks: {str(ex)}")
 
+            sleep(watch_interval)
+
     @staticmethod
     def get_aggregated_project_execution_parameters(
         company_id,