ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-02-01 01:26:49 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add last worker time to AutoScaler

Browse Source
This commit is contained in:
allegroai 2021-11-30 21:13:02 +02:00
parent b3a2b3425c
commit f620c6302a
1 changed files with 13 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
clearml/automation/auto_scaler.py
View File

@ -333,7 +333,8 @@ class AutoScaler(object):
if not task:
if worker.id not in idle_workers:
resource_name = WorkerId(worker.id).instance_type
idle_workers[worker.id] = (time(), resource_name, worker)
worker_time = worker_last_time(worker)
idle_workers[worker.id] = (worker_time, resource_name, worker)
elif worker.id in idle_workers:
idle_workers.pop(worker.id, None)
@ -373,3 +374,14 @@ def has_duplicate_resource(queues: dict):
return True
seen.add(name)
return False
def worker_last_time(worker):
"""Last time we heard from a worker. Current time if we can't find"""
time_attrs = [
'register_time',
'last_activity_time',
'last_report_time',
]
times = [getattr(worker, attr).timestamp() for attr in time_attrs if getattr(worker, attr)]
return max(times) if times else time()