ClearML/clearml-server
1
0
Fork 0
mirror of https://github.com/clearml/clearml-server synced 2025-03-03 10:43:10 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Improve visibility for distributed lock hanging

Browse Source
This commit is contained in:
allegroai 2021-05-03 17:35:17 +03:00
parent 7f4ad0d1ca
commit 6411954002
1 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
apiserver/sync.py
View File

@ -1,3 +1,4 @@
import os
import time import time
from contextlib import contextmanager from contextlib import contextmanager
from time import sleep from time import sleep
@ -18,10 +19,12 @@ def distributed_lock(name: str, timeout: int, max_wait: int = 0):
lock_name = f"dist_lock_{name}" lock_name = f"dist_lock_{name}"
start = time.time() start = time.time()
max_wait = max_wait or timeout * 2 max_wait = max_wait or timeout * 2
while not _redis.set(lock_name, value="", ex=timeout, nx=True): pid = os.getpid()
while _redis.set(lock_name, value=pid, ex=timeout, nx=True) is None:
sleep(1) sleep(1)
if time.time() - start > max_wait: if time.time() - start > max_wait:
raise Exception(f"Could not acquire {name} lock for {max_wait} seconds") holder = _redis.get(lock_name)
raise Exception(f"Could not acquire {name} lock for {max_wait} seconds. The lock is hold by {holder}")
try: try:
yield yield
finally: finally: