feat : Retry acquiring usage cleanup lock to handle potential stale locks

This commit is contained in:
안찬우[AI Product] 2025-06-26 22:24:57 +09:00
parent 21d616f8ed
commit 789e6a0db3

View File

@ -1,4 +1,6 @@
import asyncio
import random
import socketio
import logging
import sys
@ -105,10 +107,26 @@ else:
async def periodic_usage_pool_cleanup():
if not aquire_func():
log.debug("Usage pool cleanup lock already exists. Not running it.")
return
log.debug("Running periodic_usage_pool_cleanup")
max_retries = 2
retry_delay = random.uniform(
WEBSOCKET_REDIS_LOCK_TIMEOUT / 2, WEBSOCKET_REDIS_LOCK_TIMEOUT
)
for attempt in range(max_retries + 1):
if aquire_func():
break
else:
if attempt < max_retries:
log.debug(
f"Cleanup lock already exists. Retry {attempt + 1} after {retry_delay}s..."
)
await asyncio.sleep(retry_delay)
else:
log.warning(
"Failed to acquire cleanup lock after retries. Skipping cleanup."
)
return
log.debug("Running periodic_cleanup")
try:
while True:
if not renew_func():