ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-13 23:28:29 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Black formatting

Browse Source
This commit is contained in:
clearml 2024-09-22 22:31:58 +03:00
parent d069339fe9
commit eaeadb18e3
4 changed files with 75 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
clearml/utilities/locks/constants.py
View File

@ -18,7 +18,7 @@ import os
# The actual tests will execute the code anyhow so the following code can
# safely be ignored from the coverage tests
if os.name == 'nt': # pragma: no cover
if os.name == "nt": # pragma: no cover
import msvcrt
LOCK_EX = 0x1 #: exclusive lock
@ -29,7 +29,7 @@ if os.name == 'nt': # pragma: no cover
LOCKFILE_FAIL_IMMEDIATELY = 1
LOCKFILE_EXCLUSIVE_LOCK = 2
elif os.name == 'posix': # pragma: no cover
elif os.name == "posix": # pragma: no cover
import fcntl
LOCK_EX = fcntl.LOCK_EX #: exclusive lock
@ -38,4 +38,4 @@ elif os.name == 'posix': # pragma: no cover
LOCK_UN = fcntl.LOCK_UN #: unlock
else: # pragma: no cover
raise RuntimeError('PortaLocker only defined for nt and posix platforms')
raise RuntimeError("PortaLocker only defined for nt and posix platforms")

2
clearml/utilities/locks/exceptions.py
View File

@ -3,7 +3,7 @@ class BaseLockException(Exception):
LOCK_FAILED = 1
def __init__(self, *args, **kwargs):
self.fh = kwargs.pop('fh', None)
self.fh = kwargs.pop("fh", None)
Exception.__init__(self, *args, **kwargs)

40
clearml/utilities/locks/portalocker.py
View File

@ -4,7 +4,7 @@ from . import exceptions
from . import constants
if os.name == 'nt': # pragma: no cover
if os.name == "nt": # pragma: no cover
import msvcrt
if sys.version_info.major == 2:
@ -17,6 +17,7 @@ if os.name == 'nt': # pragma: no cover
import win32file
import pywintypes
import winerror
__overlapped = pywintypes.OVERLAPPED()
if sys.version_info.major == 2:
if flags & constants.LOCK_NB:
@ -38,10 +39,7 @@ if os.name == 'nt': # pragma: no cover
# error: (33, 'LockFileEx', 'The process cannot access the file
# because another process has locked a portion of the file.')
if exc_value.winerror == winerror.ERROR_LOCK_VIOLATION:
raise exceptions.LockException(
exceptions.LockException.LOCK_FAILED,
exc_value.strerror,
fh=file_)
raise exceptions.LockException(exceptions.LockException.LOCK_FAILED, exc_value.strerror, fh=file_)
else:
# Q: Are there exceptions/codes we should be dealing with
# here?
@ -72,17 +70,12 @@ if os.name == 'nt': # pragma: no cover
msvcrt.locking(file_.fileno(), mode, lock_length)
except IOError as exc_value:
# [ ] be more specific here
raise exceptions.LockException(
exceptions.LockException.LOCK_FAILED,
exc_value.strerror,
fh=file_)
raise exceptions.LockException(exceptions.LockException.LOCK_FAILED, exc_value.strerror, fh=file_)
finally:
if savepos:
file_.seek(savepos)
except IOError as exc_value:
raise exceptions.LockException(
exceptions.LockException.LOCK_FAILED, exc_value.strerror,
fh=file_)
raise exceptions.LockException(exceptions.LockException.LOCK_FAILED, exc_value.strerror, fh=file_)
def unlock(file_):
try:
@ -93,15 +86,15 @@ if os.name == 'nt': # pragma: no cover
try:
msvcrt.locking(file_.fileno(), constants.LOCK_UN, lock_length)
except IOError as exc_value:
if exc_value.strerror == 'Permission denied':
if exc_value.strerror == "Permission denied":
import pywintypes
import win32file
import winerror
__overlapped = pywintypes.OVERLAPPED()
hfile = win32file._get_osfhandle(file_.fileno())
try:
win32file.UnlockFileEx(
hfile, 0, -0x10000, __overlapped)
win32file.UnlockFileEx(hfile, 0, -0x10000, __overlapped)
except pywintypes.error as exc_value:
if exc_value.winerror == winerror.ERROR_NOT_LOCKED:
# error: (158, 'UnlockFileEx',
@ -114,25 +107,20 @@ if os.name == 'nt': # pragma: no cover
# dealing with here?
raise
else:
raise exceptions.LockException(
exceptions.LockException.LOCK_FAILED,
exc_value.strerror,
fh=file_)
raise exceptions.LockException(exceptions.LockException.LOCK_FAILED, exc_value.strerror, fh=file_)
finally:
if savepos:
file_.seek(savepos)
except IOError as exc_value:
raise exceptions.LockException(
exceptions.LockException.LOCK_FAILED, exc_value.strerror,
fh=file_)
raise exceptions.LockException(exceptions.LockException.LOCK_FAILED, exc_value.strerror, fh=file_)
elif os.name == 'posix': # pragma: no cover
elif os.name == "posix": # pragma: no cover
import fcntl
def lock(file_, flags):
locking_exceptions = IOError,
locking_exceptions = (IOError,)
try: # pragma: no cover
locking_exceptions += BlockingIOError,
locking_exceptions += (BlockingIOError,)
except NameError: # pragma: no cover
pass
@ -147,4 +135,4 @@ elif os.name == 'posix': # pragma: no cover
fcntl.flock(file_.fileno(), constants.LOCK_UN)
else: # pragma: no cover
raise RuntimeError('PortaLocker only defined for nt and posix platforms')
raise RuntimeError("PortaLocker only defined for nt and posix platforms")

98
clearml/utilities/locks/utils.py
View File

@ -10,20 +10,20 @@ from . import portalocker
current_time = getattr(time, "monotonic", time.time)
DEFAULT_TIMEOUT = 10 ** 8
DEFAULT_TIMEOUT = 10**8
DEFAULT_CHECK_INTERVAL = 0.25
LOCK_METHOD = constants.LOCK_EX | constants.LOCK_NB
__all__ = [
'Lock',
'RLock',
'open_atomic',
"Lock",
"RLock",
"open_atomic",
]
@contextlib.contextmanager
def open_atomic(filename, binary=True):
'''Open a file for atomic writing. Instead of locking this method allows
"""Open a file for atomic writing. Instead of locking this method allows
you to write the entire file and move it to the actual location. Note that
this makes the assumption that a rename is atomic on your platform which
is generally the case but not a guarantee.
@ -39,8 +39,8 @@ def open_atomic(filename, binary=True):
>>> assert os.path.exists(filename)
>>> os.remove(filename)
'''
assert not os.path.exists(filename), '%r exists' % filename
"""
assert not os.path.exists(filename), "%r exists" % filename
path, name = os.path.split(filename)
# Create the parent directory if it doesn't exist
@ -48,7 +48,7 @@ def open_atomic(filename, binary=True):
os.makedirs(path)
temp_fh = tempfile.NamedTemporaryFile(
mode=binary and 'wb' or 'w',
mode=binary and "wb" or "w",
dir=path,
delete=False,
)
@ -66,12 +66,17 @@ def open_atomic(filename, binary=True):
class Lock(object):
def __init__(
self, filename, mode='a', timeout=DEFAULT_TIMEOUT,
check_interval=DEFAULT_CHECK_INTERVAL, fail_when_locked=False,
flags=LOCK_METHOD, **file_open_kwargs):
'''Lock manager with build-in timeout
self,
filename,
mode="a",
timeout=DEFAULT_TIMEOUT,
check_interval=DEFAULT_CHECK_INTERVAL,
fail_when_locked=False,
flags=LOCK_METHOD,
**file_open_kwargs
):
"""Lock manager with build-in timeout
filename -- filename
mode -- the open mode, 'a' or 'ab' should be used for writing
@ -89,11 +94,11 @@ class Lock(object):
Note that the file is opened first and locked later. So using 'w' as
mode will result in truncate _BEFORE_ the lock is checked.
'''
"""
if 'w' in mode:
if "w" in mode:
truncate = True
mode = mode.replace('w', 'a')
mode = mode.replace("w", "a")
else:
truncate = False
@ -107,9 +112,8 @@ class Lock(object):
self.flags = flags
self.file_open_kwargs = file_open_kwargs
def acquire(
self, timeout=None, check_interval=None, fail_when_locked=None):
'''Acquire the locked filehandle'''
def acquire(self, timeout=None, check_interval=None, fail_when_locked=None):
"""Acquire the locked filehandle"""
if timeout is None:
timeout = self.timeout
if timeout is None:
@ -165,7 +169,7 @@ class Lock(object):
return fh
def release(self):
'''Releases the currently locked file handle'''
"""Releases the currently locked file handle"""
if self.fh:
# noinspection PyBroadException
try:
@ -196,7 +200,7 @@ class Lock(object):
return True
def _get_fh(self):
'''Get a new filehandle'''
"""Get a new filehandle"""
# Create the parent directory if it doesn't exist
path, name = os.path.split(self.filename)
if path and not os.path.isdir(path): # pragma: no cover
@ -205,20 +209,20 @@ class Lock(object):
return open(self.filename, self.mode, **self.file_open_kwargs)
def _get_lock(self, fh):
'''
"""
Try to lock the given filehandle
returns LockException if it fails'''
returns LockException if it fails"""
portalocker.lock(fh, self.flags)
return fh
def _prepare_fh(self, fh):
'''
"""
Prepare the filehandle for usage
If truncate is a number, the file will be truncated to that amount of
bytes
'''
"""
if self.truncate:
fh.seek(0)
fh.truncate(0)
@ -243,11 +247,15 @@ class RLock(Lock):
"""
def __init__(
self, filename, mode='a', timeout=DEFAULT_TIMEOUT,
check_interval=DEFAULT_CHECK_INTERVAL, fail_when_locked=False,
flags=LOCK_METHOD):
super(RLock, self).__init__(filename, mode, timeout, check_interval,
fail_when_locked, flags)
self,
filename,
mode="a",
timeout=DEFAULT_TIMEOUT,
check_interval=DEFAULT_CHECK_INTERVAL,
fail_when_locked=False,
flags=LOCK_METHOD
):
super(RLock, self).__init__(filename, mode, timeout, check_interval, fail_when_locked, flags)
self._acquire_count = 0
self._lock = ProcessRLock()
self._pid = os.getpid()
@ -286,16 +294,14 @@ class RLock(Lock):
if self._acquire_count >= 1:
fh = self.fh
else:
fh = super(RLock, self).acquire(timeout, check_interval,
fail_when_locked)
fh = super(RLock, self).acquire(timeout, check_interval, fail_when_locked)
self._acquire_count += 1
return fh
def release(self):
if self._acquire_count == 0:
raise exceptions.LockException(
"Cannot release more times than acquired")
raise exceptions.LockException("Cannot release more times than acquired")
if self._acquire_count == 1:
super(RLock, self).release()
@ -327,14 +333,24 @@ class RLock(Lock):
class TemporaryFileLock(Lock):
def __init__(
self,
filename=".lock",
timeout=DEFAULT_TIMEOUT,
check_interval=DEFAULT_CHECK_INTERVAL,
fail_when_locked=True,
flags=LOCK_METHOD
):
def __init__(self, filename='.lock', timeout=DEFAULT_TIMEOUT,
check_interval=DEFAULT_CHECK_INTERVAL, fail_when_locked=True,
flags=LOCK_METHOD):
Lock.__init__(self, filename=filename, mode='w', timeout=timeout,
check_interval=check_interval,
fail_when_locked=fail_when_locked, flags=flags)
Lock.__init__(
self,
filename=filename,
mode="w",
timeout=timeout,
check_interval=check_interval,
fail_when_locked=fail_when_locked,
flags=flags,
)
atexit.register(self.release)
def release(self):