ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-16 11:28:31 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Black formatting

Browse Source
This commit is contained in:
clearml 2024-09-22 22:31:58 +03:00
parent d069339fe9
commit eaeadb18e3
4 changed files with 75 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
clearml/utilities/locks/constants.py
View File

@ -18,7 +18,7 @@ import os
# The actual tests will execute the code anyhow so the following code can # The actual tests will execute the code anyhow so the following code can
# safely be ignored from the coverage tests # safely be ignored from the coverage tests
if os.name == 'nt': # pragma: no cover if os.name == "nt": # pragma: no cover
import msvcrt import msvcrt
LOCK_EX = 0x1 #: exclusive lock LOCK_EX = 0x1 #: exclusive lock
@ -29,7 +29,7 @@ if os.name == 'nt': # pragma: no cover
LOCKFILE_FAIL_IMMEDIATELY = 1 LOCKFILE_FAIL_IMMEDIATELY = 1
LOCKFILE_EXCLUSIVE_LOCK = 2 LOCKFILE_EXCLUSIVE_LOCK = 2
elif os.name == 'posix': # pragma: no cover elif os.name == "posix": # pragma: no cover
import fcntl import fcntl
LOCK_EX = fcntl.LOCK_EX #: exclusive lock LOCK_EX = fcntl.LOCK_EX #: exclusive lock
@ -38,4 +38,4 @@ elif os.name == 'posix': # pragma: no cover
LOCK_UN = fcntl.LOCK_UN #: unlock LOCK_UN = fcntl.LOCK_UN #: unlock
else: # pragma: no cover else: # pragma: no cover
raise RuntimeError('PortaLocker only defined for nt and posix platforms') raise RuntimeError("PortaLocker only defined for nt and posix platforms")

2
clearml/utilities/locks/exceptions.py
View File

@ -3,7 +3,7 @@ class BaseLockException(Exception):
LOCK_FAILED = 1 LOCK_FAILED = 1
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
self.fh = kwargs.pop('fh', None) self.fh = kwargs.pop("fh", None)
Exception.__init__(self, *args, **kwargs) Exception.__init__(self, *args, **kwargs)

40
clearml/utilities/locks/portalocker.py
View File

@ -4,7 +4,7 @@ from . import exceptions
from . import constants from . import constants
if os.name == 'nt': # pragma: no cover if os.name == "nt": # pragma: no cover
import msvcrt import msvcrt
if sys.version_info.major == 2: if sys.version_info.major == 2:
@ -17,6 +17,7 @@ if os.name == 'nt': # pragma: no cover
import win32file import win32file
import pywintypes import pywintypes
import winerror import winerror
__overlapped = pywintypes.OVERLAPPED() __overlapped = pywintypes.OVERLAPPED()
if sys.version_info.major == 2: if sys.version_info.major == 2:
if flags & constants.LOCK_NB: if flags & constants.LOCK_NB:
@ -38,10 +39,7 @@ if os.name == 'nt': # pragma: no cover
# error: (33, 'LockFileEx', 'The process cannot access the file # error: (33, 'LockFileEx', 'The process cannot access the file
# because another process has locked a portion of the file.') # because another process has locked a portion of the file.')
if exc_value.winerror == winerror.ERROR_LOCK_VIOLATION: if exc_value.winerror == winerror.ERROR_LOCK_VIOLATION:
raise exceptions.LockException( raise exceptions.LockException(exceptions.LockException.LOCK_FAILED, exc_value.strerror, fh=file_)
exceptions.LockException.LOCK_FAILED,
exc_value.strerror,
fh=file_)
else: else:
# Q: Are there exceptions/codes we should be dealing with # Q: Are there exceptions/codes we should be dealing with
# here? # here?
@ -72,17 +70,12 @@ if os.name == 'nt': # pragma: no cover
msvcrt.locking(file_.fileno(), mode, lock_length) msvcrt.locking(file_.fileno(), mode, lock_length)
except IOError as exc_value: except IOError as exc_value:
# [ ] be more specific here # [ ] be more specific here
raise exceptions.LockException( raise exceptions.LockException(exceptions.LockException.LOCK_FAILED, exc_value.strerror, fh=file_)
exceptions.LockException.LOCK_FAILED,
exc_value.strerror,
fh=file_)
finally: finally:
if savepos: if savepos:
file_.seek(savepos) file_.seek(savepos)
except IOError as exc_value: except IOError as exc_value:
raise exceptions.LockException( raise exceptions.LockException(exceptions.LockException.LOCK_FAILED, exc_value.strerror, fh=file_)
exceptions.LockException.LOCK_FAILED, exc_value.strerror,
fh=file_)
def unlock(file_): def unlock(file_):
try: try:
@ -93,15 +86,15 @@ if os.name == 'nt': # pragma: no cover
try: try:
msvcrt.locking(file_.fileno(), constants.LOCK_UN, lock_length) msvcrt.locking(file_.fileno(), constants.LOCK_UN, lock_length)
except IOError as exc_value: except IOError as exc_value:
if exc_value.strerror == 'Permission denied': if exc_value.strerror == "Permission denied":
import pywintypes import pywintypes
import win32file import win32file
import winerror import winerror
__overlapped = pywintypes.OVERLAPPED() __overlapped = pywintypes.OVERLAPPED()
hfile = win32file._get_osfhandle(file_.fileno()) hfile = win32file._get_osfhandle(file_.fileno())
try: try:
win32file.UnlockFileEx( win32file.UnlockFileEx(hfile, 0, -0x10000, __overlapped)
hfile, 0, -0x10000, __overlapped)
except pywintypes.error as exc_value: except pywintypes.error as exc_value:
if exc_value.winerror == winerror.ERROR_NOT_LOCKED: if exc_value.winerror == winerror.ERROR_NOT_LOCKED:
# error: (158, 'UnlockFileEx', # error: (158, 'UnlockFileEx',
@ -114,25 +107,20 @@ if os.name == 'nt': # pragma: no cover
# dealing with here? # dealing with here?
raise raise
else: else:
raise exceptions.LockException( raise exceptions.LockException(exceptions.LockException.LOCK_FAILED, exc_value.strerror, fh=file_)
exceptions.LockException.LOCK_FAILED,
exc_value.strerror,
fh=file_)
finally: finally:
if savepos: if savepos:
file_.seek(savepos) file_.seek(savepos)
except IOError as exc_value: except IOError as exc_value:
raise exceptions.LockException( raise exceptions.LockException(exceptions.LockException.LOCK_FAILED, exc_value.strerror, fh=file_)
exceptions.LockException.LOCK_FAILED, exc_value.strerror,
fh=file_)
elif os.name == 'posix': # pragma: no cover elif os.name == "posix": # pragma: no cover
import fcntl import fcntl
def lock(file_, flags): def lock(file_, flags):
locking_exceptions = IOError, locking_exceptions = (IOError,)
try: # pragma: no cover try: # pragma: no cover
locking_exceptions += BlockingIOError, locking_exceptions += (BlockingIOError,)
except NameError: # pragma: no cover except NameError: # pragma: no cover
pass pass
@ -147,4 +135,4 @@ elif os.name == 'posix': # pragma: no cover
fcntl.flock(file_.fileno(), constants.LOCK_UN) fcntl.flock(file_.fileno(), constants.LOCK_UN)
else: # pragma: no cover else: # pragma: no cover
raise RuntimeError('PortaLocker only defined for nt and posix platforms') raise RuntimeError("PortaLocker only defined for nt and posix platforms")

94
clearml/utilities/locks/utils.py
View File

@ -15,15 +15,15 @@ DEFAULT_CHECK_INTERVAL = 0.25
LOCK_METHOD = constants.LOCK_EX | constants.LOCK_NB LOCK_METHOD = constants.LOCK_EX | constants.LOCK_NB
__all__ = [ __all__ = [
'Lock', "Lock",
'RLock', "RLock",
'open_atomic', "open_atomic",
] ]
@contextlib.contextmanager @contextlib.contextmanager
def open_atomic(filename, binary=True): def open_atomic(filename, binary=True):
'''Open a file for atomic writing. Instead of locking this method allows """Open a file for atomic writing. Instead of locking this method allows
you to write the entire file and move it to the actual location. Note that you to write the entire file and move it to the actual location. Note that
this makes the assumption that a rename is atomic on your platform which this makes the assumption that a rename is atomic on your platform which
is generally the case but not a guarantee. is generally the case but not a guarantee.
@ -39,8 +39,8 @@ def open_atomic(filename, binary=True):
>>> assert os.path.exists(filename) >>> assert os.path.exists(filename)
>>> os.remove(filename) >>> os.remove(filename)
''' """
assert not os.path.exists(filename), '%r exists' % filename assert not os.path.exists(filename), "%r exists" % filename
path, name = os.path.split(filename) path, name = os.path.split(filename)
# Create the parent directory if it doesn't exist # Create the parent directory if it doesn't exist
@ -48,7 +48,7 @@ def open_atomic(filename, binary=True):
os.makedirs(path) os.makedirs(path)
temp_fh = tempfile.NamedTemporaryFile( temp_fh = tempfile.NamedTemporaryFile(
mode=binary and 'wb' or 'w', mode=binary and "wb" or "w",
dir=path, dir=path,
delete=False, delete=False,
) )
@ -66,12 +66,17 @@ def open_atomic(filename, binary=True):
class Lock(object): class Lock(object):
def __init__( def __init__(
self, filename, mode='a', timeout=DEFAULT_TIMEOUT, self,
check_interval=DEFAULT_CHECK_INTERVAL, fail_when_locked=False, filename,
flags=LOCK_METHOD, **file_open_kwargs): mode="a",
'''Lock manager with build-in timeout timeout=DEFAULT_TIMEOUT,
check_interval=DEFAULT_CHECK_INTERVAL,
fail_when_locked=False,
flags=LOCK_METHOD,
**file_open_kwargs
):
"""Lock manager with build-in timeout
filename -- filename filename -- filename
mode -- the open mode, 'a' or 'ab' should be used for writing mode -- the open mode, 'a' or 'ab' should be used for writing
@ -89,11 +94,11 @@ class Lock(object):
Note that the file is opened first and locked later. So using 'w' as Note that the file is opened first and locked later. So using 'w' as
mode will result in truncate _BEFORE_ the lock is checked. mode will result in truncate _BEFORE_ the lock is checked.
''' """
if 'w' in mode: if "w" in mode:
truncate = True truncate = True
mode = mode.replace('w', 'a') mode = mode.replace("w", "a")
else: else:
truncate = False truncate = False
@ -107,9 +112,8 @@ class Lock(object):
self.flags = flags self.flags = flags
self.file_open_kwargs = file_open_kwargs self.file_open_kwargs = file_open_kwargs
def acquire( def acquire(self, timeout=None, check_interval=None, fail_when_locked=None):
self, timeout=None, check_interval=None, fail_when_locked=None): """Acquire the locked filehandle"""
'''Acquire the locked filehandle'''
if timeout is None: if timeout is None:
timeout = self.timeout timeout = self.timeout
if timeout is None: if timeout is None:
@ -165,7 +169,7 @@ class Lock(object):
return fh return fh
def release(self): def release(self):
'''Releases the currently locked file handle''' """Releases the currently locked file handle"""
if self.fh: if self.fh:
# noinspection PyBroadException # noinspection PyBroadException
try: try:
@ -196,7 +200,7 @@ class Lock(object):
return True return True
def _get_fh(self): def _get_fh(self):
'''Get a new filehandle''' """Get a new filehandle"""
# Create the parent directory if it doesn't exist # Create the parent directory if it doesn't exist
path, name = os.path.split(self.filename) path, name = os.path.split(self.filename)
if path and not os.path.isdir(path): # pragma: no cover if path and not os.path.isdir(path): # pragma: no cover
@ -205,20 +209,20 @@ class Lock(object):
return open(self.filename, self.mode, **self.file_open_kwargs) return open(self.filename, self.mode, **self.file_open_kwargs)
def _get_lock(self, fh): def _get_lock(self, fh):
''' """
Try to lock the given filehandle Try to lock the given filehandle
returns LockException if it fails''' returns LockException if it fails"""
portalocker.lock(fh, self.flags) portalocker.lock(fh, self.flags)
return fh return fh
def _prepare_fh(self, fh): def _prepare_fh(self, fh):
''' """
Prepare the filehandle for usage Prepare the filehandle for usage
If truncate is a number, the file will be truncated to that amount of If truncate is a number, the file will be truncated to that amount of
bytes bytes
''' """
if self.truncate: if self.truncate:
fh.seek(0) fh.seek(0)
fh.truncate(0) fh.truncate(0)
@ -243,11 +247,15 @@ class RLock(Lock):
""" """
def __init__( def __init__(
self, filename, mode='a', timeout=DEFAULT_TIMEOUT, self,
check_interval=DEFAULT_CHECK_INTERVAL, fail_when_locked=False, filename,
flags=LOCK_METHOD): mode="a",
super(RLock, self).__init__(filename, mode, timeout, check_interval, timeout=DEFAULT_TIMEOUT,
fail_when_locked, flags) check_interval=DEFAULT_CHECK_INTERVAL,
fail_when_locked=False,
flags=LOCK_METHOD
):
super(RLock, self).__init__(filename, mode, timeout, check_interval, fail_when_locked, flags)
self._acquire_count = 0 self._acquire_count = 0
self._lock = ProcessRLock() self._lock = ProcessRLock()
self._pid = os.getpid() self._pid = os.getpid()
@ -286,16 +294,14 @@ class RLock(Lock):
if self._acquire_count >= 1: if self._acquire_count >= 1:
fh = self.fh fh = self.fh
else: else:
fh = super(RLock, self).acquire(timeout, check_interval, fh = super(RLock, self).acquire(timeout, check_interval, fail_when_locked)
fail_when_locked)
self._acquire_count += 1 self._acquire_count += 1
return fh return fh
def release(self): def release(self):
if self._acquire_count == 0: if self._acquire_count == 0:
raise exceptions.LockException( raise exceptions.LockException("Cannot release more times than acquired")
"Cannot release more times than acquired")
if self._acquire_count == 1: if self._acquire_count == 1:
super(RLock, self).release() super(RLock, self).release()
@ -327,14 +333,24 @@ class RLock(Lock):
class TemporaryFileLock(Lock): class TemporaryFileLock(Lock):
def __init__(
self,
filename=".lock",
timeout=DEFAULT_TIMEOUT,
check_interval=DEFAULT_CHECK_INTERVAL,
fail_when_locked=True,
flags=LOCK_METHOD
):
def __init__(self, filename='.lock', timeout=DEFAULT_TIMEOUT, Lock.__init__(
check_interval=DEFAULT_CHECK_INTERVAL, fail_when_locked=True, self,
flags=LOCK_METHOD): filename=filename,
mode="w",
Lock.__init__(self, filename=filename, mode='w', timeout=timeout, timeout=timeout,
check_interval=check_interval, check_interval=check_interval,
fail_when_locked=fail_when_locked, flags=flags) fail_when_locked=fail_when_locked,
flags=flags,
)
atexit.register(self.release) atexit.register(self.release)
def release(self): def release(self):