ClearML/clearml-agent
1
0
Fork 0
mirror of https://github.com/clearml/clearml-agent synced 2025-02-12 07:38:04 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix CPU mode

Browse Source
This commit is contained in:
allegroai 2021-02-14 13:44:00 +02:00
parent 1260e3d942
commit 14589aa094
1 changed files with 30 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
clearml_agent/commands/worker.py
View File

@ -1119,35 +1119,38 @@ class Worker(ServiceCommandSection):
safe_remove_file(self.temp_config_path) safe_remove_file(self.temp_config_path)
def _parse_dynamic_gpus(self, kwargs, queues): def _parse_dynamic_gpus(self, kwargs, queues):
gpu_indexes = kwargs.get('gpus')
dynamic_gpus = kwargs.get('dynamic_gpus', None) dynamic_gpus = kwargs.get('dynamic_gpus', None)
if dynamic_gpus: if not dynamic_gpus:
# test gpus were passed correctly return None, None, queues
if not gpu_indexes or len(gpu_indexes.split('-')) > 2 or (',' in gpu_indexes and '-' in gpu_indexes):
raise ValueError('--gpus must be provided, in one of two ways: '
'comma separated \'0,1,2,3\' or range \'0-3\'')
try:
if '-' in gpu_indexes:
gpu_indexes = list(range(int(gpu_indexes.split('-')[0]), 1 + int(gpu_indexes.split('-')[1])))
else:
gpu_indexes = [int(g) for g in gpu_indexes.split(',')]
except Exception:
raise ValueError('Failed parsing --gpus {}'.format(kwargs.get('gpus')))
dynamic_gpus = [(s[:-1 - len(s.split('=')[-1])], int(s.split('=')[-1])) for s in dynamic_gpus] gpu_indexes = kwargs.get('gpus')
# resolve queue ids
dynamic_gpus_q = self._resolve_queue_names([q for q, _ in dynamic_gpus], create_if_missing=False) # test gpus were passed correctly
dynamic_gpus = list(zip(dynamic_gpus_q, [i for _, i in dynamic_gpus])) if not gpu_indexes or len(gpu_indexes.split('-')) > 2 or (',' in gpu_indexes and '-' in gpu_indexes):
if set(dynamic_gpus_q) != set(queues): raise ValueError('--gpus must be provided, in one of two ways: '
raise ValueError( 'comma separated \'0,1,2,3\' or range \'0-3\'')
"--dynamic-gpus [{}] and --queues [{}] must contain the same queues".format( try:
dynamic_gpus, queues)) if '-' in gpu_indexes:
dynamic_gpus = sorted( gpu_indexes = list(range(int(gpu_indexes.split('-')[0]), 1 + int(gpu_indexes.split('-')[1])))
dynamic_gpus, reverse=True, key=cmp_to_key( else:
lambda x, y: -1 if x[1] < y[1] or x[1] == y[1] and queues.index(x[0]) > queues.index(y[0]) gpu_indexes = [int(g) for g in gpu_indexes.split(',')]
else +1)) except Exception:
# order queue priority based on the combination we have raise ValueError('Failed parsing --gpus {}'.format(kwargs.get('gpus')))
queues = [q for q, _ in dynamic_gpus]
dynamic_gpus = [(s[:-1 - len(s.split('=')[-1])], int(s.split('=')[-1])) for s in dynamic_gpus]
# resolve queue ids
dynamic_gpus_q = self._resolve_queue_names([q for q, _ in dynamic_gpus], create_if_missing=False)
dynamic_gpus = list(zip(dynamic_gpus_q, [i for _, i in dynamic_gpus]))
if set(dynamic_gpus_q) != set(queues):
raise ValueError(
"--dynamic-gpus [{}] and --queues [{}] must contain the same queues".format(
dynamic_gpus, queues))
dynamic_gpus = sorted(
dynamic_gpus, reverse=True, key=cmp_to_key(
lambda x, y: -1 if x[1] < y[1] or x[1] == y[1] and queues.index(x[0]) > queues.index(y[0])
else +1))
# order queue priority based on the combination we have
queues = [q for q, _ in dynamic_gpus]
# test server support # test server support
available_gpus = self._dynamic_gpu_get_available(gpu_indexes) available_gpus = self._dynamic_gpu_get_available(gpu_indexes)