ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-23 01:55:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix python 2.7/3.5 support, PEP8 and redundant code

Browse Source
This commit is contained in:
allegroai 2020-11-25 11:05:40 +02:00
parent 313e99f156
commit 5d1c477e10
6 changed files with 27 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
trains/backend_interface/task/hyperparams.py
View File

@ -54,7 +54,7 @@ class HyperParams(object):
def edit_hyper_params( def edit_hyper_params(
self, self,
*iterables, # type: Union[Mapping[str, Union[str, dict, None]], Iterable[dict, tasks.ParamsItem]] iterables, # type: Union[Mapping[str, Union[str, dict, None]], Iterable[dict, tasks.ParamsItem]]
replace=None, # type: Optional[str] replace=None, # type: Optional[str]
default_section=None, # type: Optional[str] default_section=None, # type: Optional[str]
force_section=None # type: Optional[str] force_section=None # type: Optional[str]
@ -89,8 +89,11 @@ class HyperParams(object):
item = value item = value
elif isinstance(value, dict): elif isinstance(value, dict):
item = tasks.ParamsItem(**value) item = tasks.ParamsItem(**value)
elif isinstance(value, tuple):
item = tasks.ParamsItem(name=str(value[0]), value=str(value[1]))
else: else:
item = tasks.ParamsItem(value=str(value)) item = tasks.ParamsItem(value=str(value))
if name: if name:
item.name = str(name) item.name = str(name)
if not item.name: if not item.name:
@ -105,11 +108,15 @@ class HyperParams(object):
return item return item
props = {} props = {}
if isinstance(iterables, dict):
iterables = [iterables]
for i in iterables: for i in iterables:
if isinstance(i, dict): if isinstance(i, dict):
props.update({name: make_item(value, name) for name, value in i.items()}) props.update({name: make_item(value, name) for name, value in i.items()})
else: else:
props.update({item.name: item for item in map(make_item, i)}) item = make_item(i)
props.update({item.name: item})
res = self.task.session.send( res = self.task.session.send(
tasks.EditHyperParamsRequest( tasks.EditHyperParamsRequest(

4
trains/backend_interface/task/repo/freeze.py
View File

@ -21,8 +21,8 @@ def pip_freeze(combine_conda_with_pip=False):
# check if this is a pypi package, if it is, leave it outside # check if this is a pypi package, if it is, leave it outside
if not r.get('channel') or r.get('channel') == 'pypi': if not r.get('channel') or r.get('channel') == 'pypi':
name = (r['name'].replace('-', '_'), r['name']) name = (r['name'].replace('-', '_'), r['name'])
pip_req_line = [l for l in pip_lines pip_req_line = [pip_l for pip_l in pip_lines
if l.split('==', 1)[0].strip() in name or l.split('@', 1)[0].strip() in name] if pip_l.split('==', 1)[0].strip() in name or pip_l.split('@', 1)[0].strip() in name]
if pip_req_line and \ if pip_req_line and \
('@' not in pip_req_line[0] or ('@' not in pip_req_line[0] or
not pip_req_line[0].split('@', 1)[1].strip().startswith('file://')): not pip_req_line[0].split('@', 1)[1].strip().startswith('file://')):

7
trains/backend_interface/task/task.py
View File

@ -1109,7 +1109,7 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
# type: (Optional[str], Optional[str]) -> () # type: (Optional[str], Optional[str]) -> ()
# if running remotely and we are the main task, skip setting ourselves. # if running remotely and we are the main task, skip setting ourselves.
if self._is_main_remote_task(): if self._is_remote_main_task():
return return
if not project_id: if not project_id:
@ -1202,7 +1202,7 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
self._edit(comment=comment) self._edit(comment=comment)
def set_task_type(self, task_type): def set_task_type(self, task_type):
# type: (Union[str, TaskTypes]) -> () # type: (Union[str, Task.TaskTypes]) -> ()
""" """
Set the task_type for the Task. Set the task_type for the Task.
@ -1628,9 +1628,6 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
return None return None
return text_to_config_dict(config_text) return text_to_config_dict(config_text)
def _is_main_remote_task(self):
return running_remotely() and get_remote_task_id() == self.id
def get_offline_mode_folder(self): def get_offline_mode_folder(self):
# type: () -> (Optional[Path]) # type: () -> (Optional[Path])
""" """

4
trains/binding/artifacts.py
View File

@ -715,10 +715,10 @@ class Artifacts(object):
# send for upload # send for upload
# noinspection PyProtectedMember # noinspection PyProtectedMember
if wait_on_upload: if wait_on_upload:
StorageManager.upload_file(local_file, uri) StorageManager.upload_file(local_file.as_posix(), uri)
if delete_after_upload: if delete_after_upload:
try: try:
os.unlink(local_file) os.unlink(local_file.as_posix())
except OSError: except OSError:
LoggerRoot.get_base_logger().warning('Failed removing temporary {}'.format(local_file)) LoggerRoot.get_base_logger().warning('Failed removing temporary {}'.format(local_file))
else: else:

23
trains/storage/manager.py
View File

@ -133,12 +133,12 @@ class StorageManager(object):
target_folder.name, time() * 1000, str(random()).replace('.', '')) target_folder.name, time() * 1000, str(random()).replace('.', ''))
temp_target_folder.mkdir(parents=True, exist_ok=True) temp_target_folder.mkdir(parents=True, exist_ok=True)
if suffix == ".zip": if suffix == ".zip":
ZipFile(cached_file).extractall(path=temp_target_folder.as_posix()) ZipFile(cached_file.as_posix()).extractall(path=temp_target_folder.as_posix())
elif suffix == ".tar.gz": elif suffix == ".tar.gz":
with tarfile.open(cached_file) as file: with tarfile.open(cached_file.as_posix()) as file:
file.extractall(temp_target_folder.as_posix()) file.extractall(temp_target_folder.as_posix())
elif suffix == ".tgz": elif suffix == ".tgz":
with tarfile.open(cached_file, mode='r:gz') as file: with tarfile.open(cached_file.as_posix(), mode='r:gz') as file:
file.extractall(temp_target_folder.as_posix()) file.extractall(temp_target_folder.as_posix())
# we assume we will have such folder if we already extract the file # we assume we will have such folder if we already extract the file
@ -152,29 +152,22 @@ class StorageManager(object):
target_folder.touch(exist_ok=True) target_folder.touch(exist_ok=True)
else: else:
base_logger.warning( base_logger.warning(
"Failed renaming {0} to {1}".format( "Failed renaming {0} to {1}".format(temp_target_folder.as_posix(), target_folder.as_posix()))
temp_target_folder, target_folder
)
)
try: try:
shutil.rmtree(temp_target_folder) shutil.rmtree(temp_target_folder.as_posix())
except Exception as ex: except Exception as ex:
base_logger.warning( base_logger.warning(
"Exception {}\nFailed deleting folder {}".format( "Exception {}\nFailed deleting folder {}".format(ex, temp_target_folder.as_posix()))
ex, temp_target_folder
)
)
except Exception as ex: except Exception as ex:
# failed extracting the file: # failed extracting the file:
base_logger.warning( base_logger.warning(
"Exception {}\nFailed extracting zip file {}".format(ex, str(cached_file)) "Exception {}\nFailed extracting zip file {}".format(ex, cached_file.as_posix()))
)
# noinspection PyBroadException # noinspection PyBroadException
try: try:
target_folder.rmdir() target_folder.rmdir()
except Exception: except Exception:
pass pass
return cached_file return cached_file.as_posix()
return target_folder.as_posix() return target_folder.as_posix()
@classmethod @classmethod

6
trains/task.py
View File

@ -1584,7 +1584,7 @@ class Task(_Task):
def set_user_properties( def set_user_properties(
self, self,
*iterables, # type: Union[Mapping[str, Union[str, dict, None]], Iterable[dict]] *iterables, # type: Union[Mapping[str, Union[str, dict, None]], Iterable[dict]]
**properties # type: Union[str, dict, None] **properties # type: Union[str, dict, int, float, None]
): ):
# type: (...) -> bool # type: (...) -> bool
""" """
@ -1646,8 +1646,8 @@ class Task(_Task):
return False return False
return self._hyper_params_manager.edit_hyper_params( return self._hyper_params_manager.edit_hyper_params(
properties, iterables=list(properties.items()) + (
*iterables, list(iterables.items()) if isinstance(iterables, dict) else list(iterables)),
replace='none', replace='none',
force_section="properties", force_section="properties",
) )