ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-09 21:40:51 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Fix python 2.7/3.5 support, PEP8 and redundant code

Browse Source
This commit is contained in:
allegroai 2020-11-25 11:05:40 +02:00
parent 313e99f156
commit 5d1c477e10
6 changed files with 27 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
trains/backend_interface/task/hyperparams.py
View File

@ -54,7 +54,7 @@ class HyperParams(object):
def edit_hyper_params(
self,
*iterables, # type: Union[Mapping[str, Union[str, dict, None]], Iterable[dict, tasks.ParamsItem]]
iterables, # type: Union[Mapping[str, Union[str, dict, None]], Iterable[dict, tasks.ParamsItem]]
replace=None, # type: Optional[str]
default_section=None, # type: Optional[str]
force_section=None # type: Optional[str]
@ -89,8 +89,11 @@ class HyperParams(object):
item = value
elif isinstance(value, dict):
item = tasks.ParamsItem(**value)
elif isinstance(value, tuple):
item = tasks.ParamsItem(name=str(value[0]), value=str(value[1]))
else:
item = tasks.ParamsItem(value=str(value))
if name:
item.name = str(name)
if not item.name:
@ -105,11 +108,15 @@ class HyperParams(object):
return item
props = {}
if isinstance(iterables, dict):
iterables = [iterables]
for i in iterables:
if isinstance(i, dict):
props.update({name: make_item(value, name) for name, value in i.items()})
else:
props.update({item.name: item for item in map(make_item, i)})
item = make_item(i)
props.update({item.name: item})
res = self.task.session.send(
tasks.EditHyperParamsRequest(
@ -213,4 +220,4 @@ UNSAFE_NAMES_2_10 = {
"exact",
"iexact",
"match",
}
}

4
trains/backend_interface/task/repo/freeze.py
View File

@ -21,8 +21,8 @@ def pip_freeze(combine_conda_with_pip=False):
# check if this is a pypi package, if it is, leave it outside
if not r.get('channel') or r.get('channel') == 'pypi':
name = (r['name'].replace('-', '_'), r['name'])
pip_req_line = [l for l in pip_lines
if l.split('==', 1)[0].strip() in name or l.split('@', 1)[0].strip() in name]
pip_req_line = [pip_l for pip_l in pip_lines
if pip_l.split('==', 1)[0].strip() in name or pip_l.split('@', 1)[0].strip() in name]
if pip_req_line and \
('@' not in pip_req_line[0] or
not pip_req_line[0].split('@', 1)[1].strip().startswith('file://')):

7
trains/backend_interface/task/task.py
View File

@ -1109,7 +1109,7 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
# type: (Optional[str], Optional[str]) -> ()
# if running remotely and we are the main task, skip setting ourselves.
if self._is_main_remote_task():
if self._is_remote_main_task():
return
if not project_id:
@ -1202,7 +1202,7 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
self._edit(comment=comment)
def set_task_type(self, task_type):
# type: (Union[str, TaskTypes]) -> ()
# type: (Union[str, Task.TaskTypes]) -> ()
"""
Set the task_type for the Task.
@ -1628,9 +1628,6 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
return None
return text_to_config_dict(config_text)
def _is_main_remote_task(self):
return running_remotely() and get_remote_task_id() == self.id
def get_offline_mode_folder(self):
# type: () -> (Optional[Path])
"""

4
trains/binding/artifacts.py
View File

@ -715,10 +715,10 @@ class Artifacts(object):
# send for upload
# noinspection PyProtectedMember
if wait_on_upload:
StorageManager.upload_file(local_file, uri)
StorageManager.upload_file(local_file.as_posix(), uri)
if delete_after_upload:
try:
os.unlink(local_file)
os.unlink(local_file.as_posix())
except OSError:
LoggerRoot.get_base_logger().warning('Failed removing temporary {}'.format(local_file))
else:

23
trains/storage/manager.py
View File

@ -133,12 +133,12 @@ class StorageManager(object):
target_folder.name, time() * 1000, str(random()).replace('.', ''))
temp_target_folder.mkdir(parents=True, exist_ok=True)
if suffix == ".zip":
ZipFile(cached_file).extractall(path=temp_target_folder.as_posix())
ZipFile(cached_file.as_posix()).extractall(path=temp_target_folder.as_posix())
elif suffix == ".tar.gz":
with tarfile.open(cached_file) as file:
with tarfile.open(cached_file.as_posix()) as file:
file.extractall(temp_target_folder.as_posix())
elif suffix == ".tgz":
with tarfile.open(cached_file, mode='r:gz') as file:
with tarfile.open(cached_file.as_posix(), mode='r:gz') as file:
file.extractall(temp_target_folder.as_posix())
# we assume we will have such folder if we already extract the file
@ -152,29 +152,22 @@ class StorageManager(object):
target_folder.touch(exist_ok=True)
else:
base_logger.warning(
"Failed renaming {0} to {1}".format(
temp_target_folder, target_folder
)
)
"Failed renaming {0} to {1}".format(temp_target_folder.as_posix(), target_folder.as_posix()))
try:
shutil.rmtree(temp_target_folder)
shutil.rmtree(temp_target_folder.as_posix())
except Exception as ex:
base_logger.warning(
"Exception {}\nFailed deleting folder {}".format(
ex, temp_target_folder
)
)
"Exception {}\nFailed deleting folder {}".format(ex, temp_target_folder.as_posix()))
except Exception as ex:
# failed extracting the file:
base_logger.warning(
"Exception {}\nFailed extracting zip file {}".format(ex, str(cached_file))
)
"Exception {}\nFailed extracting zip file {}".format(ex, cached_file.as_posix()))
# noinspection PyBroadException
try:
target_folder.rmdir()
except Exception:
pass
return cached_file
return cached_file.as_posix()
return target_folder.as_posix()
@classmethod

6
trains/task.py
View File

@ -1584,7 +1584,7 @@ class Task(_Task):
def set_user_properties(
self,
*iterables, # type: Union[Mapping[str, Union[str, dict, None]], Iterable[dict]]
**properties # type: Union[str, dict, None]
**properties # type: Union[str, dict, int, float, None]
):
# type: (...) -> bool
"""
@ -1646,8 +1646,8 @@ class Task(_Task):
return False
return self._hyper_params_manager.edit_hyper_params(
properties,
*iterables,
iterables=list(properties.items()) + (
list(iterables.items()) if isinstance(iterables, dict) else list(iterables)),
replace='none',
force_section="properties",
)