ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-04-23 07:45:24 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix dataset create single root plot

Browse Source
This commit is contained in:
allegroai 2021-04-10 22:20:11 +03:00
parent adf199b43b
commit 6415722798
1 changed files with 36 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
clearml/datasets/dataset.py
View File

@ -467,7 +467,7 @@ class Dataset(object):
:return: A the target folder containing the entire dataset :return: A the target folder containing the entire dataset
""" """
assert self._id assert self._id
target_folder = Path(target_folder) target_folder = Path(target_folder).absolute()
target_folder.mkdir(parents=True, exist_ok=True) target_folder.mkdir(parents=True, exist_ok=True)
# noinspection PyBroadException # noinspection PyBroadException
try: try:
@ -1139,7 +1139,7 @@ class Dataset(object):
raise ValueError("Dataset merging failed: {}".format([e for e in errors if e is not None])) raise ValueError("Dataset merging failed: {}".format([e for e in errors if e is not None]))
pool.close() pool.close()
return target_base_folder.as_posix() return target_base_folder.absolute().as_posix()
def _get_dependencies_by_order(self, include_unused=False): def _get_dependencies_by_order(self, include_unused=False):
# type: (bool) -> List[str] # type: (bool) -> List[str]
@ -1285,8 +1285,8 @@ class Dataset(object):
removed = len(self.list_removed_files(node)) removed = len(self.list_removed_files(node))
modified = len(self.list_modified_files(node)) modified = len(self.list_modified_files(node))
table_values += [[node, node_names.get(node, ''), table_values += [[node, node_names.get(node, ''),
removed, modified, count-modified, format_size(size)]] removed, modified, max(0, count-modified), format_size(size)]]
node_details[node] = [removed, modified, count-modified, format_size(size)] node_details[node] = [removed, modified, max(0, count-modified), format_size(size)]
# create DAG # create DAG
visited = [] visited = []
@ -1311,6 +1311,7 @@ class Dataset(object):
sankey_link['target'].append(idx) sankey_link['target'].append(idx)
sankey_link['value'].append(max(1, node_details[visited[p]][-2])) sankey_link['value'].append(max(1, node_details[visited[p]][-2]))
if len(nodes) > 1:
# create the sankey graph # create the sankey graph
dag_flow = dict( dag_flow = dict(
link=sankey_link, link=sankey_link,
@ -1320,8 +1321,29 @@ class Dataset(object):
orientation='h' orientation='h'
) )
fig = dict(data=[dag_flow], layout={'xaxis': {'visible': False}, 'yaxis': {'visible': False}}) fig = dict(data=[dag_flow], layout={'xaxis': {'visible': False}, 'yaxis': {'visible': False}})
elif len(nodes) == 1:
# hack, show single node sankey
singles_flow = dict(
x=list(range(len(nodes))), y=[1] * len(nodes),
text=sankey_node['label'],
customdata=sankey_node['customdata'],
mode='markers',
hovertemplate='%{customdata}<extra></extra>',
marker=dict(
color=[v for i, v in enumerate(sankey_node['color']) if i in nodes],
size=[40] * len(nodes),
),
showlegend=False,
type='scatter',
)
# only single nodes
fig = dict(data=[singles_flow], layout={
'hovermode': 'closest', 'xaxis': {'visible': False}, 'yaxis': {'visible': False}})
else:
fig = None
if len(nodes) > 1: # report genealogy
if fig:
self._task.get_logger().report_plotly( self._task.get_logger().report_plotly(
title='Dataset Genealogy', series='', iteration=0, figure=fig) title='Dataset Genealogy', series='', iteration=0, figure=fig)