ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-04-28 10:11:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix dataset genealogy, graph and restoring data

Browse Source
This commit is contained in:
allegroai 2021-01-10 13:06:50 +02:00
parent 4beb21eb2b
commit bb648bdb02
1 changed files with 12 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
clearml/datasets/dataset.py
View File

@ -1092,7 +1092,8 @@ class Dataset(object):
# noinspection DuplicatedCode
while roots:
r = roots.pop(0)
dependencies.append(r)
if r not in dependencies:
dependencies.append(r)
# add the parents of the current node, only if the parents are in the general graph node list
if include_unused and r not in self._dependency_graph:
roots.extend(list(reversed(
@ -1109,7 +1110,8 @@ class Dataset(object):
# noinspection DuplicatedCode
while roots:
r = roots.pop(0)
dependencies.append(r)
if r not in dependencies:
dependencies.append(r)
# add the parents of the current node, only if the parents are in the general graph node list
if include_unused and r not in self._dependency_graph:
roots.extend(list(reversed(
@ -1222,19 +1224,22 @@ class Dataset(object):
# create DAG
visited = []
# add nodes
for idx, node in enumerate(nodes):
visited.append(node)
if node in self._dependency_graph:
parents = [visited.index(p) for p in self._dependency_graph[node] or [] if p in visited]
else:
parents = [visited.index(p) for p in self.get(dataset_id=node)._get_parents() or [] if p in visited]
sankey_node['color'].append("mediumpurple" if node == self.id else "lightblue")
sankey_node['label'].append('{}'.format(node))
sankey_node['customdata'].append(
"name {}<br />removed {}<br />modified {}<br />added {}<br />size {}".format(
node_names.get(node, ''), *node_details[node]))
# add edges
for idx, node in enumerate(nodes):
if node in self._dependency_graph:
parents = [visited.index(p) for p in self._dependency_graph[node] or [] if p in visited]
else:
parents = [visited.index(p) for p in self.get(dataset_id=node)._get_parents() or [] if p in visited]
for p in parents:
sankey_link['source'].append(p)
sankey_link['target'].append(idx)