ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-04-16 21:42:10 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix dataset genealogy, graph and restoring data

Browse Source
This commit is contained in:
allegroai 2021-01-10 13:06:50 +02:00
parent 4beb21eb2b
commit bb648bdb02
1 changed files with 12 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
clearml/datasets/dataset.py
View File

@ -1092,7 +1092,8 @@ class Dataset(object):
# noinspection DuplicatedCode # noinspection DuplicatedCode
while roots: while roots:
r = roots.pop(0) r = roots.pop(0)
dependencies.append(r) if r not in dependencies:
dependencies.append(r)
# add the parents of the current node, only if the parents are in the general graph node list # add the parents of the current node, only if the parents are in the general graph node list
if include_unused and r not in self._dependency_graph: if include_unused and r not in self._dependency_graph:
roots.extend(list(reversed( roots.extend(list(reversed(
@ -1109,7 +1110,8 @@ class Dataset(object):
# noinspection DuplicatedCode # noinspection DuplicatedCode
while roots: while roots:
r = roots.pop(0) r = roots.pop(0)
dependencies.append(r) if r not in dependencies:
dependencies.append(r)
# add the parents of the current node, only if the parents are in the general graph node list # add the parents of the current node, only if the parents are in the general graph node list
if include_unused and r not in self._dependency_graph: if include_unused and r not in self._dependency_graph:
roots.extend(list(reversed( roots.extend(list(reversed(
@ -1222,19 +1224,22 @@ class Dataset(object):
# create DAG # create DAG
visited = [] visited = []
# add nodes
for idx, node in enumerate(nodes): for idx, node in enumerate(nodes):
visited.append(node) visited.append(node)
if node in self._dependency_graph:
parents = [visited.index(p) for p in self._dependency_graph[node] or [] if p in visited]
else:
parents = [visited.index(p) for p in self.get(dataset_id=node)._get_parents() or [] if p in visited]
sankey_node['color'].append("mediumpurple" if node == self.id else "lightblue") sankey_node['color'].append("mediumpurple" if node == self.id else "lightblue")
sankey_node['label'].append('{}'.format(node)) sankey_node['label'].append('{}'.format(node))
sankey_node['customdata'].append( sankey_node['customdata'].append(
"name {}<br />removed {}<br />modified {}<br />added {}<br />size {}".format( "name {}<br />removed {}<br />modified {}<br />added {}<br />size {}".format(
node_names.get(node, ''), *node_details[node])) node_names.get(node, ''), *node_details[node]))
# add edges
for idx, node in enumerate(nodes):
if node in self._dependency_graph:
parents = [visited.index(p) for p in self._dependency_graph[node] or [] if p in visited]
else:
parents = [visited.index(p) for p in self.get(dataset_id=node)._get_parents() or [] if p in visited]
for p in parents: for p in parents:
sankey_link['source'].append(p) sankey_link['source'].append(p)
sankey_link['target'].append(idx) sankey_link['target'].append(idx)