ClearML/clearml-server
1
0
Fork 0
mirror of https://github.com/clearml/clearml-server synced 2025-06-26 23:15:47 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Support tags-per-project in tags related services

Browse Source
This commit is contained in:
allegroai
2020-06-21 23:54:05 +03:00
parent 5e095af3aa
commit 1ea6408d41
15 changed files with 571 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
server/tests/automated/test_organization.py
View File

@@ -1,36 +0,0 @@
from tests.automated import TestService
class TestOrganization(TestService):
def setUp(self, version="2.8"):
super().setUp(version=version)
def test_tags(self):
tag1 = "Orgtest tag1"
tag2 = "Orgtest tag2"
system_tag = "Orgtest system tag"
model = self.create_temp(
"models", name="test_org", uri="file:///a", tags=[tag1]
)
task = self.create_temp(
"tasks", name="test org", type="training", input=dict(view={}), tags=[tag1]
)
data = self.api.organization.get_tags()
self.assertTrue(tag1 in data.tags)
self.api.tasks.edit(task=task, tags=[tag2], system_tags=[system_tag])
data = self.api.organization.get_tags(include_system=True)
self.assertTrue({tag1, tag2}.issubset(set(data.tags)))
self.assertTrue(system_tag in data.system_tags)
data = self.api.organization.get_tags(
filter={"system_tags": ["__$not", system_tag]}
)
self.assertTrue(tag1 in data.tags)
self.assertFalse(tag2 in data.tags)
self.api.models.delete(model=model)
data = self.api.organization.get_tags()
self.assertFalse(tag1 in data.tags)
self.assertTrue(tag2 in data.tags)

82
server/tests/automated/test_project_tags.py Normal file
View File

@@ -0,0 +1,82 @@
from tests.automated import TestService
class TestProjectTags(TestService):
def setUp(self, version="2.8"):
super().setUp(version=version)
def test_project_tags(self):
tags_1 = ["Test tag 1", "Test tag 2"]
tags_2 = ["Test tag 3", "Test tag 4"]
p1 = self.create_temp("projects", name="Test tags1", description="test")
task1_1 = self.new_task(project=p1, tags=tags_1[:1])
task1_2 = self.new_task(project=p1, tags=tags_1[1:])
p2 = self.create_temp("projects", name="Test tasks2", description="test")
task2 = self.new_task(project=p2, tags=tags_2)
# test tags per project
data = self.api.projects.get_task_tags(projects=[p1])
self.assertEqual(set(tags_1), set(data.tags))
data = self.api.projects.get_model_tags(projects=[p1])
self.assertEqual(set(), set(data.tags))
data = self.api.projects.get_task_tags(projects=[p2])
self.assertEqual(set(tags_2), set(data.tags))
# test tags for projects list
data = self.api.projects.get_task_tags(projects=[p1, p2])
self.assertEqual(set(tags_1) | set(tags_2), set(data.tags))
# test tags for all projects
data = self.api.projects.get_task_tags(projects=[p1, p2])
self.assertTrue((set(tags_1) | set(tags_2)).issubset(data.tags))
# test move to another project
self.api.tasks.edit(task=task1_2, project=p2)
data = self.api.projects.get_task_tags(projects=[p1])
self.assertEqual(set(tags_1[:1]), set(data.tags))
data = self.api.projects.get_task_tags(projects=[p2])
self.assertEqual(set(tags_1[1:]) | set(tags_2), set(data.tags))
# test tags update
self.api.tasks.delete(task=task1_1, force=True)
self.api.tasks.delete(task=task2, force=True)
data = self.api.projects.get_task_tags(projects=[p1, p2])
self.assertEqual(set(tags_1[1:]), set(data.tags))
def test_organization_tags(self):
tag1 = "Orgtest tag1"
tag2 = "Orgtest tag2"
system_tag = "Orgtest system tag"
model = self.new_model(tags=[tag1])
task = self.new_task(tags=[tag1])
data = self.api.organization.get_tags()
self.assertTrue(tag1 in data.tags)
self.api.tasks.edit(task=task, tags=[tag2], system_tags=[system_tag])
data = self.api.organization.get_tags(include_system=True)
self.assertTrue({tag1, tag2}.issubset(set(data.tags)))
self.assertTrue(system_tag in data.system_tags)
data = self.api.organization.get_tags(
filter={"system_tags": ["__$not", system_tag]}
)
self.assertTrue(tag1 in data.tags)
self.assertFalse(tag2 in data.tags)
self.api.models.delete(model=model)
data = self.api.organization.get_tags()
self.assertFalse(tag1 in data.tags)
self.assertTrue(tag2 in data.tags)
def new_task(self, **kwargs):
self.update_missing(
kwargs, type="testing", name="test project tags", input=dict(view=dict())
)
return self.create_temp("tasks", **kwargs)
def new_model(self, **kwargs):
self.update_missing(kwargs, name="test project tags", uri="file:///a")
return self.create_temp("models", **kwargs)

27
server/tests/automated/test_task_events.py
View File

@@ -8,6 +8,8 @@ from functools import partial
from statistics import mean
from typing import Sequence
from boltons.iterutils import first
import es_factory
from apierrors.errors.bad_request import EventsNotAdded
from tests.automated import TestService
@@ -72,6 +74,31 @@ class TestTaskEvents(TestService):
),
)
def test_last_scalar_metrics(self):
metric = "Metric1"
variant = "Variant1"
iter_count = 100
task = self._temp_task()
events = [
{
**self._create_task_event("training_stats_scalar", task, iteration),
"metric": metric,
"variant": variant,
"value": iteration,
}
for iteration in range(iter_count)
]
# send 2 batches to check the interaction with already stored db value
# each batch contains multiple iterations
self.send_batch(events[:50])
self.send_batch(events[50:])
task_data = self.api.tasks.get_by_id(task=task).task
metric_data = first(first(task_data.last_metrics.values()).values())
self.assertEqual(iter_count - 1, metric_data.value)
self.assertEqual(iter_count - 1, metric_data.max_value)
self.assertEqual(0, metric_data.min_value)
def test_task_debug_images(self):
task = self._temp_task()
metric = "Metric1"