Initial commit

server/tests/api_client.conf

@@ -0,0 +1,4 @@
+{
+        api_key: "EGRTCO8JMSIGI6S39GTP43NFWXDQOW"
+        secret_key: "x!XTov_G-#vspE*Y(h$Anm&DIc5Ou-F)jsl$PdOyj5wG1&E!Z8"
+}

server/tests/api_client.py

@@ -0,0 +1,301 @@
+import json
+import logging
+import os
+import time
+from contextlib import contextmanager
+from typing import Sequence
+
+import requests
+import six
+from boltons.typeutils import issubclass
+from boltons.iterutils import remap
+from requests.adapters import HTTPAdapter
+from requests.auth import HTTPBasicAuth
+from requests.packages.urllib3.util.retry import Retry
+from typing import Union, Tuple, Type
+
+from apierrors.base import BaseError
+from pyhocon import ConfigFactory
+
+config = ConfigFactory.parse_file("api_client.conf")
+
+log = logging.getLogger("api_client")
+
+
+class APICallResult:
+    def __init__(self, res):
+        if isinstance(res, str):
+            self._dict = json.loads(res)
+        else:
+            self._dict = res
+
+        self.data = self._dict["data"]
+        self.meta = APICallResultMeta(self._dict["meta"])
+
+    def as_dict(self):
+        return self._dict
+
+
+class APICallResultMeta:
+    def __init__(self, meta_dict):
+        m = meta_dict
+        self._dict = m
+        self.call_id = m["id"]
+        self.trx_id = m["trx"]
+        self.result_code = m["result_code"]
+        self.result_msg = m["result_msg"]
+        self.result_subcode = m["result_subcode"]
+        self.error_stack = m.get("error_stack") or ""
+        self.endpoint_name = m["endpoint"]["name"]
+        self.endpoint_version = m["endpoint"]["actual_version"]
+        self.requested_endpoint_version = m["endpoint"]["requested_version"]
+        self.codes = self.result_code, self.result_subcode
+
+
+def format_duration(duration):
+    return "" if duration is None else f" ({int(duration)} sec)"
+
+
+class APIError(Exception):
+    def __init__(self, result: APICallResult, duration=None):
+        super().__init__(result)
+        self.result = result
+        self.duration = duration
+
+    def __str__(self):
+        meta = self.result.meta
+        message = (
+            f"APIClient got {meta.result_code}/{meta.result_subcode} from {meta.endpoint_name}: "
+            f"{meta.result_msg}{format_duration(self.duration)}"
+        )
+        if meta.error_stack:
+            header = "\n--- SERVER ERROR {} ---\n"
+            formatted_traceback = "\n{}{}{}\n".format(header.format("START"), meta.error_stack, header.format("END"))
+            message += formatted_traceback
+        return message
+
+
+class AttrDict(dict):
+    """
+    ``dict`` which supports attribute lookup syntax.
+    Use to implement polymorphism over ``dict``s and database objects, which don't support subscription syntax.
+    """
+
+    def __init__(self, dct=None, **kwargs):
+        super().__init__(
+            remap(
+                {**(dct or {}), **kwargs},
+                lambda _, key, value: (
+                    key,
+                    type(self)(value)
+                    if isinstance(value, dict) and not isinstance(value, type(self))
+                    else value,
+                ),
+            )
+        )
+
+    def __getattr__(self, item):
+        return self[item]
+
+
+class APIClient:
+    def __init__(
+        self,
+        api_key=None,
+        secret_key=None,
+        base_url=None,
+        impersonated_user_id=None,
+        session_token=None,
+    ):
+        if not session_token:
+            self.api_key = (
+                api_key
+                or os.environ.get("SM_API_KEY")
+                or config.get("api_key")
+            )
+            if not self.api_key:
+                raise ValueError("APIClient requires api_key in constructor or config")
+
+            self.secret_key = (
+                secret_key
+                or os.environ.get("SM_API_SECRET")
+                or config.get("secret_key")
+            )
+            if not self.secret_key:
+                raise ValueError(
+                    "APIClient requires secret_key in constructor or config"
+                )
+
+        self.base_url = (
+            base_url or os.environ.get("SM_API_URL") or config.get("base_url")
+        )
+        if not self.base_url:
+            raise ValueError("APIClient requires base_url in constructor or config")
+
+        if self.base_url.endswith("/"):
+            self.base_url = self.base_url[:-1]
+
+        self.session_token = session_token
+
+        # create http session
+        self.http_session = requests.session()
+        retries = config.get("retries", 7)
+        backoff_factor = config.get("backoff_factor", 0.3)
+        status_forcelist = config.get("status_forcelist", (500, 502, 504))
+        retry = Retry(
+            total=retries,
+            read=retries,
+            connect=retries,
+            backoff_factor=backoff_factor,
+            status_forcelist=status_forcelist,
+        )
+        adapter = HTTPAdapter(max_retries=retry)
+        self.http_session.mount("http://", adapter)
+        self.http_session.mount("https://", adapter)
+
+        if impersonated_user_id:
+            self.http_session.headers["X-Trains-Impersonate-As"] = impersonated_user_id
+
+        if not self.session_token:
+            self.login()
+
+    def login(self):
+        res, self.session_token = self.send("auth.login", extract="token")
+
+    def impersonate(self, user_id):
+        return type(self)(
+            impersonated_user_id=user_id,
+            **{
+                key: getattr(self, key)
+                for key in ("api_key", "secret_key", "base_url", "session_token")
+            },
+        )
+
+    def send_batch(
+        self,
+        endpoint,
+        items: Sequence[dict],
+        headers_overrides=None,
+        raise_errors=True,
+        log_response=False,
+        extract=None,
+    ):
+        headers_overrides = headers_overrides or {}
+        headers_overrides.update({"Content-Type": "application/json-lines"})
+        assert isinstance(items, (list, tuple)), type(items)
+        json_lines = (json.dumps(item) for item in items)
+        data = "\n".join(json_lines)
+        return self.send(
+            endpoint,
+            data=data,
+            headers_overrides=headers_overrides,
+            raise_errors=raise_errors,
+            log_response=log_response,
+            extract=extract,
+        )
+
+    def send(
+        self,
+        endpoint,
+        data=None,
+        headers_overrides=None,
+        raise_errors=True,
+        log_response=False,
+        is_async=False,
+        extract=None,
+    ):
+        if headers_overrides is None:
+            headers_overrides = {}
+        if data is None:
+            data = {}
+        headers = {"Content-Type": "application/json"}
+        headers.update(headers_overrides)
+        if is_async:
+            headers["X-Trains-Async"] = "1"
+
+        if not isinstance(data, six.string_types):
+            data = json.dumps(data)
+
+        if not self.session_token:
+            auth = HTTPBasicAuth(self.api_key, self.secret_key)
+        else:
+            auth = None
+            headers["Authorization"] = "Bearer %s" % self.session_token
+
+        url = "%s/%s" % (self.base_url, endpoint)
+        start = time.time()
+
+        http_res = self.http_session.post(url, headers=headers, data=data, auth=auth)
+        if not http_res.text:
+            msg = "APIClient got non standard response from %s. http_status=%s " % (
+                url,
+                http_res.status_code,
+            )
+            log.error(msg)
+            raise Exception(msg)
+
+        res = APICallResult(http_res.text)
+        if res.meta.result_code == 202:
+            # poll server for async result
+            got_result = False
+            call_id = res.meta.call_id
+            async_res_url = "%s/async.result?id=%s" % (self.base_url, call_id)
+            async_res_headers = headers.copy()
+            async_res_headers.pop("X-Trains-Async")
+            while not got_result:
+                log.info("Got 202. Checking async result for %s (%s)" % (url, call_id))
+                http_res = self.http_session.get(
+                    async_res_url, headers=async_res_headers
+                )
+                if http_res.status_code == 202:
+                    time.sleep(5)
+                else:
+                    res = APICallResult(http_res.text)
+                    got_result = True
+
+        duration = time.time() - start
+
+        if res.meta.result_code != 200:
+            error = APIError(res, duration)
+            log.error(error)
+            if raise_errors:
+                raise error
+        else:
+            msg = "APIClient got {} from {}{}".format(
+                res.meta.result_code, url, format_duration(duration)
+            )
+            log.info(msg)
+            if log_response:
+                log.debug(res.as_dict())
+
+        if extract is None:
+            return res, res.data
+        else:
+            return res, res.data.get(extract)
+
+    class Service(object):
+        def __init__(self, api, name):
+            self.api = api
+            self.name = name
+
+        def __getattr__(self, item):
+            return lambda **kwargs: AttrDict(
+                self.api.send("{}.{}".format(self.name, item), kwargs)[1]
+            )
+
+    def __getattr__(self, item):
+        return self.Service(self, item)
+
+    @staticmethod
+    @contextmanager
+    def raises(codes: Union[Type[BaseError], Tuple[int, int]]):
+        if issubclass(codes, BaseError):
+            codes = codes.codes
+        log.info("Expecting failure: %s, %s", *codes)
+        try:
+            yield
+        except APIError as e:
+            if e.result.meta.codes != codes:
+                raise
+        else:
+            assert False, "call should fail"

server/tests/automated/__init__.py

@@ -0,0 +1,93 @@
+import abc
+import sys
+from functools import partial
+from unittest import TestCase
+
+from tests.api_client import APIClient
+from config import config
+
+log = config.logger(__file__)
+
+
+class TestServiceInterface(metaclass=abc.ABCMeta):
+    api = abc.abstractproperty()
+
+    @abc.abstractmethod
+    def defer(self, func, *args, can_fail=False, **kwargs):
+        pass
+
+
+class TestService(TestCase, TestServiceInterface):
+    @property
+    def api(self):
+        return self._api
+
+    @api.setter
+    def api(self, value):
+        self._api = value
+
+    def defer(self, func, *args, can_fail=False, **kwargs):
+        self._deferred.append((can_fail, partial(func, *args, **kwargs)))
+
+    def _create_temp_helper(
+        self,
+        service,
+        object_name,
+        create_endpoint,
+        delete_endpoint,
+        create_params,
+        *,
+        client=None,
+        delete_params=None,
+    ):
+        client = client or self.api
+        res, data = client.send(f"{service}.{create_endpoint}", create_params)
+
+        object_id = data["id"]
+        self.defer(
+            client.send,
+            f"{service}.{delete_endpoint}",
+            can_fail=True,
+            data={object_name: object_id, "force": True, **(delete_params or {})},
+        )
+        return object_id
+
+    def create_temp(self, service, *, client=None, delete_params=None, **kwargs) -> str:
+        return self._create_temp_helper(
+            service=service,
+            create_endpoint="create",
+            delete_endpoint="delete",
+            object_name=service.rstrip("s"),
+            create_params=kwargs,
+            client=client,
+            delete_params=delete_params,
+        )
+
+    def create_temp_version(self, *, client=None, **kwargs) -> str:
+        return self._create_temp_helper(
+            service="datasets",
+            create_endpoint="create_version",
+            delete_endpoint="delete_version",
+            object_name="version",
+            create_params=kwargs,
+            client=client,
+        )
+
+    def setUp(self, version="1.7"):
+        self._api = APIClient(base_url=f"http://localhost:8008/v{version}")
+        self._deferred = []
+        header(self.id())
+
+    def tearDown(self):
+        log.info("Cleanup...")
+        for can_fail, func in reversed(self._deferred):
+            try:
+                func()
+            except Exception as ex:
+                if not can_fail:
+                    log.exception(ex)
+        self._deferred = []
+
+
+def header(info, title="=" * 20):
+    print(title, info, title, file=sys.stderr)

server/tests/automated/model_tests.py

@@ -0,0 +1,222 @@
+from tests.automated import TestService
+
+MODEL_CANNOT_BE_UPDATED_CODES = (400, 203)
+TASK_CANNOT_BE_UPDATED_CODES = (400, 110)
+PUBLISHED = "published"
+IN_PROGRESS = "in_progress"
+
+
+class TestModelsService(TestService):
+    def test_publish_output_model_running_task(self):
+        task_id, model_id = self._create_task_and_model()
+        self._assert_model_ready(model_id, False)
+
+        with self._assert_update_task_failure():
+            self.api.models.set_ready(model=model_id)
+
+        self._assert_model_ready(model_id, False)
+        self._assert_task_status(task_id, IN_PROGRESS)
+
+    def test_publish_output_model_running_task_no_task_publish(self):
+        task_id, model_id = self._create_task_and_model()
+        self._assert_model_ready(model_id, False)
+
+        res = self.api.models.set_ready(model=model_id, publish_task=False)
+        assert res.updated == 1  # model updated
+        assert res.get("published_task", None) is None
+
+        self._assert_model_ready(model_id, True)
+        self._assert_task_status(task_id, IN_PROGRESS)
+
+    def test_publish_output_model_running_task_force_task_publish(self):
+        task_id, model_id = self._create_task_and_model()
+        self._assert_model_ready(model_id, False)
+
+        self.api.models.set_ready(model=model_id, force_publish_task=True)
+
+        self._assert_model_ready(model_id, True)
+        self._assert_task_status(task_id, PUBLISHED)
+
+    def test_publish_output_model_published_task(self):
+        task_id, model_id = self._create_task_and_model()
+        self._assert_model_ready(model_id, False)
+
+        self.api.tasks.stopped(task=task_id)
+        self.api.tasks.publish(task=task_id, publish_model=False)
+        self._assert_model_ready(model_id, False)
+        self._assert_task_status(task_id, PUBLISHED)
+
+        res = self.api.models.set_ready(model=model_id)
+        assert res.updated == 1  # model updated
+        assert res.get("published_task", None) is None
+
+        self._assert_model_ready(model_id, True)
+        self._assert_task_status(task_id, PUBLISHED)
+
+    def test_publish_output_model_stopped_task(self):
+        task_id, model_id = self._create_task_and_model()
+        self._assert_model_ready(model_id, False)
+
+        self.api.tasks.stopped(task=task_id)
+
+        res = self.api.models.set_ready(model=model_id)
+        assert res.updated == 1  # model updated
+        assert res.published_task.id == task_id
+        assert res.published_task.data.updated == 1
+        self._assert_model_ready(model_id, True)
+        self._assert_task_status(task_id, PUBLISHED)
+
+        # cannot publish already published model
+        with self._assert_update_model_failure():
+            self.api.models.set_ready(model=model_id)
+        self._assert_model_ready(model_id, True)
+
+    def test_publish_output_model_no_task(self):
+        model_id = self.create_temp(
+            service="models", name='test', uri='file:///a', labels={}, ready=False
+        )
+        self._assert_model_ready(model_id, False)
+
+        res = self.api.models.set_ready(model=model_id)
+        assert res.updated == 1  # model updated
+        assert res.get("task", None) is None
+        self._assert_model_ready(model_id, True)
+
+    def test_publish_task_with_output_model(self):
+        task_id, model_id = self._create_task_and_model()
+        self._assert_model_ready(model_id, False)
+
+        self.api.tasks.stopped(task=task_id)
+        res = self.api.tasks.publish(task=task_id)
+        assert res.updated == 1  # model updated
+        self._assert_model_ready(model_id, True)
+        self._assert_task_status(task_id, PUBLISHED)
+
+    def test_publish_task_with_published_model(self):
+        task_id, model_id = self._create_task_and_model()
+        self.api.models.set_ready(model=model_id, publish_task=False)
+        self._assert_model_ready(model_id, True)
+
+        self.api.tasks.stopped(task=task_id)
+        res = self.api.tasks.publish(task=task_id)
+        assert res.updated == 1  # model updated
+        self._assert_task_status(task_id, PUBLISHED)
+        self._assert_model_ready(model_id, True)
+
+    def test_publish_task_no_output_model(self):
+        task_id = self.create_temp(
+            service="tasks", type='testing', name='server-test', input=dict(view={})
+        )
+        self.api.tasks.started(task=task_id)
+        self.api.tasks.stopped(task=task_id)
+
+        res = self.api.tasks.publish(task=task_id)
+        assert res.updated == 1  # model updated
+        self._assert_task_status(task_id, PUBLISHED)
+
+    def test_update_model_iteration_with_task(self):
+        task_id = self._create_task()
+        model_id = self._create_model()
+        self.api.models.update(model=model_id, task=task_id, iteration=1000, labels={"foo": 1})
+
+        self.assertEqual(
+            self.api.tasks.get_by_id(task=task_id).task.last_iteration,
+            1000
+        )
+
+        self.api.models.update(model=model_id, task=task_id, iteration=500)
+
+        self.assertEqual(
+            self.api.tasks.get_by_id(task=task_id).task.last_iteration,
+            1000
+        )
+
+    def test_update_model_for_task_iteration(self):
+        task_id = self._create_task()
+
+        res = self.api.models.update_for_task(
+            task=task_id,
+            name="test model",
+            uri="file:///b",
+            iteration=999,
+        )
+
+        model_id = res.id
+
+        self.defer(self.api.models.delete, can_fail=True, model=model_id, force=True)
+
+        self.assertEqual(
+            self.api.tasks.get_by_id(task=task_id).task.last_iteration,
+            999
+        )
+
+        self.api.models.update_for_task(task=task_id, uri="file:///c", iteration=1000)
+
+        self.assertEqual(
+            self.api.tasks.get_by_id(task=task_id).task.last_iteration,
+            1000
+        )
+
+        self.api.models.update_for_task(task=task_id, uri="file:///d", iteration=888)
+
+        self.assertEqual(
+            self.api.tasks.get_by_id(task=task_id).task.last_iteration,
+            1000
+        )
+
+    def _assert_task_status(self, task_id, status):
+        task = self.api.tasks.get_by_id(task=task_id).task
+        assert task.status == status
+
+    def _assert_model_ready(self, model_id, ready):
+        model = self.api.models.get_by_id(model=model_id)["model"]
+        assert model.ready == ready
+
+    def _assert_update_model_failure(self):
+        return self.api.raises(MODEL_CANNOT_BE_UPDATED_CODES)
+
+    def _assert_update_task_failure(self):
+        return self.api.raises(TASK_CANNOT_BE_UPDATED_CODES)
+
+    def _create_model(self):
+        model_id = self.create_temp(
+            service="models",
+            name='test',
+            uri='file:///a',
+            labels={}
+        )
+
+        self.defer(self.api.models.delete, can_fail=True, model=model_id, force=True)
+
+        return model_id
+
+    def _create_task(self):
+        task_id = self.create_temp(
+            service="tasks",
+            type='testing',
+            name='server-test',
+            input=dict(view={}),
+        )
+
+        return task_id
+
+    def _create_task_and_model(self):
+        execution_model_id = self.create_temp(
+            service="models",
+            name='test',
+            uri='file:///a',
+            labels={}
+        )
+        task_id = self.create_temp(
+            service="tasks",
+            type='testing',
+            name='server-test',
+            input=dict(view={}),
+            execution=dict(model=execution_model_id)
+        )
+        self.api.tasks.started(task=task_id)
+        output_model_id = self.api.models.update_for_task(
+            task=task_id, uri='file:///b'
+        )["id"]
+
+        return task_id, output_model_id

server/tests/automated/projection_tests.py

@@ -0,0 +1,19 @@
+from config import config
+from database.model.task.task import TaskStatus
+from tests.automated import TestService
+
+log = config.logger(__file__)
+
+
+class TestProjection(TestService):
+    def test_overlapping_fields(self):
+        message = "task started"
+        task_id = self.create_temp(
+            "tasks", name="test", type="testing", input=dict(view=dict())
+        )
+        self.api.tasks.started(task=task_id, status_message=message)
+        task = self.api.tasks.get_all_ex(
+            id=[task_id], only_fields=["status", "status_message"]
+        ).tasks[0]
+        assert task["status"] == TaskStatus.in_progress
+        assert task["status_message"] == message

server/tests/automated/requirements.txt

@@ -0,0 +1 @@
+parameterized

server/tests/automated/task_events_tests.py

@@ -0,0 +1,159 @@
+"""
+Comprehensive test of all(?) use cases of datasets and frames
+"""
+import json
+import unittest
+
+import es_factory
+from tests.api_client import APIClient
+from config import config
+
+log = config.logger(__file__)
+
+
+class TestDatasetsService(unittest.TestCase):
+
+    def setUp(self):
+        self.api = APIClient(base_url="http://localhost:5100/v1.0")
+        self.created_tasks = []
+
+        self.task = dict(
+            name="test task events",
+            type="training",
+        )
+        res, self.task_id = self.api.send('tasks.create', self.task, extract="id")
+        assert (res.meta.result_code == 200)
+        self.created_tasks.append(self.task_id)
+
+    def tearDown(self):
+        log.info("Cleanup...")
+        for task_id in self.created_tasks:
+            try:
+                self.api.send('tasks.delete', dict(task=task_id, force=True))
+            except Exception as ex:
+                log.exception(ex)
+
+    def create_task_event(self, type, iteration):
+        return {
+            "worker": "test",
+            "type": type,
+            "task": self.task_id,
+            "iter": iteration,
+            "timestamp": es_factory.get_timestamp_millis()
+        }
+
+    def copy_and_update(self, src_obj, new_data):
+        obj = src_obj.copy()
+        obj.update(new_data)
+        return obj
+
+    def test_task_logs(self):
+        events = []
+        for iter in range(10):
+            log_event = self.create_task_event("log", iteration=iter)
+            events.append(self.copy_and_update(log_event, {
+                "msg": "This is a log message from test task iter " + str(iter)
+            }))
+            # sleep so timestamp is not the same
+            import time
+            time.sleep(0.01)
+        self.send_batch(events)
+
+        data = self.api.events.get_task_log(task=self.task_id)
+        assert len(data["events"]) == 10
+
+        self.api.tasks.reset(task=self.task_id)
+        data = self.api.events.get_task_log(task=self.task_id)
+        assert len(data["events"]) == 0
+
+    def test_task_plots(self):
+        event = self.create_task_event("plot", 0)
+        event["metric"] = "roc"
+        event.update({
+            "plot_str": json.dumps({
+                "data": [
+                    {
+                        "x": [0, 1, 2, 3, 4, 5, 6, 7, 8],
+                        "y": [0, 1, 2, 3, 4, 5, 6, 7, 8],
+                        "text": ["Th=0.1", "Th=0.2", "Th=0.3", "Th=0.4", "Th=0.5", "Th=0.6", "Th=0.7", "Th=0.8"],
+                        "name": 'class1'
+                    },
+                    {
+                        "x": [0, 1, 2, 3, 4, 5, 6, 7, 8],
+                        "y": [2.0, 3.0, 5.0, 8.2, 6.4, 7.5, 9.2, 8.1, 10.0],
+                        "text": ["Th=0.1", "Th=0.2", "Th=0.3", "Th=0.4", "Th=0.5", "Th=0.6", "Th=0.7", "Th=0.8"],
+                        "name": 'class2',
+                    }
+                ],
+                "layout": {
+                    "title": "ROC for iter 0",
+                    "xaxis": {
+                        "title": 'my x axis'
+                    },
+                    "yaxis": {
+                        "title": 'my y axis'
+                    }
+                }
+            })
+        })
+        self.send(event)
+
+        event = self.create_task_event("plot", 100)
+        event["metric"] = "confusion"
+        event.update({
+            "plot_str": json.dumps({
+                "data": [
+                    {
+                        "y": [
+                            "lying",
+                            "sitting",
+                            "standing",
+                            "people",
+                            "backgroun"
+                        ],
+                        "x": [
+                            "lying",
+                            "sitting",
+                            "standing",
+                            "people",
+                            "backgroun"
+                        ],
+                        "z": [
+                            [758, 163, 0, 0, 23],
+                            [63, 858, 3, 0, 0],
+                            [0, 50, 188, 21, 35],
+                            [0, 22, 8, 40, 4, ],
+                            [12, 91, 26, 29, 368]
+                        ],
+                        "type": "heatmap"
+                    }
+                ],
+                "layout": {
+                    "title": "Confusion Matrix for iter 100",
+                    "xaxis": {
+                        "title": "Predicted value"
+                    },
+                    "yaxis": {
+                        "title": "Real value"
+                    }
+                }
+            })
+        })
+        self.send(event)
+
+        data = self.api.events.get_task_plots(task=self.task_id)
+        assert len(data["plots"]) == 2
+
+        self.api.tasks.reset(task=self.task_id)
+        data = self.api.events.get_task_plots(task=self.task_id)
+        assert len(data["plots"]) == 0
+
+    def send_batch(self, events):
+        self.api.send_batch('events.add_batch', events)
+
+    def send(self, event):
+        self.api.send('events.add', event)
+
+
+if __name__ == '__main__':
+    unittest.main()

server/tests/automated/tasks_delete_tests.py

@@ -0,0 +1,95 @@
+from parameterized import parameterized
+
+from config import config
+from tests.automated import TestService
+
+log = config.logger(__file__)
+
+
+continuations = (
+    (lambda self, task: self.tasks.reset(task=task),),
+    (lambda self, task: self.tasks.delete(task=task),),
+)
+
+
+def reset_and_delete():
+    """
+    Parametrize a test for both delete and reset operations,
+    which should yield the same results.
+    NOTE: "parameterized" engages in call stack manipulation,
+        so be careful when changing the application of the decorator.
+        For example, receiving "func" as a parameter and passing it to
+        "expand" doesn't work.
+    """
+    return parameterized.expand(
+        [
+            (lambda self, task: self.tasks.delete(task=task),),
+            (lambda self, task: self.tasks.reset(task=task),),
+        ],
+        name_func=lambda func, num, _: "{}_{}".format(
+            func.__name__, ["delete", "reset"][int(num)]
+        ),
+    )
+
+
+class TestTasksResetDelete(TestService):
+
+    TASK_CANNOT_BE_DELETED_CODES = (400, 123)
+
+    def setUp(self):
+        super(TestTasksResetDelete, self).setUp()
+        self.tasks = self.api.tasks
+        self.models = self.api.models
+
+    def new_task(self, **kwargs):
+        task_id = self.tasks.create(
+            type='testing', name='server-test', input=dict(view=dict()), **kwargs
+        )['id']
+        self.defer(self.tasks.delete, can_fail=True, task=task_id, force=True)
+        return task_id
+
+    def new_model(self, **kwargs):
+        model_id = self.models.create(name='test', uri='file:///a', labels={}, **kwargs)['id']
+        self.defer(self.models.delete, can_fail=True, model=model_id, force=True)
+        return model_id
+
+    def delete_failure(self):
+        return self.api.raises(self.TASK_CANNOT_BE_DELETED_CODES)
+
+    def publish_created_task(self, task_id):
+        self.tasks.started(task=task_id)
+        self.tasks.stopped(task=task_id)
+        self.tasks.publish(task=task_id)
+
+    @reset_and_delete()
+    def test_plain(self, cont):
+        cont(self, self.new_task())
+
+    @reset_and_delete()
+    def test_draft_child(self, cont):
+        parent = self.new_task()
+        self.new_task(parent=parent)
+        cont(self, parent)
+
+    @reset_and_delete()
+    def test_published_child(self, cont):
+        parent = self.new_task()
+        child = self.new_task(parent=parent)
+        self.publish_created_task(child)
+        with self.delete_failure():
+            cont(self, parent)
+
+    @reset_and_delete()
+    def test_draft_model(self, cont):
+        task = self.new_task()
+        model = self.new_model()
+        self.models.edit(model=model, task=task, ready=False)
+        cont(self, task)
+
+    @reset_and_delete()
+    def test_published_model(self, cont):
+        task = self.new_task()
+        model = self.new_model()
+        self.models.edit(model=model, task=task, ready=True)
+        with self.delete_failure():
+            cont(self, task)

server/tests/automated/tasks_edit_tests.py

@@ -0,0 +1,55 @@
+from apierrors.errors.bad_request import ModelNotReady
+from config import config
+from tests.automated import TestService
+
+
+log = config.logger(__file__)
+
+
+class TestTasksEdit(TestService):
+    def new_task(self, **kwargs):
+        return self.create_temp(
+            "tasks", type="testing", name="test", input=dict(view=dict()), **kwargs
+        )
+
+    def new_model(self):
+        return self.create_temp("models", name="test", uri="file:///a/b", labels={})
+
+    def test_edit_model_ready(self):
+        task = self.new_task()
+        model = self.new_model()
+        self.api.tasks.edit(task=task, execution=dict(model=model))
+
+    def test_edit_model_not_ready(self):
+        task = self.new_task()
+        model = self.new_model()
+        self.api.models.edit(model=model, ready=False)
+        self.assertFalse(self.api.models.get_by_id(model=model).model.ready)
+        with self.api.raises(ModelNotReady):
+            self.api.tasks.edit(task=task, execution=dict(model=model))
+
+    def test_edit_model_not_ready_force(self):
+        task = self.new_task()
+        model = self.new_model()
+        self.api.models.edit(model=model, ready=False)
+        self.assertFalse(self.api.models.get_by_id(model=model).model.ready)
+        self.api.tasks.edit(task=task, execution=dict(model=model), force=True)
+
+    def test_edit_had_model_model_not_ready(self):
+        ready_model = self.new_model()
+        self.assert_(self.api.models.get_by_id(model=ready_model).model.ready)
+        task = self.new_task(execution=dict(model=ready_model))
+        not_ready_model = self.new_model()
+        self.api.models.edit(model=not_ready_model, ready=False)
+        self.assertFalse(self.api.models.get_by_id(model=not_ready_model).model.ready)
+        with self.api.raises(ModelNotReady):
+            self.api.tasks.edit(task=task, execution=dict(model=not_ready_model))
+
+    def test_edit_had_model_model_not_ready_force(self):
+        ready_model = self.new_model()
+        self.assert_(self.api.models.get_by_id(model=ready_model).model.ready)
+        task = self.new_task(execution=dict(model=ready_model))
+        not_ready_model = self.new_model()
+        self.api.models.edit(model=not_ready_model, ready=False)
+        self.assertFalse(self.api.models.get_by_id(model=not_ready_model).model.ready)
+        self.api.tasks.edit(task=task, execution=dict(model=not_ready_model), force=True)

server/tests/requirements.txt

@@ -0,0 +1 @@
+numpy>=1.12.1