Add new task types and support for Server API version 2.8

trains/backend_api/api_proxy.py

@@ -9,7 +9,7 @@ from ..utilities.check_updates import Version
 
 class ApiServiceProxy(object):
     _main_services_module = "trains.backend_api.services"
-    _max_available_version = None
+    _available_versions = None
 
     def __init__(self, module):
         self.__wrapped_name__ = module
@@ -20,17 +20,18 @@ class ApiServiceProxy(object):
             return self.__dict__.get(attr)
 
         if not self.__dict__.get("__wrapped__") or self.__dict__.get("__wrapped_version__") != Session.api_version:
-            if not ApiServiceProxy._max_available_version:
+            if not ApiServiceProxy._available_versions:
                 from ..backend_api import services
-                ApiServiceProxy._max_available_version = max([
+                ApiServiceProxy._available_versions = [
                     Version(name[1:].replace("_", "."))
                     for name in [
                         module_name
                         for _, module_name, _ in pkgutil.iter_modules(services.__path__)
                         if re.match(r"^v[0-9]+_[0-9]+$", module_name)
-                    ]])
+                    ]]
 
-            version = str(min(Version(Session.api_version), ApiServiceProxy._max_available_version))
+            # get the most advanced service version that supports our api
+            version = [str(v) for v in ApiServiceProxy._available_versions if Version(Session.api_version) >= v][-1]
             self.__dict__["__wrapped_version__"] = Session.api_version
             name = ".v{}.{}".format(
                 version.replace(".", "_"), self.__dict__.get("__wrapped_name__")

trains/backend_api/services/v2_8/auth.py

@@ -0,0 +1,598 @@
+"""
+auth service
+
+This service provides authentication management and authorization
+validation for the entire system.
+"""
+import six
+import types
+from datetime import datetime
+import enum
+
+from dateutil.parser import parse as parse_datetime
+
+from ....backend_api.session import Request, BatchRequest, Response, DataModel, NonStrictDataModel, CompoundRequest, schema_property, StringEnum
+
+
+class Credentials(NonStrictDataModel):
+    """
+    :param access_key: Credentials access key
+    :type access_key: str
+    :param secret_key: Credentials secret key
+    :type secret_key: str
+    """
+    _schema = {
+        'properties': {
+            'access_key': {
+                'description': 'Credentials access key',
+                'type': ['string', 'null'],
+            },
+            'secret_key': {
+                'description': 'Credentials secret key',
+                'type': ['string', 'null'],
+            },
+        },
+        'type': 'object',
+    }
+    def __init__(
+            self, access_key=None, secret_key=None, **kwargs):
+        super(Credentials, self).__init__(**kwargs)
+        self.access_key = access_key
+        self.secret_key = secret_key
+
+    @schema_property('access_key')
+    def access_key(self):
+        return self._property_access_key
+
+    @access_key.setter
+    def access_key(self, value):
+        if value is None:
+            self._property_access_key = None
+            return
+
+        self.assert_isinstance(value, "access_key", six.string_types)
+        self._property_access_key = value
+
+    @schema_property('secret_key')
+    def secret_key(self):
+        return self._property_secret_key
+
+    @secret_key.setter
+    def secret_key(self, value):
+        if value is None:
+            self._property_secret_key = None
+            return
+
+        self.assert_isinstance(value, "secret_key", six.string_types)
+        self._property_secret_key = value
+
+
+class CredentialKey(NonStrictDataModel):
+    """
+    :param access_key:
+    :type access_key: str
+    :param last_used:
+    :type last_used: datetime.datetime
+    :param last_used_from:
+    :type last_used_from: str
+    """
+    _schema = {
+        'properties': {
+            'access_key': {'description': '', 'type': ['string', 'null']},
+            'last_used': {
+                'description': '',
+                'format': 'date-time',
+                'type': ['string', 'null'],
+            },
+            'last_used_from': {'description': '', 'type': ['string', 'null']},
+        },
+        'type': 'object',
+    }
+    def __init__(
+            self, access_key=None, last_used=None, last_used_from=None, **kwargs):
+        super(CredentialKey, self).__init__(**kwargs)
+        self.access_key = access_key
+        self.last_used = last_used
+        self.last_used_from = last_used_from
+
+    @schema_property('access_key')
+    def access_key(self):
+        return self._property_access_key
+
+    @access_key.setter
+    def access_key(self, value):
+        if value is None:
+            self._property_access_key = None
+            return
+
+        self.assert_isinstance(value, "access_key", six.string_types)
+        self._property_access_key = value
+
+    @schema_property('last_used')
+    def last_used(self):
+        return self._property_last_used
+
+    @last_used.setter
+    def last_used(self, value):
+        if value is None:
+            self._property_last_used = None
+            return
+
+        self.assert_isinstance(value, "last_used", six.string_types + (datetime,))
+        if not isinstance(value, datetime):
+            value = parse_datetime(value)
+        self._property_last_used = value
+
+    @schema_property('last_used_from')
+    def last_used_from(self):
+        return self._property_last_used_from
+
+    @last_used_from.setter
+    def last_used_from(self, value):
+        if value is None:
+            self._property_last_used_from = None
+            return
+
+        self.assert_isinstance(value, "last_used_from", six.string_types)
+        self._property_last_used_from = value
+
+
+
+
+class CreateCredentialsRequest(Request):
+    """
+    Creates a new set of credentials for the authenticated user.
+                            New key/secret is returned.
+                            Note: Secret will never be returned in any other API call.
+                            If a secret is lost or compromised, the key should be revoked
+                            and a new set of credentials can be created.
+
+    """
+
+    _service = "auth"
+    _action = "create_credentials"
+    _version = "2.8"
+    _schema = {
+        'additionalProperties': False,
+        'definitions': {},
+        'properties': {},
+        'type': 'object',
+    }
+
+
+class CreateCredentialsResponse(Response):
+    """
+    Response of auth.create_credentials endpoint.
+
+    :param credentials: Created credentials
+    :type credentials: Credentials
+    """
+    _service = "auth"
+    _action = "create_credentials"
+    _version = "2.8"
+
+    _schema = {
+        'definitions': {
+            'credentials': {
+                'properties': {
+                    'access_key': {
+                        'description': 'Credentials access key',
+                        'type': ['string', 'null'],
+                    },
+                    'secret_key': {
+                        'description': 'Credentials secret key',
+                        'type': ['string', 'null'],
+                    },
+                },
+                'type': 'object',
+            },
+        },
+        'properties': {
+            'credentials': {
+                'description': 'Created credentials',
+                'oneOf': [{'$ref': '#/definitions/credentials'}, {'type': 'null'}],
+            },
+        },
+        'type': 'object',
+    }
+    def __init__(
+            self, credentials=None, **kwargs):
+        super(CreateCredentialsResponse, self).__init__(**kwargs)
+        self.credentials = credentials
+
+    @schema_property('credentials')
+    def credentials(self):
+        return self._property_credentials
+
+    @credentials.setter
+    def credentials(self, value):
+        if value is None:
+            self._property_credentials = None
+            return
+        if isinstance(value, dict):
+            value = Credentials.from_dict(value)
+        else:
+            self.assert_isinstance(value, "credentials", Credentials)
+        self._property_credentials = value
+
+
+
+
+class EditUserRequest(Request):
+    """
+     Edit a users' auth data properties
+
+    :param user: User ID
+    :type user: str
+    :param role: The new user's role within the company
+    :type role: str
+    """
+
+    _service = "auth"
+    _action = "edit_user"
+    _version = "2.8"
+    _schema = {
+        'definitions': {},
+        'properties': {
+            'role': {
+                'description': "The new user's role within the company",
+                'enum': ['admin', 'superuser', 'user', 'annotator'],
+                'type': ['string', 'null'],
+            },
+            'user': {'description': 'User ID', 'type': ['string', 'null']},
+        },
+        'type': 'object',
+    }
+    def __init__(
+            self, user=None, role=None, **kwargs):
+        super(EditUserRequest, self).__init__(**kwargs)
+        self.user = user
+        self.role = role
+
+    @schema_property('user')
+    def user(self):
+        return self._property_user
+
+    @user.setter
+    def user(self, value):
+        if value is None:
+            self._property_user = None
+            return
+
+        self.assert_isinstance(value, "user", six.string_types)
+        self._property_user = value
+
+    @schema_property('role')
+    def role(self):
+        return self._property_role
+
+    @role.setter
+    def role(self, value):
+        if value is None:
+            self._property_role = None
+            return
+
+        self.assert_isinstance(value, "role", six.string_types)
+        self._property_role = value
+
+
+class EditUserResponse(Response):
+    """
+    Response of auth.edit_user endpoint.
+
+    :param updated: Number of users updated (0 or 1)
+    :type updated: float
+    :param fields: Updated fields names and values
+    :type fields: dict
+    """
+    _service = "auth"
+    _action = "edit_user"
+    _version = "2.8"
+
+    _schema = {
+        'definitions': {},
+        'properties': {
+            'fields': {
+                'additionalProperties': True,
+                'description': 'Updated fields names and values',
+                'type': ['object', 'null'],
+            },
+            'updated': {
+                'description': 'Number of users updated (0 or 1)',
+                'enum': [0, 1],
+                'type': ['number', 'null'],
+            },
+        },
+        'type': 'object',
+    }
+    def __init__(
+            self, updated=None, fields=None, **kwargs):
+        super(EditUserResponse, self).__init__(**kwargs)
+        self.updated = updated
+        self.fields = fields
+
+    @schema_property('updated')
+    def updated(self):
+        return self._property_updated
+
+    @updated.setter
+    def updated(self, value):
+        if value is None:
+            self._property_updated = None
+            return
+
+        self.assert_isinstance(value, "updated", six.integer_types + (float,))
+        self._property_updated = value
+
+    @schema_property('fields')
+    def fields(self):
+        return self._property_fields
+
+    @fields.setter
+    def fields(self, value):
+        if value is None:
+            self._property_fields = None
+            return
+
+        self.assert_isinstance(value, "fields", (dict,))
+        self._property_fields = value
+
+
+class GetCredentialsRequest(Request):
+    """
+    Returns all existing credential keys for the authenticated user.
+            Note: Only credential keys are returned.
+
+    """
+
+    _service = "auth"
+    _action = "get_credentials"
+    _version = "2.8"
+    _schema = {
+        'additionalProperties': False,
+        'definitions': {},
+        'properties': {},
+        'type': 'object',
+    }
+
+
+class GetCredentialsResponse(Response):
+    """
+    Response of auth.get_credentials endpoint.
+
+    :param credentials: List of credentials, each with an empty secret field.
+    :type credentials: Sequence[CredentialKey]
+    """
+    _service = "auth"
+    _action = "get_credentials"
+    _version = "2.8"
+
+    _schema = {
+        'definitions': {
+            'credential_key': {
+                'properties': {
+                    'access_key': {'description': '', 'type': ['string', 'null']},
+                    'last_used': {
+                        'description': '',
+                        'format': 'date-time',
+                        'type': ['string', 'null'],
+                    },
+                    'last_used_from': {
+                        'description': '',
+                        'type': ['string', 'null'],
+                    },
+                },
+                'type': 'object',
+            },
+        },
+        'properties': {
+            'credentials': {
+                'description': 'List of credentials, each with an empty secret field.',
+                'items': {'$ref': '#/definitions/credential_key'},
+                'type': ['array', 'null'],
+            },
+        },
+        'type': 'object',
+    }
+    def __init__(
+            self, credentials=None, **kwargs):
+        super(GetCredentialsResponse, self).__init__(**kwargs)
+        self.credentials = credentials
+
+    @schema_property('credentials')
+    def credentials(self):
+        return self._property_credentials
+
+    @credentials.setter
+    def credentials(self, value):
+        if value is None:
+            self._property_credentials = None
+            return
+
+        self.assert_isinstance(value, "credentials", (list, tuple))
+        if any(isinstance(v, dict) for v in value):
+            value = [CredentialKey.from_dict(v) if isinstance(v, dict) else v for v in value]
+        else:
+            self.assert_isinstance(value, "credentials", CredentialKey, is_array=True)
+        self._property_credentials = value
+
+
+
+
+class LoginRequest(Request):
+    """
+    Get a token based on supplied credentials (key/secret).
+            Intended for use by users with key/secret credentials that wish to obtain a token
+            for use with other services. Token will be limited by the same permissions that
+            exist for the credentials used in this call.
+
+    :param expiration_sec: Requested token expiration time in seconds. Not
+        guaranteed,  might be overridden by the service
+    :type expiration_sec: int
+    """
+
+    _service = "auth"
+    _action = "login"
+    _version = "2.8"
+    _schema = {
+        'definitions': {},
+        'properties': {
+            'expiration_sec': {
+                'description': 'Requested token expiration time in seconds. \n                        Not guaranteed,  might be overridden by the service',
+                'type': ['integer', 'null'],
+            },
+        },
+        'type': 'object',
+    }
+    def __init__(
+            self, expiration_sec=None, **kwargs):
+        super(LoginRequest, self).__init__(**kwargs)
+        self.expiration_sec = expiration_sec
+
+    @schema_property('expiration_sec')
+    def expiration_sec(self):
+        return self._property_expiration_sec
+
+    @expiration_sec.setter
+    def expiration_sec(self, value):
+        if value is None:
+            self._property_expiration_sec = None
+            return
+        if isinstance(value, float) and value.is_integer():
+            value = int(value)
+
+        self.assert_isinstance(value, "expiration_sec", six.integer_types)
+        self._property_expiration_sec = value
+
+
+class LoginResponse(Response):
+    """
+    Response of auth.login endpoint.
+
+    :param token: Token string
+    :type token: str
+    """
+    _service = "auth"
+    _action = "login"
+    _version = "2.8"
+
+    _schema = {
+        'definitions': {},
+        'properties': {
+            'token': {'description': 'Token string', 'type': ['string', 'null']},
+        },
+        'type': 'object',
+    }
+    def __init__(
+            self, token=None, **kwargs):
+        super(LoginResponse, self).__init__(**kwargs)
+        self.token = token
+
+    @schema_property('token')
+    def token(self):
+        return self._property_token
+
+    @token.setter
+    def token(self, value):
+        if value is None:
+            self._property_token = None
+            return
+
+        self.assert_isinstance(value, "token", six.string_types)
+        self._property_token = value
+
+
+
+
+class RevokeCredentialsRequest(Request):
+    """
+    Revokes (and deletes) a set (key, secret) of credentials for
+            the authenticated user.
+
+    :param access_key: Credentials key
+    :type access_key: str
+    """
+
+    _service = "auth"
+    _action = "revoke_credentials"
+    _version = "2.8"
+    _schema = {
+        'definitions': {},
+        'properties': {
+            'access_key': {
+                'description': 'Credentials key',
+                'type': ['string', 'null'],
+            },
+        },
+        'required': ['key_id'],
+        'type': 'object',
+    }
+    def __init__(
+            self, access_key=None, **kwargs):
+        super(RevokeCredentialsRequest, self).__init__(**kwargs)
+        self.access_key = access_key
+
+    @schema_property('access_key')
+    def access_key(self):
+        return self._property_access_key
+
+    @access_key.setter
+    def access_key(self, value):
+        if value is None:
+            self._property_access_key = None
+            return
+
+        self.assert_isinstance(value, "access_key", six.string_types)
+        self._property_access_key = value
+
+
+class RevokeCredentialsResponse(Response):
+    """
+    Response of auth.revoke_credentials endpoint.
+
+    :param revoked: Number of credentials revoked
+    :type revoked: int
+    """
+    _service = "auth"
+    _action = "revoke_credentials"
+    _version = "2.8"
+
+    _schema = {
+        'definitions': {},
+        'properties': {
+            'revoked': {
+                'description': 'Number of credentials revoked',
+                'enum': [0, 1],
+                'type': ['integer', 'null'],
+            },
+        },
+        'type': 'object',
+    }
+    def __init__(
+            self, revoked=None, **kwargs):
+        super(RevokeCredentialsResponse, self).__init__(**kwargs)
+        self.revoked = revoked
+
+    @schema_property('revoked')
+    def revoked(self):
+        return self._property_revoked
+
+    @revoked.setter
+    def revoked(self, value):
+        if value is None:
+            self._property_revoked = None
+            return
+        if isinstance(value, float) and value.is_integer():
+            value = int(value)
+
+        self.assert_isinstance(value, "revoked", six.integer_types)
+        self._property_revoked = value
+
+
+response_mapping = {
+    LoginRequest: LoginResponse,
+    CreateCredentialsRequest: CreateCredentialsResponse,
+    GetCredentialsRequest: GetCredentialsResponse,
+    RevokeCredentialsRequest: RevokeCredentialsResponse,
+    EditUserRequest: EditUserResponse,
+}

trains/backend_api/services/v2_8/organization.py

@@ -0,0 +1,154 @@
+"""
+organization service
+
+This service provides organization level operations
+"""
+import six
+import types
+from datetime import datetime
+import enum
+
+from dateutil.parser import parse as parse_datetime
+
+from ....backend_api.session import Request, Response, NonStrictDataModel, schema_property, StringEnum
+
+
+class GetTagsRequest(Request):
+    """
+    Get all the user and system tags used for the company tasks and models
+
+    :param include_system: If set to 'true' then the list of the system tags is
+        also returned. The default value is 'false'
+    :type include_system: bool
+    :param filter: Filter on entities to collect tags from
+    :type filter: dict
+    """
+
+    _service = "organization"
+    _action = "get_tags"
+    _version = "2.8"
+    _schema = {
+        'definitions': {},
+        'properties': {
+            'filter': {
+                'description': 'Filter on entities to collect tags from',
+                'properties': {
+                    'system_tags': {
+                        'description': "The list of system tag values to filter by. Use 'null' value to specify empty tags. Use '__Snot' value to specify that the following value should be excluded",
+                        'items': {'type': 'string'},
+                        'type': 'array',
+                    },
+                },
+                'type': ['object', 'null'],
+            },
+            'include_system': {
+                'default': False,
+                'description': "If set to 'true' then the list of the system tags is also returned. The default value is 'false'",
+                'type': ['boolean', 'null'],
+            },
+        },
+        'type': 'object',
+    }
+    def __init__(
+            self, include_system=False, filter=None, **kwargs):
+        super(GetTagsRequest, self).__init__(**kwargs)
+        self.include_system = include_system
+        self.filter = filter
+
+    @schema_property('include_system')
+    def include_system(self):
+        return self._property_include_system
+
+    @include_system.setter
+    def include_system(self, value):
+        if value is None:
+            self._property_include_system = None
+            return
+        
+        self.assert_isinstance(value, "include_system", (bool,))
+        self._property_include_system = value
+
+    @schema_property('filter')
+    def filter(self):
+        return self._property_filter
+
+    @filter.setter
+    def filter(self, value):
+        if value is None:
+            self._property_filter = None
+            return
+        
+        self.assert_isinstance(value, "filter", (dict,))
+        self._property_filter = value
+
+
+class GetTagsResponse(Response):
+    """
+    Response of organization.get_tags endpoint.
+
+    :param tags: The list of unique tag values
+    :type tags: Sequence[str]
+    :param system_tags: The list of unique system tag values. Returned only if
+        'include_system' is set to 'true' in the request
+    :type system_tags: Sequence[str]
+    """
+    _service = "organization"
+    _action = "get_tags"
+    _version = "2.8"
+
+    _schema = {
+        'definitions': {},
+        'properties': {
+            'system_tags': {
+                'description': "The list of unique system tag values. Returned only if 'include_system' is set to 'true' in the request",
+                'items': {'type': 'string'},
+                'type': ['array', 'null'],
+            },
+            'tags': {
+                'description': 'The list of unique tag values',
+                'items': {'type': 'string'},
+                'type': ['array', 'null'],
+            },
+        },
+        'type': 'object',
+    }
+    def __init__(
+            self, tags=None, system_tags=None, **kwargs):
+        super(GetTagsResponse, self).__init__(**kwargs)
+        self.tags = tags
+        self.system_tags = system_tags
+
+    @schema_property('tags')
+    def tags(self):
+        return self._property_tags
+
+    @tags.setter
+    def tags(self, value):
+        if value is None:
+            self._property_tags = None
+            return
+        
+        self.assert_isinstance(value, "tags", (list, tuple))
+        
+        self.assert_isinstance(value, "tags", six.string_types, is_array=True)
+        self._property_tags = value
+
+    @schema_property('system_tags')
+    def system_tags(self):
+        return self._property_system_tags
+
+    @system_tags.setter
+    def system_tags(self, value):
+        if value is None:
+            self._property_system_tags = None
+            return
+        
+        self.assert_isinstance(value, "system_tags", (list, tuple))
+        
+        self.assert_isinstance(value, "system_tags", six.string_types, is_array=True)
+        self._property_system_tags = value
+
+
+response_mapping = {
+    GetTagsRequest: GetTagsResponse,
+}

trains/backend_api/session/session.py

@@ -162,6 +162,7 @@ class Session(TokenManager):
             if not api_version:
                 api_version = '2.2' if token_dict.get('env', '') == 'prod' else Session.api_version
             if token_dict.get('server_version'):
+                if not any(True for c in Session._client if c[0] == 'trains-server'):
                     Session._client.append(('trains-server', token_dict.get('server_version'), ))
 
             Session.api_version = str(api_version)

trains/backend_interface/task/task.py

@@ -64,6 +64,15 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
 
         training = 'training'
         testing = 'testing'
+        inference = "inference"
+        data_processing = "data_processing"
+        application = "application"
+        monitor = "monitor"
+        controller = "controller"
+        optimizer = "optimizer"
+        service = "service"
+        qc = "qc"
+        custom = "custom"
 
     class TaskStatusEnum(Enum):
         def __str__(self):
@@ -292,6 +301,13 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
     def _auto_generate(self, project_name=None, task_name=None, task_type=TaskTypes.training):
         created_msg = make_message('Auto-generated at %(time)s by %(user)s@%(host)s')
 
+        if task_type.value not in (self.TaskTypes.training, self.TaskTypes.testing) and \
+                not Session.check_min_api_version('2.8'):
+            print('WARNING: Changing task type to "{}" : '
+                  'trains-server does not support task type "{}", '
+                  'please upgrade trains-server.'.format(self.TaskTypes.training, task_type.value))
+            task_type = self.TaskTypes.training
+
         project_id = None
         if project_name:
             project_id = get_or_create_project(self, project_name, created_msg)

trains/model.py

@@ -6,7 +6,7 @@ from tempfile import mkdtemp, mkstemp
 
 import pyparsing
 import six
-from typing import List, Dict, Union, Optional, TYPE_CHECKING
+from typing import List, Dict, Union, Optional, TYPE_CHECKING, Sequence
 
 from .backend_api import Session
 from .backend_api.services import models
@@ -1150,7 +1150,7 @@ class OutputModel(BaseModel):
 
     def update_weights_package(
         self,
-        weights_filenames=None,  # type: Optional[str]
+        weights_filenames=None,  # type: Optional[Sequence[str]]
         weights_path=None,  # type: Optional[str]
         upload_uri=None,  # type: Optional[str]
         target_filename=None,  # type: Optional[str]

trains/task.py

@@ -506,6 +506,15 @@ class Task(_Task):
 
             - ``TaskTypes.training`` (default)
             - ``TaskTypes.testing``
+            - ``TaskTypes.inference``
+            - ``TaskTypes.data_processing``
+            - ``TaskTypes.application``
+            - ``TaskTypes.monitor``
+            - ``TaskTypes.controller``
+            - ``TaskTypes.optimizer``
+            - ``TaskTypes.service``
+            - ``TaskTypes.qc``
+            - ``TaskTypes.custom``
 
         :type task_type: TaskTypeEnum(value)
         :return: A new experiment.
@@ -2350,6 +2359,14 @@ class Task(_Task):
             if project:
                 project_name = project.name
 
+        if task_data.get('type') and \
+                task_data.get('type') not in (cls.TaskTypes.training, cls.TaskTypes.testing) and \
+                not Session.check_min_api_version(2.8):
+            print('WARNING: Changing task type to "{}" : '
+                  'trains-server does not support task type "{}", '
+                  'please upgrade trains-server.'.format(cls.TaskTypes.training, task_data['type'].value))
+            task_data['type'] = cls.TaskTypes.training
+
         compares = (
             (task.name, 'name'),
             (project_name, 'project'),