Add Artifacts support, changed tags to system_tags and added user tags

Add hyper parameter sorting
Add min/max value for all time series metrics

server/service_repo/apicall.py

@@ -104,7 +104,7 @@ class DataContainer(object):
         if self._batched_data:
             try:
                 data_model = [cls(**item) for item in self._batched_data]
-            except TypeError as ex:
+            except (ValueError, TypeError) as ex:
                 raise CallParsingError(str(ex))
 
             for m in data_model:
@@ -112,7 +112,7 @@ class DataContainer(object):
         else:
             try:
                 data_model = cls(**self.data)
-            except TypeError as ex:
+            except (ValueError, TypeError) as ex:
                 raise CallParsingError(str(ex))
 
             if not self.schema_validator.enabled:
@@ -182,8 +182,6 @@ class APICallResult(DataContainer):
             traceback=self._traceback,
             extra=self._extra,
         )
-        if self.log_data:
-            res["data"] = self.data
         return res
 
     def copy_from(self, result):

server/service_repo/auth/auth.py

@@ -1,19 +1,19 @@
 import base64
+from datetime import datetime
+
 import jwt
 from mongoengine import Q
 
-from database.errors import translate_errors_context
-from database.model.company import Company
-from database.utils import get_options
-from database.model.auth import User, Entities, Credentials
 from apierrors import errors
 from config import config
+from database.errors import translate_errors_context
+from database.model.auth import User, Entities, Credentials
+from database.model.company import Company
+from database.utils import get_options
 from timing_context import TimingContext
-
-from .payload import Payload, Token, Basic, AuthType
-from .identity import Identity
 from .fixed_user import FixedUser
-
+from .identity import Identity
+from .payload import Payload, Token, Basic, AuthType
 
 log = config.logger(__file__)
 
@@ -38,12 +38,16 @@ def authorize_token(jwt_token, *_, **__):
         return Token.from_encoded_token(jwt_token)
 
     except jwt.exceptions.InvalidKeyError as ex:
-        raise errors.unauthorized.InvalidToken('jwt invalid key error', reason=ex.args[0])
+        raise errors.unauthorized.InvalidToken(
+            "jwt invalid key error", reason=ex.args[0]
+        )
     except jwt.InvalidTokenError as ex:
-        raise errors.unauthorized.InvalidToken('invalid jwt token', reason=ex.args[0])
+        raise errors.unauthorized.InvalidToken("invalid jwt token", reason=ex.args[0])
     except ValueError as ex:
-        log.exception('Failed while processing token: %s' % ex.args[0])
-        raise errors.unauthorized.InvalidToken('failed processing token', reason=ex.args[0])
+        log.exception("Failed while processing token: %s" % ex.args[0])
+        raise errors.unauthorized.InvalidToken(
+            "failed processing token", reason=ex.args[0]
+        )
 
 
 def authorize_credentials(auth_data, service, action, call_data_items):
@@ -67,9 +71,14 @@ def authorize_credentials(auth_data, service, action, call_data_items):
 
     with TimingContext("mongo", "user_by_cred"), translate_errors_context('authorizing request'):
         user = User.objects(query).first()
+        if not user:
+            raise errors.unauthorized.InvalidCredentials('failed to locate provided credentials')
 
-    if not user:
-        raise errors.unauthorized.InvalidCredentials('failed to locate provided credentials')
+        if not FixedUser.enabled():
+            # In case these are proper credentials, update last used time
+            User.objects(id=user.id, credentials__key=access_key).update(
+                **{"set__credentials__$__last_used": datetime.utcnow()}
+            )
 
     with TimingContext("mongo", "company_by_id"):
         company = Company.objects(id=user.company).only('id', 'name').first()
@@ -85,13 +94,13 @@ def authorize_credentials(auth_data, service, action, call_data_items):
     return basic
 
 
-def authorize_impersonation(user, identity, service, action, call_data_items):
+def authorize_impersonation(user, identity, service, action, call):
     """ Returns a new basic object (auth payload)"""
     if not user:
-        raise ValueError('missing user')
+        raise ValueError("missing user")
 
-    company = Company.objects(id=user.company).only('id', 'name').first()
+    company = Company.objects(id=user.company).only("id", "name").first()
     if not company:
-        raise errors.unauthorized.InvalidCredentials('invalid user company')
+        raise errors.unauthorized.InvalidCredentials("invalid user company")
 
     return Payload(auth_type=None, identity=identity)

server/service_repo/auth/utils.py

@@ -30,6 +30,8 @@ def get_secret_key(length=50):
     Create a random secret key.
 
     Taken from the Django project.
+    NOTE: asterisk is not supported due to issues with environment variables containing
+     asterisks (in case the secret key is stored in an environment variable)
     """
-    chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&*(-_=+)'
+    chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&(-_=+)'
     return get_random_string(length, chars)

server/service_repo/endpoint.py

@@ -76,7 +76,7 @@ class Endpoint(object):
         Provided endpoints and their schemas on a best-effort basis.
         """
         d = {
-            "min_version": self.min_version,
+            "min_version": str(self.min_version),
             "required_fields": self.required_fields,
             "request_data_model": None,
             "response_data_model": None,

server/service_repo/schema_validator.py

@@ -12,7 +12,7 @@ from config import config
 log = config.logger(__file__)
 
 
-@attr.s(auto_attribs=True, auto_exc=True)
+@attr.s(auto_attribs=True, cmp=False)
 class FastValidationError(Exception):
     error: fastjsonschema.JsonSchemaException
     data: dict

server/service_repo/service_repo.py

@@ -1,10 +1,10 @@
 import re
 from importlib import import_module
 from itertools import chain
-from typing import cast, Iterable, List, MutableMapping
+from pathlib import Path
+from typing import cast, Iterable, List, MutableMapping, Optional, Tuple
 
 import jsonmodels.models
-from pathlib import Path
 
 import timing_context
 from apierrors import APIError
@@ -30,7 +30,11 @@ class ServiceRepo(object):
     _version_required = config.get("apiserver.version.required")
     """ If version is required, parsing will fail for endpoint paths that do not contain a valid version """
 
-    _max_version = PartialVersion("2.1")
+    _check_max_version = config.get("apiserver.version.check_max_version")
+    """If the check is set, parsing will fail for endpoint request with the version that is grater than the current 
+    maximum """
+
+    _max_version = PartialVersion("2.3")
     """ Maximum version number (the highest min_version value across all endpoints) """
 
     _endpoint_exp = (
@@ -133,7 +137,7 @@ class ServiceRepo(object):
         return cls._max_version
 
     @classmethod
-    def _get_endpoint(cls, name, version):
+    def _get_endpoint(cls, name, version) -> Optional[Endpoint]:
         versions = cls._endpoints.get(name)
         if not versions:
             return None
@@ -144,7 +148,7 @@ class ServiceRepo(object):
             return None
 
     @classmethod
-    def _resolve_endpoint_from_call(cls, call):
+    def _resolve_endpoint_from_call(cls, call: APICall) -> Optional[Endpoint]:
         assert isinstance(call, APICall)
         endpoint = cls._get_endpoint(
             call.endpoint_name, call.requested_endpoint_version
@@ -167,7 +171,7 @@ class ServiceRepo(object):
         return endpoint
 
     @classmethod
-    def parse_endpoint_path(cls, path):
+    def parse_endpoint_path(cls, path: str) -> Tuple[PartialVersion, str]:
         """ Parse endpoint version, service and action from request path. """
         m = cls._endpoint_exp.match(path)
         if not m:
@@ -182,14 +186,14 @@ class ServiceRepo(object):
                 version = PartialVersion(version)
             except ValueError as e:
                 raise RequestPathHasInvalidVersion(version=version, reason=e)
-            if version > cls._max_version:
+            if cls._check_max_version and version > cls._max_version:
                 raise InvalidVersionError(
                     f"Invalid API version (max. supported version is {cls._max_version})"
                 )
         return version, endpoint_name
 
     @classmethod
-    def _should_return_stack(cls, code, subcode):
+    def _should_return_stack(cls, code: int, subcode: int) -> bool:
         if not cls._return_stack or code not in cls._return_stack_on_code:
             return False
         if subcode is None:
@@ -202,7 +206,7 @@ class ServiceRepo(object):
         return subcode in subcode_list
 
     @classmethod
-    def _validate_call(cls, call):
+    def _validate_call(cls, call: APICall) -> Optional[Endpoint]:
         endpoint = cls._resolve_endpoint_from_call(call)
         if call.failed:
             return
@@ -210,11 +214,13 @@ class ServiceRepo(object):
         return endpoint
 
     @classmethod
-    def validate_call(cls, call):
+    def validate_call(cls, call: APICall):
         cls._validate_call(call)
 
     @classmethod
-    def _get_company(cls, call, endpoint=None, ignore_error=False):
+    def _get_company(
+        cls, call: APICall, endpoint: Endpoint = None, ignore_error: bool = False
+    ) -> Optional[str]:
         authorize = endpoint and endpoint.authorize
         if ignore_error or not authorize:
             try:
@@ -224,7 +230,7 @@ class ServiceRepo(object):
         return call.identity.company
 
     @classmethod
-    def handle_call(cls, call):
+    def handle_call(cls, call: APICall):
         try:
             assert isinstance(call, APICall)
 

server/service_repo/validators.py

@@ -150,7 +150,7 @@ def validate_impersonation(endpoint, call):
                 ),
                 service=service,
                 action=action,
-                call_data_items=call.batched_data,
+                call=call,
             )
         else:
             return False