Allow reporting a pre-uploaded image url in Logger.report_image using the url parameter

trains/backend_interface/metrics/events.py

@@ -156,8 +156,8 @@ class PlotEvent(MetricsEventAdapter):
 class ImageEventNoUpload(MetricsEventAdapter):
 
     def __init__(self, metric, variant, src, iter=0, **kwargs):
+        self._url = src
         parts = urlparse(src)
-        self._url = urlunparse((parts.scheme, parts.netloc, '', '', '', ''))
         self._key = urlunparse(('', '', parts.path, parts.params, parts.query, parts.fragment))
         super(ImageEventNoUpload, self).__init__(metric, variant, iter=iter, **kwargs)
 

trains/backend_interface/util.py

@@ -79,15 +79,15 @@ def get_single_result(entity, query, results, log=None, show_results=10, raise_o
     return results[0]
 
 
-def at_least_one(_exception_cls=Exception, **kwargs):
-    actual = [k for k, v in kwargs.items() if v]
+def at_least_one(_exception_cls=Exception, _check_none=False, **kwargs):
+    actual = [k for k, v in kwargs.items() if (v is not None if _check_none else v)]
     if len(actual) < 1:
         raise _exception_cls('At least one of (%s) is required' % ', '.join(kwargs.keys()))
 
 
-def mutually_exclusive(_exception_cls=Exception, _require_at_least_one=True, **kwargs):
+def mutually_exclusive(_exception_cls=Exception, _require_at_least_one=True, _check_none=False, **kwargs):
     """ Helper for checking mutually exclusive options """
-    actual = [k for k, v in kwargs.items() if v]
+    actual = [k for k, v in kwargs.items() if (v is not None if _check_none else v)]
     if _require_at_least_one:
         at_least_one(_exception_cls=_exception_cls, **kwargs)
     if len(actual) > 1:
@@ -106,4 +106,3 @@ def validate_dict(obj, key_types, value_types, desc=''):
 def exact_match_regex(name):
     """ Convert string to a regex representing an exact match """
     return '^%s$' % re.escape(name)
-

trains/logger.py

@@ -5,15 +5,17 @@ import numpy as np
 from PIL import Image
 from pathlib2 import Path
 
+from .backend_api.services import tasks
 from .backend_interface.logger import StdStreamPatch, LogFlusher
-from .debugging.log import LoggerRoot
+from .backend_interface.task import Task as _Task
 from .backend_interface.task.development.worker import DevWorker
 from .backend_interface.task.log import TaskHandler
+from .backend_interface.util import mutually_exclusive
+from .config import running_remotely, get_cache_dir
+from .debugging.log import LoggerRoot
+from .errors import UsageError
 from .storage import StorageHelper
 from .utilities.plotly_reporter import SeriesInfo
-from .backend_api.services import tasks
-from .backend_interface.task import Task as _Task
-from .config import running_remotely, get_cache_dir
 
 # Make sure that DeprecationWarning within this package always gets printed
 warnings.filterwarnings('always', category=DeprecationWarning, module=__name__)
@@ -353,17 +355,22 @@ class Logger(object):
         )
 
     def report_image(self, title, series, iteration, local_path=None, image=None, matrix=None, max_image_history=None,
-                     delete_after_upload=False):
+                     delete_after_upload=False, url=None):
         """
         Report an image and upload its contents.
 
         Image is uploaded to a preconfigured bucket (see setup_upload()) with a key (filename)
         describing the task ID, title, series and iteration.
 
+        .. note::
+            :paramref:`~.Logger.report_image.local_path`, :paramref:`~.Logger.report_image.url`, :paramref:`~.Logger.report_image.image` and :paramref:`~.Logger.report_image.matrix`
+            are mutually exclusive, and at least one must be provided.
+
         :param str title: Title (AKA metric)
         :param str series: Series (AKA variant)
         :param int iteration: Iteration number
-        :param str local_path: A path to an image file. Required unless matrix is provided.
+        :param str local_path: A path to an image file.
+        :param str url: A URL to the location of a pre-uploaded image.
         :param np.ndarray or PIL.Image.Image image: Could be a PIL.Image.Image object or a 3D numpy.ndarray
                 object containing image data (RGB).
         :param np.ndarray matrix: A 3D numpy.ndarray object containing image data (RGB).
@@ -372,10 +379,12 @@ class Logger(object):
             use negative value for unlimited. default is set in global configuration (default=5)
         :param bool delete_after_upload: if True, one the file was uploaded the local copy will be deleted
         """
+        mutually_exclusive(
+            UsageError, _check_none=True,
+            local_path=local_path or None, url=url or None, image=image, matrix=matrix
+        )
         if matrix is not None:
             warnings.warn("'matrix' variable is deprecated; use 'image' instead.", DeprecationWarning)
-        if len([x for x in (matrix, image, local_path) if x is not None]) != 1:
-            raise ValueError('Expected only one of [image, matrix, local_path]')
         if image is None:
             image = matrix
         if image is not None and not isinstance(image, (np.ndarray, Image.Image)):
@@ -383,29 +392,40 @@ class Logger(object):
 
         # if task was not started, we have to start it
         self._start_task_if_needed()
-        upload_uri = self.get_default_upload_destination()
-        if not upload_uri:
-            upload_uri = Path(get_cache_dir()) / 'debug_images'
-            upload_uri.mkdir(parents=True, exist_ok=True)
-            # Verify that we can upload to this destination
-            upload_uri = str(upload_uri)
-            storage = StorageHelper.get(upload_uri)
-            upload_uri = storage.verify_upload(folder_uri=upload_uri)
+
         self._touch_title_series(title, series)
 
-        if isinstance(image, Image.Image):
-            image = np.array(image)
+        if url:
+            self._task.reporter.report_image(
+                title=title,
+                series=series,
+                src=url,
+                iter=iteration,
+            )
 
-        self._task.reporter.report_image_and_upload(
-            title=title,
-            series=series,
-            path=local_path,
-            image=image,
-            iter=iteration,
-            upload_uri=upload_uri,
-            max_image_history=max_image_history,
-            delete_after_upload=delete_after_upload,
-        )
+        else:
+            upload_uri = self.get_default_upload_destination()
+            if not upload_uri:
+                upload_uri = Path(get_cache_dir()) / 'debug_images'
+                upload_uri.mkdir(parents=True, exist_ok=True)
+                # Verify that we can upload to this destination
+                upload_uri = str(upload_uri)
+                storage = StorageHelper.get(upload_uri)
+                upload_uri = storage.verify_upload(folder_uri=upload_uri)
+
+            if isinstance(image, Image.Image):
+                image = np.array(image)
+
+            self._task.reporter.report_image_and_upload(
+                title=title,
+                series=series,
+                path=local_path,
+                image=image,
+                iter=iteration,
+                upload_uri=upload_uri,
+                max_image_history=max_image_history,
+                delete_after_upload=delete_after_upload,
+            )
 
     def set_default_upload_destination(self, uri):
         """

trains/utilities/check_updates.py

@@ -336,6 +336,11 @@ class CheckPackageUpdates(object):
 
     @staticmethod
     def get_version_from_updates_server(cur_version):
+        """
+        Get the latest version for trains from updates server
+        :param cur_version: The current running version of trains
+        :type cur_version: Version
+        """
         try:
             _ = requests.get('https://updates.trains.allegro.ai/updates',
                              data=json.dumps({"versions": {"trains": str(cur_version)}}),