Fix git diff larger than 500kb , git section will contain warning, and entire git diff will be uploaded as artifact named auxiliary_git_diff

trains/backend_interface/task/repo/scriptinfo.py

@@ -592,6 +592,7 @@ class ScriptInfo(object):
         script_dir = scripts_dir[0]
         script_path = scripts_path[0]
         messages = []
+        auxiliary_git_diff = None
 
         if not plugin:
             log.info("No repository found, storing script code instead")
@@ -625,7 +626,10 @@ class ScriptInfo(object):
                 messages.append(
                     "======> WARNING! Git diff to large to store "
                     "({}kb), skipping uncommitted changes <======".format(len(diff)//1024))
-                diff = ''
+                auxiliary_git_diff = diff
+                diff = '# WARNING! git diff too large to store, clear this section to execute without it.\n' \
+                       '# full git diff available in Artifacts/auxiliary_git_diff\n' \
+                       '# Clear the section before enqueueing Task!\n'
 
         else:
             diff = ''
@@ -665,7 +669,7 @@ class ScriptInfo(object):
         if not any(script_info.values()):
             script_info = None
 
-        return (ScriptInfoResult(script=script_info, warning_messages=messages),
+        return (ScriptInfoResult(script=script_info, warning_messages=messages, auxiliary_git_diff=auxiliary_git_diff),
                 script_requirements)
 
     @classmethod
@@ -724,6 +728,7 @@ class ScriptInfo(object):
 class ScriptInfoResult(object):
     script = attr.ib(default=None)
     warning_messages = attr.ib(factory=list)
+    auxiliary_git_diff = attr.ib(default=None)
 
 
 class _JupyterHistoryLogger(object):

trains/backend_interface/task/task.py

@@ -21,6 +21,7 @@ from collections import OrderedDict
 from six.moves.urllib.parse import quote
 
 from ...utilities.locks import RLock as FileRLock
+from ...binding.artifacts import Artifacts
 from ...backend_interface.task.development.worker import DevWorker
 from ...backend_api import Session
 from ...backend_api.services import tasks, models, events, projects
@@ -154,6 +155,7 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
             log_to_backend = False
         self._log_to_backend = log_to_backend
         self._setup_log(default_log_to_backend=log_to_backend)
+        self._artifacts_manager = Artifacts(self)
 
     def _setup_log(self, default_log_to_backend=None, replace_existing=False):
         """
@@ -264,6 +266,11 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
             for msg in result.warning_messages:
                 self.get_logger().report_text(msg)
 
+            # if the git is too large to store on the task, we must store it as artifact:
+            if result.auxiliary_git_diff:
+                self._artifacts_manager.upload_artifact(
+                    name='auxiliary_git_diff', artifact_object=result.auxiliary_git_diff)
+
             # store original entry point
             entry_point = result.script.get('entry_point') if result.script else None