Support deleting external artifacts when deleting projects

apiserver/apimodels/__init__.py

@@ -61,12 +61,6 @@ class ListField(fields.ListField):
             item.validate()
 
 
-# since there is no distinction between None and empty DictField
-# this value can be used as sentinel in order to distinguish
-# between not set and empty DictField
-DictFieldNotSet = {}
-
-
 class DictField(fields.BaseField):
     types = (dict,)
 

apiserver/bll/project/project_cleanup.py

@@ -8,6 +8,7 @@ from apiserver.bll.task.task_cleanup import (
     collect_debug_image_urls,
     collect_plot_image_urls,
     TaskUrls,
+    _schedule_for_delete,
 )
 from apiserver.config_repo import config
 from apiserver.database.model import EntityVisibility
@@ -58,13 +59,22 @@ def validate_project_delete(company: str, project_id: str):
 
 
 def delete_project(
-    company: str, project_id: str, force: bool, delete_contents: bool
+    company: str,
+    user: str,
+    project_id: str,
+    force: bool,
+    delete_contents: bool,
+    delete_external_artifacts=True,
 ) -> Tuple[DeleteProjectResult, Set[str]]:
     project = Project.get_for_writing(
         company=company, id=project_id, _only=("id", "path", "system_tags")
     )
     if not project:
         raise errors.bad_request.InvalidProjectId(id=project_id)
+
+    delete_external_artifacts = delete_external_artifacts and config.get(
+        "services.async_urls_delete.enabled", False
+    )
     is_pipeline = "pipeline" in (project.system_tags or [])
     project_ids = _ids_with_children([project_id])
     if not force:
@@ -95,6 +105,16 @@ def delete_project(
         deleted_tasks, event_urls, artifact_urls = _delete_tasks(
             company=company, projects=project_ids
         )
+        if delete_external_artifacts:
+            scheduled = _schedule_for_delete(
+                task_id=project_id,
+                company=company,
+                user=user,
+                urls=event_urls | model_urls | artifact_urls,
+                can_delete_folders=True,
+            )
+            for urls in (event_urls, model_urls, artifact_urls):
+                urls.difference_update(scheduled)
         res = DeleteProjectResult(
             deleted_tasks=deleted_tasks,
             deleted_models=deleted_models,

apiserver/jobs/async_urls_delete.py

@@ -90,13 +90,14 @@ def delete_fileserver_urls(urls_query: Q, fileserver_host: str):
     except Exception as ex:
         err = str(ex)
         log.warn(f"Error deleting {len(paths)} files from fileserver: {err}")
-        mark_failed(Q(id__in=list(ids_to_delete)), err)
+        mark_retry_failed(list(ids_to_delete), err)
         return
 
     res_data = res.json()
     deleted_ids = set(
         chain.from_iterable(
-            path_to_id_mapping.get(path, []) for path in list(res_data.get("deleted", {}))
+            path_to_id_mapping.get(path, [])
+            for path in list(res_data.get("deleted", {}))
         )
     )
     if deleted_ids:

apiserver/services/projects.py

@@ -273,6 +273,7 @@ def validate_delete(call: APICall, company_id: str, request: ProjectRequest):
 def delete(call: APICall, company_id: str, request: DeleteRequest):
     res, affected_projects = delete_project(
         company=company_id,
+        user=call.identity.user,
         project_id=request.project,
         force=request.force,
         delete_contents=request.delete_contents,