diff --git a/apiserver/apimodels/__init__.py b/apiserver/apimodels/__init__.py index 50dcb48..153263f 100644 --- a/apiserver/apimodels/__init__.py +++ b/apiserver/apimodels/__init__.py @@ -61,12 +61,6 @@ class ListField(fields.ListField): item.validate() -# since there is no distinction between None and empty DictField -# this value can be used as sentinel in order to distinguish -# between not set and empty DictField -DictFieldNotSet = {} - - class DictField(fields.BaseField): types = (dict,) diff --git a/apiserver/bll/project/project_cleanup.py b/apiserver/bll/project/project_cleanup.py index 9d36194..ce1e87b 100644 --- a/apiserver/bll/project/project_cleanup.py +++ b/apiserver/bll/project/project_cleanup.py @@ -8,6 +8,7 @@ from apiserver.bll.task.task_cleanup import ( collect_debug_image_urls, collect_plot_image_urls, TaskUrls, + _schedule_for_delete, ) from apiserver.config_repo import config from apiserver.database.model import EntityVisibility @@ -58,13 +59,22 @@ def validate_project_delete(company: str, project_id: str): def delete_project( - company: str, project_id: str, force: bool, delete_contents: bool + company: str, + user: str, + project_id: str, + force: bool, + delete_contents: bool, + delete_external_artifacts=True, ) -> Tuple[DeleteProjectResult, Set[str]]: project = Project.get_for_writing( company=company, id=project_id, _only=("id", "path", "system_tags") ) if not project: raise errors.bad_request.InvalidProjectId(id=project_id) + + delete_external_artifacts = delete_external_artifacts and config.get( + "services.async_urls_delete.enabled", False + ) is_pipeline = "pipeline" in (project.system_tags or []) project_ids = _ids_with_children([project_id]) if not force: @@ -95,6 +105,16 @@ def delete_project( deleted_tasks, event_urls, artifact_urls = _delete_tasks( company=company, projects=project_ids ) + if delete_external_artifacts: + scheduled = _schedule_for_delete( + task_id=project_id, + company=company, + user=user, + urls=event_urls | model_urls | artifact_urls, + can_delete_folders=True, + ) + for urls in (event_urls, model_urls, artifact_urls): + urls.difference_update(scheduled) res = DeleteProjectResult( deleted_tasks=deleted_tasks, deleted_models=deleted_models, diff --git a/apiserver/jobs/async_urls_delete.py b/apiserver/jobs/async_urls_delete.py index 1bcddb7..ef547d9 100644 --- a/apiserver/jobs/async_urls_delete.py +++ b/apiserver/jobs/async_urls_delete.py @@ -90,13 +90,14 @@ def delete_fileserver_urls(urls_query: Q, fileserver_host: str): except Exception as ex: err = str(ex) log.warn(f"Error deleting {len(paths)} files from fileserver: {err}") - mark_failed(Q(id__in=list(ids_to_delete)), err) + mark_retry_failed(list(ids_to_delete), err) return res_data = res.json() deleted_ids = set( chain.from_iterable( - path_to_id_mapping.get(path, []) for path in list(res_data.get("deleted", {})) + path_to_id_mapping.get(path, []) + for path in list(res_data.get("deleted", {})) ) ) if deleted_ids: diff --git a/apiserver/services/projects.py b/apiserver/services/projects.py index a03b138..5e7bbd1 100644 --- a/apiserver/services/projects.py +++ b/apiserver/services/projects.py @@ -273,6 +273,7 @@ def validate_delete(call: APICall, company_id: str, request: ProjectRequest): def delete(call: APICall, company_id: str, request: DeleteRequest): res, affected_projects = delete_project( company=company_id, + user=call.identity.user, project_id=request.project, force=request.force, delete_contents=request.delete_contents,