ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-03 10:42:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add object-storage support in cleanup-service

Browse Source
This commit is contained in:
allegroai 2021-05-12 15:39:34 +03:00
parent f46561629f
commit 0e9538d42a
1 changed files with 8 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
examples/services/cleanup/cleanup_service.py
View File

@ -14,10 +14,7 @@ Requirements:
""" """
import logging import logging
import os
from datetime import datetime from datetime import datetime
from glob import glob
from shutil import rmtree
from time import sleep, time from time import sleep, time
from clearml.backend_api.session.client import APIClient from clearml.backend_api.session.client import APIClient
@ -33,12 +30,6 @@ task = Task.init(
reuse_last_task_id=False, reuse_last_task_id=False,
) )
# set the base docker including the mount point for the file server data data
file_server_mount = "/opt/clearml/data/fileserver/"
task.set_base_docker(
"ubuntu:18.04 -v /opt/clearml/data/fileserver/:{}".format(file_server_mount)
)
# args for the running task # args for the running task
args = { args = {
"delete_threshold_days": 30.0, "delete_threshold_days": 30.0,
@ -80,23 +71,19 @@ while True:
) )
page += 1 page += 1
if tasks:
print("Deleting {} tasks".format(len(tasks)))
# delete and cleanup tasks # delete and cleanup tasks
for task in tasks: for task in tasks:
# noinspection PyBroadException # noinspection PyBroadException
try: try:
# try delete task frm system deleted_task = Task.get_task(task_id=task.id)
client.tasks.delete(task=task.id, force=args["force_delete"]) deleted_task.delete(
# if we succeeded, delete the task output content delete_artifacts_and_models=True,
task_folders = glob( skip_models_used_by_other_tasks=True,
os.path.join(file_server_mount, "*/*.{}/".format(task.id)) raise_on_error=False
) )
for folder in task_folders:
print("Deleting Task id={} data folder {}".format(task.id, folder))
# noinspection PyBroadException
try:
rmtree(folder)
except Exception:
logging.warning("Failed removing folder {}".format(folder))
except Exception as ex: except Exception as ex:
logging.warning( logging.warning(
"Could not delete Task ID={}, {}".format( "Could not delete Task ID={}, {}".format(