ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-03 10:42:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add object-storage support in cleanup-service

Browse Source
This commit is contained in:
allegroai 2021-05-12 15:39:34 +03:00
parent f46561629f
commit 0e9538d42a
1 changed files with 8 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
examples/services/cleanup/cleanup_service.py
View File

@ -14,10 +14,7 @@ Requirements:
"""
import logging
import os
from datetime import datetime
from glob import glob
from shutil import rmtree
from time import sleep, time
from clearml.backend_api.session.client import APIClient
@ -33,12 +30,6 @@ task = Task.init(
reuse_last_task_id=False,
)
# set the base docker including the mount point for the file server data data
file_server_mount = "/opt/clearml/data/fileserver/"
task.set_base_docker(
"ubuntu:18.04 -v /opt/clearml/data/fileserver/:{}".format(file_server_mount)
)
# args for the running task
args = {
"delete_threshold_days": 30.0,
@ -80,23 +71,19 @@ while True:
)
page += 1
if tasks:
print("Deleting {} tasks".format(len(tasks)))
# delete and cleanup tasks
for task in tasks:
# noinspection PyBroadException
try:
# try delete task frm system
client.tasks.delete(task=task.id, force=args["force_delete"])
# if we succeeded, delete the task output content
task_folders = glob(
os.path.join(file_server_mount, "*/*.{}/".format(task.id))
deleted_task = Task.get_task(task_id=task.id)
deleted_task.delete(
delete_artifacts_and_models=True,
skip_models_used_by_other_tasks=True,
raise_on_error=False
)
for folder in task_folders:
print("Deleting Task id={} data folder {}".format(task.id, folder))
# noinspection PyBroadException
try:
rmtree(folder)
except Exception:
logging.warning("Failed removing folder {}".format(folder))
except Exception as ex:
logging.warning(
"Could not delete Task ID={}, {}".format(