clearml-agent/examples/archive_experiments.py

#!/usr/bin/python3
"""
An example script that cleans up failed experiments by moving them to the archive
"""
import argparse
from datetime import datetime

from trains_agent import APIClient

parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--project", "-P", help="Project ID. Only clean up experiments from this project")
parser.add_argument("--user", "-U", help="User ID. Only clean up experiments assigned to this user")
parser.add_argument(
    "--status", "-S", default="failed",
    help="Experiment status. Only clean up experiments with this status (default %(default)s)"
)
parser.add_argument(
    "--iterations", "-I", type=int,
    help="Number of iterations. Only clean up experiments with less or equal number of iterations"
)
parser.add_argument(
    "--sec-from-start", "-T", type=int,
    help="Seconds from start time. "
         "Only clean up experiments if less or equal number of seconds have elapsed since started"
)

args = parser.parse_args()

client = APIClient()

tasks = client.tasks.get_all(
    project=[args.project] if args.project else None,
    user=[args.user] if args.user else None,
    status=[args.status] if args.status else None,
    system_tags=["-archived"]
)

count = 0

for task in tasks:
    if args.iterations and (task.last_iteration or 0) > args.iterations:
        continue
    if args.sec_from_start:
        if not task.started:
            continue
        if (datetime.utcnow() - task.started.replace(tzinfo=None)).total_seconds() > args.sec_from_start:
            continue

    try:
        client.tasks.edit(
            task=task.id,
            system_tags=(task.system_tags or []) + ["archived"],
            force=True
        )
        count += 1
    except Exception as ex:
        print("Failed editing experiment: {}".format(ex))

print("Cleaned up {} experiments".format(count))