ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-05-10 15:40:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Black formatting

Browse Source
This commit is contained in:
allegroai 2024-07-15 13:39:00 +03:00
parent d75564d514
commit cbb90a8bd5
1 changed files with 143 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

222
clearml/cli/task/__main__.py
View File

@ -12,75 +12,139 @@ clearml.backend_api.session.Session.add_client("clearml-task", __version__)
def setup_parser(parser): def setup_parser(parser):
parser.add_argument('--version', action='store_true', default=None, parser.add_argument("--version", action="store_true", default=None, help="Display the clearml-task utility version")
help='Display the clearml-task utility version') parser.add_argument(
parser.add_argument('--project', type=str, default=None, "--project",
help='Required: set the project name for the task. ' type=str,
'If --base-task-id is used, this arguments is optional.') default=None,
parser.add_argument('--name', type=str, default=None, help="Required: set the project name for the task. " "If --base-task-id is used, this arguments is optional.",
help='Required: select a name for the remote task') )
parser.add_argument('--tags', default=None, nargs='*', parser.add_argument("--name", type=str, default=None, help="Required: select a name for the remote task")
help='Optional: add tags to the newly created Task. ' parser.add_argument(
'Example: --tags "base" "job"') "--tags",
parser.add_argument('--repo', type=str, default=None, default=None,
help='remote URL for the repository to use. ' nargs="*",
'Example: --repo https://github.com/allegroai/clearml.git') help="Optional: add tags to the newly created Task. " 'Example: --tags "base" "job"',
parser.add_argument('--branch', type=str, default=None, )
help='Select specific repository branch/tag (implies the latest commit from the branch)') parser.add_argument(
parser.add_argument('--commit', type=str, default=None, "--repo",
help='Select specific commit id to use (default: latest commit, ' type=str,
'or when used with local repository matching the local commit id)') default=None,
parser.add_argument('--folder', type=str, default=None, help="remote URL for the repository to use. " "Example: --repo https://github.com/allegroai/clearml.git",
help='Remotely execute the code in the local folder. ' )
'Notice! It assumes a git repository already exists. ' parser.add_argument(
'Current state of the repo (commit id and uncommitted changes) is logged ' "--branch",
'and will be replicated on the remote machine') type=str,
parser.add_argument('--script', type=str, default=None, default=None,
help='Specify the entry point script for the remote execution. ' help="Select specific repository branch/tag (implies the latest commit from the branch)",
'Currently support .py .ipynb and .sh scripts (python, jupyter notebook, bash) ' )
'When used in tandem with --repo the script should be a relative path inside ' parser.add_argument(
'the repository, for example: --script source/train.py ' "--commit",
'When used with --folder it supports a direct path to a file inside the local ' type=str,
'repository itself, for example: --script ~/project/source/train.py') default=None,
parser.add_argument('--module', type=str, default=None, help="Select specific commit id to use (default: latest commit, "
help='Instead of a script entry point, specify a python module to be remotely executed. ' "or when used with local repository matching the local commit id)",
'Notice: It cannot be used with --script at the same time. ' )
'for example: --module "torch.distributed.launch train_script.py"') parser.add_argument(
parser.add_argument('--cwd', type=str, default=None, "--folder",
help='Working directory to launch the script from. Default: repository root folder. ' type=str,
'Relative to repo root or local folder') default=None,
parser.add_argument('--args', default=None, nargs='*', help="Remotely execute the code in the local folder. "
help='Arguments to pass to the remote execution, list of <argument>=<value> strings.' "Notice! It assumes a git repository already exists. "
'Currently only argparse arguments are supported. ' "Current state of the repo (commit id and uncommitted changes) is logged "
'Example: --args lr=0.003 batch_size=64') "and will be replicated on the remote machine",
parser.add_argument('--queue', type=str, default=None, )
help='Select the queue to launch the task. ' parser.add_argument(
'If not provided a Task will be created but it will not be launched.') "--script",
parser.add_argument('--requirements', type=str, default=None, type=str,
help='Specify requirements.txt file to install when setting the session. ' default=None,
'If not provided, the requirements.txt from the repository will be used.') help="Specify the entry point script for the remote execution. "
parser.add_argument('--packages', default=None, nargs='*', "Currently support .py .ipynb and .sh scripts (python, jupyter notebook, bash) "
help='Manually specify a list of required packages. ' "When used in tandem with --repo the script should be a relative path inside "
'Example: --packages "tqdm>=2.1" "scikit-learn"') "the repository, for example: --script source/train.py "
parser.add_argument('--docker', type=str, default=None, "When used with --folder it supports a direct path to a file inside the local "
help='Select the docker image to use in the remote session') "repository itself, for example: --script ~/project/source/train.py",
parser.add_argument('--docker_args', type=str, default=None, )
help='Add docker arguments, pass a single string') parser.add_argument(
parser.add_argument('--docker_bash_setup_script', type=str, default=None, "--module",
help="Add bash script to be executed inside the docker before setting up " type=str,
"the Task's environment") default=None,
parser.add_argument('--output-uri', type=str, default=None, required=False, help="Instead of a script entry point, specify a python module to be remotely executed. "
help='Optional: set the Task `output_uri` (automatically upload model destination)') "Notice: It cannot be used with --script at the same time. "
parser.add_argument('--task-type', type=str, default=None, 'for example: --module "torch.distributed.launch train_script.py"',
help='Set the Task type, optional values: ' )
'training, testing, inference, data_processing, application, monitor, ' parser.add_argument(
'controller, optimizer, service, qc, custom') "--cwd",
parser.add_argument('--skip-task-init', action='store_true', default=None, type=str,
help='If set, Task.init() call is not added to the entry point, and is assumed ' default=None,
'to be called in within the script. Default: add Task.init() call entry point script') help="Working directory to launch the script from. Default: repository root folder. "
parser.add_argument('--base-task-id', type=str, default=None, "Relative to repo root or local folder",
help='Use a pre-existing task in the system, instead of a local repo/script. ' )
'Essentially clones an existing task and overrides arguments/requirements.') parser.add_argument(
"--args",
default=None,
nargs="*",
help="Arguments to pass to the remote execution, list of <argument>=<value> strings."
"Currently only argparse arguments are supported. "
"Example: --args lr=0.003 batch_size=64",
)
parser.add_argument(
"--queue",
type=str,
default=None,
help="Select the queue to launch the task. "
"If not provided a Task will be created but it will not be launched.",
)
parser.add_argument(
"--requirements",
type=str,
default=None,
help="Specify requirements.txt file to install when setting the session. "
"If not provided, the requirements.txt from the repository will be used.",
)
parser.add_argument(
"--packages",
default=None,
nargs="*",
help="Manually specify a list of required packages. " 'Example: --packages "tqdm>=2.1" "scikit-learn"',
)
parser.add_argument("--docker", type=str, default=None, help="Select the docker image to use in the remote session")
parser.add_argument("--docker_args", type=str, default=None, help="Add docker arguments, pass a single string")
parser.add_argument(
"--docker_bash_setup_script",
type=str,
default=None,
help="Add bash script to be executed inside the docker before setting up " "the Task's environment",
)
parser.add_argument(
"--output-uri",
type=str,
default=None,
required=False,
help="Optional: set the Task `output_uri` (automatically upload model destination)",
)
parser.add_argument(
"--task-type",
type=str,
default=None,
help="Set the Task type, optional values: "
"training, testing, inference, data_processing, application, monitor, "
"controller, optimizer, service, qc, custom",
)
parser.add_argument(
"--skip-task-init",
action="store_true",
default=None,
help="If set, Task.init() call is not added to the entry point, and is assumed "
"to be called in within the script. Default: add Task.init() call entry point script",
)
parser.add_argument(
"--base-task-id",
type=str,
default=None,
help="Use a pre-existing task in the system, instead of a local repo/script. "
"Essentially clones an existing task and overrides arguments/requirements.",
)
parser.add_argument( parser.add_argument(
"--import-offline-session", "--import-offline-session",
type=str, type=str,
@ -90,7 +154,7 @@ def setup_parser(parser):
def cli(): def cli():
title = 'ClearML launch - launch any codebase on remote machine running clearml-agent' title = "ClearML launch - launch any codebase on remote machine running clearml-agent"
print(title) print(title)
parser = ArgumentParser(description=title) parser = ArgumentParser(description=title)
setup_parser(parser) setup_parser(parser)
@ -103,7 +167,7 @@ def cli():
exit(0) exit(0)
if args.version: if args.version:
print('Version {}'.format(__version__)) print("Version {}".format(__version__))
exit(0) exit(0)
if not args.name and not args.import_offline_session: if not args.name and not args.import_offline_session:
@ -145,7 +209,7 @@ def cli():
) )
# verify args before creating the Task # verify args before creating the Task
create_populate.update_task_args(args.args) create_populate.update_task_args(args.args)
print('Creating new task') print("Creating new task")
create_populate.create_task() create_populate.create_task()
# update Task args # update Task args
create_populate.update_task_args(args.args) create_populate.update_task_args(args.args)
@ -156,25 +220,25 @@ def cli():
# noinspection PyProtectedMember # noinspection PyProtectedMember
create_populate.task._set_runtime_properties({"_CLEARML_TASK": True}) create_populate.task._set_runtime_properties({"_CLEARML_TASK": True})
print('New task created id={}'.format(create_populate.get_id())) print("New task created id={}".format(create_populate.get_id()))
if not args.queue: if not args.queue:
print('Warning: No queue was provided, leaving task in draft-mode.') print("Warning: No queue was provided, leaving task in draft-mode.")
exit(0) exit(0)
Task.enqueue(create_populate.task, queue_name=args.queue) Task.enqueue(create_populate.task, queue_name=args.queue)
print('Task id={} sent for execution on queue {}'.format(create_populate.get_id(), args.queue)) print("Task id={} sent for execution on queue {}".format(create_populate.get_id(), args.queue))
print('Execution log at: {}'.format(create_populate.task.get_output_log_web_page())) print("Execution log at: {}".format(create_populate.task.get_output_log_web_page()))
def main(): def main():
try: try:
cli() cli()
except KeyboardInterrupt: except KeyboardInterrupt:
print('\nUser aborted') print("\nUser aborted")
except Exception as ex: except Exception as ex:
print('\nError: {}'.format(ex)) print("\nError: {}".format(ex))
exit(1) exit(1)
if __name__ == '__main__': if __name__ == "__main__":
main() main()