diff --git a/clearml/cli/task/__main__.py b/clearml/cli/task/__main__.py index 05a98b42..10553d82 100644 --- a/clearml/cli/task/__main__.py +++ b/clearml/cli/task/__main__.py @@ -12,75 +12,139 @@ clearml.backend_api.session.Session.add_client("clearml-task", __version__) def setup_parser(parser): - parser.add_argument('--version', action='store_true', default=None, - help='Display the clearml-task utility version') - parser.add_argument('--project', type=str, default=None, - help='Required: set the project name for the task. ' - 'If --base-task-id is used, this arguments is optional.') - parser.add_argument('--name', type=str, default=None, - help='Required: select a name for the remote task') - parser.add_argument('--tags', default=None, nargs='*', - help='Optional: add tags to the newly created Task. ' - 'Example: --tags "base" "job"') - parser.add_argument('--repo', type=str, default=None, - help='remote URL for the repository to use. ' - 'Example: --repo https://github.com/allegroai/clearml.git') - parser.add_argument('--branch', type=str, default=None, - help='Select specific repository branch/tag (implies the latest commit from the branch)') - parser.add_argument('--commit', type=str, default=None, - help='Select specific commit id to use (default: latest commit, ' - 'or when used with local repository matching the local commit id)') - parser.add_argument('--folder', type=str, default=None, - help='Remotely execute the code in the local folder. ' - 'Notice! It assumes a git repository already exists. ' - 'Current state of the repo (commit id and uncommitted changes) is logged ' - 'and will be replicated on the remote machine') - parser.add_argument('--script', type=str, default=None, - help='Specify the entry point script for the remote execution. ' - 'Currently support .py .ipynb and .sh scripts (python, jupyter notebook, bash) ' - 'When used in tandem with --repo the script should be a relative path inside ' - 'the repository, for example: --script source/train.py ' - 'When used with --folder it supports a direct path to a file inside the local ' - 'repository itself, for example: --script ~/project/source/train.py') - parser.add_argument('--module', type=str, default=None, - help='Instead of a script entry point, specify a python module to be remotely executed. ' - 'Notice: It cannot be used with --script at the same time. ' - 'for example: --module "torch.distributed.launch train_script.py"') - parser.add_argument('--cwd', type=str, default=None, - help='Working directory to launch the script from. Default: repository root folder. ' - 'Relative to repo root or local folder') - parser.add_argument('--args', default=None, nargs='*', - help='Arguments to pass to the remote execution, list of = strings.' - 'Currently only argparse arguments are supported. ' - 'Example: --args lr=0.003 batch_size=64') - parser.add_argument('--queue', type=str, default=None, - help='Select the queue to launch the task. ' - 'If not provided a Task will be created but it will not be launched.') - parser.add_argument('--requirements', type=str, default=None, - help='Specify requirements.txt file to install when setting the session. ' - 'If not provided, the requirements.txt from the repository will be used.') - parser.add_argument('--packages', default=None, nargs='*', - help='Manually specify a list of required packages. ' - 'Example: --packages "tqdm>=2.1" "scikit-learn"') - parser.add_argument('--docker', type=str, default=None, - help='Select the docker image to use in the remote session') - parser.add_argument('--docker_args', type=str, default=None, - help='Add docker arguments, pass a single string') - parser.add_argument('--docker_bash_setup_script', type=str, default=None, - help="Add bash script to be executed inside the docker before setting up " - "the Task's environment") - parser.add_argument('--output-uri', type=str, default=None, required=False, - help='Optional: set the Task `output_uri` (automatically upload model destination)') - parser.add_argument('--task-type', type=str, default=None, - help='Set the Task type, optional values: ' - 'training, testing, inference, data_processing, application, monitor, ' - 'controller, optimizer, service, qc, custom') - parser.add_argument('--skip-task-init', action='store_true', default=None, - help='If set, Task.init() call is not added to the entry point, and is assumed ' - 'to be called in within the script. Default: add Task.init() call entry point script') - parser.add_argument('--base-task-id', type=str, default=None, - help='Use a pre-existing task in the system, instead of a local repo/script. ' - 'Essentially clones an existing task and overrides arguments/requirements.') + parser.add_argument("--version", action="store_true", default=None, help="Display the clearml-task utility version") + parser.add_argument( + "--project", + type=str, + default=None, + help="Required: set the project name for the task. " "If --base-task-id is used, this arguments is optional.", + ) + parser.add_argument("--name", type=str, default=None, help="Required: select a name for the remote task") + parser.add_argument( + "--tags", + default=None, + nargs="*", + help="Optional: add tags to the newly created Task. " 'Example: --tags "base" "job"', + ) + parser.add_argument( + "--repo", + type=str, + default=None, + help="remote URL for the repository to use. " "Example: --repo https://github.com/allegroai/clearml.git", + ) + parser.add_argument( + "--branch", + type=str, + default=None, + help="Select specific repository branch/tag (implies the latest commit from the branch)", + ) + parser.add_argument( + "--commit", + type=str, + default=None, + help="Select specific commit id to use (default: latest commit, " + "or when used with local repository matching the local commit id)", + ) + parser.add_argument( + "--folder", + type=str, + default=None, + help="Remotely execute the code in the local folder. " + "Notice! It assumes a git repository already exists. " + "Current state of the repo (commit id and uncommitted changes) is logged " + "and will be replicated on the remote machine", + ) + parser.add_argument( + "--script", + type=str, + default=None, + help="Specify the entry point script for the remote execution. " + "Currently support .py .ipynb and .sh scripts (python, jupyter notebook, bash) " + "When used in tandem with --repo the script should be a relative path inside " + "the repository, for example: --script source/train.py " + "When used with --folder it supports a direct path to a file inside the local " + "repository itself, for example: --script ~/project/source/train.py", + ) + parser.add_argument( + "--module", + type=str, + default=None, + help="Instead of a script entry point, specify a python module to be remotely executed. " + "Notice: It cannot be used with --script at the same time. " + 'for example: --module "torch.distributed.launch train_script.py"', + ) + parser.add_argument( + "--cwd", + type=str, + default=None, + help="Working directory to launch the script from. Default: repository root folder. " + "Relative to repo root or local folder", + ) + parser.add_argument( + "--args", + default=None, + nargs="*", + help="Arguments to pass to the remote execution, list of = strings." + "Currently only argparse arguments are supported. " + "Example: --args lr=0.003 batch_size=64", + ) + parser.add_argument( + "--queue", + type=str, + default=None, + help="Select the queue to launch the task. " + "If not provided a Task will be created but it will not be launched.", + ) + parser.add_argument( + "--requirements", + type=str, + default=None, + help="Specify requirements.txt file to install when setting the session. " + "If not provided, the requirements.txt from the repository will be used.", + ) + parser.add_argument( + "--packages", + default=None, + nargs="*", + help="Manually specify a list of required packages. " 'Example: --packages "tqdm>=2.1" "scikit-learn"', + ) + parser.add_argument("--docker", type=str, default=None, help="Select the docker image to use in the remote session") + parser.add_argument("--docker_args", type=str, default=None, help="Add docker arguments, pass a single string") + parser.add_argument( + "--docker_bash_setup_script", + type=str, + default=None, + help="Add bash script to be executed inside the docker before setting up " "the Task's environment", + ) + parser.add_argument( + "--output-uri", + type=str, + default=None, + required=False, + help="Optional: set the Task `output_uri` (automatically upload model destination)", + ) + parser.add_argument( + "--task-type", + type=str, + default=None, + help="Set the Task type, optional values: " + "training, testing, inference, data_processing, application, monitor, " + "controller, optimizer, service, qc, custom", + ) + parser.add_argument( + "--skip-task-init", + action="store_true", + default=None, + help="If set, Task.init() call is not added to the entry point, and is assumed " + "to be called in within the script. Default: add Task.init() call entry point script", + ) + parser.add_argument( + "--base-task-id", + type=str, + default=None, + help="Use a pre-existing task in the system, instead of a local repo/script. " + "Essentially clones an existing task and overrides arguments/requirements.", + ) parser.add_argument( "--import-offline-session", type=str, @@ -90,7 +154,7 @@ def setup_parser(parser): def cli(): - title = 'ClearML launch - launch any codebase on remote machine running clearml-agent' + title = "ClearML launch - launch any codebase on remote machine running clearml-agent" print(title) parser = ArgumentParser(description=title) setup_parser(parser) @@ -103,7 +167,7 @@ def cli(): exit(0) if args.version: - print('Version {}'.format(__version__)) + print("Version {}".format(__version__)) exit(0) if not args.name and not args.import_offline_session: @@ -145,7 +209,7 @@ def cli(): ) # verify args before creating the Task create_populate.update_task_args(args.args) - print('Creating new task') + print("Creating new task") create_populate.create_task() # update Task args create_populate.update_task_args(args.args) @@ -156,25 +220,25 @@ def cli(): # noinspection PyProtectedMember create_populate.task._set_runtime_properties({"_CLEARML_TASK": True}) - print('New task created id={}'.format(create_populate.get_id())) + print("New task created id={}".format(create_populate.get_id())) if not args.queue: - print('Warning: No queue was provided, leaving task in draft-mode.') + print("Warning: No queue was provided, leaving task in draft-mode.") exit(0) Task.enqueue(create_populate.task, queue_name=args.queue) - print('Task id={} sent for execution on queue {}'.format(create_populate.get_id(), args.queue)) - print('Execution log at: {}'.format(create_populate.task.get_output_log_web_page())) + print("Task id={} sent for execution on queue {}".format(create_populate.get_id(), args.queue)) + print("Execution log at: {}".format(create_populate.task.get_output_log_web_page())) def main(): try: cli() except KeyboardInterrupt: - print('\nUser aborted') + print("\nUser aborted") except Exception as ex: - print('\nError: {}'.format(ex)) + print("\nError: {}".format(ex)) exit(1) -if __name__ == '__main__': +if __name__ == "__main__": main()