ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-26 18:16:07 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Fix populate Task called from jupyter notebook. Added packages=True to auto populate based on locally installed packages

Browse Source
This commit is contained in:
allegroai 2021-01-24 09:11:09 +02:00
parent fef2ddb43a
commit 1988cf8adf
3 changed files with 28 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
clearml/backend_interface/task/populate.py
View File

@ -23,7 +23,7 @@ class CreateAndPopulate(object):
commit=None, # Optional[str] commit=None, # Optional[str]
script=None, # Optional[str] script=None, # Optional[str]
working_directory=None, # Optional[str] working_directory=None, # Optional[str]
packages=None, # Optional[Sequence[str]] packages=None, # Optional[Union[bool, Sequence[str]]]
requirements_file=None, # Optional[Union[str, Path]] requirements_file=None, # Optional[Union[str, Path]]
docker=None, # Optional[str] docker=None, # Optional[str]
base_task_id=None, # Optional[str] base_task_id=None, # Optional[str]
@ -53,6 +53,8 @@ class CreateAndPopulate(object):
:param working_directory: Working directory to launch the script from. Default: repository root folder. :param working_directory: Working directory to launch the script from. Default: repository root folder.
Relative to repo root or local folder. Relative to repo root or local folder.
:param packages: Manually specify a list of required packages. Example: ["tqdm>=2.1", "scikit-learn"] :param packages: Manually specify a list of required packages. Example: ["tqdm>=2.1", "scikit-learn"]
or `True` to automatically create requirements
based on locally installed packages (repository must be local).
:param requirements_file: Specify requirements.txt file to install when setting the session. :param requirements_file: Specify requirements.txt file to install when setting the session.
If not provided, the requirements.txt from the repository will be used. If not provided, the requirements.txt from the repository will be used.
:param docker: Select the docker image to be executed in by the remote session :param docker: Select the docker image to be executed in by the remote session
@ -88,8 +90,9 @@ class CreateAndPopulate(object):
self.repo = repo self.repo = repo
self.script = script self.script = script
self.cwd = working_directory self.cwd = working_directory
assert not packages or isinstance(packages, (tuple, list)) assert not packages or isinstance(packages, (tuple, list, bool))
self.packages = list(packages) if packages else None self.packages = list(packages) if packages is not None and not isinstance(packages, bool) \
else (packages or None)
self.requirements_file = Path(requirements_file) if requirements_file else None self.requirements_file = Path(requirements_file) if requirements_file else None
self.base_task_id = base_task_id self.base_task_id = base_task_id
self.docker = docker self.docker = docker
@ -126,7 +129,8 @@ class CreateAndPopulate(object):
repo_info, requirements = ScriptInfo.get( repo_info, requirements = ScriptInfo.get(
filepaths=[entry_point], filepaths=[entry_point],
log=getLogger(), log=getLogger(),
create_requirements=False, uncommitted_from_remote=True) create_requirements=self.packages is True, uncommitted_from_remote=True,
detect_jupyter_notebook=False)
# check if we have no repository and no requirements raise error # check if we have no repository and no requirements raise error
if self.raise_on_missing_entries and (not self.requirements_file and not self.packages) \ if self.raise_on_missing_entries and (not self.requirements_file and not self.packages) \
@ -163,7 +167,7 @@ class CreateAndPopulate(object):
task_state['script']['working_dir'] = repo_info.script['working_dir'] task_state['script']['working_dir'] = repo_info.script['working_dir']
task_state['script']['entry_point'] = repo_info.script['entry_point'] task_state['script']['entry_point'] = repo_info.script['entry_point']
task_state['script']['binary'] = repo_info.script['binary'] task_state['script']['binary'] = repo_info.script['binary']
task_state['script']['requirements'] = {} task_state['script']['requirements'] = repo_info.script.get('requirements') or {}
if self.cwd: if self.cwd:
self.cwd = self.cwd self.cwd = self.cwd
cwd = self.cwd if Path(self.cwd).is_dir() else ( cwd = self.cwd if Path(self.cwd).is_dir() else (
@ -199,7 +203,7 @@ class CreateAndPopulate(object):
if self.requirements_file: if self.requirements_file:
with open(self.requirements_file.as_posix(), 'rt') as f: with open(self.requirements_file.as_posix(), 'rt') as f:
reqs = [line.strip() for line in f.readlines()] reqs = [line.strip() for line in f.readlines()]
if self.packages: if self.packages and self.packages is not True:
reqs += self.packages reqs += self.packages
if reqs: if reqs:
# make sure we have clearml. # make sure we have clearml.
@ -214,7 +218,7 @@ class CreateAndPopulate(object):
if not clearml_found: if not clearml_found:
reqs.append('clearml') reqs.append('clearml')
task_state['script']['requirements'] = {'pip': '\n'.join(reqs)} task_state['script']['requirements'] = {'pip': '\n'.join(reqs)}
elif not self.repo and repo_info: elif not self.repo and repo_info and not repo_info.script.get('requirements'):
# we are in local mode, make sure we have "requirements.txt" it is a must # we are in local mode, make sure we have "requirements.txt" it is a must
reqs_txt_file = Path(repo_info.script['repo_root']) / "requirements.txt" reqs_txt_file = Path(repo_info.script['repo_root']) / "requirements.txt"
if self.raise_on_missing_entries and not reqs_txt_file.is_file(): if self.raise_on_missing_entries and not reqs_txt_file.is_file():

21
clearml/backend_interface/task/repo/scriptinfo.py
View File

@ -643,9 +643,10 @@ class ScriptInfo(object):
return '' return ''
@classmethod @classmethod
def _get_script_info(cls, filepaths, check_uncommitted=True, create_requirements=True, log=None, def _get_script_info(
uncommitted_from_remote=False): cls, filepaths, check_uncommitted=True, create_requirements=True, log=None,
jupyter_filepath = cls._get_jupyter_notebook_filename() uncommitted_from_remote=False, detect_jupyter_notebook=True):
jupyter_filepath = cls._get_jupyter_notebook_filename() if detect_jupyter_notebook else None
if jupyter_filepath: if jupyter_filepath:
scripts_path = [Path(os.path.normpath(jupyter_filepath)).absolute()] scripts_path = [Path(os.path.normpath(jupyter_filepath)).absolute()]
else: else:
@ -756,14 +757,20 @@ class ScriptInfo(object):
@classmethod @classmethod
def get(cls, filepaths=None, check_uncommitted=True, create_requirements=True, log=None, def get(cls, filepaths=None, check_uncommitted=True, create_requirements=True, log=None,
uncommitted_from_remote=False): uncommitted_from_remote=False, detect_jupyter_notebook=True):
try: try:
if not filepaths: if not filepaths:
filepaths = [sys.argv[0], ] filepaths = [sys.argv[0], ]
return cls._get_script_info( return cls._get_script_info(
filepaths=filepaths, check_uncommitted=check_uncommitted, filepaths=filepaths,
create_requirements=create_requirements, log=log, uncommitted_from_remote=uncommitted_from_remote) check_uncommitted=check_uncommitted,
except Exception as ex: create_requirements=create_requirements, log=log,
uncommitted_from_remote=uncommitted_from_remote,
detect_jupyter_notebook=detect_jupyter_notebook,
)
except SystemExit:
pass
except BaseException as ex:
if log: if log:
log.warning("Failed auto-detecting task repository: {}".format(ex)) log.warning("Failed auto-detecting task repository: {}".format(ex))
return ScriptInfoResult(), None return ScriptInfoResult(), None

4
clearml/task.py
View File

@ -599,7 +599,7 @@ class Task(_Task):
commit=None, # Optional[str] commit=None, # Optional[str]
script=None, # Optional[str] script=None, # Optional[str]
working_directory=None, # Optional[str] working_directory=None, # Optional[str]
packages=None, # Optional[Sequence[str]] packages=None, # Optional[Union[bool, Sequence[str]]]
requirements_file=None, # Optional[Union[str, Path]] requirements_file=None, # Optional[Union[str, Path]]
docker=None, # Optional[str] docker=None, # Optional[str]
argparse_args=None, # Optional[Sequence[Tuple[str, str]]] argparse_args=None, # Optional[Sequence[Tuple[str, str]]]
@ -633,6 +633,8 @@ class Task(_Task):
:param working_directory: Working directory to launch the script from. Default: repository root folder. :param working_directory: Working directory to launch the script from. Default: repository root folder.
Relative to repo root or local folder. Relative to repo root or local folder.
:param packages: Manually specify a list of required packages. Example: ["tqdm>=2.1", "scikit-learn"] :param packages: Manually specify a list of required packages. Example: ["tqdm>=2.1", "scikit-learn"]
or `True` to automatically create requirements
based on locally installed packages (repository must be local).
:param requirements_file: Specify requirements.txt file to install when setting the session. :param requirements_file: Specify requirements.txt file to install when setting the session.
If not provided, the requirements.txt from the repository will be used. If not provided, the requirements.txt from the repository will be used.
:param docker: Select the docker image to be executed in by the remote session :param docker: Select the docker image to be executed in by the remote session