ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-03 10:42:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix support for auto package detection for standalone scripts

Browse Source
This commit is contained in:
allegroai 2019-11-15 21:59:50 +02:00
parent d19fde7041
commit 15683b5b43
1 changed files with 33 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
trains/backend_interface/task/repo/scriptinfo.py
View File

@ -76,7 +76,6 @@ class ScriptRequirements(object):
modules = ImportedModules() modules = ImportedModules()
try_imports = set() try_imports = set()
local_mods = list() local_mods = list()
cur_dir = project_path # os.getcwd()
ignore_paths = collections.defaultdict(set) ignore_paths = collections.defaultdict(set)
if not ignores: if not ignores:
ignore_paths[project_path].add('.git') ignore_paths[project_path].add('.git')
@ -85,28 +84,36 @@ class ScriptRequirements(object):
parent_dir = os.path.dirname(path) parent_dir = os.path.dirname(path)
ignore_paths[parent_dir].add(os.path.basename(path)) ignore_paths[parent_dir].add(os.path.basename(path))
for dirpath, dirnames, files in os.walk(project_path, followlinks=True): if os.path.isfile(project_path):
if dirpath in ignore_paths: fake_path = Path(project_path).name
dirnames[:] = [d for d in dirnames with open(project_path, 'rb') as f:
if d not in ignore_paths[dirpath]] fmodules, try_ipts = file_import_modules(fake_path, f.read())
py_files = list() modules |= fmodules
for fn in files: try_imports |= try_ipts
# C extension. else:
if fn.endswith('.so'): cur_dir = project_path # os.getcwd()
local_mods.append(fn[:-3]) for dirpath, dirnames, files in os.walk(project_path, followlinks=True):
# Normal Python file. if dirpath in ignore_paths:
if fn.endswith('.py'): dirnames[:] = [d for d in dirnames
local_mods.append(fn[:-3]) if d not in ignore_paths[dirpath]]
py_files.append(fn) py_files = list()
if '__init__.py' in files: for fn in files:
local_mods.append(os.path.basename(dirpath)) # C extension.
for file in py_files: if fn.endswith('.so'):
fpath = os.path.join(dirpath, file) local_mods.append(fn[:-3])
fake_path = fpath.split(cur_dir)[1][1:] # Normal Python file.
with open(fpath, 'rb') as f: if fn.endswith('.py'):
fmodules, try_ipts = file_import_modules(fake_path, f.read()) local_mods.append(fn[:-3])
modules |= fmodules py_files.append(fn)
try_imports |= try_ipts if '__init__.py' in files:
local_mods.append(os.path.basename(dirpath))
for file in py_files:
fpath = os.path.join(dirpath, file)
fake_path = fpath.split(cur_dir)[1][1:]
with open(fpath, 'rb') as f:
fmodules, try_ipts = file_import_modules(fake_path, f.read())
modules |= fmodules
try_imports |= try_ipts
# hack: forcefully insert storage modules if we have them # hack: forcefully insert storage modules if we have them
# noinspection PyBroadException # noinspection PyBroadException
@ -452,12 +459,13 @@ class ScriptInfo(object):
if not plugin or not repo_info.commit else repo_info.diff if not plugin or not repo_info.commit else repo_info.diff
else: else:
diff = '' diff = ''
# if this is not jupyter, get the requirements.txt # if this is not jupyter, get the requirements.txt
requirements = '' requirements = ''
# create requirements if backend supports requirements # create requirements if backend supports requirements
# if jupyter is present, requirements will be created in the background, when saving a snapshot # if jupyter is present, requirements will be created in the background, when saving a snapshot
if not jupyter_filepath and Session.check_min_api_version('2.2'): if not jupyter_filepath and Session.check_min_api_version('2.2'):
script_requirements = ScriptRequirements(Path(repo_root).as_posix()) script_requirements = ScriptRequirements(
Path(repo_root).as_posix() if repo_info.url else script_path.as_posix())
if create_requirements: if create_requirements:
requirements = script_requirements.get_requirements() requirements = script_requirements.get_requirements()
else: else: