ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-04-28 10:11:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Ignore virtual-environment folder that might be inside the project's directory

Browse Source
This commit is contained in:
allegroai 2020-04-01 19:02:54 +03:00
parent 6de3d4b6fd
commit 01772430d6
2 changed files with 20 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
trains/backend_interface/task/repo/scriptinfo.py
View File

@ -32,7 +32,8 @@ class ScriptRequirements(object):
from ....utilities.pigar.__main__ import GenerateReqs from ....utilities.pigar.__main__ import GenerateReqs
installed_pkgs = get_installed_pkgs_detail() installed_pkgs = get_installed_pkgs_detail()
gr = GenerateReqs(save_path='', project_path=self._root_folder, installed_pkgs=installed_pkgs, gr = GenerateReqs(save_path='', project_path=self._root_folder, installed_pkgs=installed_pkgs,
ignores=['.git', '.hg', '.idea', '__pycache__', '.ipynb_checkpoints']) ignores=['.git', '.hg', '.idea', '__pycache__', '.ipynb_checkpoints',
'site-packages', 'dist-packages'])
reqs, try_imports, guess, local_pks = gr.extract_reqs(module_callback=ScriptRequirements.add_trains_used_packages) reqs, try_imports, guess, local_pks = gr.extract_reqs(module_callback=ScriptRequirements.add_trains_used_packages)
return self.create_requirements_txt(reqs, local_pks) return self.create_requirements_txt(reqs, local_pks)
except Exception: except Exception:

27
trains/utilities/pigar/reqs.py
View File

@ -29,13 +29,14 @@ def project_import_modules(project_path, ignores):
modules = ImportedModules() modules = ImportedModules()
try_imports = set() try_imports = set()
local_mods = list() local_mods = list()
ignore_paths = collections.defaultdict(set) ignore_paths = set()
venv_subdirs = set(('bin', 'etc', 'include', 'lib', 'lib64', 'share'))
ignore_absolute = []
if not ignores: if not ignores:
ignore_paths[project_path].add('.git') ignore_paths.add('.git')
else: else:
for path in ignores: for path in ignores:
parent_dir = os.path.dirname(path) ignore_paths.add(Path(path).name)
ignore_paths[parent_dir].add(os.path.basename(path))
if os.path.isfile(project_path): if os.path.isfile(project_path):
fake_path = Path(project_path).name fake_path = Path(project_path).name
@ -44,11 +45,19 @@ def project_import_modules(project_path, ignores):
modules |= fmodules modules |= fmodules
try_imports |= try_ipts try_imports |= try_ipts
else: else:
cur_dir = project_path # os.getcwd() cur_dir = project_path
for dirpath, dirnames, files in os.walk(project_path, followlinks=True): for dirpath, dirnames, files in os.walk(os.path.abspath(project_path), followlinks=True):
if dirpath in ignore_paths: # see if we have a parent folder in the ignore list
dirnames[:] = [d for d in dirnames if ignore_paths & set(Path(dirpath).relative_to(cur_dir).parts):
if d not in ignore_paths[dirpath]] continue
# check if we are in a subfolder of ignore list
if any(True for prefix in ignore_absolute if dirpath.startswith(prefix)):
continue
# Hack detect if this is a virtual-env folder, if so add it to the uignore list
if set(dirnames) == venv_subdirs:
ignore_absolute.append(Path(dirpath).as_posix()+os.sep)
continue
py_files = list() py_files = list()
for fn in files: for fn in files:
# C extension. # C extension.