ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-14 07:38:25 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add support to python package namespace detection

Browse Source
This commit is contained in:
allegroai 2023-03-27 13:40:26 +03:00
parent 4cd8857c0d
commit fa92c75ffc
2 changed files with 26 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
clearml/utilities/pigar/__main__.py
View File

@ -30,6 +30,14 @@ class GenerateReqs(object):
def extract_reqs(self, module_callback=None, entry_point_filename=None): def extract_reqs(self, module_callback=None, entry_point_filename=None):
"""Extract requirements from project.""" """Extract requirements from project."""
def _internal_create_req(_version, _pkg_name, _name, _modules):
if _name not in _modules:
_modules.add(_name, _name, 0)
if not _version and _pkg_name and _pkg_name.startswith('-e '):
return ('{} @ {}'.format(_name, _pkg_name.replace('-e ', '', 1)), _version, _modules[_name])
else:
return (_pkg_name, _version, _modules[_name])
reqs = ReqsModules() reqs = ReqsModules()
guess = ReqsModules() guess = ReqsModules()
local = ReqsModules() local = ReqsModules()
@ -80,14 +88,11 @@ class GenerateReqs(object):
for name in candidates: for name in candidates:
logger.info('Checking module: %s', name) logger.info('Checking module: %s', name)
if name in self._installed_pkgs: if name in self._installed_pkgs:
pkg_name, version = self._installed_pkgs[name] pkginfo = self._installed_pkgs[name]
if name not in modules: for pkg, (pkg_name, version) in ({"": pkginfo} if not isinstance(pkginfo, dict) else pkginfo).items():
modules.add(name, name, 0) _name = pkg or name
if not version and pkg_name and pkg_name.startswith('-e '): reqs.add(*_internal_create_req(version, pkg_name, _name, modules))
reqs.add('{} @ {}'.format(name, pkg_name.replace('-e ', '', 1)), version, modules[name]) reqs_module_name.append(_name)
else:
reqs.add(pkg_name, version, modules[name])
reqs_module_name.append(name)
elif name in modules: elif name in modules:
guess.add(name, 0, modules[name]) guess.add(name, 0, modules[name])

14
clearml/utilities/pigar/reqs.py
View File

@ -320,6 +320,9 @@ def is_std_or_local_lib(name):
return True return True
return False return False
mpath = module_info.origin mpath = module_info.origin
# this is a subpackage
if not mpath and module_info.loader is not None:
return False
# this is std # this is std
if mpath == 'built-in': if mpath == 'built-in':
mpath = None mpath = None
@ -443,7 +446,16 @@ def _search_path(path):
continue continue
with open(top_level, 'r') as f: with open(top_level, 'r') as f:
for line in f: for line in f:
mapping[line.strip()] = (pkg_name, version) top_package = line.strip()
# NOTICE: this is a namespace package
if top_package and mapping_pkg_name.startswith("{}_".format(top_package)):
top = mapping.get(top_package, dict())
if not isinstance(top, dict):
top = {top_package: top}
top[mapping_pkg_name] = (pkg_name, version)
mapping[top_package] = top
else:
mapping[top_package] = (pkg_name, version)
# Install from local and available in GitHub. # Install from local and available in GitHub.
elif fnmatch.fnmatch(file, '*-link'): elif fnmatch.fnmatch(file, '*-link'):