ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-26 18:16:07 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add sdk.development.detect_with_conda_freeze (default true) for full conda freeze (requires trains-agent >= 16.2)

Browse Source
This commit is contained in:
allegroai 2020-10-12 11:09:45 +03:00
parent aeb1a8e64b
commit c22eacd3fc
3 changed files with 72 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/trains.conf
View File

@ -166,6 +166,7 @@ sdk {
# If this flag is true (default is false), instead of analyzing the code with Pigar, analyze with `pip freeze`
detect_with_pip_freeze: false
detect_with_conda_freeze: false
# Log specific environment variables. OS environments are enlisted in the "Environment" section
# of the Hyper-Parameters.

92
trains/backend_interface/task/repo/freeze.py
View File

@ -1,33 +1,73 @@
import sys
import os
import json
from .util import get_command_output
def pip_freeze():
def pip_freeze(combine_conda_with_pip=False):
req_lines = []
local_packages = []
try:
req_lines = get_command_output([sys.executable, "-m", "pip", "freeze"]).splitlines()
# fix "package @ file://" from pip freeze to "package"
for i, r in enumerate(req_lines):
parts = r.split('@', 1)
if parts and len(parts) == 2 and parts[1].strip().lower().startswith('file://'):
req_lines[i] = parts[0]
local_packages.append((i, parts[0].strip()))
# if we found local packages, at least get their versions (using pip list)
if local_packages:
# noinspection PyBroadException
try:
list_lines = get_command_output(
[sys.executable, "-m", "pip", "list", "--format", "freeze"]).splitlines()
for index, name in local_packages:
line = [r for r in list_lines if r.strip().startswith(name+'==')]
if not line:
continue
line = line[0]
req_lines[index] = line.strip()
except Exception:
pass
except Exception as ex:
print('Failed calling "pip freeze": {}'.format(str(ex)))
return req_lines
conda_lines = []
conda_prefix = os.environ.get('CONDA_PREFIX')
if conda_prefix and not conda_prefix.endswith(os.path.sep):
conda_prefix += os.path.sep
if conda_prefix and sys.executable.startswith(conda_prefix):
pip_lines = get_command_output([sys.executable, "-m", "pip", "freeze"]).splitlines()
conda_packages_json = get_command_output(['conda', 'list', '--json'])
conda_packages_json = json.loads(conda_packages_json)
for r in conda_packages_json:
# check if this is a pypi package, if it is, leave it outside
if not r.get('channel') or r.get('channel') == 'pypi':
name = (r['name'].replace('-', '_'), r['name'])
pip_req_line = [l for l in pip_lines
if l.split('==', 1)[0].strip() in name or l.split('@', 1)[0].strip() in name]
if pip_req_line and \
('@' not in pip_req_line[0] or
not pip_req_line[0].split('@', 1)[1].strip().startswith('file://')):
req_lines.append(pip_req_line[0])
continue
req_lines.append('{}=={}'.format(name[0], r['version']) if r.get('version') else '{}'.format(name[0]))
continue
# check if we have it in our required packages
name = r['name']
# hack support pytorch/torch different naming convention
if name == 'pytorch':
name = 'torch'
# skip over packages with _
if name.startswith('_'):
continue
conda_lines.append('{}=={}'.format(name, r['version']) if r.get('version') else '{}'.format(name))
# make sure we see the conda packages, put them into the pip as well
if combine_conda_with_pip and conda_lines:
req_lines += ['', '# Conda Packages', ''] + conda_lines
else:
try:
req_lines = get_command_output([sys.executable, "-m", "pip", "freeze"]).splitlines()
# fix "package @ file://" from pip freeze to "package"
for i, r in enumerate(req_lines):
parts = r.split('@', 1)
if parts and len(parts) == 2 and parts[1].strip().lower().startswith('file://'):
req_lines[i] = parts[0]
local_packages.append((i, parts[0].strip()))
# if we found local packages, at least get their versions (using pip list)
if local_packages:
# noinspection PyBroadException
try:
list_lines = get_command_output(
[sys.executable, "-m", "pip", "list", "--format", "freeze"]).splitlines()
for index, name in local_packages:
line = [r for r in list_lines if r.strip().startswith(name+'==')]
if not line:
continue
line = line[0]
req_lines[index] = line.strip()
except Exception:
pass
except Exception as ex:
print('Failed calling "pip freeze": {}'.format(str(ex)))
return "\n".join(req_lines), "\n".join(conda_lines)

8
trains/backend_interface/task/task.py
View File

@ -309,10 +309,12 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
if result.script and script_requirements:
entry_point_filename = None if config.get('development.force_analyze_entire_repo', False) else \
os.path.join(result.script['working_dir'], entry_point)
if config.get('development.detect_with_pip_freeze', False):
conda_requirements = ""
if config.get('development.detect_with_pip_freeze', False) or \
config.get('development.detect_with_conda_freeze', False):
requirements, conda_requirements = pip_freeze(
config.get('development.detect_with_conda_freeze', False))
requirements = '# Python ' + sys.version.replace('\n', ' ').replace('\r', ' ') + '\n\n'\
+ "\n".join(pip_freeze())
+ requirements
else:
requirements, conda_requirements = script_requirements.get_requirements(
entry_point_filename=entry_point_filename)