ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-02-07 21:33:25 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Removed automagic reuse Model objects

Browse Source
This commit is contained in:
allegroai 2020-06-19 00:46:43 +03:00
parent 71003e02b2
commit 7ab93e7dba
1 changed files with 38 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
trains/binding/frameworks/__init__.py
View File

@ -50,8 +50,8 @@ class _Empty(object):
class WeightsFileHandler(object): class WeightsFileHandler(object):
_model_out_store_lookup = {} # _model_out_store_lookup = {}
_model_in_store_lookup = {} # _model_in_store_lookup = {}
_model_store_lookup_lock = threading.Lock() _model_store_lookup_lock = threading.Lock()
_model_pre_callbacks = {} _model_pre_callbacks = {}
_model_post_callbacks = {} _model_post_callbacks = {}
@ -185,13 +185,15 @@ class WeightsFileHandler(object):
if model_info.model: if model_info.model:
trains_in_model = model_info.model trains_in_model = model_info.model
else: else:
trains_in_model, ref_model = WeightsFileHandler._model_in_store_lookup.get( # # disable model reuse, let Model module try to find it for use
id(model) if model is not None else None, (None, None)) trains_in_model, ref_model = None, None
# noinspection PyCallingNonCallable # trains_in_model, ref_model = WeightsFileHandler._model_in_store_lookup.get(
if ref_model is not None and model != ref_model(): # id(model) if model is not None else None, (None, None))
# old id pop it - it was probably reused because the object is dead # # noinspection PyCallingNonCallable
WeightsFileHandler._model_in_store_lookup.pop(id(model)) # if ref_model is not None and model != ref_model():
trains_in_model, ref_model = None, None # # old id pop it - it was probably reused because the object is dead
# WeightsFileHandler._model_in_store_lookup.pop(id(model))
# trains_in_model, ref_model = None, None
# check if object already has InputModel # check if object already has InputModel
model_name_id = getattr(model, 'name', '') if model else '' model_name_id = getattr(model, 'name', '') if model else ''
@ -241,13 +243,15 @@ class WeightsFileHandler(object):
pass pass
trains_in_model = model_info.model trains_in_model = model_info.model
if model is not None: # # disable model reuse, let Model module try to find it for use
# noinspection PyBroadException # if model is not None:
try: # # noinspection PyBroadException
ref_model = weakref.ref(model) # try:
except Exception: # ref_model = weakref.ref(model)
ref_model = None # except Exception:
WeightsFileHandler._model_in_store_lookup[id(model)] = (trains_in_model, ref_model) # ref_model = None
# WeightsFileHandler._model_in_store_lookup[id(model)] = (trains_in_model, ref_model)
# todo: support multiple models for the same task # todo: support multiple models for the same task
task.connect(trains_in_model) task.connect(trains_in_model)
# if we are running remotely we should deserialize the object # if we are running remotely we should deserialize the object
@ -285,15 +289,17 @@ class WeightsFileHandler(object):
try: try:
WeightsFileHandler._model_store_lookup_lock.acquire() WeightsFileHandler._model_store_lookup_lock.acquire()
# # disable model reuse, let Model module try to find it for use
trains_out_model, ref_model = None, None
# check if object already has InputModel # check if object already has InputModel
trains_out_model, ref_model = WeightsFileHandler._model_out_store_lookup.get( # trains_out_model, ref_model = WeightsFileHandler._model_out_store_lookup.get(
id(model) if model is not None else None, (None, None)) # id(model) if model is not None else None, (None, None))
# notice ref_model() is not an error/typo this is a weakref object call # # notice ref_model() is not an error/typo this is a weakref object call
# noinspection PyCallingNonCallable # # noinspection PyCallingNonCallable
if ref_model is not None and model != ref_model(): # if ref_model is not None and model != ref_model():
# old id pop it - it was probably reused because the object is dead # # old id pop it - it was probably reused because the object is dead
WeightsFileHandler._model_out_store_lookup.pop(id(model)) # WeightsFileHandler._model_out_store_lookup.pop(id(model))
trains_out_model, ref_model = None, None # trains_out_model, ref_model = None, None
model_info = WeightsFileHandler.ModelInfo( model_info = WeightsFileHandler.ModelInfo(
model=trains_out_model, upload_filename=None, local_model_path=saved_path, model=trains_out_model, upload_filename=None, local_model_path=saved_path,
@ -372,13 +378,14 @@ class WeightsFileHandler(object):
framework=framework, framework=framework,
base_model_id=in_model_id base_model_id=in_model_id
) )
if model is not None: # # disable model reuse, let Model module try to find it for use
# noinspection PyBroadException # if model is not None:
try: # # noinspection PyBroadException
ref_model = weakref.ref(model) # try:
except Exception: # ref_model = weakref.ref(model)
ref_model = None # except Exception:
WeightsFileHandler._model_out_store_lookup[id(model)] = (trains_out_model, ref_model) # ref_model = None
# WeightsFileHandler._model_out_store_lookup[id(model)] = (trains_out_model, ref_model)
model_info.model = trains_out_model model_info.model = trains_out_model
# call post model callback functions # call post model callback functions