Better multi-processing, switching to to multiprocess Lock

This commit is contained in:
allegroai
2019-10-04 01:34:22 +03:00
parent 9e930a6894
commit 9f73dbabdb
5 changed files with 7 additions and 6 deletions

View File

@@ -1,6 +1,6 @@
import abc
import time
from threading import Lock
from multiprocessing import Lock
import attr
import numpy as np

View File

@@ -1,6 +1,6 @@
from functools import partial
from multiprocessing.pool import ThreadPool
from threading import Lock
from multiprocessing import Lock
from time import time
from humanfriendly import format_timespan

View File

@@ -386,12 +386,12 @@ class Model(IdObjectBase, AsyncManagerMixin, _StorageUriMixin):
if not uri or not uri.strip():
return None
helper = StorageHelper.get(uri, logger=self._log, verbose=True)
helper = StorageHelper.get(uri)
filename = uri.split('/')[-1]
ext = '.'.join(filename.split('.')[1:])
fd, local_filename = mkstemp(suffix='.'+ext)
os.close(fd)
local_download = helper.download_to_file(uri, local_path=local_filename, overwrite_existing=True)
local_download = helper.download_to_file(uri, local_path=local_filename, overwrite_existing=True, verbose=True)
# save local model, so we can later query what was the original one
Model._local_model_to_id_uri[str(local_download)] = (self.model_id, uri)
return local_download