Better multi-processing, switching to to multiprocess Lock

This commit is contained in:
allegroai 2019-10-04 01:34:22 +03:00
parent 9e930a6894
commit 9f73dbabdb
5 changed files with 7 additions and 6 deletions

View File

@ -1,6 +1,6 @@
import abc
import time
from threading import Lock
from multiprocessing import Lock
import attr
import numpy as np

View File

@ -1,6 +1,6 @@
from functools import partial
from multiprocessing.pool import ThreadPool
from threading import Lock
from multiprocessing import Lock
from time import time
from humanfriendly import format_timespan

View File

@ -386,12 +386,12 @@ class Model(IdObjectBase, AsyncManagerMixin, _StorageUriMixin):
if not uri or not uri.strip():
return None
helper = StorageHelper.get(uri, logger=self._log, verbose=True)
helper = StorageHelper.get(uri)
filename = uri.split('/')[-1]
ext = '.'.join(filename.split('.')[1:])
fd, local_filename = mkstemp(suffix='.'+ext)
os.close(fd)
local_download = helper.download_to_file(uri, local_path=local_filename, overwrite_existing=True)
local_download = helper.download_to_file(uri, local_path=local_filename, overwrite_existing=True, verbose=True)
# save local model, so we can later query what was the original one
Model._local_model_to_id_uri[str(local_download)] = (self.model_id, uri)
return local_download

View File

@ -7,7 +7,8 @@ from copy import deepcopy
from datetime import datetime
from multiprocessing.pool import ThreadPool
from tempfile import mkdtemp, mkstemp
from threading import Thread, Event, RLock
from threading import Thread, Event
from multiprocessing import RLock
from time import time
import humanfriendly

View File

@ -1,6 +1,6 @@
import os
import time
from threading import Lock
from multiprocessing import Lock
import six