mirror of
https://github.com/clearml/clearml
synced 2025-04-06 05:35:32 +00:00
Better multi-processing, switching to to multiprocess Lock
This commit is contained in:
parent
9e930a6894
commit
9f73dbabdb
@ -1,6 +1,6 @@
|
||||
import abc
|
||||
import time
|
||||
from threading import Lock
|
||||
from multiprocessing import Lock
|
||||
|
||||
import attr
|
||||
import numpy as np
|
||||
|
@ -1,6 +1,6 @@
|
||||
from functools import partial
|
||||
from multiprocessing.pool import ThreadPool
|
||||
from threading import Lock
|
||||
from multiprocessing import Lock
|
||||
from time import time
|
||||
|
||||
from humanfriendly import format_timespan
|
||||
|
@ -386,12 +386,12 @@ class Model(IdObjectBase, AsyncManagerMixin, _StorageUriMixin):
|
||||
if not uri or not uri.strip():
|
||||
return None
|
||||
|
||||
helper = StorageHelper.get(uri, logger=self._log, verbose=True)
|
||||
helper = StorageHelper.get(uri)
|
||||
filename = uri.split('/')[-1]
|
||||
ext = '.'.join(filename.split('.')[1:])
|
||||
fd, local_filename = mkstemp(suffix='.'+ext)
|
||||
os.close(fd)
|
||||
local_download = helper.download_to_file(uri, local_path=local_filename, overwrite_existing=True)
|
||||
local_download = helper.download_to_file(uri, local_path=local_filename, overwrite_existing=True, verbose=True)
|
||||
# save local model, so we can later query what was the original one
|
||||
Model._local_model_to_id_uri[str(local_download)] = (self.model_id, uri)
|
||||
return local_download
|
||||
|
@ -7,7 +7,8 @@ from copy import deepcopy
|
||||
from datetime import datetime
|
||||
from multiprocessing.pool import ThreadPool
|
||||
from tempfile import mkdtemp, mkstemp
|
||||
from threading import Thread, Event, RLock
|
||||
from threading import Thread, Event
|
||||
from multiprocessing import RLock
|
||||
from time import time
|
||||
|
||||
import humanfriendly
|
||||
|
@ -1,6 +1,6 @@
|
||||
import os
|
||||
import time
|
||||
from threading import Lock
|
||||
from multiprocessing import Lock
|
||||
|
||||
import six
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user