ClearML/clearml-serving
1
0
Fork 0
mirror of https://github.com/clearml/clearml-serving synced 2025-02-07 13:22:16 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add support for Preprocess class inside a module (i.e. __init__.py with subfolders)

Browse Source
This commit is contained in:
allegroai 2022-09-02 21:50:41 +03:00
parent 0588f58405
commit c6c40c9a36
1 changed files with 21 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
clearml_serving/serving/preprocess_service.py
View File

@ -1,4 +1,6 @@
import os import os
import sys
from pathlib import Path
from typing import Optional, Any, Callable, List from typing import Optional, Any, Callable, List
import numpy as np import numpy as np
@ -45,7 +47,10 @@ class BasePreprocessRequest(object):
self.model_endpoint.preprocess_artifact, ex)) self.model_endpoint.preprocess_artifact, ex))
def _instantiate_custom_preprocess_cls(self, task: Task) -> None: def _instantiate_custom_preprocess_cls(self, task: Task) -> None:
path = task.artifacts[self.model_endpoint.preprocess_artifact].get_local_copy() path = task.artifacts[self.model_endpoint.preprocess_artifact].get_local_copy(extract_archive=False)
if not path or not Path(path).exists():
raise ValueError("Artifact '{}' could not be downloaded".format(self.model_endpoint.preprocess_artifact))
# check file content hash, should only happen once?! # check file content hash, should only happen once?!
# noinspection PyProtectedMember # noinspection PyProtectedMember
file_hash, _ = sha256sum(path, block_size=Artifacts._hash_block_size) file_hash, _ = sha256sum(path, block_size=Artifacts._hash_block_size)
@ -63,9 +68,21 @@ class BasePreprocessRequest(object):
) )
import importlib.util import importlib.util
if Path(path).is_file():
spec = importlib.util.spec_from_file_location("Preprocess", path) spec = importlib.util.spec_from_file_location("Preprocess", path)
_preprocess = importlib.util.module_from_spec(spec) _preprocess = importlib.util.module_from_spec(spec)
spec.loader.exec_module(_preprocess) spec.loader.exec_module(_preprocess)
else:
submodules_path = [Path(path).as_posix()] + sys.path
module_name = str(self.model_endpoint.preprocess_artifact).replace(".", "_")
spec = importlib.util.spec_from_file_location(
module_name, location=(Path(path) / "__init__.py").as_posix(),
submodule_search_locations=submodules_path,
)
_preprocess = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = _preprocess
spec.loader.exec_module(_preprocess)
Preprocess = _preprocess.Preprocess # noqa Preprocess = _preprocess.Preprocess # noqa
# override `send_request` method # override `send_request` method
Preprocess.send_request = BasePreprocessRequest._preprocess_send_request Preprocess.send_request = BasePreprocessRequest._preprocess_send_request