Skip to content

📄 Awesome OCR multiple programing languages toolkits based on ONNX Runtime, OpenVINO, MNN, PaddlePaddle, TensorRT and PyTorch.

feat: adapt CANN of onnxruntime (PR #428) (#447)

SWHLd5b0738fdb5a
diff --git a/python/rapidocr/config.yaml b/python/rapidocr/config.yaml
index 52b95f6..29065df 100644
--- a/python/rapidocr/config.yaml
+++ b/python/rapidocr/config.yaml
@@ -21,6 +21,7 @@ EngineConfig:
enable_cpu_mem_arena: false
use_cuda: false
use_dml: false
+ use_cann: false
openvino:
inference_num_threads: -1
diff --git a/python/rapidocr/inference_engine/onnxruntime.py b/python/rapidocr/inference_engine/onnxruntime.py
deleted file mode 100644
index 4f03567..0000000
--- a/python/rapidocr/inference_engine/onnxruntime.py
+++ /dev/null
@@ -1,247 +0,0 @@
-# -*- encoding: utf-8 -*-
-# @Author: SWHL
-# @Contact: [email protected]
-import os
-import platform
-import traceback
-from enum import Enum
-from pathlib import Path
-from typing import Any, Dict, List, Tuple
-
-import numpy as np
-from onnxruntime import (
- GraphOptimizationLevel,
- InferenceSession,
- SessionOptions,
- get_available_providers,
- get_device,
-)
-
-from ..utils import DownloadFile, DownloadFileInput
-from ..utils.logger import Logger
-from .base import FileInfo, InferSession
-
-
-class EP(Enum):
- CPU_EP = "CPUExecutionProvider"
- CUDA_EP = "CUDAExecutionProvider"
- DIRECTML_EP = "DmlExecutionProvider"
-
-
-class OrtInferSession(InferSession):
- def __init__(self, cfg: Dict[str, Any]):
- self.logger = Logger(logger_name=__name__).get_log()
-
- model_path = cfg.get("model_path", None)
- if model_path is None:
- # 说明用户没有指定自己模型,使用默认模型
- model_info = self.get_model_url(
- FileInfo(
- engine_type=cfg.engine_type,
- ocr_version=cfg.ocr_version,
- task_type=cfg.task_type,
- lang_type=cfg.lang_type,
- model_type=cfg.model_type,
- )
- )
- model_path = self.DEFAULT_MODEL_PATH / Path(model_info["model_dir"]).name
- download_params = DownloadFileInput(
- file_url=model_info["model_dir"],
- sha256=model_info["SHA256"],
- save_path=model_path,
- logger=self.logger,
- )
- DownloadFile.run(download_params)
-
- self.logger.info(f"Using {model_path}")
- model_path = Path(model_path)
- self._verify_model(model_path)
-
- self.cfg_use_cuda = cfg.engine_cfg.get("use_cuda", None)
- self.cfg_use_dml = cfg.engine_cfg.get("use_dml", None)
-
- self.had_providers: List[str] = get_available_providers()
- EP_list = self._get_ep_list()
-
- sess_opt = self._init_sess_opts(cfg.engine_cfg)
- self.session = InferenceSession(
- model_path,
- sess_options=sess_opt,
- providers=EP_list,
- )
- self._verify_providers()
-
- @staticmethod
- def _init_sess_opts(cfg: Dict[str, Any]) -> SessionOptions:
- sess_opt = SessionOptions()
- sess_opt.log_severity_level = 4
- sess_opt.enable_cpu_mem_arena = cfg.enable_cpu_mem_arena
- sess_opt.graph_optimization_level = GraphOptimizationLevel.ORT_ENABLE_ALL
-
- cpu_nums = os.cpu_count()
- intra_op_num_threads = cfg.get("intra_op_num_threads", -1)
- if intra_op_num_threads != -1 and 1 <= intra_op_num_threads <= cpu_nums:
- sess_opt.intra_op_num_threads = intra_op_num_threads
-
- inter_op_num_threads = cfg.get("inter_op_num_threads", -1)
- if inter_op_num_threads != -1 and 1 <= inter_op_num_threads <= cpu_nums:
- sess_opt.inter_op_num_threads = inter_op_num_threads
-
- return sess_opt
-
- def _get_ep_list(self) -> List[Tuple[str, Dict[str, Any]]]:
- cpu_provider_opts = {
- "arena_extend_strategy": "kSameAsRequested",
- }
- EP_list = [(EP.CPU_EP.value, cpu_provider_opts)]
-
- cuda_provider_opts = {
- "device_id": 0,
- "arena_extend_strategy": "kNextPowerOfTwo",
- "cudnn_conv_algo_search": "EXHAUSTIVE",
- "do_copy_in_default_stream": True,
- }
- self.use_cuda = self._check_cuda()
- if self.use_cuda:
- EP_list.insert(0, (EP.CUDA_EP.value, cuda_provider_opts))
-
- self.use_directml = self._check_dml()
- if self.use_directml:
- self.logger.info(
- "Windows 10 or above detected, try to use DirectML as primary provider"
- )
- directml_options = (
- cuda_provider_opts if self.use_cuda else cpu_provider_opts
- )
- EP_list.insert(0, (EP.DIRECTML_EP.value, directml_options))
- return EP_list
-
- def _check_cuda(self) -> bool:
- if not self.cfg_use_cuda:
- return False
-
- cur_device = get_device()
- if cur_device == "GPU" and EP.CUDA_EP.value in self.had_providers:
- return True
-
- self.logger.warning(
- "%s is not in available providers (%s). Use %s inference by default.",
- EP.CUDA_EP.value,
- self.had_providers,
- self.had_providers[0],
- )
- self.logger.info("!!!Recommend to use rapidocr_paddle for inference on GPU.")
- self.logger.info(
- "(For reference only) If you want to use GPU acceleration, you must do:"
- )
- self.logger.info(
- "First, uninstall all onnxruntime pakcages in current environment."
- )
- self.logger.info(
- "Second, install onnxruntime-gpu by `pip install onnxruntime-gpu`."
- )
- self.logger.info(
- "\tNote the onnxruntime-gpu version must match your cuda and cudnn version."
- )
- self.logger.info(
- "\tYou can refer this link: https://onnxruntime.ai/docs/execution-providers/CUDA-EP.html"
- )
- self.logger.info(
- "Third, ensure %s is in available providers list. e.g. ['CUDAExecutionProvider', 'CPUExecutionProvider']",
- EP.CUDA_EP.value,
- )
- return False
-
- def _check_dml(self) -> bool:
- if not self.cfg_use_dml:
- return False
-
- cur_os = platform.system()
- if cur_os != "Windows":
- self.logger.warning(
- "DirectML is only supported in Windows OS. The current OS is %s. Use %s inference by default.",
- cur_os,
- self.had_providers[0],
- )
- return False
-
- window_build_number_str = platform.version().split(".")[-1]
- window_build_number = (
- int(window_build_number_str) if window_build_number_str.isdigit() else 0
- )
-
- if window_build_number < 18362:
- self.logger.warning(
- "DirectML is only supported in Windows 10 Build 18362 and above OS. The current Windows Build is %s. Use %s inference by default.",
- window_build_number,
- self.had_providers[0],
- )
- return False
-
- if EP.DIRECTML_EP.value in self.had_providers:
- return True
-
- self.logger.warning(
- "%s is not in available providers (%s). Use %s inference by default.",
- EP.DIRECTML_EP.value,
- self.had_providers,
- self.had_providers[0],
- )
- self.logger.info("If you want to use DirectML acceleration, you must do:")
- self.logger.info(
- "First, uninstall all onnxruntime pakcages in current environment."
- )
- self.logger.info(
- "Second, install onnxruntime-directml by `pip install onnxruntime-directml`"
- )
- self.logger.info(
- "Third, ensure %s is in available providers list. e.g. ['DmlExecutionProvider', 'CPUExecutionProvider']",
- EP.DIRECTML_EP.value,
- )
- return False
-
- def _verify_providers(self):
- session_providers = self.session.get_providers()
- first_provider = session_providers[0]
-
- if self.use_cuda and first_provider != EP.CUDA_EP.value:
- self.logger.warning(
- "%s is not avaiable for current env, the inference part is automatically shifted to be executed under %s.",
- EP.CUDA_EP.value,
- first_provider,
- )
-
- if self.use_directml and first_provider != EP.DIRECTML_EP.value:
- self.logger.warning(
- "%s is not available for current env, the inference part is automatically shifted to be executed under %s.",
- EP.DIRECTML_EP.value,
- first_provider,
- )
-
- def __call__(self, input_content: np.ndarray) -> np.ndarray:
- input_dict = dict(zip(self.get_input_names(), [input_content]))
- try:
- return self.session.run(self.get_output_names(), input_dict)[0]
- except Exception as e:
- error_info = traceback.format_exc()
- raise ONNXRuntimeError(error_info) from e
-
- def get_input_names(self) -> List[str]:
- return [v.name for v in self.session.get_inputs()]
-
- def get_output_names(self) -> List[str]:
- return [v.name for v in self.session.get_outputs()]
-
- def get_character_list(self, key: str = "character") -> List[str]:
- meta_dict = self.session.get_modelmeta().custom_metadata_map
- return meta_dict[key].splitlines()
-
- def have_key(self, key: str = "character") -> bool:
- meta_dict = self.session.get_modelmeta().custom_metadata_map
- if key in meta_dict.keys():
- return True
- return False
-
-
-class ONNXRuntimeError(Exception):
- pass
diff --git a/python/rapidocr/inference_engine/onnxruntime/__init__.py b/python/rapidocr/inference_engine/onnxruntime/__init__.py
new file mode 100644
index 0000000..fe32edf
--- /dev/null
+++ b/python/rapidocr/inference_engine/onnxruntime/__init__.py
@@ -0,0 +1,4 @@
+# -*- encoding: utf-8 -*-
+# @Author: SWHL
+# @Contact: [email protected]
+from .main import OrtInferSession
diff --git a/python/rapidocr/inference_engine/onnxruntime/main.py b/python/rapidocr/inference_engine/onnxruntime/main.py
new file mode 100644
index 0000000..4627bcb
--- /dev/null
+++ b/python/rapidocr/inference_engine/onnxruntime/main.py
@@ -0,0 +1,104 @@
+# -*- encoding: utf-8 -*-
+# @Author: SWHL
+# @Contact: [email protected]
+import os
+import traceback
+from pathlib import Path
+from typing import Any, Dict, List
+
+import numpy as np
+from onnxruntime import GraphOptimizationLevel, InferenceSession, SessionOptions
+
+from ...utils import DownloadFile, DownloadFileInput
+from ...utils.logger import Logger
+from ..base import FileInfo, InferSession
+from .provider_config import ProviderConfig
+
+
+class OrtInferSession(InferSession):
+ def __init__(self, cfg: Dict[str, Any]):
+ self.logger = Logger(logger_name=__name__).get_log()
+
+ model_path = cfg.get("model_path", None)
+ if model_path is None:
+ # 说明用户没有指定自己模型,使用默认模型
+ model_info = self.get_model_url(
+ FileInfo(
+ engine_type=cfg.engine_type,
+ ocr_version=cfg.ocr_version,
+ task_type=cfg.task_type,
+ lang_type=cfg.lang_type,
+ model_type=cfg.model_type,
+ )
+ )
+ model_path = self.DEFAULT_MODEL_PATH / Path(model_info["model_dir"]).name
+ download_params = DownloadFileInput(
+ file_url=model_info["model_dir"],
+ sha256=model_info["SHA256"],
+ save_path=model_path,
+ logger=self.logger,
+ )
+ DownloadFile.run(download_params)
+
+ self.logger.info(f"Using {model_path}")
+ model_path = Path(model_path)
+ self._verify_model(model_path)
+
+ sess_opt = self._init_sess_opts(cfg.engine_cfg)
+ provider_cfg = ProviderConfig(
+ cfg_use_cuda=cfg.engine_cfg.get("use_cuda", False),
+ cfg_use_dml=cfg.engine_cfg.get("use_dml", False),
+ cfg_use_cann=cfg.engine_cfg.get("use_cann", False),
+ )
+ self.session = InferenceSession(
+ model_path,
+ sess_options=sess_opt,
+ providers=provider_cfg.get_ep_list(),
+ )
+ provider_cfg.verify_providers(self.session.get_providers())
+
+ @staticmethod
+ def _init_sess_opts(cfg: Dict[str, Any]) -> SessionOptions:
+ sess_opt = SessionOptions()
+ sess_opt.log_severity_level = 4
+ sess_opt.enable_cpu_mem_arena = cfg.enable_cpu_mem_arena
+ sess_opt.graph_optimization_level = GraphOptimizationLevel.ORT_ENABLE_ALL
+
+ cpu_nums = os.cpu_count()
+ intra_op_num_threads = cfg.get("intra_op_num_threads", -1)
+ if intra_op_num_threads != -1 and 1 <= intra_op_num_threads <= cpu_nums:
+ sess_opt.intra_op_num_threads = intra_op_num_threads
+
+ inter_op_num_threads = cfg.get("inter_op_num_threads", -1)
+ if inter_op_num_threads != -1 and 1 <= inter_op_num_threads <= cpu_nums:
+ sess_opt.inter_op_num_threads = inter_op_num_threads
+
+ return sess_opt
+
+ def __call__(self, input_content: np.ndarray) -> np.ndarray:
+ input_dict = dict(zip(self.get_input_names(), [input_content]))
+ try:
+ return self.session.run(self.get_output_names(), input_dict)[0]
+ except Exception as e:
+ error_info = traceback.format_exc()
+ raise ONNXRuntimeError(error_info) from e
+
+ def get_input_names(self) -> List[str]:
+ return [v.name for v in self.session.get_inputs()]
+
+ def get_output_names(self) -> List[str]:
+ return [v.name for v in self.session.get_outputs()]
+
+ def get_character_list(self, key: str = "character") -> List[str]:
+ meta_dict = self.session.get_modelmeta().custom_metadata_map
+ return meta_dict[key].splitlines()
+
+ def have_key(self, key: str = "character") -> bool:
+ meta_dict = self.session.get_modelmeta().custom_metadata_map
+ if key in meta_dict.keys():
+ return True
+ return False
+
+
+class ONNXRuntimeError(Exception):
+ pass
diff --git a/python/rapidocr/inference_engine/onnxruntime/provider_config.py b/python/rapidocr/inference_engine/onnxruntime/provider_config.py
new file mode 100644
index 0000000..a8ca3de
--- /dev/null
+++ b/python/rapidocr/inference_engine/onnxruntime/provider_config.py
@@ -0,0 +1,201 @@
+# -*- encoding: utf-8 -*-
+# @Author: SWHL
+# @Contact: [email protected]
+import platform
+from enum import Enum
+from typing import Any, Dict, List
+
+from onnxruntime import get_available_providers, get_device
+
+from ...utils.logger import Logger
+
+
+class EP(Enum):
+ CPU_EP = "CPUExecutionProvider"
+ CUDA_EP = "CUDAExecutionProvider"
+ DIRECTML_EP = "DmlExecutionProvider"
+ CANN_EP = "CANNExecutionProvider"
+
+
+class ProviderConfig:
+ def __init__(self, cfg_use_cuda: bool, cfg_use_dml: bool, cfg_use_cann: bool):
+ self.logger = Logger(logger_name=__name__).get_log()
+
+ self.had_providers: List[str] = get_available_providers()
+ self.default_provider = self.had_providers[0]
+
+ self.cfg_use_cuda = cfg_use_cuda
+ self.cfg_use_dml = cfg_use_dml
+ self.cfg_use_cann = cfg_use_cann
+
+ def get_ep_list(self):
+ results = [(EP.CPU_EP.value, self.cpu_ep_cfg())]
+
+ if self.is_cuda_available():
+ results.insert(0, (EP.CUDA_EP.value, self.cuda_ep_cfg()))
+
+ if self.is_dml_available():
+ self.logger.info(
+ "Windows 10 or above detected, try to use DirectML as primary provider"
+ )
+ results.insert(0, (EP.DIRECTML_EP.value, self.dml_ep_cfg()))
+
+ if self.is_cann_available():
+ self.logger.info("Try to use CANNExecutionProvider to infer")
+ results.insert(0, (EP.CANN_EP.value, self.cann_ep_cfg()))
+
+ return results
+
+ def cpu_ep_cfg(self) -> Dict[str, Any]:
+ return {
+ "arena_extend_strategy": "kSameAsRequested",
+ }
+
+ def cuda_ep_cfg(self) -> Dict[str, Any]:
+ return {
+ "device_id": 0,
+ "arena_extend_strategy": "kNextPowerOfTwo",
+ "cudnn_conv_algo_search": "EXHAUSTIVE",
+ "do_copy_in_default_stream": True,
+ }
+
+ def dml_ep_cfg(self) -> Dict[str, Any]:
+ if self.use_cuda:
+ return self.cuda_ep_cfg()
+ return self.cpu_ep_cfg()
+
+ def cann_ep_cfg(self) -> Dict[str, Any]:
+ return {
+ "device_id": 0,
+ "arena_extend_strategy": "kNextPowerOfTwo",
+ "npu_mem_limit": 20 * 1024 * 1024 * 1024,
+ "op_select_impl_mode": "high_performance",
+ "optypelist_for_implmode": "Gelu",
+ "enable_cann_graph": True,
+ }
+
+ def verify_providers(self, session_providers: List[str]):
+ if not session_providers:
+ raise ValueError("Session Providers is empty")
+
+ first_provider = session_providers[0]
+
+ providers_to_check = {
+ EP.CUDA_EP: self.is_cuda_available,
+ EP.DIRECTML_EP: self.is_dml_available,
+ EP.CANN_EP: self.is_cann_available,
+ }
+
+ for ep, check_func in providers_to_check.items():
+ if check_func() and first_provider != ep.value:
+ self.logger.warning(
+ f"{ep.value} is not available for current env, the inference part is automatically shifted to be executed under {first_provider}. "
+ )
+ self.logger.warning(f"The available lists are {session_providers}")
+
+ def is_cuda_available(self) -> bool:
+ if not self.cfg_use_cuda:
+ return False
+
+ CUDA_EP = EP.CUDA_EP.value
+ if get_device() == "GPU" and CUDA_EP in self.had_providers:
+ return True
+
+ self.logger.warning(
+ f"{CUDA_EP} is not in available providers ({self.had_providers}). Use {self.default_provider} inference by default."
+ )
+ self.logger.info("!!! Recommend to use rapidocr_paddle for inference on GPU.")
+ self.logger.info(
+ "(For reference only) If you want to use GPU acceleration, you must do:"
+ )
+ self.logger.info(
+ "First, uninstall all onnxruntime packages in current environment."
+ )
+ self.logger.info(
+ "Second, install onnxruntime-gpu by `pip install onnxruntime-gpu`."
+ )
+ self.logger.info(
+ "Note the onnxruntime-gpu version must match your cuda and cudnn version."
+ )
+ self.logger.info(
+ "You can refer this link: https://onnxruntime.ai/docs/execution-providers/CUDA-EP.html"
+ )
+ self.logger.info(
+ "Third, ensure %s is in available providers list. e.g. ['CUDAExecutionProvider', 'CPUExecutionProvider']",
+ CUDA_EP,
+ )
+ return False
+
+ def is_dml_available(self) -> bool:
+ if not self.cfg_use_dml:
+ return False
+
+ cur_os = platform.system()
+ if cur_os != "Windows":
+ self.logger.warning(
+ f"DirectML is only supported in Windows OS. The current OS is {cur_os}. Use {self.default_provider} inference by default.",
+ )
+ return False
+
+ window_build_number_str = platform.version().split(".")[-1]
+ window_build_number = (
+ int(window_build_number_str) if window_build_number_str.isdigit() else 0
+ )
+ if window_build_number < 18362:
+ self.logger.warning(
+ f"DirectML is only supported in Windows 10 Build 18362 and above OS. The current Windows Build is {window_build_number}. Use {self.default_provider} inference by default.",
+ )
+ return False
+
+ DML_EP = EP.DIRECTML_EP.value
+ if DML_EP in self.had_providers:
+ return True
+
+ self.logger.warning(
+ "%s is not in available providers (%s). Use %s inference by default.",
+ DML_EP,
+ self.had_providers,
+ self.default_provider,
+ )
+ self.logger.info("If you want to use DirectML acceleration, you must do:")
+ self.logger.info(
+ "First, uninstall all onnxruntime packages in current environment."
+ )
+ self.logger.info(
+ "Second, install onnxruntime-directml by `pip install onnxruntime-directml`"
+ )
+ self.logger.info(
+ "Third, ensure %s is in available providers list. e.g. ['DmlExecutionProvider', 'CPUExecutionProvider']",
+ DML_EP,
+ )
+ return False
+
+ def is_cann_available(self) -> bool:
+ if not self.cfg_use_cann:
+ return False
+
+ CANN_EP = EP.CANN_EP.value
+ if CANN_EP in self.had_providers:
+ return True
+
+ self.logger.warning(
+ "%s is not in available providers (%s). Use %s inference by default.",
+ CANN_EP,
+ self.had_providers,
+ self.default_provider,
+ )
+ self.logger.info("If you want to use CANN acceleration, you must do:")
+ self.logger.info(
+ "First, ensure you have installed Huawei Ascend software stack."
+ )
+ self.logger.info(
+ "Second, install onnxruntime with CANN support by following the instructions at:"
+ )
+ self.logger.info(
+ "\thttps://onnxruntime.ai/docs/execution-providers/CANN-ExecutionProvider.html"
+ )
+ self.logger.info(
+ "Third, ensure %s is in available providers list. e.g. ['CANNExecutionProvider', 'CPUExecutionProvider']",
+ CANN_EP,
+ )
+ return False