Skip to content

High performance self-hosted photo and video management solution.

feat(ml): enable openvino for cpu (#22948)

Aleksander Pejcicaaf34fa7d4fb
diff --git a/machine-learning/immich_ml/sessions/ort.py b/machine-learning/immich_ml/sessions/ort.py
index 5b728fce6f..bebd235970 100644
--- a/machine-learning/immich_ml/sessions/ort.py
+++ b/machine-learning/immich_ml/sessions/ort.py
@@ -64,14 +64,6 @@ class OrtSession:
def _providers_default(self) -> list[str]:
available_providers = set(ort.get_available_providers())
log.debug(f"Available ORT providers: {available_providers}")
- if (openvino := "OpenVINOExecutionProvider") in available_providers:
- device_ids: list[str] = ort.capi._pybind_state.get_available_openvino_device_ids()
- log.debug(f"Available OpenVINO devices: {device_ids}")
-
- gpu_devices = [device_id for device_id in device_ids if device_id.startswith("GPU")]
- if not gpu_devices:
- log.warning("No GPU device found in OpenVINO. Falling back to CPU.")
- available_providers.remove(openvino)
return [provider for provider in SUPPORTED_PROVIDERS if provider in available_providers]
@property
@@ -102,12 +94,19 @@ class OrtSession:
"migraphx_fp16_enable": "1" if settings.rocm_precision == ModelPrecision.FP16 else "0",
}
case "OpenVINOExecutionProvider":
- openvino_dir = self.model_path.parent / "openvino"
- device = f"GPU.{settings.device_id}"
+ device_ids: list[str] = ort.capi._pybind_state.get_available_openvino_device_ids()
+ # Check for available devices, preferring GPU over CPU
+ gpu_devices = [d for d in device_ids if d.startswith("GPU")]
+ if gpu_devices:
+ device_type = f"GPU.{settings.device_id}"
+ log.debug(f"OpenVINO: Using GPU device {device_type}")
+ else:
+ device_type = "CPU"
+ log.debug("OpenVINO: No GPU found, using CPU")
options = {
- "device_type": device,
+ "device_type": device_type,
"precision": settings.openvino_precision.value,
- "cache_dir": openvino_dir.as_posix(),
+ "cache_dir": (self.model_path.parent / "openvino").as_posix(),
}
case "CoreMLExecutionProvider":
options = {
@@ -139,12 +138,14 @@ class OrtSession:
sess_options.enable_cpu_mem_arena = settings.model_arena
# avoid thread contention between models
+ # Set inter_op threads
if settings.model_inter_op_threads > 0:
sess_options.inter_op_num_threads = settings.model_inter_op_threads
# these defaults work well for CPU, but bottleneck GPU
elif settings.model_inter_op_threads == 0 and self.providers == ["CPUExecutionProvider"]:
sess_options.inter_op_num_threads = 1
+ # Set intra_op threads
if settings.model_intra_op_threads > 0:
sess_options.intra_op_num_threads = settings.model_intra_op_threads
elif settings.model_intra_op_threads == 0 and self.providers == ["CPUExecutionProvider"]:
diff --git a/machine-learning/test_main.py b/machine-learning/test_main.py
index a5cf1acc2e..0182c57c67 100644
--- a/machine-learning/test_main.py
+++ b/machine-learning/test_main.py
@@ -204,13 +204,6 @@ class TestOrtSession:
assert session.providers == self.OV_EP
- @pytest.mark.ov_device_ids(["CPU"])
- @pytest.mark.providers(OV_EP)
- def test_avoids_openvino_if_gpu_not_available(self, providers: list[str], ov_device_ids: list[str]) -> None:
- session = OrtSession("ViT-B-32__openai")
-
- assert session.providers == self.CPU_EP
-
@pytest.mark.providers(CUDA_EP_OUT_OF_ORDER)
def test_sets_providers_in_correct_order(self, providers: list[str]) -> None:
session = OrtSession("ViT-B-32__openai")
@@ -256,7 +249,8 @@ class TestOrtSession:
{"arena_extend_strategy": "kSameAsRequested"},
]
- def test_sets_provider_options_for_openvino(self) -> None:
+ @pytest.mark.ov_device_ids(["GPU.0", "GPU.1", "CPU"])
+ def test_sets_provider_options_for_openvino(self, ov_device_ids: list[str]) -> None:
model_path = "/cache/ViT-B-32__openai/textual/model.onnx"
os.environ["MACHINE_LEARNING_DEVICE_ID"] = "1"
@@ -270,7 +264,8 @@ class TestOrtSession:
}
]
- def test_sets_openvino_to_fp16_if_enabled(self, mocker: MockerFixture) -> None:
+ @pytest.mark.ov_device_ids(["GPU.0", "GPU.1", "CPU"])
+ def test_sets_openvino_to_fp16_if_enabled(self, ov_device_ids: list[str], mocker: MockerFixture) -> None:
model_path = "/cache/ViT-B-32__openai/textual/model.onnx"
os.environ["MACHINE_LEARNING_DEVICE_ID"] = "1"
mocker.patch.object(settings, "openvino_precision", ModelPrecision.FP16)
@@ -285,6 +280,19 @@ class TestOrtSession:
}
]
+ @pytest.mark.ov_device_ids(["CPU"])
+ def test_sets_provider_options_for_openvino_cpu(self, ov_device_ids: list[str]) -> None:
+ model_path = "/cache/ViT-B-32__openai/model.onnx"
+ session = OrtSession(model_path, providers=["OpenVINOExecutionProvider"])
+
+ assert session.provider_options == [
+ {
+ "device_type": "CPU",
+ "precision": "FP32",
+ "cache_dir": "/cache/ViT-B-32__openai/openvino",
+ }
+ ]
+
def test_sets_provider_options_for_cuda(self) -> None:
os.environ["MACHINE_LEARNING_DEVICE_ID"] = "1"
@@ -341,6 +349,23 @@ class TestOrtSession:
assert session.sess_options.inter_op_num_threads == 1
assert session.sess_options.intra_op_num_threads == 2
+ @pytest.mark.ov_device_ids(["CPU"])
+ def test_sets_default_sess_options_if_openvino_cpu(self, ov_device_ids: list[str]) -> None:
+ model_path = "/cache/ViT-B-32__openai/model.onnx"
+ session = OrtSession(model_path, providers=["OpenVINOExecutionProvider"])
+
+ assert session.sess_options.execution_mode == ort.ExecutionMode.ORT_SEQUENTIAL
+ assert session.sess_options.inter_op_num_threads == 0
+ assert session.sess_options.intra_op_num_threads == 0
+
+ @pytest.mark.ov_device_ids(["GPU.0", "CPU"])
+ def test_sets_default_sess_options_if_openvino_gpu(self, ov_device_ids: list[str]) -> None:
+ model_path = "/cache/ViT-B-32__openai/model.onnx"
+ session = OrtSession(model_path, providers=["OpenVINOExecutionProvider"])
+
+ assert session.sess_options.inter_op_num_threads == 0
+ assert session.sess_options.intra_op_num_threads == 0
+
def test_sets_default_sess_options_does_not_set_threads_if_non_cpu_and_default_threads(self) -> None:
session = OrtSession("ViT-B-32__openai", providers=["CUDAExecutionProvider", "CPUExecutionProvider"])