High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
feat(ml): enable openvino for cpu (#22948)
Aleksander Pejcicaaf34fa7d4fb
diff --git a/machine-learning/immich_ml/sessions/ort.py b/machine-learning/immich_ml/sessions/ort.pyindex 5b728fce6f..bebd235970 100644--- a/machine-learning/immich_ml/sessions/ort.py+++ b/machine-learning/immich_ml/sessions/ort.py@@ -64,14 +64,6 @@ class OrtSession:def _providers_default(self) -> list[str]:available_providers = set(ort.get_available_providers())log.debug(f"Available ORT providers: {available_providers}")- if (openvino := "OpenVINOExecutionProvider") in available_providers:- device_ids: list[str] = ort.capi._pybind_state.get_available_openvino_device_ids()- log.debug(f"Available OpenVINO devices: {device_ids}")-- gpu_devices = [device_id for device_id in device_ids if device_id.startswith("GPU")]- if not gpu_devices:- log.warning("No GPU device found in OpenVINO. Falling back to CPU.")- available_providers.remove(openvino)return [provider for provider in SUPPORTED_PROVIDERS if provider in available_providers]@property@@ -102,12 +94,19 @@ class OrtSession:"migraphx_fp16_enable": "1" if settings.rocm_precision == ModelPrecision.FP16 else "0",}case "OpenVINOExecutionProvider":- openvino_dir = self.model_path.parent / "openvino"- device = f"GPU.{settings.device_id}"+ device_ids: list[str] = ort.capi._pybind_state.get_available_openvino_device_ids()+ # Check for available devices, preferring GPU over CPU+ gpu_devices = [d for d in device_ids if d.startswith("GPU")]+ if gpu_devices:+ device_type = f"GPU.{settings.device_id}"+ log.debug(f"OpenVINO: Using GPU device {device_type}")+ else:+ device_type = "CPU"+ log.debug("OpenVINO: No GPU found, using CPU")options = {- "device_type": device,+ "device_type": device_type,"precision": settings.openvino_precision.value,- "cache_dir": openvino_dir.as_posix(),+ "cache_dir": (self.model_path.parent / "openvino").as_posix(),}case "CoreMLExecutionProvider":options = {@@ -139,12 +138,14 @@ class OrtSession:sess_options.enable_cpu_mem_arena = settings.model_arena# avoid thread contention between models+ # Set inter_op threadsif settings.model_inter_op_threads > 0:sess_options.inter_op_num_threads = settings.model_inter_op_threads# these defaults work well for CPU, but bottleneck GPUelif settings.model_inter_op_threads == 0 and self.providers == ["CPUExecutionProvider"]:sess_options.inter_op_num_threads = 1+ # Set intra_op threadsif settings.model_intra_op_threads > 0:sess_options.intra_op_num_threads = settings.model_intra_op_threadselif settings.model_intra_op_threads == 0 and self.providers == ["CPUExecutionProvider"]:diff --git a/machine-learning/test_main.py b/machine-learning/test_main.pyindex a5cf1acc2e..0182c57c67 100644--- a/machine-learning/test_main.py+++ b/machine-learning/test_main.py@@ -204,13 +204,6 @@ class TestOrtSession:assert session.providers == self.OV_EP- @pytest.mark.ov_device_ids(["CPU"])- @pytest.mark.providers(OV_EP)- def test_avoids_openvino_if_gpu_not_available(self, providers: list[str], ov_device_ids: list[str]) -> None:- session = OrtSession("ViT-B-32__openai")-- assert session.providers == self.CPU_EP-@pytest.mark.providers(CUDA_EP_OUT_OF_ORDER)def test_sets_providers_in_correct_order(self, providers: list[str]) -> None:session = OrtSession("ViT-B-32__openai")@@ -256,7 +249,8 @@ class TestOrtSession:{"arena_extend_strategy": "kSameAsRequested"},]- def test_sets_provider_options_for_openvino(self) -> None:+ @pytest.mark.ov_device_ids(["GPU.0", "GPU.1", "CPU"])+ def test_sets_provider_options_for_openvino(self, ov_device_ids: list[str]) -> None:model_path = "/cache/ViT-B-32__openai/textual/model.onnx"os.environ["MACHINE_LEARNING_DEVICE_ID"] = "1"@@ -270,7 +264,8 @@ class TestOrtSession:}]- def test_sets_openvino_to_fp16_if_enabled(self, mocker: MockerFixture) -> None:+ @pytest.mark.ov_device_ids(["GPU.0", "GPU.1", "CPU"])+ def test_sets_openvino_to_fp16_if_enabled(self, ov_device_ids: list[str], mocker: MockerFixture) -> None:model_path = "/cache/ViT-B-32__openai/textual/model.onnx"os.environ["MACHINE_LEARNING_DEVICE_ID"] = "1"mocker.patch.object(settings, "openvino_precision", ModelPrecision.FP16)@@ -285,6 +280,19 @@ class TestOrtSession:}]+ @pytest.mark.ov_device_ids(["CPU"])+ def test_sets_provider_options_for_openvino_cpu(self, ov_device_ids: list[str]) -> None:+ model_path = "/cache/ViT-B-32__openai/model.onnx"+ session = OrtSession(model_path, providers=["OpenVINOExecutionProvider"])++ assert session.provider_options == [+ {+ "device_type": "CPU",+ "precision": "FP32",+ "cache_dir": "/cache/ViT-B-32__openai/openvino",+ }+ ]+def test_sets_provider_options_for_cuda(self) -> None:os.environ["MACHINE_LEARNING_DEVICE_ID"] = "1"@@ -341,6 +349,23 @@ class TestOrtSession:assert session.sess_options.inter_op_num_threads == 1assert session.sess_options.intra_op_num_threads == 2+ @pytest.mark.ov_device_ids(["CPU"])+ def test_sets_default_sess_options_if_openvino_cpu(self, ov_device_ids: list[str]) -> None:+ model_path = "/cache/ViT-B-32__openai/model.onnx"+ session = OrtSession(model_path, providers=["OpenVINOExecutionProvider"])++ assert session.sess_options.execution_mode == ort.ExecutionMode.ORT_SEQUENTIAL+ assert session.sess_options.inter_op_num_threads == 0+ assert session.sess_options.intra_op_num_threads == 0++ @pytest.mark.ov_device_ids(["GPU.0", "CPU"])+ def test_sets_default_sess_options_if_openvino_gpu(self, ov_device_ids: list[str]) -> None:+ model_path = "/cache/ViT-B-32__openai/model.onnx"+ session = OrtSession(model_path, providers=["OpenVINOExecutionProvider"])++ assert session.sess_options.inter_op_num_threads == 0+ assert session.sess_options.intra_op_num_threads == 0+def test_sets_default_sess_options_does_not_set_threads_if_non_cpu_and_default_threads(self) -> None:session = OrtSession("ViT-B-32__openai", providers=["CUDAExecutionProvider", "CPUExecutionProvider"])