High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
feat(ml): improve model loading (#31742)
Mertde13aab713fd
diff --git a/machine-learning/Dockerfile b/machine-learning/Dockerfileindex fa24b0f87d..b1ca31c888 100644--- a/machine-learning/Dockerfile+++ b/machine-learning/Dockerfile@@ -4,7 +4,7 @@ FROM python:3.13-slim-trixie@sha256:9d2e5553305c7c7b0097999bb17187c69b921ccd6bc9FROM builder-cpu AS builder-openvino-FROM builder-cpu AS builder-cuda+FROM python:3.13-slim-bookworm@sha256:2325bb286ec344af3e5898cc224b5844e2707ac6e26b1632516fd3edc84a5e26 AS builder-cudaFROM python:3.13-trixie@sha256:4a0820d8b1b1a1e012a23ff9a1296a3e28b48896a4d27a328a91ee197ce619d9 AS builder-armnn@@ -108,17 +108,23 @@ ENV MACHINE_LEARNING_MODEL_ARENA=falseADD --chmod=644 --checksum=sha256:73993ed4b440460825f21611731564503cc1d5a0c123746477da6cd574f34885 "https://github.com/airockchip/rknn-toolkit2/raw/refs/tags/${RKNN_TOOLKIT_VERSION}/rknpu2/runtime/Linux/librknn_api/aarch64/librknnrt.so" /usr/lib/+FROM python:3.13-slim-trixie@sha256:9d2e5553305c7c7b0097999bb17187c69b921ccd6bc9d40e4bb5ebe652c00285 AS mimalloc+RUN echo "deb http://deb.debian.org/debian trixie-backports main" > /etc/apt/sources.list.d/backports.list && \+ apt-get update && \+ apt-get install -y --no-install-recommends libmimalloc3/trixie-backports && \+ cp /usr/lib/*-linux-gnu/libmimalloc.so.3 /libmimalloc.so+FROM prod-${DEVICE} AS prodARG DEVICE-RUN MIMALLOC=$(case "$DEVICE" in cuda) echo "libmimalloc2.0";; openvino|rocm);; *) echo "libmimalloc3";; esac); \- apt-get update && \- apt-get install -y --no-install-recommends tini ccache libgl1 libglib2.0-0 libgomp1 $MIMALLOC && \+COPY --from=mimalloc /libmimalloc.so /usr/lib/libmimalloc.so+RUN apt-get update && \+ apt-get install -y --no-install-recommends tini ccache libgomp1 && \apt-get autoremove -yqq && \apt-get clean && \rm -rf /var/lib/apt/lists/* && \- { [ -z "$MIMALLOC" ] || ls /usr/lib/*-linux-gnu/libmimalloc.so.? > /etc/ld.so.preload; }+ case "$DEVICE" in openvino|rocm);; *) echo /usr/lib/libmimalloc.so > /etc/ld.so.preload;; esacWORKDIR /usr/srcENV TRANSFORMERS_CACHE=/cache \@@ -157,6 +163,10 @@ ENV IMMICH_SOURCE_URL=https://github.com/immich-app/immich/commit/${BUILD_SOURCE# store huggingface in /cache subfolder to enable rootless deployment with a single volumeENV HF_HOME=/cache/hf-cache+ENV CUDA_CACHE_PATH=/cache/jit/cuda \+ AMD_COMGR_CACHE_DIR=/cache/jit/comgr \+ NEO_CACHE_DIR=/cache/jit/neo+ENTRYPOINT ["tini", "--"]CMD ["python", "-m", "immich_ml"]diff --git a/machine-learning/conftest.py b/machine-learning/conftest.pyindex a5acc059d1..a968d6906a 100644--- a/machine-learning/conftest.py+++ b/machine-learning/conftest.py@@ -9,6 +9,7 @@ from PIL import Imagefrom immich_ml.config import logfrom immich_ml.main import app+from immich_ml.models.base import InferenceModel@pytest.fixture@@ -18,7 +19,7 @@ def pil_image() -> Image.Image:@pytest.fixturedef mock_get_model() -> Iterator[mock.Mock]:- with mock.patch("immich_ml.models.cache.from_model_type", autospec=True) as mocked:+ with mock.patch("immich_ml.models.cache.get_model_class") as mocked:yield mocked@@ -187,5 +188,8 @@ def exception() -> Iterator[mock.Mock]:@pytest.fixture(scope="function")def snapshot_download() -> Iterator[mock.Mock]:- with mock.patch("immich_ml.models.base.snapshot_download") as mocked:+ with (+ mock.patch("immich_ml.models.base.snapshot_download") as mocked,+ mock.patch.object(InferenceModel, "cached", new_callable=mock.PropertyMock, side_effect=lambda: mocked.called),+ ):yield mockeddiff --git a/machine-learning/immich_ml/__main__.py b/machine-learning/immich_ml/__main__.pyindex f305c77657..49e37aae94 100644--- a/machine-learning/immich_ml/__main__.py+++ b/machine-learning/immich_ml/__main__.py@@ -13,6 +13,11 @@ else:module_dir = Path(__file__).parent+# on i915, Intel's runtime submits every allocation in the process with each run it gives OpenVINO's GPU plugin+if any(driver.resolve().name == "i915" for driver in Path("/sys/class/drm").glob("renderD*/device/driver")):+ os.environ.setdefault("NEOReadDebugKeys", "1")+ os.environ.setdefault("EnableDirectSubmission", "0")+def is_ipv6(host: str) -> bool:try:diff --git a/machine-learning/immich_ml/allocator.py b/machine-learning/immich_ml/allocator.pynew file mode 100644index 0000000000..7cb3338bc6--- /dev/null+++ b/machine-learning/immich_ml/allocator.py@@ -0,0 +1,10 @@+import ctypes++_process = ctypes.CDLL(None)+++def release() -> None:+ if hasattr(_process, "mi_collect"):+ _process.mi_collect(True)+ elif hasattr(_process, "malloc_trim"):+ _process.malloc_trim(0)diff --git a/machine-learning/immich_ml/config.py b/machine-learning/immich_ml/config.pyindex d1e44fd70a..747d1a4988 100644--- a/machine-learning/immich_ml/config.py+++ b/machine-learning/immich_ml/config.py@@ -13,7 +13,7 @@ from rich.logging import RichHandlerfrom uvicorn import Serverfrom uvicorn.workers import UvicornWorker-from .schemas import ModelPrecision+from .schemas import ModelOrganization, ModelPrecisionclass ClipSettings(BaseModel):@@ -74,6 +74,12 @@ class Settings(BaseSettings):max_batch_size: MaxBatchSize | None = Noneopenvino_precision: ModelPrecision = ModelPrecision.FP32rocm_precision: ModelPrecision = ModelPrecision.FP32+ model_organization: ModelOrganization = ModelOrganization.APP+ model_revision: str = "main"++ @property+ def legacy_models(self) -> bool:+ return self.model_revision == "main"@propertydef device_id(self) -> str:diff --git a/machine-learning/immich_ml/main.py b/machine-learning/immich_ml/main.pyindex 4ac5b020ae..fdad03eae3 100644--- a/machine-learning/immich_ml/main.py+++ b/machine-learning/immich_ml/main.py@@ -2,25 +2,26 @@ import asyncioimport gcimport osimport signal-import threadingimport timefrom concurrent.futures import ThreadPoolExecutorfrom contextlib import asynccontextmanagerfrom functools import partial-from typing import Any, AsyncGenerator, Callable, Iterator+from typing import Any, AsyncGenerator, Callablefrom zipfile import BadZipFileimport orjsonfrom fastapi import Depends, FastAPI, File, Form, HTTPException-from fastapi.responses import PlainTextResponse+from fastapi.responses import JSONResponse, PlainTextResponsefrom onnxruntime.capi.onnxruntime_pybind11_state import InvalidProtobuf, NoSuchFilefrom PIL.Image import Imagefrom pydantic import ValidationErrorfrom starlette.formparsers import MultiPartParser+from immich_ml import allocatorfrom immich_ml.models import get_model_depsfrom immich_ml.models.base import InferenceModelfrom immich_ml.models.transforms import decode_pil+from immich_ml.sessions.ort import flush_denormalsfrom .config import PreloadModelData, log, settingsfrom .models.cache import ModelCache@@ -32,18 +33,24 @@ from .schemas import (ModelIdentity,ModelTask,ModelType,- ORJSONResponse,PipelineRequest,T,)++class ORJSONResponse(JSONResponse):+ def render(self, content: Any) -> bytes:+ return orjson.dumps(content, option=orjson.OPT_SERIALIZE_NUMPY)++MultiPartParser.spool_max_size = 2**26 # spools to disk if payload is 64 MiB or larger-model_cache = ModelCache(revalidate=settings.model_ttl > 0)+model_cache = ModelCache()thread_pool: ThreadPoolExecutor | None = None-lock = threading.Lock()+MODEL_FILE_ERRORS = (InvalidProtobuf, NoSuchFile)active_requests = 0last_called: float | None = None+release: asyncio.TimerHandle | None = None@asynccontextmanager@@ -57,19 +64,20 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:)try:+ flush_denormals() # for work that runs on the event loop's own threadif settings.request_threads > 0:# asyncio is a huge bottleneck for performance, so we use a thread pool to run blocking code- thread_pool = ThreadPoolExecutor(settings.request_threads) if settings.request_threads > 0 else None+ thread_pool = ThreadPoolExecutor(settings.request_threads, initializer=flush_denormals)log.info(f"Initialized request thread pool with {settings.request_threads} threads.")if settings.model_ttl > 0 and settings.model_ttl_poll_s > 0:asyncio.ensure_future(idle_shutdown_task())if settings.preload is not None:await preload_models(settings.preload)+ allocator.release()yieldfinally:log.handlers.clear()- for model in model_cache.cache._cache.values():- del model+ model_cache.clear()if thread_pool is not None:thread_pool.shutdown()gc.collect()@@ -78,10 +86,10 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:async def preload_models(preload: PreloadModelData) -> None:log.info(f"Preloading models: clip:{preload.clip} facial_recognition:{preload.facial_recognition}")- async def load_models(model_string: str, model_type: ModelType, model_task: ModelTask) -> None:+ async def load_models(model_string: str, model_type: ModelType, model_task: ModelTask, **options: Any) -> None:for model_name in model_string.split(","):model_name = model_name.strip()- model = await model_cache.get(model_name, model_type, model_task)+ model = model_cache.get(model_name, model_type, model_task, **options)await load(model)if preload.clip.textual is not None:@@ -119,14 +127,19 @@ async def preload_models(preload: PreloadModelData) -> None:)-def update_state() -> Iterator[None]:- global active_requests, last_called+async def update_state() -> AsyncGenerator[None, None]:+ global active_requests, last_called, releaseactive_requests += 1last_called = time.time()+ if release is not None:+ release.cancel()+ release = Nonetry:yieldfinally:active_requests -= 1+ if not active_requests:+ release = asyncio.get_running_loop().call_later(5, allocator.release)def get_entries(entries: str = Form()) -> InferenceEntries:@@ -187,9 +200,7 @@ async def run_inference(payload: Image | str, entries: InferenceEntries) -> Inferesponse: InferenceResponse = {}async def _run_inference(entry: InferenceEntry) -> None:- model = await model_cache.get(- entry["name"], entry["type"], entry["task"], ttl=settings.model_ttl, **entry["options"]- )+ model = model_cache.get(entry["name"], entry["type"], entry["task"], ttl=settings.model_ttl, **entry["options"])inputs = [payload]for dep in model.depends:try:@@ -198,14 +209,15 @@ async def run_inference(payload: Image | str, entries: InferenceEntries) -> Infemessage = f"Task {entry['task']} of type {entry['type']} depends on output of {dep}"raise HTTPException(400, message)model = await load(model)- output = await run(model.predict, *inputs, **entry["options"])+ output = await attempt(model, partial(model.predict, *inputs, **entry["options"]), MODEL_FILE_ERRORS)outputs[model.identity] = outputresponse[entry["task"]] = output- without_deps, with_deps = entries- await asyncio.gather(*[_run_inference(entry) for entry in without_deps])- if with_deps:- await asyncio.gather(*[_run_inference(entry) for entry in with_deps])+ for stage in entries: # those that depend on another's output run after it+ results = await asyncio.gather(*[_run_inference(entry) for entry in stage], return_exceptions=True)+ for result in results:+ if isinstance(result, BaseException):+ raise resultif isinstance(payload, Image):response["imageHeight"], response["imageWidth"] = payload.height, payload.width@@ -220,42 +232,40 @@ async def run(func: Callable[..., T], *args: Any, **kwargs: Any) -> T:async def load(model: InferenceModel) -> InferenceModel:- if model.loaded:- return model+ if not model.loaded:+ await attempt(model, model.load, (OSError, BadZipFile, *MODEL_FILE_ERRORS))+ return model+- def _load(model: InferenceModel) -> InferenceModel:- if model.load_attempts > 1:+async def attempt(model: InferenceModel, func: Callable[[], T], corrupt: tuple[type[Exception], ...]) -> T:+ def _attempt() -> T:+ if not model.loaded and model.load_attempts > 1:raise HTTPException(500, f"Failed to load model '{model.model_name}'")- with lock:- try:- model.load()- except FileNotFoundError as e:- if model.model_format == ModelFormat.ONNX:- raise e- log.warning(- f"{model.model_format.upper()} is available, but model '{model.model_name}' does not support it.",- exc_info=e,- )- model.model_format = ModelFormat.ONNX- model.load()- return model+ try:+ return func()+ except FileNotFoundError as e:+ if model.model_format == ModelFormat.ONNX:+ raise e+ log.warning(+ f"{model.model_format.upper()} is available, but model '{model.model_name}' does not support it.",+ exc_info=e,+ )+ model.unload()+ model.model_format = ModelFormat.ONNX+ return func()try:- return await run(_load, model)- except (OSError, InvalidProtobuf, BadZipFile, NoSuchFile):+ return await run(_attempt)+ except corrupt:log.warning(f"Failed to load {model.model_type.replace('_', ' ')} model '{model.model_name}'. Clearing cache.")+ model.unload()model.clear_cache()- return await run(_load, model)+ return await run(_attempt)async def idle_shutdown_task() -> None:while True:- if (- last_called is not None- and not active_requests- and not lock.locked()- and time.time() - last_called > settings.model_ttl- ):+ if last_called is not None and not active_requests and time.time() - last_called > settings.model_ttl:log.info("Shutting down due to inactivity.")os.kill(os.getpid(), signal.SIGINT)breakdiff --git a/machine-learning/immich_ml/models/__init__.py b/machine-learning/immich_ml/models/__init__.pyindex 6c3a74c963..f939391307 100644--- a/machine-learning/immich_ml/models/__init__.py+++ b/machine-learning/immich_ml/models/__init__.py@@ -1,5 +1,3 @@-from typing import Any-from immich_ml.models.base import InferenceModelfrom immich_ml.models.clip.textual import MClipTextualEncoder, OpenClipTextualEncoderfrom immich_ml.models.clip.visual import OpenClipVisualEncoder@@ -40,9 +38,5 @@ def get_model_class(model_name: str, model_type: ModelType, model_task: ModelTasraise ValueError(f"Unknown model combination: {source}, {model_type}, {model_task}")-def from_model_type(model_name: str, model_type: ModelType, model_task: ModelTask, **kwargs: Any) -> InferenceModel:- return get_model_class(model_name, model_type, model_task)(model_name, **kwargs)--def get_model_deps(model_name: str, model_type: ModelType, model_task: ModelTask) -> list[tuple[ModelType, ModelTask]]:return get_model_class(model_name, model_type, model_task).dependsdiff --git a/machine-learning/immich_ml/models/base.py b/machine-learning/immich_ml/models/base.pyindex 5c40b21f32..a54a80485d 100644--- a/machine-learning/immich_ml/models/base.py+++ b/machine-learning/immich_ml/models/base.py@@ -3,6 +3,7 @@ from __future__ import annotationsfrom abc import ABC, abstractmethodfrom pathlib import Pathfrom shutil import rmtree+from threading import Lockfrom typing import Any, ClassVarfrom huggingface_hub import snapshot_download@@ -15,10 +16,17 @@ from ..config import clean_name, log, settingsfrom ..schemas import ModelFormat, ModelIdentity, ModelSession, ModelTask, ModelTypefrom ..sessions.ann import AnnSession+_IGNORED_PATTERNS: dict[ModelFormat, list[str]] = {+ ModelFormat.ONNX: ["*.armnn", "*.rknn"],+ ModelFormat.ARMNN: ["*.rknn"],+ ModelFormat.RKNN: ["*.armnn"],+}+class InferenceModel(ABC):depends: ClassVar[list[ModelIdentity]]identity: ClassVar[ModelIdentity]+ threads: ClassVar[int] = 2 # how many threads one run takes for CPU inferencedef __init__(self,@@ -30,6 +38,7 @@ class InferenceModel(ABC):) -> None:self.loaded = session is not Noneself.load_attempts = 0+ self._load_lock = Lock()self.model_name = clean_name(model_name)self.cache_dir = Path(cache_dir) if cache_dir is not None else self._cache_dir_defaultself.model_format = model_format if model_format is not None else self._model_format_default@@ -37,50 +46,47 @@ class InferenceModel(ABC):self.session = sessiondef download(self) -> None:- if not self.cached:- model_type = self.model_type.replace("-", " ")- log.info(f"Downloading {model_type} model '{self.model_name}' to {self.model_path}. This may take a while.")- self._download()-- def load(self) -> None:- if self.loaded:+ if self.cached:return- self.load_attempts += 1+ model_type = self.model_type.replace("-", " ")+ log.info(f"Downloading {model_type} model '{self.model_name}' to {self.model_dir}. This may take a while.")+ snapshot_download(+ f"{settings.model_organization}/{self.model_name}",+ revision=settings.model_revision,+ cache_dir=self.cache_dir,+ local_dir=self.cache_dir,+ ignore_patterns=_IGNORED_PATTERNS.get(self.model_format, []),+ )+ if not self.cached: # the repository has nothing in this format, which is what sends a model back to ONNX+ raise FileNotFoundError(f"Model file not found: {self.model_path}")- self.download()- attempt = f"Attempt #{self.load_attempts} to load" if self.load_attempts > 1 else "Loading"- log.info(f"{attempt} {self.model_type.replace('-', ' ')} model '{self.model_name}' to memory")- self.session = self._load()- self.loaded = True+ def load(self) -> None:+ with self._load_lock: # two requests may arrive for a model neither has loaded+ if self.loaded:+ return+ self.load_attempts += 1++ self.download()+ attempt = f"Attempt #{self.load_attempts} to load" if self.load_attempts > 1 else "Loading"+ log.info(f"{attempt} {self.model_type.replace('-', ' ')} model '{self.model_name}' to memory")+ self.session = self._load()+ self.loaded = True++ def unload(self) -> None:+ with self._load_lock:+ if self.loaded:+ del self.session+ self.loaded = Falsedef predict(self, *inputs: Any, **model_kwargs: Any) -> Any:self.load()- if model_kwargs:- self.configure(**model_kwargs)return self._predict(*inputs, **model_kwargs)@abstractmethoddef _predict(self, *inputs: Any, **model_kwargs: Any) -> Any: ...- def configure(self, **kwargs: Any) -> None:- pass-- def _download(self) -> None:- ignored_patterns: dict[ModelFormat, list[str]] = {- ModelFormat.ONNX: ["*.armnn", "*.rknn"],- ModelFormat.ARMNN: ["*.rknn"],- ModelFormat.RKNN: ["*.armnn"],- }-- snapshot_download(- f"immich-app/{clean_name(self.model_name)}",- cache_dir=self.cache_dir,- local_dir=self.cache_dir,- ignore_patterns=ignored_patterns.get(self.model_format, []),- )-def _load(self) -> ModelSession:- return self._make_session(self.model_path)+ return self._make_session()def clear_cache(self) -> None:if not self.cache_dir.exists():@@ -104,20 +110,14 @@ class InferenceModel(ABC):self.cache_dir.unlink()self.cache_dir.mkdir(parents=True, exist_ok=True)- def _make_session(self, model_path: Path) -> ModelSession:- if not model_path.is_file():- raise FileNotFoundError(f"Model file not found: {model_path}")-- match model_path.suffix:- case ".armnn":- session: ModelSession = AnnSession(model_path)- case ".onnx":- session = OrtSession(model_path)- case ".rknn":- session = rknn.RknnSession(model_path)- case _:- raise ValueError(f"Unsupported model file type: {model_path.suffix}")- return session+ def _make_session(self) -> ModelSession:+ match self.model_format:+ case ModelFormat.ARMNN:+ return AnnSession(self.model_path)+ case ModelFormat.ONNX:+ return OrtSession(self.model_path, threads=self.threads)+ case ModelFormat.RKNN:+ return rknn.RknnSession(self.model_path)def model_path_for_format(self, model_format: ModelFormat) -> Path:model_path_prefix = rknn.model_prefix if model_format == ModelFormat.RKNN else Nonediff --git a/machine-learning/immich_ml/models/cache.py b/machine-learning/immich_ml/models/cache.pyindex d8f9ca81bd..1a8dad2eb0 100644--- a/machine-learning/immich_ml/models/cache.py+++ b/machine-learning/immich_ml/models/cache.py@@ -1,60 +1,43 @@+import asynciofrom typing import Any-from aiocache.backends.memory import SimpleMemoryCache-from aiocache.lock import OptimisticLock-from aiocache.plugins import TimingPlugin--from immich_ml.models import from_model_type+from immich_ml import allocator+from immich_ml.models import get_model_classfrom immich_ml.models.base import InferenceModel-from ..schemas import ModelTask, ModelType, has_profiling+from ..schemas import ModelTask, ModelTypeclass ModelCache:- """Fetches a model from an in-memory cache, instantiating it if it's missing."""-- def __init__(- self,- revalidate: bool = False,- timeout: int | None = None,- profiling: bool = False,- ) -> None:- """- Args:- revalidate: Resets TTL on cache hit. Useful to keep models in memory while active. Defaults to False.- timeout: Maximum allowed time for model to load. Disabled if None. Defaults to None.- profiling: Collects metrics for cache operations, adding slight overhead. Defaults to False.- """-- plugins = []-- if profiling:- plugins.append(TimingPlugin())-- self.should_revalidate = revalidate+ def __init__(self) -> None:+ self._models: dict[tuple[Any, ...], InferenceModel] = {}+ self._expiries: dict[tuple[Any, ...], asyncio.TimerHandle] = {}- self.cache = SimpleMemoryCache(timeout=timeout, plugins=plugins, namespace=None)-- async def get(- self, model_name: str, model_type: ModelType, model_task: ModelTask, **model_kwargs: Any+ def get(+ self, model_name: str, model_type: ModelType, model_task: ModelTask, ttl: int | None = None, **options: Any) -> InferenceModel:- key = f"{model_name}{model_type}{model_task}"-- async with OptimisticLock(self.cache, key) as lock:- model: InferenceModel | None = await self.cache.get(key)- if model is None:- model = from_model_type(model_name, model_type, model_task, **model_kwargs)- await lock.cas(model, ttl=model_kwargs.get("ttl", None))- elif self.should_revalidate:- await self.revalidate(key, model_kwargs.get("ttl", None))- return model-- async def get_profiling(self) -> dict[str, float] | None:- if not has_profiling(self.cache):- return None-- return self.cache.profiling-- async def revalidate(self, key: str, ttl: int | None) -> None:- if ttl is not None and key in self.cache._handlers:- await self.cache.expire(key, ttl)+ model_cls = get_model_class(model_name, model_type, model_task)+ key = (model_name, model_type, model_task)+ if key not in self._models:+ self._models[key] = model_cls(model_name, **options)+ elif key not in self._expiries:+ ttl = None # a model that was preloaded stays+ if ttl:+ if key in self._expiries:+ self._expiries[key].cancel()+ self._expiries[key] = asyncio.get_running_loop().call_later(ttl, self._evict, key)+ return self._models[key]++ def clear(self) -> None:+ if not self._models:+ return+ self._models.clear()+ for key in list(self._expiries.keys()):+ self._expiries.pop(key).cancel()+ allocator.release()++ def _evict(self, key: tuple[Any, ...]) -> None:+ if key not in self._models:+ return+ del self._models[key], self._expiries[key]+ allocator.release()diff --git a/machine-learning/immich_ml/models/clip/textual.py b/machine-learning/immich_ml/models/clip/textual.pyindex 421459fc03..003d30e561 100644--- a/machine-learning/immich_ml/models/clip/textual.py+++ b/machine-learning/immich_ml/models/clip/textual.py@@ -18,6 +18,7 @@ from immich_ml.schemas import ModelSession, ModelTask, ModelTypeclass BaseCLIPTextualEncoder(InferenceModel):depends = []identity = (ModelType.TEXTUAL, ModelTask.SEARCH)+ threads = 4 # lower search latencydef _predict(self, inputs: str, language: str | None = None) -> str:tokens = self.tokenize(inputs, language=language)diff --git a/machine-learning/immich_ml/models/clip/visual.py b/machine-learning/immich_ml/models/clip/visual.pyindex 1d46d8fe10..37362fc535 100644--- a/machine-learning/immich_ml/models/clip/visual.py+++ b/machine-learning/immich_ml/models/clip/visual.py@@ -70,8 +70,16 @@ class OpenClipVisualEncoder(BaseCLIPVisualEncoder):return super()._load()def transform(self, image: Image.Image) -> dict[str, NDArray[np.float32]]:- image = resize_pil(image, self.size)- image = crop_pil(image, self.size)+ image = self._resize(image)image_np = to_numpy(image)image_np = normalize(image_np, self.mean, self.std)return {"image": np.expand_dims(image_np.transpose(2, 0, 1), 0)}++ def _resize(self, image: Image.Image) -> Image.Image:+ match self.preprocess_cfg.get("resize_mode", "shortest"): # open_clip's default for older configs+ case "squash":+ return image.resize((self.size, self.size), resample=self.resampling)+ case "shortest":+ return crop_pil(resize_pil(image, self.size, self.resampling), self.size)+ case mode:+ raise ValueError(f"Unsupported resize_mode {mode!r} in {self.preprocess_cfg_path}")diff --git a/machine-learning/immich_ml/models/facial_recognition/recognition.py b/machine-learning/immich_ml/models/facial_recognition/recognition.pyindex 2b0575ec38..09e48350a2 100644--- a/machine-learning/immich_ml/models/facial_recognition/recognition.py+++ b/machine-learning/immich_ml/models/facial_recognition/recognition.py@@ -33,10 +33,10 @@ class FaceRecognizer(InferenceModel):self.batch_size = max_batch_size if max_batch_size else self._batch_size_defaultdef _load(self) -> ModelSession:- session = self._make_session(self.model_path)+ session = self._make_session()if (not self.batch_size or self.batch_size > 1) and str(session.get_inputs()[0].shape[0]) != "batch":self._add_batch_axis(self.model_path)- session = self._make_session(self.model_path)+ session = self._make_session()return sessiondef _predict(diff --git a/machine-learning/immich_ml/schemas.py b/machine-learning/immich_ml/schemas.pyindex 0dcc540d0f..a88faac9d1 100644--- a/machine-learning/immich_ml/schemas.py+++ b/machine-learning/immich_ml/schemas.py@@ -1,19 +1,12 @@from collections.abc import Sequencefrom enum import Enum-from typing import Any, Literal, Protocol, TypeGuard, TypeVar+from typing import Any, Literal, Protocol, TypeVarimport numpy as npimport numpy.typing as npt-import orjson-from fastapi.responses import JSONResponsefrom typing_extensions import TypedDict-class ORJSONResponse(JSONResponse):- def render(self, content: Any) -> bytes:- return orjson.dumps(content, option=orjson.OPT_SERIALIZE_NUMPY)--class StrEnum(str, Enum):__str__ = str.__str__@@ -56,6 +49,11 @@ class ModelPrecision(StrEnum):FP32 = "FP32"+class ModelOrganization(StrEnum):+ APP = "immich-app"+ TESTING = "immich-testing"++ModelIdentity = tuple[ModelType, ModelTask]@@ -84,10 +82,6 @@ class ModelSession(Protocol):def get_metadata(self) -> dict[str, str]: ...-class HasProfiling(Protocol):- profiling: dict[str, float]--class FaceDetectionOutput(TypedDict):boxes: npt.NDArray[np.float32]scores: npt.NDArray[np.float32]@@ -124,8 +118,4 @@ InferenceEntries = tuple[list[InferenceEntry], list[InferenceEntry]]InferenceResponse = dict[ModelTask | Literal["imageHeight"] | Literal["imageWidth"], Any]-def has_profiling(obj: Any) -> TypeGuard[HasProfiling]:- return hasattr(obj, "profiling") and isinstance(obj.profiling, dict)--T = TypeVar("T")diff --git a/machine-learning/immich_ml/sessions/ort.py b/machine-learning/immich_ml/sessions/ort.pyindex 95a40782c7..1f887256bb 100644--- a/machine-learning/immich_ml/sessions/ort.py+++ b/machine-learning/immich_ml/sessions/ort.py@@ -1,5 +1,8 @@from __future__ import annotations+import ctypes+import platform+import sysfrom collections.abc import Sequencefrom pathlib import Pathfrom threading import Lock@@ -53,9 +56,13 @@ class OrtSession:providers: list[str] | None = None,provider_options: list[dict[str, Any]] | None = None,sess_options: ort.SessionOptions | None = None,+ threads: int = 2,):self.model_path = Path(model_path)+ self.threads = threadsself.providers = providers if providers is not None else self._providers_default+ self.disabled_optimizers = _disabled_optimizers_default(self.providers)+ log.debug(f"Setting disabled_optimizers to {self.disabled_optimizers}")self.provider_options = provider_options if provider_options is not None else self._provider_options_defaultself.sess_options = sess_options if sess_options is not None else self._sess_options_defaultself.session = ort.InferenceSession(@@ -63,6 +70,7 @@ class OrtSession:providers=self.providers,provider_options=self.provider_options,sess_options=self.sess_options,+ disabled_optimizers=self.disabled_optimizers,)def get_inputs(self) -> Sequence[SessionNode]:@@ -184,6 +192,8 @@ class OrtSession:@propertydef _sess_options_default(self) -> ort.SessionOptions:sess_options = ort.SessionOptions()+ # some CPUs slow down many times over on subnormal operands, and ORT clears the flush in its threads otherwise+ sess_options.add_session_config_entry("session.set_denormal_as_zero", "1")sess_options.enable_cpu_mem_arena = settings.model_arena# avoid thread contention between models@@ -198,9 +208,32 @@ class OrtSession:if settings.model_intra_op_threads > 0:sess_options.intra_op_num_threads = settings.model_intra_op_threadselif settings.model_intra_op_threads == 0 and self.providers == ["CPUExecutionProvider"]:- sess_options.intra_op_num_threads = 2+ sess_options.intra_op_num_threads = self.threadsif sess_options.inter_op_num_threads > 1:sess_options.execution_mode = ort.ExecutionMode.ORT_PARALLELreturn sess_options+++def _disabled_optimizers_default(providers: list[str]) -> list[str]:+ disabled_optimizers: list[str] = []+ if platform.machine() in ("arm64", "aarch64"): # as macOS and Linux name the same architecture+ disabled_optimizers.append("ConvAddActivationFusion")++ # the Gemm it makes runs slower than the pair it replaces there, and on CUDA it also keeps BiasGelu from fusing+ if "CoreMLExecutionProvider" in providers or "CUDAExecutionProvider" in providers:+ disabled_optimizers.append("MatMulAddFusion")++ return disabled_optimizers+++def flush_denormals() -> None:+ """Reads subnormal floats as zero on the calling thread, which ORT does only on its own threads."""+ if sys.platform != "linux" or platform.machine() != "x86_64":+ return+ libm = ctypes.CDLL("libm.so.6")+ env = (ctypes.c_uint32 * 8)() # glibc's fenv_t here: the x87 environment, then MXCSR+ libm.fegetenv(env)+ env[7] |= 0x8040 # denormals are zero, flush to zero+ libm.fesetenv(env)diff --git a/machine-learning/pyproject.toml b/machine-learning/pyproject.tomlindex 6af41477b8..84b1d2f4e8 100644--- a/machine-learning/pyproject.toml+++ b/machine-learning/pyproject.toml@@ -6,7 +6,6 @@ authors = [{ name = "Hau Tran", email = "[email protected]" }]requires-python = ">=3.12,<4.0"readme = "README.md"dependencies = [- "aiocache>=0.12.1,<1.0","fastapi>=0.95.2,<1.0","gunicorn>=21.1.0","huggingface-hub>=1.0,<2.0",@@ -48,7 +47,7 @@ dev = ["locust>=2.15.1", { include-group = "test" }, { include-group = "lint" }][project.optional-dependencies]cpu = ["onnxruntime>=1.23.2,<2"]-cuda = ["onnxruntime-gpu>=1.23.2,<2"]+cuda = ["onnxruntime-gpu>=1.27.1,<1.28"] # TODO: last ORT release supporting CUDA 12.2, also need to handle CUDA 13openvino = ["onnxruntime-openvino>=1.24.1,<2"]armnn = ["onnxruntime>=1.23.2,<2"]rknn = ["onnxruntime>=1.23.2,<2", "rknn-toolkit-lite2>=2.3.0,<3"]@@ -56,6 +55,16 @@ rocm = ["onnxruntime-migraphx>=1.23.2,<2"][tool.uv]compile-bytecode = true+override-dependencies = ["opencv-python ; sys_platform == 'never'"] # only install the headless version++[tool.uv.sources]+onnxruntime-gpu = { index = "onnxruntime-cuda-12" }++[[tool.uv.index]]+name = "onnxruntime-cuda-12"+url = "https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/"+explicit = true+[tool.hatch.build.targets.sdist]include = ["immich_ml"]diff --git a/machine-learning/test_main.py b/machine-learning/test_main.pyindex a50bea00b7..27191d67f6 100644--- a/machine-learning/test_main.py+++ b/machine-learning/test_main.py@@ -1,5 +1,10 @@import jsonimport os+import platform+import sys+import threading+import time+from concurrent.futures import ThreadPoolExecutorfrom io import BytesIOfrom pathlib import Pathfrom random import randint@@ -17,8 +22,10 @@ from PIL import Imagefrom pytest import MonkeyPatchfrom pytest_mock import MockerFixture-from immich_ml.config import MaxBatchSize, Settings, settings-from immich_ml.main import load, preload_models+from immich_ml import allocator+from immich_ml.config import MaxBatchSize, PreloadModelData, Settings, settings+from immich_ml.main import app, lifespan, load, preload_models, update_state+from immich_ml.main import run_inference as run_requestfrom immich_ml.models.base import InferenceModelfrom immich_ml.models.cache import ModelCachefrom immich_ml.models.clip.textual import MClipTextualEncoder, OpenClipTextualEncoder@@ -29,7 +36,7 @@ from immich_ml.models.ocr.detection import TextDetectorfrom immich_ml.models.ocr.recognition import TextRecognizerfrom immich_ml.schemas import ModelFormat, ModelPrecision, ModelTask, ModelTypefrom immich_ml.sessions.ann import AnnSession-from immich_ml.sessions.ort import OrtSession+from immich_ml.sessions.ort import OrtSession, flush_denormalsfrom immich_ml.sessions.rknn import RknnSession, run_inference@@ -162,6 +169,7 @@ class TestBase:snapshot_download.assert_called_once_with("immich-app/ViT-B-32__openai",+ revision=settings.model_revision,cache_dir=encoder.cache_dir,local_dir=encoder.cache_dir,ignore_patterns=["*.armnn", "*.rknn"],@@ -171,27 +179,18 @@ class TestBase:encoder = OpenClipTextualEncoder("ViT-B-32__openai", model_format=ModelFormat.ARMNN)encoder.download()- snapshot_download.assert_called_once_with(- "immich-app/ViT-B-32__openai",- cache_dir=encoder.cache_dir,- local_dir=encoder.cache_dir,- ignore_patterns=["*.rknn"],- )+ assert snapshot_download.call_args.kwargs["ignore_patterns"] == ["*.rknn"]def test_download_downloads_rknn_if_preferred_format(self, snapshot_download: mock.Mock) -> None:encoder = OpenClipTextualEncoder("ViT-B-32__openai", model_format=ModelFormat.RKNN)encoder.download()- snapshot_download.assert_called_once_with(- "immich-app/ViT-B-32__openai",- cache_dir=encoder.cache_dir,- local_dir=encoder.cache_dir,- ignore_patterns=["*.armnn"],- )+ assert snapshot_download.call_args.kwargs["ignore_patterns"] == ["*.armnn"]def test_throws_exception_if_model_path_does_not_exist(- self, snapshot_download: mock.Mock, ort_session: mock.Mock, path: mock.Mock+ self, ort_session: mock.Mock, path: mock.Mock, mocker: MockerFixture) -> None:+ snapshot_download = mocker.patch("immich_ml.models.base.snapshot_download") # which brings no such filepath.return_value.__truediv__.return_value.__truediv__.return_value.is_file.return_value = Falseencoder = OpenClipTextualEncoder("ViT-B-32__openai", cache_dir=path)@@ -202,6 +201,18 @@ class TestBase:snapshot_download.assert_called_once()ort_session.assert_not_called()+ @pytest.mark.parametrize(+ ("symbols", "called"), [(["mi_collect", "malloc_trim"], "mi_collect"), (["malloc_trim"], "malloc_trim")]+ )+ def test_returns_memory_through_the_allocator_the_process_runs_on(+ self, mocker: MockerFixture, symbols: list[str], called: str+ ) -> None:+ process = mocker.patch("immich_ml.allocator._process", mock.Mock(spec=symbols))++ allocator.release()++ getattr(process, called).assert_called_once()+@pytest.mark.usefixtures("ort_session")class TestOrtSession:@@ -370,12 +381,63 @@ class TestOrtSession:assert session.provider_options == []+ @pytest.mark.parametrize(+ ("machine", "disabled"),+ [("arm64", ["ConvAddActivationFusion"]), ("aarch64", ["ConvAddActivationFusion"]), ("x86_64", [])],+ )+ def test_disables_the_fusion_that_is_slower_on_arm_whatever_the_os_calls_it(+ self, ort_session: mock.Mock, mocker: MockerFixture, machine: str, disabled: list[str]+ ) -> None:+ mocker.patch("immich_ml.sessions.ort.platform.machine", return_value=machine)++ OrtSession("ViT-B-32__openai", providers=["CPUExecutionProvider"])++ assert ort_session.call_args.kwargs["disabled_optimizers"] == disabled++ @pytest.mark.parametrize(+ ("providers", "disabled"),+ [+ (["CUDAExecutionProvider", "CPUExecutionProvider"], ["MatMulAddFusion"]),+ (["CoreMLExecutionProvider", "CPUExecutionProvider"], ["MatMulAddFusion"]),+ (["CPUExecutionProvider"], []),+ ],+ )+ def test_disables_the_fusion_that_is_slower_on_cuda_and_coreml(+ self, ort_session: mock.Mock, mocker: MockerFixture, providers: list[str], disabled: list[str]+ ) -> None:+ mocker.patch("immich_ml.sessions.ort.platform.machine", return_value="x86_64")++ OrtSession("ViT-B-32__openai", providers=providers)++ assert ort_session.call_args.kwargs["disabled_optimizers"] == disabled+def test_sets_default_sess_options_if_cpu(self) -> None:session = OrtSession("ViT-B-32__openai", providers=["CPUExecutionProvider"])assert session.sess_options.execution_mode == ort.ExecutionMode.ORT_SEQUENTIALassert session.sess_options.inter_op_num_threads == 1assert session.sess_options.intra_op_num_threads == 2+ assert session.sess_options.get_session_config_entry("session.set_denormal_as_zero") == "1"++ @pytest.mark.skipif(sys.platform != "linux" or platform.machine() != "x86_64", reason="an x86 register")+ def test_flushes_denormals_on_the_calling_thread(self) -> None:+ results: list[float] = []++ def probe() -> None: # a fresh thread, as a request thread is+ flush_denormals()+ results.append(float((np.array([1e-39], np.float32) * np.float32(1))[0]))++ thread = threading.Thread(target=probe)+ thread.start()+ thread.join()++ assert results == [0.0]++ def test_gives_a_model_the_cpu_threads_it_asks_for(self) -> None:+ session = OrtSession("ViT-B-32__openai", providers=["CPUExecutionProvider"], threads=4)++ assert session.sess_options.intra_op_num_threads == 4+ assert OpenClipTextualEncoder.threads == 4 and OpenClipVisualEncoder.threads == 2@pytest.mark.ov_device_ids(["CPU"])def test_sets_default_sess_options_if_openvino_cpu(self, ov_device_ids: list[str]) -> None:@@ -595,6 +657,22 @@ class TestCLIP:assert len(embedding) == clip_model_cfg["embed_dim"]mocked.run.assert_called_once()+ def test_visual_squashes_when_the_tower_was_calibrated_that_way(+ self, mocker: MockerFixture, clip_model_cfg: dict[str, Any], clip_preprocess_cfg: dict[str, Any]+ ) -> None:+ mocker.patch.object(OpenClipVisualEncoder, "download")+ mocker.patch.object(OpenClipVisualEncoder, "model_cfg", clip_model_cfg)+ mocker.patch.object(OpenClipVisualEncoder, "preprocess_cfg", clip_preprocess_cfg | {"resize_mode": "squash"})+ session = mocker.patch.object(InferenceModel, "_make_session", autospec=True).return_value+ session.run.return_value = [[self.embedding]]++ # a stripe far enough left that a shortest-side crop of this frame would discard it+ image = Image.new("RGB", (600, 200), "black")+ image.paste(Image.new("RGB", (20, 200), "white"), (0, 0))+ OpenClipVisualEncoder("ViT-B-32__openai", cache_dir="test_cache").predict(image)++ assert (session.run.call_args.args[1]["image"][0, :, 0, 0] > 0).all() # white, where black normalizes below 0+def test_basic_text(self,mocker: MockerFixture,@@ -1179,56 +1257,76 @@ class TestOcr:class TestCache:async def test_caches(self, mock_get_model: mock.Mock) -> None:model_cache = ModelCache()- await model_cache.get("test_model_name", ModelType.RECOGNITION, ModelTask.FACIAL_RECOGNITION)- await model_cache.get("test_model_name", ModelType.RECOGNITION, ModelTask.FACIAL_RECOGNITION)- assert len(model_cache.cache._cache) == 1- mock_get_model.assert_called_once()+ model_cache.get("test_model_name", ModelType.RECOGNITION, ModelTask.FACIAL_RECOGNITION)+ model_cache.get("test_model_name", ModelType.RECOGNITION, ModelTask.FACIAL_RECOGNITION)+ assert len(model_cache._models) == 1+ mock_get_model.return_value.assert_called_once()async def test_kwargs_used(self, mock_get_model: mock.Mock) -> None:model_cache = ModelCache()- await model_cache.get(- "test_model_name", ModelType.RECOGNITION, ModelTask.FACIAL_RECOGNITION, cache_dir="test_cache"- )- mock_get_model.assert_called_once_with(- "test_model_name", ModelType.RECOGNITION, ModelTask.FACIAL_RECOGNITION, cache_dir="test_cache"- )+ model_cache.get("test_model_name", ModelType.RECOGNITION, ModelTask.FACIAL_RECOGNITION, cache_dir="test_cache")+ mock_get_model.return_value.assert_called_once_with("test_model_name", cache_dir="test_cache")async def test_different_clip(self, mock_get_model: mock.Mock) -> None:model_cache = ModelCache()- await model_cache.get("test_model_name", ModelType.VISUAL, ModelTask.SEARCH)- await model_cache.get("test_model_name", ModelType.TEXTUAL, ModelTask.SEARCH)- mock_get_model.assert_has_calls(- [- mock.call("test_model_name", ModelType.VISUAL, ModelTask.SEARCH),- mock.call("test_model_name", ModelType.TEXTUAL, ModelTask.SEARCH),- ]- )- assert len(model_cache.cache._cache) == 2+ model_cache.get("test_model_name", ModelType.VISUAL, ModelTask.SEARCH)+ model_cache.get("test_model_name", ModelType.TEXTUAL, ModelTask.SEARCH)+ assert mock_get_model.call_args_list == [+ mock.call("test_model_name", ModelType.VISUAL, ModelTask.SEARCH),+ mock.call("test_model_name", ModelType.TEXTUAL, ModelTask.SEARCH),+ ]+ assert len(model_cache._models) == 2- @mock.patch("immich_ml.models.cache.OptimisticLock", autospec=True)- async def test_model_ttl(self, mock_lock_cls: mock.Mock, mock_get_model: mock.Mock) -> None:+ async def test_lets_go_of_a_model_unused_for_its_ttl_and_returns_its_memory(+ self, mock_get_model: mock.Mock, mocker: MockerFixture+ ) -> None:+ events: list[str] = []++ class Model:+ def __init__(self, *args: Any, **kwargs: Any) -> None:+ pass++ def __del__(self) -> None:+ events.append("destroyed")++ mock_get_model.return_value = Model+ mocker.patch("immich_ml.models.cache.allocator.release", side_effect=lambda: events.append("released"))+ loop = mocker.patch("immich_ml.models.cache.asyncio.get_running_loop").return_value+ model_cache = ModelCache()++ model_cache.get("test_model_name", ModelType.RECOGNITION, ModelTask.FACIAL_RECOGNITION, ttl=100)+ delay, evict, key = loop.call_later.call_args.args+ evict(key)++ assert delay == 100+ assert events == ["destroyed", "released"]+ assert not model_cache._models++ async def test_a_use_starts_the_ttl_over(self, mock_get_model: mock.Mock, mocker: MockerFixture) -> None:+ loop = mocker.patch("immich_ml.models.cache.asyncio.get_running_loop").return_valuemodel_cache = ModelCache()- await model_cache.get("test_model_name", ModelType.RECOGNITION, ModelTask.FACIAL_RECOGNITION, ttl=100)- mock_lock_cls.return_value.__aenter__.return_value.cas.assert_called_with(mock.ANY, ttl=100)-- @mock.patch("immich_ml.models.cache.SimpleMemoryCache.expire")- async def test_revalidate_get(self, mock_cache_expire: mock.Mock, mock_get_model: mock.Mock) -> None:- model_cache = ModelCache(revalidate=True)- await model_cache.get("test_model_name", ModelType.RECOGNITION, ModelTask.FACIAL_RECOGNITION, ttl=100)- await model_cache.get("test_model_name", ModelType.RECOGNITION, ModelTask.FACIAL_RECOGNITION, ttl=100)- mock_cache_expire.assert_called_once_with(mock.ANY, 100)-- async def test_profiling(self, mock_get_model: mock.Mock) -> None:- model_cache = ModelCache(profiling=True)- await model_cache.get("test_model_name", ModelType.RECOGNITION, ModelTask.FACIAL_RECOGNITION, ttl=100)- profiling = await model_cache.get_profiling()- assert isinstance(profiling, dict)- assert profiling == model_cache.cache.profiling++ model_cache.get("test_model_name", ModelType.RECOGNITION, ModelTask.FACIAL_RECOGNITION, ttl=100)+ model_cache.get("test_model_name", ModelType.RECOGNITION, ModelTask.FACIAL_RECOGNITION, ttl=100)++ loop.call_later.return_value.cancel.assert_called_once()+ assert loop.call_later.call_count == 2++ async def test_keeps_a_preloaded_model_whatever_ttl_a_request_names(+ self, mock_get_model: mock.Mock, mocker: MockerFixture+ ) -> None:+ loop = mocker.patch("immich_ml.models.cache.asyncio.get_running_loop").return_value+ model_cache = ModelCache()++ model_cache.get("test_model_name", ModelType.RECOGNITION, ModelTask.FACIAL_RECOGNITION)+ model_cache.get("test_model_name", ModelType.RECOGNITION, ModelTask.FACIAL_RECOGNITION, ttl=100)++ loop.call_later.assert_not_called()async def test_loads_mclip(self) -> None:model_cache = ModelCache()- model = await model_cache.get("XLM-Roberta-Large-Vit-B-32", ModelType.TEXTUAL, ModelTask.SEARCH)+ model = model_cache.get("XLM-Roberta-Large-Vit-B-32", ModelType.TEXTUAL, ModelTask.SEARCH)assert isinstance(model, MClipTextualEncoder)assert model.model_name == "XLM-Roberta-Large-Vit-B-32"@@ -1238,13 +1336,13 @@ class TestCache:model_cache = ModelCache()with pytest.raises(ValueError):- await model_cache.get("XLM-Roberta-Large-Vit-B-32", ModelType.TEXTUAL, invalid)+ model_cache.get("XLM-Roberta-Large-Vit-B-32", ModelType.TEXTUAL, invalid)async def test_raises_exception_if_unknown_model_name(self) -> None:model_cache = ModelCache()with pytest.raises(ValueError):- await model_cache.get("test_model_name", ModelType.TEXTUAL, ModelTask.SEARCH)+ model_cache.get("test_model_name", ModelType.TEXTUAL, ModelTask.SEARCH)async def test_preloads_clip_models(self, monkeypatch: MonkeyPatch, mock_get_model: mock.Mock) -> None:os.environ["MACHINE_LEARNING_PRELOAD__CLIP__TEXTUAL"] = "ViT-B-32__openai"@@ -1528,3 +1626,64 @@ class TestPredictionEndpoints:parsed_embedding = orjson.loads(embedding)assert np.allclose(expected_face["embedding"], parsed_embedding)assert np.allclose(expected_face["score"], actual_face["score"])+++async def test_waits_for_every_entry_before_failing_the_request(mocker: MockerFixture) -> None:+ finished = threading.Event()++ def model(task: ModelTask, load: Callable[[], None]) -> mock.Mock:+ stub = mock.Mock(+ spec=InferenceModel, depends=[], loaded=False, load_attempts=0, identity=(ModelType.VISUAL, task)+ )+ stub.load.side_effect = load+ return stub++ def slow() -> None:+ time.sleep(0.2)+ finished.set()++ models = {+ ModelTask.SEARCH: model(ModelTask.SEARCH, slow),+ ModelTask.OCR: model(ModelTask.OCR, mock.Mock(side_effect=RuntimeError("fails at once"))),+ }+ mocker.patch("immich_ml.main.model_cache.get", side_effect=lambda name, type, task, **options: models[task])+ entries: Any = [+ {"name": "a", "task": ModelTask.SEARCH, "type": ModelType.VISUAL, "options": {}},+ {"name": "b", "task": ModelTask.OCR, "type": ModelType.DETECTION, "options": {}},+ ]++ with ThreadPoolExecutor(2) as pool, pytest.raises(RuntimeError, match="fails at once"):+ mocker.patch("immich_ml.main.thread_pool", pool)+ await run_request("text", (entries, []))++ assert finished.is_set() # or it would still be loading with no request counted as active+++async def test_returns_what_a_preload_freed(mocker: MockerFixture) -> None:+ events: list[str] = []+ mocker.patch.object(settings, "preload", PreloadModelData())+ mocker.patch("immich_ml.main.preload_models", side_effect=lambda _: events.append("preloaded"))+ mocker.patch("immich_ml.main.allocator.release", side_effect=lambda: events.append("released"))++ async with lifespan(app):+ assert events == ["preloaded", "released"]+++async def test_returns_memory_once_requests_stop_unless_another_arrives(mocker: MockerFixture) -> None:+ mocker.patch("immich_ml.main.release", None)+ loop = mocker.patch("immich_ml.main.asyncio.get_running_loop").return_value+ first, second, third = update_state(), update_state(), update_state()+ await anext(first)+ await anext(second)++ await first.aclose()+ loop.call_later.assert_not_called()+ await second.aclose()+ loop.call_later.assert_called_once_with(5, allocator.release)++ await anext(third)+ loop.call_later.return_value.cancel.assert_called_once()+ await third.aclose()diff --git a/machine-learning/uv.lock b/machine-learning/uv.lockindex 6ca9530355..5c84de4a42 100644--- a/machine-learning/uv.lock+++ b/machine-learning/uv.lock@@ -13,14 +13,8 @@ resolution-markers = ["(python_full_version < '3.13' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.13' and sys_platform != 'darwin' and sys_platform != 'linux')",]-[[package]]-name = "aiocache"-version = "0.12.3"-source = { registry = "https://pypi.org/simple" }-sdist = { url = "https://files.pythonhosted.org/packages/7a/64/b945b8025a9d1e6e2138845f4022165d3b337f55f50984fbc6a4c0a1e355/aiocache-0.12.3.tar.gz", hash = "sha256:f528b27bf4d436b497a1d0d1a8f59a542c153ab1e37c3621713cb376d44c4713", size = 132196, upload-time = "2024-09-25T13:20:23.823Z" }-wheels = [- { url = "https://files.pythonhosted.org/packages/37/d7/15d67e05b235d1ed8c3ce61688fe4d84130e72af1657acadfaac3479f4cf/aiocache-0.12.3-py2.py3-none-any.whl", hash = "sha256:889086fc24710f431937b87ad3720a289f7fc31c4fd8b68e9f918b9bacd8270d", size = 28199, upload-time = "2024-09-25T13:20:22.688Z" },-]+[manifest]+overrides = [{ name = "opencv-python", marker = "sys_platform == 'never'" }][[package]]name = "annotated-doc"@@ -685,7 +679,6 @@ name = "immich-ml"version = "3.2.0"source = { editable = "." }dependencies = [- { name = "aiocache" },{ name = "fastapi" },{ name = "gunicorn" },{ name = "huggingface-hub" },@@ -766,7 +759,6 @@ types = [[package.metadata]requires-dist = [- { name = "aiocache", specifier = ">=0.12.1,<1.0" },{ name = "fastapi", specifier = ">=0.95.2,<1.0" },{ name = "gunicorn", specifier = ">=21.1.0" },{ name = "huggingface-hub", specifier = ">=1.0,<2.0" },@@ -775,7 +767,7 @@ requires-dist = [{ name = "onnxruntime", marker = "extra == 'armnn'", specifier = ">=1.23.2,<2" },{ name = "onnxruntime", marker = "extra == 'cpu'", specifier = ">=1.23.2,<2" },{ name = "onnxruntime", marker = "extra == 'rknn'", specifier = ">=1.23.2,<2" },- { name = "onnxruntime-gpu", marker = "extra == 'cuda'", specifier = ">=1.23.2,<2" },+ { name = "onnxruntime-gpu", marker = "extra == 'cuda'", specifier = ">=1.27.1,<1.28", index = "https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/" },{ name = "onnxruntime-migraphx", marker = "extra == 'rocm'", specifier = ">=1.23.2,<2" },{ name = "onnxruntime-openvino", marker = "extra == 'openvino'", specifier = ">=1.24.1,<2" },{ name = "opencv-python-headless", specifier = ">=4.7.0.72,<5.0" },@@ -1326,8 +1318,8 @@ wheels = [[[package]]name = "onnxruntime-gpu"-version = "1.26.0"-source = { registry = "https://pypi.org/simple" }+version = "1.27.1"+source = { registry = "https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/" }dependencies = [{ name = "flatbuffers" },{ name = "numpy" },@@ -1335,14 +1327,14 @@ dependencies = [{ name = "protobuf" },]wheels = [- { url = "https://files.pythonhosted.org/packages/94/fd/59bee7cffaa435da44fefdeb63e29c61de4dbfa4b279852f59cd02c042ae/onnxruntime_gpu-1.26.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3c01119ed4d9449d60367fa8ccffcd02bd3fe736754284e4b198d131f54edad6", size = 276971796, upload-time = "2026-05-08T19:15:46.192Z" },- { url = "https://files.pythonhosted.org/packages/a4/e4/9b378a5466ea0bed65e5beb8e09254973c580a6522810a38afbcc45e5105/onnxruntime_gpu-1.26.0-cp312-cp312-win_amd64.whl", hash = "sha256:5f49c44689894650990e4c8a857d2edafc276fbd79bba57ceb224bd18d25d491", size = 226548963, upload-time = "2026-05-08T19:09:34.925Z" },- { url = "https://files.pythonhosted.org/packages/dd/97/fe8979f44b9275654b42f7bb556e30789b71a1b22998c83b540df2b1b774/onnxruntime_gpu-1.26.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cfda2fad535595bfc3e570eb588092717711dcb2957656d814695e0c9ceb1508", size = 276974871, upload-time = "2026-05-08T19:15:58.052Z" },- { url = "https://files.pythonhosted.org/packages/67/3f/59f1777a394625ecc9a85636de57dc47c25dbb5f888da050f1463955a0ce/onnxruntime_gpu-1.26.0-cp313-cp313-win_amd64.whl", hash = "sha256:6ab9f9c741d2e239b2e321ab0d389c04329d4ab7f11e3b92dd3aa7db1c59dee4", size = 226548083, upload-time = "2026-05-08T19:09:44.408Z" },- { url = "https://files.pythonhosted.org/packages/89/96/360328e3c463f7ea08e853c4239c397e83363dd0204de71a710dc1a544bd/onnxruntime_gpu-1.26.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bcf6f347cad9f88a9a625c2b352cf9de927528aedb627ebbc089a201f1990b94", size = 276992052, upload-time = "2026-05-08T19:16:09.892Z" },- { url = "https://files.pythonhosted.org/packages/fd/c8/aa2dc0e79bba577f37d5448bcb32fea79977e07506684d8138c19a0f1077/onnxruntime_gpu-1.26.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e6e4fb1ec9ae1cf456534d9115f106ab2a1ae96fa513b4ed0f4795302b4a2c6", size = 276978254, upload-time = "2026-05-08T19:16:22.096Z" },- { url = "https://files.pythonhosted.org/packages/41/e7/923298431e669567d7ccc2a4c898b6534a47641a051569fd97165fe6d9b8/onnxruntime_gpu-1.26.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e592439b0183d303c2374517b5b392599a3d50b2dc9de949b9b15731ac921c9", size = 229142768, upload-time = "2026-05-08T19:09:54.589Z" },- { url = "https://files.pythonhosted.org/packages/97/91/93ffe5431d154989f5e04864a25a97eea480997d771232bcbbc538188241/onnxruntime_gpu-1.26.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:56dc7b73954ff4bdc71f5b8ab306b6f61be5d007881b6ef423a609e2b9cd088b", size = 276991545, upload-time = "2026-05-08T19:16:33.347Z" },+ { url = "https://aiinfra.pkgs.visualstudio.com/2692857e-05ef-43b4-ba9c-ccf1c22c437c/_packaging/9387c3aa-d9ad-4513-968c-383f6f7f53b8/pypi/download/onnxruntime-gpu/1.27.1/onnxruntime_gpu-1.27.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b258525fb253b97a5c215aeb9bc40f32fd2f562b7cf64c0992bc09793af7e4ba", size = 304202950, upload-time = "2026-07-09T19:59:11Z" },+ { url = "https://aiinfra.pkgs.visualstudio.com/2692857e-05ef-43b4-ba9c-ccf1c22c437c/_packaging/9387c3aa-d9ad-4513-968c-383f6f7f53b8/pypi/download/onnxruntime-gpu/1.27.1/onnxruntime_gpu-1.27.1-cp312-cp312-win_amd64.whl", hash = "sha256:1f4a0b56de1510db3d418bf00760224a00dbbbbb52114ff6a85bb3a5e2e279d4", size = 240898739, upload-time = "2026-07-09T19:56:11Z" },+ { url = "https://aiinfra.pkgs.visualstudio.com/2692857e-05ef-43b4-ba9c-ccf1c22c437c/_packaging/9387c3aa-d9ad-4513-968c-383f6f7f53b8/pypi/download/onnxruntime-gpu/1.27.1/onnxruntime_gpu-1.27.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cccabc89bc1c146849e09e482ee5435709c5d7f2e6bc743f4ad68c1733c23c15", size = 304194132, upload-time = "2026-07-09T19:57:24Z" },+ { url = "https://aiinfra.pkgs.visualstudio.com/2692857e-05ef-43b4-ba9c-ccf1c22c437c/_packaging/9387c3aa-d9ad-4513-968c-383f6f7f53b8/pypi/download/onnxruntime-gpu/1.27.1/onnxruntime_gpu-1.27.1-cp313-cp313-win_amd64.whl", hash = "sha256:da603fef4bc9ffbcf057714f086757c54a295c8bb13976be37077b8bff1810b7", size = 240893326, upload-time = "2026-07-09T19:56:30Z" },+ { url = "https://aiinfra.pkgs.visualstudio.com/2692857e-05ef-43b4-ba9c-ccf1c22c437c/_packaging/9387c3aa-d9ad-4513-968c-383f6f7f53b8/pypi/download/onnxruntime-gpu/1.27.1/onnxruntime_gpu-1.27.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:684cc8b3114da29927db3db7d9b007b579ec3cd283af4e9bc213ee00984d4be2", size = 304224252, upload-time = "2026-07-09T19:58:48Z" },+ { url = "https://aiinfra.pkgs.visualstudio.com/2692857e-05ef-43b4-ba9c-ccf1c22c437c/_packaging/9387c3aa-d9ad-4513-968c-383f6f7f53b8/pypi/download/onnxruntime-gpu/1.27.1/onnxruntime_gpu-1.27.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:72a0d1490c8c6ca8458178dd125968045bbed1b43f6bb617cab54ed74e147a2e", size = 304202506, upload-time = "2026-07-09T19:58:00Z" },+ { url = "https://aiinfra.pkgs.visualstudio.com/2692857e-05ef-43b4-ba9c-ccf1c22c437c/_packaging/9387c3aa-d9ad-4513-968c-383f6f7f53b8/pypi/download/onnxruntime-gpu/1.27.1/onnxruntime_gpu-1.27.1-cp314-cp314-win_amd64.whl", hash = "sha256:e879c96037fe282157fce15cedbc0e998dfdac17f75ee6cd455d08fad6b49dd3", size = 243537318, upload-time = "2026-07-09T19:55:48Z" },+ { url = "https://aiinfra.pkgs.visualstudio.com/2692857e-05ef-43b4-ba9c-ccf1c22c437c/_packaging/9387c3aa-d9ad-4513-968c-383f6f7f53b8/pypi/download/onnxruntime-gpu/1.27.1/onnxruntime_gpu-1.27.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0c32338c64cab70fa6fddc906564f26c7a6ebd8b5ffd7704c6be396e31965b09", size = 304220218, upload-time = "2026-07-09T19:58:25Z" },][[package]]@@ -1384,17 +1376,9 @@ name = "opencv-python"version = "4.11.0.86"source = { registry = "https://pypi.org/simple" }dependencies = [- { name = "numpy" },+ { name = "numpy", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },]sdist = { url = "https://files.pythonhosted.org/packages/17/06/68c27a523103dad5837dc5b87e71285280c4f098c60e4fe8a8db6486ab09/opencv-python-4.11.0.86.tar.gz", hash = "sha256:03d60ccae62304860d232272e4a4fda93c39d595780cb40b161b310244b736a4", size = 95171956, upload-time = "2025-01-16T13:52:24.737Z" }-wheels = [- { url = "https://files.pythonhosted.org/packages/05/4d/53b30a2a3ac1f75f65a59eb29cf2ee7207ce64867db47036ad61743d5a23/opencv_python-4.11.0.86-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:432f67c223f1dc2824f5e73cdfcd9db0efc8710647d4e813012195dc9122a52a", size = 37326322, upload-time = "2025-01-16T13:52:25.887Z" },- { url = "https://files.pythonhosted.org/packages/3b/84/0a67490741867eacdfa37bc18df96e08a9d579583b419010d7f3da8ff503/opencv_python-4.11.0.86-cp37-abi3-macosx_13_0_x86_64.whl", hash = "sha256:9d05ef13d23fe97f575153558653e2d6e87103995d54e6a35db3f282fe1f9c66", size = 56723197, upload-time = "2025-01-16T13:55:21.222Z" },- { url = "https://files.pythonhosted.org/packages/f3/bd/29c126788da65c1fb2b5fb621b7fed0ed5f9122aa22a0868c5e2c15c6d23/opencv_python-4.11.0.86-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b92ae2c8852208817e6776ba1ea0d6b1e0a1b5431e971a2a0ddd2a8cc398202", size = 42230439, upload-time = "2025-01-16T13:51:35.822Z" },- { url = "https://files.pythonhosted.org/packages/2c/8b/90eb44a40476fa0e71e05a0283947cfd74a5d36121a11d926ad6f3193cc4/opencv_python-4.11.0.86-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b02611523803495003bd87362db3e1d2a0454a6a63025dc6658a9830570aa0d", size = 62986597, upload-time = "2025-01-16T13:52:08.836Z" },- { url = "https://files.pythonhosted.org/packages/fb/d7/1d5941a9dde095468b288d989ff6539dd69cd429dbf1b9e839013d21b6f0/opencv_python-4.11.0.86-cp37-abi3-win32.whl", hash = "sha256:810549cb2a4aedaa84ad9a1c92fbfdfc14090e2749cedf2c1589ad8359aa169b", size = 29384337, upload-time = "2025-01-16T13:52:13.549Z" },- { url = "https://files.pythonhosted.org/packages/a4/7d/f1c30a92854540bf789e9cd5dde7ef49bbe63f855b85a2e6b3db8135c591/opencv_python-4.11.0.86-cp37-abi3-win_amd64.whl", hash = "sha256:085ad9b77c18853ea66283e98affefe2de8cc4c1f43eda4c100cf9b2721142ec", size = 39488044, upload-time = "2025-01-16T13:52:21.928Z" },-][[package]]name = "opencv-python-headless"@@ -1933,7 +1917,7 @@ dependencies = [{ name = "colorlog" },{ name = "numpy" },{ name = "omegaconf" },- { name = "opencv-python" },+ { name = "opencv-python", marker = "sys_platform == 'never'" },{ name = "pillow" },{ name = "pyclipper" },{ name = "pyyaml" },