Skip to content

High performance self-hosted photo and video management solution.

fix(ml): read CLIP model configs as UTF-8 (#31075)

Aditya Raj Singhc1a88dd9a33f
diff --git a/machine-learning/immich_ml/models/clip/textual.py b/machine-learning/immich_ml/models/clip/textual.py
index 989f46a2af..421459fc03 100644
--- a/machine-learning/immich_ml/models/clip/textual.py
+++ b/machine-learning/immich_ml/models/clip/textual.py
@@ -58,7 +58,7 @@ class BaseCLIPTextualEncoder(InferenceModel):
@cached_property
def model_cfg(self) -> dict[str, Any]:
log.debug(f"Loading model config for CLIP model '{self.model_name}'")
- model_cfg: dict[str, Any] = json.load(self.model_cfg_path.open())
+ model_cfg: dict[str, Any] = json.load(self.model_cfg_path.open(encoding="utf-8"))
log.debug(f"Loaded model config for CLIP model '{self.model_name}'")
return model_cfg
@@ -70,14 +70,14 @@ class BaseCLIPTextualEncoder(InferenceModel):
@cached_property
def tokenizer_file(self) -> dict[str, Any]:
log.debug(f"Loading tokenizer file for CLIP model '{self.model_name}'")
- tokenizer_file: dict[str, Any] = json.load(self.tokenizer_file_path.open())
+ tokenizer_file: dict[str, Any] = json.load(self.tokenizer_file_path.open(encoding="utf-8"))
log.debug(f"Loaded tokenizer file for CLIP model '{self.model_name}'")
return tokenizer_file
@cached_property
def tokenizer_cfg(self) -> dict[str, Any]:
log.debug(f"Loading tokenizer config for CLIP model '{self.model_name}'")
- tokenizer_cfg: dict[str, Any] = json.load(self.tokenizer_cfg_path.open())
+ tokenizer_cfg: dict[str, Any] = json.load(self.tokenizer_cfg_path.open(encoding="utf-8"))
log.debug(f"Loaded tokenizer config for CLIP model '{self.model_name}'")
return tokenizer_cfg
diff --git a/machine-learning/immich_ml/models/clip/visual.py b/machine-learning/immich_ml/models/clip/visual.py
index 16993f9c01..1d46d8fe10 100644
--- a/machine-learning/immich_ml/models/clip/visual.py
+++ b/machine-learning/immich_ml/models/clip/visual.py
@@ -46,14 +46,14 @@ class BaseCLIPVisualEncoder(InferenceModel):
@cached_property
def model_cfg(self) -> dict[str, Any]:
log.debug(f"Loading model config for CLIP model '{self.model_name}'")
- model_cfg: dict[str, Any] = json.load(self.model_cfg_path.open())
+ model_cfg: dict[str, Any] = json.load(self.model_cfg_path.open(encoding="utf-8"))
log.debug(f"Loaded model config for CLIP model '{self.model_name}'")
return model_cfg
@cached_property
def preprocess_cfg(self) -> dict[str, Any]:
log.debug(f"Loading visual preprocessing config for CLIP model '{self.model_name}'")
- preprocess_cfg: dict[str, Any] = json.load(self.preprocess_cfg_path.open())
+ preprocess_cfg: dict[str, Any] = json.load(self.preprocess_cfg_path.open(encoding="utf-8"))
log.debug(f"Loaded visual preprocessing config for CLIP model '{self.model_name}'")
return preprocess_cfg
diff --git a/machine-learning/test_main.py b/machine-learning/test_main.py
index 748a8840cc..9e3fe50cba 100644
--- a/machine-learning/test_main.py
+++ b/machine-learning/test_main.py
@@ -617,6 +617,38 @@ class TestCLIP:
assert len(embedding) == clip_model_cfg["embed_dim"]
mocked.run.assert_called_once()
+ def test_reads_model_configs_as_utf8(self, mocker: MockerFixture, tmp_path: Path) -> None:
+ original_open = Path.open
+
+ def locale_default_is_ascii(self: Path, mode: str = "r", *args: Any, **kwargs: Any) -> Any:
+ if "b" not in mode and kwargs.get("encoding") is None:
+ kwargs["encoding"] = "ascii"
+ return original_open(self, mode, *args, **kwargs)
+
+ mocker.patch.object(OpenClipTextualEncoder, "download")
+ mocker.patch.object(OpenClipVisualEncoder, "download")
+
+ textual = OpenClipTextualEncoder("ViT-B-32__openai", cache_dir=tmp_path)
+ visual = OpenClipVisualEncoder("ViT-B-32__openai", cache_dir=tmp_path)
+ paths = [
+ textual.model_cfg_path,
+ textual.tokenizer_file_path,
+ textual.tokenizer_cfg_path,
+ visual.model_cfg_path,
+ visual.preprocess_cfg_path,
+ ]
+ for path in paths:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(orjson.dumps({"eos_token": "<|café|>"}).decode(), encoding="utf-8")
+
+ mocker.patch.object(Path, "open", locale_default_is_ascii)
+
+ assert textual.model_cfg["eos_token"] == "<|café|>"
+ assert textual.tokenizer_file["eos_token"] == "<|café|>"
+ assert textual.tokenizer_cfg["eos_token"] == "<|café|>"
+ assert visual.model_cfg["eos_token"] == "<|café|>"
+ assert visual.preprocess_cfg["eos_token"] == "<|café|>"
+
def test_openclip_tokenizer(
self,
mocker: MockerFixture,