High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
fix(ml): read CLIP model configs as UTF-8 (#31075)
Aditya Raj Singhc1a88dd9a33f
diff --git a/machine-learning/immich_ml/models/clip/textual.py b/machine-learning/immich_ml/models/clip/textual.pyindex 989f46a2af..421459fc03 100644--- a/machine-learning/immich_ml/models/clip/textual.py+++ b/machine-learning/immich_ml/models/clip/textual.py@@ -58,7 +58,7 @@ class BaseCLIPTextualEncoder(InferenceModel):@cached_propertydef model_cfg(self) -> dict[str, Any]:log.debug(f"Loading model config for CLIP model '{self.model_name}'")- model_cfg: dict[str, Any] = json.load(self.model_cfg_path.open())+ model_cfg: dict[str, Any] = json.load(self.model_cfg_path.open(encoding="utf-8"))log.debug(f"Loaded model config for CLIP model '{self.model_name}'")return model_cfg@@ -70,14 +70,14 @@ class BaseCLIPTextualEncoder(InferenceModel):@cached_propertydef tokenizer_file(self) -> dict[str, Any]:log.debug(f"Loading tokenizer file for CLIP model '{self.model_name}'")- tokenizer_file: dict[str, Any] = json.load(self.tokenizer_file_path.open())+ tokenizer_file: dict[str, Any] = json.load(self.tokenizer_file_path.open(encoding="utf-8"))log.debug(f"Loaded tokenizer file for CLIP model '{self.model_name}'")return tokenizer_file@cached_propertydef tokenizer_cfg(self) -> dict[str, Any]:log.debug(f"Loading tokenizer config for CLIP model '{self.model_name}'")- tokenizer_cfg: dict[str, Any] = json.load(self.tokenizer_cfg_path.open())+ tokenizer_cfg: dict[str, Any] = json.load(self.tokenizer_cfg_path.open(encoding="utf-8"))log.debug(f"Loaded tokenizer config for CLIP model '{self.model_name}'")return tokenizer_cfgdiff --git a/machine-learning/immich_ml/models/clip/visual.py b/machine-learning/immich_ml/models/clip/visual.pyindex 16993f9c01..1d46d8fe10 100644--- a/machine-learning/immich_ml/models/clip/visual.py+++ b/machine-learning/immich_ml/models/clip/visual.py@@ -46,14 +46,14 @@ class BaseCLIPVisualEncoder(InferenceModel):@cached_propertydef model_cfg(self) -> dict[str, Any]:log.debug(f"Loading model config for CLIP model '{self.model_name}'")- model_cfg: dict[str, Any] = json.load(self.model_cfg_path.open())+ model_cfg: dict[str, Any] = json.load(self.model_cfg_path.open(encoding="utf-8"))log.debug(f"Loaded model config for CLIP model '{self.model_name}'")return model_cfg@cached_propertydef preprocess_cfg(self) -> dict[str, Any]:log.debug(f"Loading visual preprocessing config for CLIP model '{self.model_name}'")- preprocess_cfg: dict[str, Any] = json.load(self.preprocess_cfg_path.open())+ preprocess_cfg: dict[str, Any] = json.load(self.preprocess_cfg_path.open(encoding="utf-8"))log.debug(f"Loaded visual preprocessing config for CLIP model '{self.model_name}'")return preprocess_cfgdiff --git a/machine-learning/test_main.py b/machine-learning/test_main.pyindex 748a8840cc..9e3fe50cba 100644--- a/machine-learning/test_main.py+++ b/machine-learning/test_main.py@@ -617,6 +617,38 @@ class TestCLIP:assert len(embedding) == clip_model_cfg["embed_dim"]mocked.run.assert_called_once()+ def test_reads_model_configs_as_utf8(self, mocker: MockerFixture, tmp_path: Path) -> None:+ original_open = Path.open++ def locale_default_is_ascii(self: Path, mode: str = "r", *args: Any, **kwargs: Any) -> Any:+ if "b" not in mode and kwargs.get("encoding") is None:+ kwargs["encoding"] = "ascii"+ return original_open(self, mode, *args, **kwargs)++ mocker.patch.object(OpenClipTextualEncoder, "download")+ mocker.patch.object(OpenClipVisualEncoder, "download")++ textual = OpenClipTextualEncoder("ViT-B-32__openai", cache_dir=tmp_path)+ visual = OpenClipVisualEncoder("ViT-B-32__openai", cache_dir=tmp_path)+ paths = [+ textual.model_cfg_path,+ textual.tokenizer_file_path,+ textual.tokenizer_cfg_path,+ visual.model_cfg_path,+ visual.preprocess_cfg_path,+ ]+ for path in paths:+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(orjson.dumps({"eos_token": "<|café|>"}).decode(), encoding="utf-8")++ mocker.patch.object(Path, "open", locale_default_is_ascii)++ assert textual.model_cfg["eos_token"] == "<|café|>"+ assert textual.tokenizer_file["eos_token"] == "<|café|>"+ assert textual.tokenizer_cfg["eos_token"] == "<|café|>"+ assert visual.model_cfg["eos_token"] == "<|café|>"+ assert visual.preprocess_cfg["eos_token"] == "<|café|>"+def test_openclip_tokenizer(self,mocker: MockerFixture,