Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
fix: warn when loading FAISS indexes that require pickle deserialization
Frankie-Xue5a3eea85d8d
diff --git a/docs/en/cli/commands/build-index.md b/docs/en/cli/commands/build-index.mdindex dddf333..a2a798c 100644--- a/docs/en/cli/commands/build-index.md+++ b/docs/en/cli/commands/build-index.md@@ -116,6 +116,8 @@ The index is stored in the knowledge abstract directory:└── docstore.json # Document store mapping```+Load an existing `index/` only from Knowledge Abstract directories you created or fully trust. Local FAISS indexes require deserialization on load.+---## Performancediff --git a/docs/en/python/guides/saving-loading.md b/docs/en/python/guides/saving-loading.mdindex 30f91ef..1e1cf8d 100644--- a/docs/en/python/guides/saving-loading.md+++ b/docs/en/python/guides/saving-loading.md@@ -53,6 +53,9 @@ result.dump("./my_ka/")## Loading Knowledge Abstracts+!!! warning "Trusted directories only"+ Local FAISS indexes under `index/` are deserialized when you load a Knowledge Abstract. Only call `load()` (or CLI/MCP commands that load a KA) on directories you generated yourself or fully trust.+### Basic Load```pythondiff --git a/docs/zh/cli/commands/build-index.md b/docs/zh/cli/commands/build-index.mdindex 19bb9dc..1d51c1b 100644--- a/docs/zh/cli/commands/build-index.md+++ b/docs/zh/cli/commands/build-index.md@@ -116,6 +116,8 @@ he build-index ./ka/ -f└── docstore.json # 文档存储映射```+请只加载自己生成或完全信任的知识库目录中的 `index/`。本地 FAISS 索引在加载时需要反序列化。+---## 性能diff --git a/docs/zh/python/guides/saving-loading.md b/docs/zh/python/guides/saving-loading.mdindex 5329f88..191bc50 100644--- a/docs/zh/python/guides/saving-loading.md+++ b/docs/zh/python/guides/saving-loading.md@@ -52,6 +52,9 @@ result.dump("./my_ka/")## 加载知识库+!!! warning "只加载受信任的目录"+ 加载知识库时,`index/` 下的本地 FAISS 索引需要反序列化。请只对自己生成或完全信任的知识库目录调用 `load()`(以及会加载 KA 的 CLI/MCP 命令)。+### 基本加载```pythondiff --git a/hyperextract/types/_faiss.py b/hyperextract/types/_faiss.pynew file mode 100644index 0000000..5dd463a--- /dev/null+++ b/hyperextract/types/_faiss.py@@ -0,0 +1,27 @@+"""Shared helpers for local FAISS index loading."""++from __future__ import annotations++import logging+from pathlib import Path++logger = logging.getLogger(__name__)++_warned_index_paths: set[str] = set()+++def _warn_untrusted_faiss_load(folder_path: str | Path) -> None:+ """Warn once per path that a local FAISS index requires deserialization.++ Local FAISS indexes are loaded with deserialization enabled. Only load+ indexes from Knowledge Abstract directories you created or fully trust.+ """+ path = str(Path(folder_path).resolve())+ if path in _warned_index_paths:+ return+ _warned_index_paths.add(path)+ logger.warning(+ "Loading a local FAISS index from %s requires deserialization. "+ "Only load Knowledge Abstract directories that you created or fully trust.",+ path,+ )diff --git a/hyperextract/types/list.py b/hyperextract/types/list.pyindex f66230c..c6a5bee 100644--- a/hyperextract/types/list.py+++ b/hyperextract/types/list.py@@ -21,6 +21,7 @@ from pydantic import BaseModel, Field, create_modelfrom hyperextract.utils.logging import get_logger+from ._faiss import _warn_untrusted_faiss_loadfrom .base import BaseAutoTypelogger = get_logger(__name__)@@ -320,6 +321,7 @@ class AutoList(BaseAutoType[AutoListSchema[ItemSchema]], Generic[ItemSchema]):folder = Path(folder_path)if not folder.is_dir():raise ValueError(f"Folder does not exist: {folder_path}")+ _warn_untrusted_faiss_load(folder)self._index = FAISS.load_local(str(folder), self.embedder, allow_dangerous_deserialization=True)diff --git a/hyperextract/types/model.py b/hyperextract/types/model.pyindex c2971a9..308280e 100644--- a/hyperextract/types/model.py+++ b/hyperextract/types/model.py@@ -16,6 +16,7 @@ from ontosight import view_nodesfrom hyperextract.utils.logging import get_logger+from ._faiss import _warn_untrusted_faiss_loadfrom .base import BaseAutoType, Tlogger = get_logger(__name__)@@ -311,6 +312,7 @@ class AutoModel(BaseAutoType[T]):folder = Path(folder_path)if not folder.is_dir():raise ValueError(f"Folder does not exist: {folder_path}")+ _warn_untrusted_faiss_load(folder)self._index = FAISS.load_local(str(folder), self.embedder, allow_dangerous_deserialization=True)diff --git a/tests/types/test_faiss_load.py b/tests/types/test_faiss_load.pynew file mode 100644index 0000000..1f26f52--- /dev/null+++ b/tests/types/test_faiss_load.py@@ -0,0 +1,128 @@+"""Tests for FAISS local-index load warnings on AutoModel and AutoList."""++from __future__ import annotations++import logging+from unittest.mock import MagicMock, patch++import pytest+from pydantic import BaseModel++from hyperextract.types import AutoList, AutoModel+from hyperextract.types._faiss import _warned_index_paths+from tests.fixtures import PersonSchema++_FORBIDDEN_WARNING_SNIPPETS = (+ "malicious",+ "gadget",+ "construct",+ "payload",+ "exploit",+ "rce",+)+++class PersonItemSchema(BaseModel):+ name: str+ age: int | None = None++[email protected](autouse=True)+def _reset_faiss_load_warnings():+ _warned_index_paths.clear()+ yield+ _warned_index_paths.clear()+++def _assert_trusted_load_warning(caplog: pytest.LogCaptureFixture, folder) -> None:+ warning_text = " ".join(+ record.getMessage() for record in caplog.records if record.levelno >= logging.WARNING+ )+ assert warning_text, "expected a warning when loading a local FAISS index"+ assert str(folder) in warning_text or folder.name in warning_text+ assert "deserializ" in warning_text.lower()+ assert "trust" in warning_text.lower()+ lowered = warning_text.lower()+ for snippet in _FORBIDDEN_WARNING_SNIPPETS:+ assert snippet not in lowered+++def _assert_dangerous_deserialization_enabled(mock_load_local: MagicMock) -> None:+ mock_load_local.assert_called_once()+ _, kwargs = mock_load_local.call_args+ assert kwargs.get("allow_dangerous_deserialization") is True+++class TestAutoModelFaissLoad:+ @patch("hyperextract.types.model.FAISS.load_local")+ def test_load_index_warns_and_keeps_deserialization_flag(+ self, mock_load_local, llm_client, embedder, tmp_path, caplog+ ):+ mock_load_local.return_value = MagicMock(name="faiss_index")+ folder = tmp_path / "model-index"+ folder.mkdir()++ model = AutoModel(+ data_schema=PersonSchema,+ llm_client=llm_client,+ embedder=embedder,+ )++ with caplog.at_level(logging.WARNING, logger="hyperextract.types._faiss"):+ model.load_index(folder)++ _assert_dangerous_deserialization_enabled(mock_load_local)+ assert mock_load_local.call_args.args[0] == str(folder)+ assert mock_load_local.call_args.args[1] is embedder+ assert model._index is mock_load_local.return_value+ _assert_trusted_load_warning(caplog, folder)++ @patch("hyperextract.types.model.FAISS.load_local")+ def test_load_index_warns_once_per_path(+ self, mock_load_local, llm_client, embedder, tmp_path, caplog+ ):+ mock_load_local.return_value = MagicMock(name="faiss_index")+ folder = tmp_path / "model-index"+ folder.mkdir()+ model = AutoModel(+ data_schema=PersonSchema,+ llm_client=llm_client,+ embedder=embedder,+ )++ with caplog.at_level(logging.WARNING, logger="hyperextract.types._faiss"):+ model.load_index(folder)+ model.load_index(folder)++ warning_records = [+ record+ for record in caplog.records+ if record.levelno >= logging.WARNING and "FAISS" in record.getMessage()+ ]+ assert len(warning_records) == 1+ assert mock_load_local.call_count == 2+++class TestAutoListFaissLoad:+ @patch("hyperextract.types.list.FAISS.load_local")+ def test_load_index_warns_and_keeps_deserialization_flag(+ self, mock_load_local, llm_client, embedder, tmp_path, caplog+ ):+ mock_load_local.return_value = MagicMock(name="faiss_index")+ folder = tmp_path / "list-index"+ folder.mkdir()++ auto_list = AutoList(+ item_schema=PersonItemSchema,+ llm_client=llm_client,+ embedder=embedder,+ )++ with caplog.at_level(logging.WARNING, logger="hyperextract.types._faiss"):+ auto_list.load_index(folder)++ _assert_dangerous_deserialization_enabled(mock_load_local)+ assert mock_load_local.call_args.args[0] == str(folder)+ assert mock_load_local.call_args.args[1] is embedder+ assert auto_list._index is mock_load_local.return_value+ _assert_trusted_load_warning(caplog, folder)