Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

fix: warn when loading FAISS indexes that require pickle deserialization

Frankie-Xue5a3eea85d8d
diff --git a/docs/en/cli/commands/build-index.md b/docs/en/cli/commands/build-index.md
index dddf333..a2a798c 100644
--- a/docs/en/cli/commands/build-index.md
+++ b/docs/en/cli/commands/build-index.md
@@ -116,6 +116,8 @@ The index is stored in the knowledge abstract directory:
└── docstore.json # Document store mapping
```
+Load an existing `index/` only from Knowledge Abstract directories you created or fully trust. Local FAISS indexes require deserialization on load.
+
---
## Performance
diff --git a/docs/en/python/guides/saving-loading.md b/docs/en/python/guides/saving-loading.md
index 30f91ef..1e1cf8d 100644
--- a/docs/en/python/guides/saving-loading.md
+++ b/docs/en/python/guides/saving-loading.md
@@ -53,6 +53,9 @@ result.dump("./my_ka/")
## Loading Knowledge Abstracts
+!!! warning "Trusted directories only"
+ Local FAISS indexes under `index/` are deserialized when you load a Knowledge Abstract. Only call `load()` (or CLI/MCP commands that load a KA) on directories you generated yourself or fully trust.
+
### Basic Load
```python
diff --git a/docs/zh/cli/commands/build-index.md b/docs/zh/cli/commands/build-index.md
index 19bb9dc..1d51c1b 100644
--- a/docs/zh/cli/commands/build-index.md
+++ b/docs/zh/cli/commands/build-index.md
@@ -116,6 +116,8 @@ he build-index ./ka/ -f
└── docstore.json # 文档存储映射
```
+请只加载自己生成或完全信任的知识库目录中的 `index/`。本地 FAISS 索引在加载时需要反序列化。
+
---
## 性能
diff --git a/docs/zh/python/guides/saving-loading.md b/docs/zh/python/guides/saving-loading.md
index 5329f88..191bc50 100644
--- a/docs/zh/python/guides/saving-loading.md
+++ b/docs/zh/python/guides/saving-loading.md
@@ -52,6 +52,9 @@ result.dump("./my_ka/")
## 加载知识库
+!!! warning "只加载受信任的目录"
+ 加载知识库时,`index/` 下的本地 FAISS 索引需要反序列化。请只对自己生成或完全信任的知识库目录调用 `load()`(以及会加载 KA 的 CLI/MCP 命令)。
+
### 基本加载
```python
diff --git a/hyperextract/types/_faiss.py b/hyperextract/types/_faiss.py
new file mode 100644
index 0000000..5dd463a
--- /dev/null
+++ b/hyperextract/types/_faiss.py
@@ -0,0 +1,27 @@
+"""Shared helpers for local FAISS index loading."""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+_warned_index_paths: set[str] = set()
+
+
+def _warn_untrusted_faiss_load(folder_path: str | Path) -> None:
+ """Warn once per path that a local FAISS index requires deserialization.
+
+ Local FAISS indexes are loaded with deserialization enabled. Only load
+ indexes from Knowledge Abstract directories you created or fully trust.
+ """
+ path = str(Path(folder_path).resolve())
+ if path in _warned_index_paths:
+ return
+ _warned_index_paths.add(path)
+ logger.warning(
+ "Loading a local FAISS index from %s requires deserialization. "
+ "Only load Knowledge Abstract directories that you created or fully trust.",
+ path,
+ )
diff --git a/hyperextract/types/list.py b/hyperextract/types/list.py
index f66230c..c6a5bee 100644
--- a/hyperextract/types/list.py
+++ b/hyperextract/types/list.py
@@ -21,6 +21,7 @@ from pydantic import BaseModel, Field, create_model
from hyperextract.utils.logging import get_logger
+from ._faiss import _warn_untrusted_faiss_load
from .base import BaseAutoType
logger = get_logger(__name__)
@@ -320,6 +321,7 @@ class AutoList(BaseAutoType[AutoListSchema[ItemSchema]], Generic[ItemSchema]):
folder = Path(folder_path)
if not folder.is_dir():
raise ValueError(f"Folder does not exist: {folder_path}")
+ _warn_untrusted_faiss_load(folder)
self._index = FAISS.load_local(
str(folder), self.embedder, allow_dangerous_deserialization=True
)
diff --git a/hyperextract/types/model.py b/hyperextract/types/model.py
index c2971a9..308280e 100644
--- a/hyperextract/types/model.py
+++ b/hyperextract/types/model.py
@@ -16,6 +16,7 @@ from ontosight import view_nodes
from hyperextract.utils.logging import get_logger
+from ._faiss import _warn_untrusted_faiss_load
from .base import BaseAutoType, T
logger = get_logger(__name__)
@@ -311,6 +312,7 @@ class AutoModel(BaseAutoType[T]):
folder = Path(folder_path)
if not folder.is_dir():
raise ValueError(f"Folder does not exist: {folder_path}")
+ _warn_untrusted_faiss_load(folder)
self._index = FAISS.load_local(
str(folder), self.embedder, allow_dangerous_deserialization=True
)
diff --git a/tests/types/test_faiss_load.py b/tests/types/test_faiss_load.py
new file mode 100644
index 0000000..1f26f52
--- /dev/null
+++ b/tests/types/test_faiss_load.py
@@ -0,0 +1,128 @@
+"""Tests for FAISS local-index load warnings on AutoModel and AutoList."""
+
+from __future__ import annotations
+
+import logging
+from unittest.mock import MagicMock, patch
+
+import pytest
+from pydantic import BaseModel
+
+from hyperextract.types import AutoList, AutoModel
+from hyperextract.types._faiss import _warned_index_paths
+from tests.fixtures import PersonSchema
+
+_FORBIDDEN_WARNING_SNIPPETS = (
+ "malicious",
+ "gadget",
+ "construct",
+ "payload",
+ "exploit",
+ "rce",
+)
+
+
+class PersonItemSchema(BaseModel):
+ name: str
+ age: int | None = None
+
+
[email protected](autouse=True)
+def _reset_faiss_load_warnings():
+ _warned_index_paths.clear()
+ yield
+ _warned_index_paths.clear()
+
+
+def _assert_trusted_load_warning(caplog: pytest.LogCaptureFixture, folder) -> None:
+ warning_text = " ".join(
+ record.getMessage() for record in caplog.records if record.levelno >= logging.WARNING
+ )
+ assert warning_text, "expected a warning when loading a local FAISS index"
+ assert str(folder) in warning_text or folder.name in warning_text
+ assert "deserializ" in warning_text.lower()
+ assert "trust" in warning_text.lower()
+ lowered = warning_text.lower()
+ for snippet in _FORBIDDEN_WARNING_SNIPPETS:
+ assert snippet not in lowered
+
+
+def _assert_dangerous_deserialization_enabled(mock_load_local: MagicMock) -> None:
+ mock_load_local.assert_called_once()
+ _, kwargs = mock_load_local.call_args
+ assert kwargs.get("allow_dangerous_deserialization") is True
+
+
+class TestAutoModelFaissLoad:
+ @patch("hyperextract.types.model.FAISS.load_local")
+ def test_load_index_warns_and_keeps_deserialization_flag(
+ self, mock_load_local, llm_client, embedder, tmp_path, caplog
+ ):
+ mock_load_local.return_value = MagicMock(name="faiss_index")
+ folder = tmp_path / "model-index"
+ folder.mkdir()
+
+ model = AutoModel(
+ data_schema=PersonSchema,
+ llm_client=llm_client,
+ embedder=embedder,
+ )
+
+ with caplog.at_level(logging.WARNING, logger="hyperextract.types._faiss"):
+ model.load_index(folder)
+
+ _assert_dangerous_deserialization_enabled(mock_load_local)
+ assert mock_load_local.call_args.args[0] == str(folder)
+ assert mock_load_local.call_args.args[1] is embedder
+ assert model._index is mock_load_local.return_value
+ _assert_trusted_load_warning(caplog, folder)
+
+ @patch("hyperextract.types.model.FAISS.load_local")
+ def test_load_index_warns_once_per_path(
+ self, mock_load_local, llm_client, embedder, tmp_path, caplog
+ ):
+ mock_load_local.return_value = MagicMock(name="faiss_index")
+ folder = tmp_path / "model-index"
+ folder.mkdir()
+ model = AutoModel(
+ data_schema=PersonSchema,
+ llm_client=llm_client,
+ embedder=embedder,
+ )
+
+ with caplog.at_level(logging.WARNING, logger="hyperextract.types._faiss"):
+ model.load_index(folder)
+ model.load_index(folder)
+
+ warning_records = [
+ record
+ for record in caplog.records
+ if record.levelno >= logging.WARNING and "FAISS" in record.getMessage()
+ ]
+ assert len(warning_records) == 1
+ assert mock_load_local.call_count == 2
+
+
+class TestAutoListFaissLoad:
+ @patch("hyperextract.types.list.FAISS.load_local")
+ def test_load_index_warns_and_keeps_deserialization_flag(
+ self, mock_load_local, llm_client, embedder, tmp_path, caplog
+ ):
+ mock_load_local.return_value = MagicMock(name="faiss_index")
+ folder = tmp_path / "list-index"
+ folder.mkdir()
+
+ auto_list = AutoList(
+ item_schema=PersonItemSchema,
+ llm_client=llm_client,
+ embedder=embedder,
+ )
+
+ with caplog.at_level(logging.WARNING, logger="hyperextract.types._faiss"):
+ auto_list.load_index(folder)
+
+ _assert_dangerous_deserialization_enabled(mock_load_local)
+ assert mock_load_local.call_args.args[0] == str(folder)
+ assert mock_load_local.call_args.args[1] is embedder
+ assert auto_list._index is mock_load_local.return_value
+ _assert_trusted_load_warning(caplog, folder)