Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

4.5 KB
Raw Blame
"""JSON-backed FAISS index persistence (no pickle deserialization).

Legacy ``FAISS.save_local`` writes ``index.faiss`` + ``index.pkl``; loading
the pickle executes code inside it, so a KA directory is only as safe as
its source. This module stores the raw vectors and documents as plain JSON
and rebuilds the flat index in memory on load — same search behavior, zero
code execution.

Layout (``index.json``)::

    {
      "version": 1,
      "metric": 1,              # faiss metric_type of the saved index
      "dimension": 768,
      "vectors": [[...], ...],  # one vector per item, index order
      "texts": [...],           # page_content per item
      "metadatas": [...],       # metadata per item
      "ids": [...]              # docstore ids per item
    }

Only flat indexes (as built by ``FAISS.from_documents`` with default
settings) are supported — that is what AutoModel/AutoList build.
"""

import json
from pathlib import Path
from typing import Any

import numpy as np
from langchain_community.vectorstores import FAISS

from hyperextract.utils.logging import get_logger

logger = get_logger(__name__)

INDEX_JSON_NAME = "index.json"
LEGACY_INDEX_NAME = "index.faiss"
LEGACY_PICKLE_NAME = "index.pkl"


def has_json_index(folder: str | Path) -> bool:
    """True when the folder contains a JSON index."""
    return (Path(folder) / INDEX_JSON_NAME).is_file()


def has_legacy_index(folder: str | Path) -> bool:
    """True when the folder contains a legacy pickle-based FAISS index."""
    folder = Path(folder)
    return (folder / LEGACY_INDEX_NAME).is_file() and (
        folder / LEGACY_PICKLE_NAME
    ).is_file()


def dump_index_json(index: FAISS, folder: str | Path) -> Path:
    """Persist a FAISS vector index as JSON (vectors + documents, no pickle).

    Returns the written ``index.json`` path. Legacy ``index.faiss`` /
    ``index.pkl`` files in the folder are removed after a successful write
    so a stale pickle index can never shadow the JSON one.
    """
    folder = Path(folder)
    folder.mkdir(parents=True, exist_ok=True)

    ntotal = int(index.index.ntotal)
    vectors = (
        index.index.reconstruct_n(0, ntotal)
        if ntotal
        else np.zeros((0, int(index.index.d)), dtype="float32")
    )

    items = []
    for i in range(ntotal):
        doc_id = index.index_to_docstore_id[i]
        doc = index.docstore.search(doc_id)
        items.append(
            {
                "id": doc_id,
                "text": doc.page_content,
                "metadata": doc.metadata,
            }
        )

    payload = {
        "version": 1,
        "metric": int(index.index.metric_type),
        "dimension": int(index.index.d),
        "vectors": np.asarray(vectors, dtype="float32").tolist(),
        "texts": [item["text"] for item in items],
        "metadatas": [item["metadata"] for item in items],
        "ids": [item["id"] for item in items],
    }
    path = folder / INDEX_JSON_NAME
    path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
    logger.info("json_index_dumped", path=str(path), items=ntotal)

    for legacy in (LEGACY_INDEX_NAME, LEGACY_PICKLE_NAME):
        stale = folder / legacy
        if stale.exists():
            stale.unlink()
            logger.info("legacy_index_removed", path=str(stale))
    return path


def load_index_json(folder: str | Path, embedder: Any) -> FAISS | None:
    """Load a JSON index and rebuild the FAISS flat index in memory.

    Returns ``None`` when the folder has no ``index.json``. Vectors are
    restored exactly as saved — no re-embedding happens.
    """
    path = Path(folder) / INDEX_JSON_NAME
    if not path.is_file():
        return None

    payload = json.loads(path.read_text(encoding="utf-8"))
    vectors = payload.get("vectors", [])
    texts = payload.get("texts", [])
    metadatas = payload.get("metadatas", [])
    ids = payload.get("ids", [])
    if not (len(vectors) == len(texts) == len(metadatas) == len(ids)):
        raise ValueError(f"Corrupt index.json: field lengths differ in {path}")
    if not vectors:
        raise ValueError(f"Corrupt index.json: no entries in {path}")

    metric = int(payload.get("metric", 0))
    if metric:  # only flat L2 indexes are written today
        saved = metric
        restored = 1  # faiss.METRIC_L2
        if saved != restored:
            logger.warning("json_index_metric_mismatch saved=%s rebuilt=L2", saved)

    index = FAISS.from_embeddings(
        list(zip(texts, vectors)), embedder, metadatas=metadatas, ids=ids
    )
    logger.info("json_index_loaded", path=str(path), items=len(ids))
    return index