Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

6.2 KB
Raw Blame
"""Integration test: provenance lifecycle against REAL LLM APIs.

Provider-configurable via environment variables (defaults to the
chatanywhere-compatible config in .env):

    HE_TEST_LLM_BASE_URL     LLM endpoint   (default: OPENAI_BASE_URL)
    HE_TEST_LLM_API_KEY      LLM key        (default: OPENAI_API_KEY)
    HE_TEST_LLM_MODEL        LLM model      (default: gpt-4o-mini)
    HE_TEST_EMBED_BASE_URL   embed endpoint (default: LLM base URL)
    HE_TEST_EMBED_API_KEY    embed key      (default: LLM key)
    HE_TEST_EMBED_MODEL      embed model    (default: text-embedding-3-small)

Example — DeepSeek LLM + chatanywhere embeddings:
    HE_TEST_LLM_BASE_URL=https://api.deepseek.com \\
    HE_TEST_LLM_API_KEY=sk-... HE_TEST_LLM_MODEL=deepseek-chat \\
    HE_TEST_EMBED_BASE_URL=https://api.chatanywhere.tech/v1 \\
    HE_TEST_EMBED_API_KEY=sk-... \\
    pytest tests/integration/test_provenance_real.py -v

Exercises the exact-replay question mocks cannot answer: when a document
is rolled back and its keys re-merged from surviving sources, the
LLM.BALANCED merger replays with real non-determinism — we assert
semantic preservation (entity/fact survival), not wording.
"""

import hashlib
import os

import pytest
from langchain_openai import ChatOpenAI, OpenAIEmbeddings
from pydantic import BaseModel

from hyperextract.types import AutoGraph

pytestmark = [
    pytest.mark.integration,
    pytest.mark.skipif(
        not os.environ.get("OPENAI_API_KEY"),
        reason="OPENAI_API_KEY not set",
    ),
]


class Entity(BaseModel):
    name: str
    type: str = "organization"
    description: str = ""


class Relation(BaseModel):
    source: str
    target: str
    relation_type: str
    description: str = ""


DOC1_V1 = (
    "Apple was founded by Steve Jobs in 1976. "
    "Google was founded by Larry Page and Sergey Brin in 1998. "
    "Apple is headquartered in Cupertino, California."
)
DOC1_V2 = (
    "Apple was founded by Steve Jobs in 1976. "
    "Tim Cook is the CEO of Apple. "
    "Apple is headquartered in Cupertino, California."
)  # Google fact removed from the document
DOC2 = "Microsoft was founded by Bill Gates and Paul Allen in 1975."


def _client_config(prefix: str, fallback_base: str, fallback_key: str):
    base = os.environ.get(f"{prefix}_BASE_URL", fallback_base)
    key = os.environ.get(f"{prefix}_API_KEY", fallback_key)
    return base, key


def _graph():
    llm_base, llm_key = _client_config(
        "HE_TEST_LLM",
        fallback_base=os.environ.get("OPENAI_BASE_URL", ""),
        fallback_key=os.environ.get("OPENAI_API_KEY", ""),
    )
    emb_base, emb_key = _client_config(
        "HE_TEST_EMBED",
        fallback_base=llm_base,
        fallback_key=llm_key,
    )
    llm_model = os.environ.get("HE_TEST_LLM_MODEL", "gpt-4o-mini")
    emb_model = os.environ.get("HE_TEST_EMBED_MODEL", "text-embedding-3-small")

    llm = ChatOpenAI(
        model=llm_model,
        temperature=0,
        api_key=llm_key,
        base_url=llm_base or None,
    )
    embedder = OpenAIEmbeddings(
        model=emb_model,
        api_key=emb_key,
        base_url=emb_base or None,
    )
    return AutoGraph(
        node_schema=Entity,
        edge_schema=Relation,
        node_key_extractor=lambda x: x.name,
        edge_key_extractor=lambda x: f"{x.source}-{x.relation_type}-{x.target}",
        nodes_in_edge_extractor=lambda x: (x.source, x.target),
        llm_client=llm,
        embedder=embedder,
        extraction_mode="one_stage",
    )


def _names(graph):
    return {n.name for n in graph.nodes}


class TestProvenanceRealAPI:
    def test_full_lifecycle(self):
        g = _graph()

        # 1. Attributed ingestion of doc1 v1 (real extraction)
        g.feed_text(DOC1_V1, source_id="doc1")
        assert "doc1" in g._node_memory._sources
        names_v1 = _names(g)
        assert "Apple" in names_v1
        assert "Google" in names_v1

        # 2. doc2 contributes to the shared "Apple" key
        g.feed_text(DOC2, source_id="doc2")
        assert "Microsoft" in _names(g)

        # 3. doc1 v2 removes the Google fact → auto-upsert rolls back v1:
        #    Google (sole doc1 contribution) is removed; Apple (shared with
        #    doc2) is re-merged from doc2's raw results with real LLM merge.
        g.feed_text(DOC1_V2, source_id="doc1")

        names_after = _names(g)
        assert "Google" not in names_after
        assert "Microsoft" in names_after
        assert "Apple" in names_after

        # 4. Roll back doc2 entirely → Microsoft goes, Apple stays.
        #    (Real extraction also pulls people/years from DOC2 — all of
        #    them are doc2-only contributions, removed together.)
        report = g.remove_source("doc2")
        removed = set(report["removed_nodes"])
        assert "Microsoft" in removed
        assert "Bill Gates" in removed
        names_final = _names(g)
        assert "Microsoft" not in names_final
        assert "Bill Gates" not in names_final
        assert "Paul Allen" not in names_final
        assert "Apple" in names_final

        # 5. Index patched in place throughout, consistent with storage.
        assert g._node_memory.has_index()
        doc_keys = {
            d.metadata["key"] for d in g._node_memory._index.docstore._dict.values()
        }
        storage_keys = {g.node_key_extractor(n) for n in g.nodes}
        assert doc_keys == storage_keys, (
            f"index/storage key mismatch: index_only={doc_keys - storage_keys}, "
            f"storage_only={storage_keys - doc_keys}"
        )

        # 6. Semantic search still works on the patched index.
        nodes, edges = g.search("Tim Cook CEO of Apple", top_k_nodes=3, top_k_edges=2)
        node_names = [n.name for n in nodes]
        assert "Apple" in node_names or "Tim Cook" in node_names

    def test_content_hash_change_detection(self):
        g = _graph()

        h1 = hashlib.sha256(DOC1_V1.encode()).hexdigest()
        g.feed_text(DOC1_V1, source_id="doc1", content_hash=h1)
        assert g.source_content_hash("doc1") == h1

        # Same hash re-feed is a no-op at the API level (CLI enforces skip).
        h2 = hashlib.sha256(b"changed").hexdigest()
        g.feed_text(DOC1_V2, source_id="doc1", content_hash=h2)
        assert g.source_content_hash("doc1") == h2