Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

test: provider-configurable real-API provenance lifecycle integration test

yifanfeng97a2bac6dd371f
diff --git a/tests/integration/test_provenance_real.py b/tests/integration/test_provenance_real.py
new file mode 100644
index 0000000..2ec6a87
--- /dev/null
+++ b/tests/integration/test_provenance_real.py
@@ -0,0 +1,182 @@
+"""Integration test: provenance lifecycle against REAL LLM APIs.
+
+Provider-configurable via environment variables (defaults to the
+chatanywhere-compatible config in .env):
+
+ HE_TEST_LLM_BASE_URL LLM endpoint (default: OPENAI_BASE_URL)
+ HE_TEST_LLM_API_KEY LLM key (default: OPENAI_API_KEY)
+ HE_TEST_LLM_MODEL LLM model (default: gpt-4o-mini)
+ HE_TEST_EMBED_BASE_URL embed endpoint (default: LLM base URL)
+ HE_TEST_EMBED_API_KEY embed key (default: LLM key)
+ HE_TEST_EMBED_MODEL embed model (default: text-embedding-3-small)
+
+Example — DeepSeek LLM + chatanywhere embeddings:
+ HE_TEST_LLM_BASE_URL=https://api.deepseek.com \\
+ HE_TEST_LLM_API_KEY=sk-... HE_TEST_LLM_MODEL=deepseek-chat \\
+ HE_TEST_EMBED_BASE_URL=https://api.chatanywhere.tech/v1 \\
+ HE_TEST_EMBED_API_KEY=sk-... \\
+ pytest tests/integration/test_provenance_real.py -v
+
+Exercises the exact-replay question mocks cannot answer: when a document
+is rolled back and its keys re-merged from surviving sources, the
+LLM.BALANCED merger replays with real non-determinism — we assert
+semantic preservation (entity/fact survival), not wording.
+"""
+
+import hashlib
+import os
+
+import pytest
+from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+from pydantic import BaseModel
+
+from hyperextract.types import AutoGraph
+
+pytestmark = [
+ pytest.mark.integration,
+ pytest.mark.skipif(
+ not os.environ.get("OPENAI_API_KEY"),
+ reason="OPENAI_API_KEY not set",
+ ),
+]
+
+
+class Entity(BaseModel):
+ name: str
+ type: str = "organization"
+ description: str = ""
+
+
+class Relation(BaseModel):
+ source: str
+ target: str
+ relation_type: str
+ description: str = ""
+
+
+DOC1_V1 = (
+ "Apple was founded by Steve Jobs in 1976. "
+ "Google was founded by Larry Page and Sergey Brin in 1998. "
+ "Apple is headquartered in Cupertino, California."
+)
+DOC1_V2 = (
+ "Apple was founded by Steve Jobs in 1976. "
+ "Tim Cook is the CEO of Apple. "
+ "Apple is headquartered in Cupertino, California."
+) # Google fact removed from the document
+DOC2 = "Microsoft was founded by Bill Gates and Paul Allen in 1975."
+
+
+def _client_config(prefix: str, fallback_base: str, fallback_key: str):
+ base = os.environ.get(f"{prefix}_BASE_URL", fallback_base)
+ key = os.environ.get(f"{prefix}_API_KEY", fallback_key)
+ return base, key
+
+
+def _graph():
+ llm_base, llm_key = _client_config(
+ "HE_TEST_LLM",
+ fallback_base=os.environ.get("OPENAI_BASE_URL", ""),
+ fallback_key=os.environ.get("OPENAI_API_KEY", ""),
+ )
+ emb_base, emb_key = _client_config(
+ "HE_TEST_EMBED",
+ fallback_base=llm_base,
+ fallback_key=llm_key,
+ )
+ llm_model = os.environ.get("HE_TEST_LLM_MODEL", "gpt-4o-mini")
+ emb_model = os.environ.get("HE_TEST_EMBED_MODEL", "text-embedding-3-small")
+
+ llm = ChatOpenAI(
+ model=llm_model,
+ temperature=0,
+ api_key=llm_key,
+ base_url=llm_base or None,
+ )
+ embedder = OpenAIEmbeddings(
+ model=emb_model,
+ api_key=emb_key,
+ base_url=emb_base or None,
+ )
+ return AutoGraph(
+ node_schema=Entity,
+ edge_schema=Relation,
+ node_key_extractor=lambda x: x.name,
+ edge_key_extractor=lambda x: f"{x.source}-{x.relation_type}-{x.target}",
+ nodes_in_edge_extractor=lambda x: (x.source, x.target),
+ llm_client=llm,
+ embedder=embedder,
+ extraction_mode="one_stage",
+ )
+
+
+def _names(graph):
+ return {n.name for n in graph.nodes}
+
+
+class TestProvenanceRealAPI:
+ def test_full_lifecycle(self):
+ g = _graph()
+
+ # 1. Attributed ingestion of doc1 v1 (real extraction)
+ g.feed_text(DOC1_V1, source_id="doc1")
+ assert "doc1" in g._node_memory._sources
+ names_v1 = _names(g)
+ assert "Apple" in names_v1
+ assert "Google" in names_v1
+
+ # 2. doc2 contributes to the shared "Apple" key
+ g.feed_text(DOC2, source_id="doc2")
+ assert "Microsoft" in _names(g)
+
+ # 3. doc1 v2 removes the Google fact → auto-upsert rolls back v1:
+ # Google (sole doc1 contribution) is removed; Apple (shared with
+ # doc2) is re-merged from doc2's raw results with real LLM merge.
+ g.feed_text(DOC1_V2, source_id="doc1")
+
+ names_after = _names(g)
+ assert "Google" not in names_after
+ assert "Microsoft" in names_after
+ assert "Apple" in names_after
+
+ # 4. Roll back doc2 entirely → Microsoft goes, Apple stays.
+ # (Real extraction also pulls people/years from DOC2 — all of
+ # them are doc2-only contributions, removed together.)
+ report = g.remove_source("doc2")
+ removed = set(report["removed_nodes"])
+ assert "Microsoft" in removed
+ assert "Bill Gates" in removed
+ names_final = _names(g)
+ assert "Microsoft" not in names_final
+ assert "Bill Gates" not in names_final
+ assert "Paul Allen" not in names_final
+ assert "Apple" in names_final
+
+ # 5. Index patched in place throughout, consistent with storage.
+ assert g._node_memory.has_index()
+ doc_keys = {
+ d.metadata["key"]
+ for d in g._node_memory._index.docstore._dict.values()
+ }
+ storage_keys = {g.node_key_extractor(n) for n in g.nodes}
+ assert doc_keys == storage_keys, (
+ f"index/storage key mismatch: index_only={doc_keys - storage_keys}, "
+ f"storage_only={storage_keys - doc_keys}"
+ )
+
+ # 6. Semantic search still works on the patched index.
+ nodes, edges = g.search("Tim Cook CEO of Apple", top_k_nodes=3, top_k_edges=2)
+ node_names = [n.name for n in nodes]
+ assert "Apple" in node_names or "Tim Cook" in node_names
+
+ def test_content_hash_change_detection(self):
+ g = _graph()
+
+ h1 = hashlib.sha256(DOC1_V1.encode()).hexdigest()
+ g.feed_text(DOC1_V1, source_id="doc1", content_hash=h1)
+ assert g.source_content_hash("doc1") == h1
+
+ # Same hash re-feed is a no-op at the API level (CLI enforces skip).
+ h2 = hashlib.sha256(b"changed").hexdigest()
+ g.feed_text(DOC1_V2, source_id="doc1", content_hash=h2)
+ assert g.source_content_hash("doc1") == h2