DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
"""On-disk layout for a GraphRAG-backed knowledge base.
A GraphRAG KB keeps a self-contained project inside the KB's flat ``version-N``
directory (reused from ``index_versioning`` with a ``None`` signature, exactly
like the PageIndex pipeline). The version dir doubles as GraphRAG's project
root::
<kb_dir>/version-N/
settings.yaml # generated from DeepTutor config (see config.py)
input/ # parsed *.txt fed to the indexer
output/ # GraphRAG's parquet artefacts + lancedb
cache/ logs/
meta.json # synthetic "ready" marker (see write_meta)
The synthetic ``meta.json`` is what makes the existing "is this KB initialised?"
and Index-versions UI checks treat a GraphRAG KB as ready, without teaching the
manager about GraphRAG internals.
"""
from __future__ import annotations
from datetime import datetime, timezone
import logging
from pathlib import Path
from deeptutor.services.file_io import atomic_write_json
logger = logging.getLogger(__name__)
META_FILENAME = "meta.json"
PROVIDER = "graphrag"
INPUT_DIRNAME = "input"
OUTPUT_DIRNAME = "output"
# Parquet artefacts GraphRAG writes on a successful index; their presence is our
# "the index actually built" signal (independent of the synthetic meta marker).
OUTPUT_TABLES = (
"entities",
"communities",
"community_reports",
"text_units",
"relationships",
)
def input_dir(root_dir: Path) -> Path:
return Path(root_dir) / INPUT_DIRNAME
def output_dir(root_dir: Path) -> Path:
return Path(root_dir) / OUTPUT_DIRNAME
def has_output(root_dir: Path | None) -> bool:
"""True when GraphRAG has produced at least its core parquet tables."""
if root_dir is None:
return False
out = output_dir(root_dir)
if not out.is_dir():
return False
return any((out / f"{name}.parquet").exists() for name in OUTPUT_TABLES)
def indexed_input_names(root_dir: Path) -> set[str]:
"""Input filenames with persisted GraphRAG text units; empty on unknown output."""
documents = output_dir(root_dir) / "documents.parquet"
text_units = output_dir(root_dir) / "text_units.parquet"
if not documents.is_file() or not text_units.is_file():
return set()
try:
import pandas as pd
rows = pd.read_parquet(documents, columns=["title", "text_unit_ids"])
unit_ids = set(pd.read_parquet(text_units, columns=["id"])["id"])
except Exception as exc:
logger.warning("Could not inspect GraphRAG document receipts: %s", exc)
return set()
names: set[str] = set()
for title, text_unit_ids in rows.itertuples(index=False, name=None):
if not isinstance(title, str) or isinstance(text_unit_ids, (str, bytes)):
continue
try:
has_persisted_unit = any(unit_id in unit_ids for unit_id in text_unit_ids)
except TypeError:
has_persisted_unit = False
if has_persisted_unit:
names.add(Path(title).name)
return names
def write_meta(root_dir: Path) -> None:
"""Write a flat-layout ``meta.json`` so the version is listed as ready.
Mirrors ``index_versioning.write_version_meta`` but carries a synthetic
``graphrag`` signature instead of an embedding hash. The embedding identity
is stamped alongside so an externally-linked index can be checked for
embedding compatibility at connect time (GraphRAG otherwise fails retrieval
silently on a dimension mismatch).
"""
from deeptutor.services.rag.embedding_signature import embedding_meta_fields
target = Path(root_dir)
payload = {
"version": target.name,
"signature": PROVIDER,
"provider": PROVIDER,
"layout": "flat",
"created_at": datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + "Z",
**embedding_meta_fields(),
}
atomic_write_json(target / META_FILENAME, payload)
__all__ = [
"META_FILENAME",
"PROVIDER",
"INPUT_DIRNAME",
"OUTPUT_DIRNAME",
"OUTPUT_TABLES",
"input_dir",
"output_dir",
"has_output",
"indexed_input_names",
"write_meta",
]