Skip to content
Python

refactor(memhub): remove the private retrieval vector index

alvinfeie5ce0cd96445
diff --git a/memhub/apps/cli/src/cli.ts b/memhub/apps/cli/src/cli.ts
index 4cdb1d6..cc30fd6 100644
--- a/memhub/apps/cli/src/cli.ts
+++ b/memhub/apps/cli/src/cli.ts
@@ -4869,10 +4869,10 @@ async function runMaintenanceRestore(options: {
const instanceId = randomUUID();
const child = spawn(options.coreCommand, ["serve", "--maintenance"], {
env: {
- // Do not pass the caller's Runtime tuning variables (for example
- // retrieval-index flags) into a maintenance process: a restore must
- // start the product's minimal maintenance configuration, not inherit
- // an unrelated experiment from the shell.
+ // Do not pass the caller's Runtime tuning variables into a
+ // maintenance process: a restore must start the product's minimal
+ // maintenance configuration, not inherit an unrelated experiment from
+ // the shell.
PATH: process.env.PATH,
HOME: process.env.HOME,
LANG: process.env.LANG,
diff --git a/memhub/apps/core/pyproject.toml b/memhub/apps/core/pyproject.toml
index dda79b0..3e99cc7 100644
--- a/memhub/apps/core/pyproject.toml
+++ b/memhub/apps/core/pyproject.toml
@@ -13,7 +13,8 @@ dependencies = [
"filelock>=3.32.4",
# Public SDK version validated against MemHub's permission and recall contracts.
"hy-memory==2.0.5",
- # Retrieval's derived vector index directly consumes these locked versions.
+ # chromadb arrives through the HY SDK; locked here so an unrelated upgrade
+ # cannot move the SDK's store. Built-in embedding consumes numpy directly.
"chromadb==1.5.9",
"numpy==2.5.2",
"httpx>=0.28.1",
diff --git a/memhub/apps/core/src/one_memory/api.py b/memhub/apps/core/src/one_memory/api.py
index 19d462d..dcd75c5 100644
--- a/memhub/apps/core/src/one_memory/api.py
+++ b/memhub/apps/core/src/one_memory/api.py
@@ -841,8 +841,6 @@ def create_app(
secret_store: SecretStore | None = None,
) -> FastAPI:
runtime_settings = settings or load_settings()
- if runtime_settings.retrieval_index_enabled and not manage_runtime:
- raise ValueError("retrieval_runtime_ownership_required")
@asynccontextmanager
async def lifespan(_: FastAPI) -> AsyncIterator[None]:
@@ -1456,22 +1454,7 @@ def create_app(
app.state.resource_diagnostics = watched_task(
asyncio.get_running_loop(), _emit_resource_snapshots
)
- app.state.retrieval_runtime = None
try:
- if runtime_settings.retrieval_index_enabled:
- from one_memory.retrieval.index_provisioning import provision_retrieval_index
- from one_memory.retrieval.runtime import ExistingRetrievalRuntime
-
- # The runtime only resumes an index someone initialized. Turning
- # the feature on is that someone; otherwise enabling it would
- # leave every search on the lexical lane with nothing to say why.
- await asyncio.to_thread(provision_retrieval_index, runtime_settings.data_dir)
- app.state.retrieval_runtime = ExistingRetrievalRuntime(
- runtime_settings.data_dir,
- minimum_free_bytes=runtime_settings.retrieval_minimum_free_bytes,
- service_levels=app.state.service_levels,
- )
- await app.state.retrieval_runtime.start()
# Startup writes are done. Only now let the derived-content
# rebuild take the single write checkout: it is a long
# background job, and on a large library it would otherwise
@@ -1498,15 +1481,6 @@ def create_app(
outcome="started",
instance_id=runtime_settings.instance_id,
)
- retrieval_stop_error: BaseException | None = None
- if app.state.retrieval_runtime is not None:
- try:
- await app.state.retrieval_runtime.stop()
- except (Exception, asyncio.CancelledError) as error:
- # Preserve failure while cleaning other owned services.
- # Experimental: an exceptional live-thread failure still
- # needs an outer ownership fence before production enablement.
- retrieval_stop_error = error
app.state.main_database_report.cancel()
app.state.loop_heartbeat.cancel()
with suppress(asyncio.CancelledError):
@@ -1564,8 +1538,6 @@ def create_app(
# Waited for off the loop: this is the task whose whole point
# is that the loop never waits on a disk.
await asyncio.to_thread(flush_diagnostics)
- if retrieval_stop_error is not None:
- raise retrieval_stop_error
app = FastAPI(
title="MemHub Core",
@@ -4481,9 +4453,8 @@ class _LazyASGIApp:
`create_app()` must not run at import time. The managed `serve()` entry
imports `create_app` from this module before it can construct its own app
with `manage_runtime=True`; an eager module-level app was therefore built
- first with the default `manage_runtime=False`. With retrieval index enabled
- that default app hit `retrieval_runtime_ownership_required` and killed the
- process before the managed app could bind a port.
+ first with the default `manage_runtime=False`, doing startup work for an
+ app that never serves.
"""
def __init__(self) -> None:
diff --git a/memhub/apps/core/src/one_memory/config.py b/memhub/apps/core/src/one_memory/config.py
index e68cb34..4f8543e 100644
--- a/memhub/apps/core/src/one_memory/config.py
+++ b/memhub/apps/core/src/one_memory/config.py
@@ -23,10 +23,6 @@ class Settings(BaseSettings):
live_sync_enabled: bool = True
automatic_memory_processing: bool = True
imports_enabled: bool = True
- # Experimental and unreleased: ownership/error shutdown still requires acceptance.
- # Resume only explicitly initialized generations; never migrate at startup.
- retrieval_index_enabled: bool = False
- retrieval_minimum_free_bytes: int = Field(default=256 * 1024 * 1024, ge=0)
# Bounded metadata only: query, original text and answers are never stored.
call_event_retention_days: int | None = Field(default=30, ge=1, le=365)
call_event_max_events: int | None = Field(default=10_000, ge=1, le=100_000)
diff --git a/memhub/apps/core/src/one_memory/context_routes.py b/memhub/apps/core/src/one_memory/context_routes.py
index 3096d6b..c434939 100644
--- a/memhub/apps/core/src/one_memory/context_routes.py
+++ b/memhub/apps/core/src/one_memory/context_routes.py
@@ -38,7 +38,7 @@ from one_memory.context_models import (
SdkMemoryContextItem,
SdkRawContextItem,
)
-from one_memory.context_raw import revalidate_raw_context
+from one_memory.context_raw import read_raw_context, revalidate_raw_context
from one_memory.context_search import search_context
from one_memory.diagnostic_log import diagnostic_event, diagnostic_exception
from one_memory.external_memory_access import external_access
@@ -50,7 +50,6 @@ from one_memory.hy_sdk_types import (
parse_sdk_search,
)
from one_memory.imports.storage import EvidenceStore
-from one_memory.retrieval.raw_lane import read_raw_indexed_context
from one_memory.semantic_deadline import hy_wait_budget, search_within
CALL_EVENT_RESPONSE_BUDGET_MS = 250
@@ -99,12 +98,11 @@ def context_router(dependencies: Sequence[Depends]) -> APIRouter:
read = asyncio.create_task(
in_read_pool(
raw_read_pool(),
- read_raw_indexed_context,
+ read_raw_context,
database,
value,
source_ids=authority.raw_sources,
deadline=lane_end,
- index=getattr(request.app.state, "retrieval_runtime", None),
)
)
done, _ = await asyncio.wait({read}, timeout=max(0, lane_end - time.monotonic()))
diff --git a/memhub/apps/core/src/one_memory/context_v2_coordinator.py b/memhub/apps/core/src/one_memory/context_v2_coordinator.py
index f4ca101..14d9f55 100644
--- a/memhub/apps/core/src/one_memory/context_v2_coordinator.py
+++ b/memhub/apps/core/src/one_memory/context_v2_coordinator.py
@@ -22,7 +22,7 @@ from dataclasses import dataclass
from datetime import UTC, datetime
from functools import partial
from pathlib import Path
-from typing import Any, Literal, Protocol, cast
+from typing import Any, Literal, cast
from one_memory.background_work import interactive_read_pool
from one_memory.context_expand import ExpansionAuthority, ExpansionRefSigner
@@ -36,23 +36,8 @@ from one_memory.retrieval.models import (
RawEvidence,
RetrievalPath,
RetrievalResponse,
- TimeRange,
)
from one_memory.retrieval.query_plan import plan_local_query
-from one_memory.retrieval.raw_candidates import RawCandidateBatch
-from one_memory.retrieval.raw_evidence import materialize_raw_candidates
-
-
-class RawIndexLookup(Protocol):
- def lookup_raw(
- self,
- query: str,
- *,
- source_ids: frozenset[str],
- deadline: float,
- time_range: TimeRange | None = None,
- ) -> tuple[RawCandidateBatch, bool]: ...
-
ExpandRefs = tuple[ExpansionRefSigner, ExpansionAuthority]
@@ -106,11 +91,6 @@ class LaneResult:
coverage_limited: bool = False
-def _semantic_only_raw(item: RawEvidence) -> bool:
- """Semantic overlap alone is a candidate, not a confirmed recall item."""
- return set(item.matched_by) == {"semantic"}
-
-
@dataclass(frozen=True)
class StageTimings:
raw_ms: float = 0.0
@@ -191,12 +171,6 @@ def _managed(
return selected, truncated
-def _group_missing_requirements(item: RawEvidence | GovernedEvidence) -> list[str]:
- if isinstance(item, RawEvidence) and _semantic_only_raw(item):
- return ["relevance_confirmation"]
- return []
-
-
def _path(
status: str,
reason: str | None,
@@ -296,8 +270,7 @@ def read_lexical_raw_lane(
("lexical", "object") if plan.object_keys else ("lexical",),
truncated,
# Starvation means: this lane matched, then dropped the item whole. An
- # ordinary no-match must stay empty, and an unconfigured semantic lane
- # is not starvation.
+ # ordinary no-match must stay empty.
starved and not items,
probe_saturated,
)
@@ -564,97 +537,6 @@ def warm_raw_read_path(
return False
-def read_semantic_raw_lane(
- index: RawIndexLookup | None,
- database_path: Path,
- query: CoordinatorQuery,
- *,
- source_ids: frozenset[str],
- deadline: float,
-) -> LaneResult:
- if not source_ids:
- return LaneResult((), "unavailable", "permission_required", (), False)
- # No derived index is the no-model baseline, not a failure: the lexical lane
- # must still answer. Semantic is genuinely unsearched in that state, so it
- # must not claim a channel or a coverage result it never produced.
- if index is None:
- return LaneResult((), "not_searched", None, (), False)
- try:
- batch, unpublished = index.lookup_raw(
- query.query,
- source_ids=source_ids,
- deadline=deadline,
- time_range=plan_local_query(query.query).time_range,
- )
- except Exception:
- return LaneResult((), "unavailable", "index_unavailable", ("semantic",), False)
- if not isinstance(batch, RawCandidateBatch):
- return LaneResult((), "unavailable", "index_unavailable", ("semantic",), False)
- if batch.reason == "not_authorized":
- return LaneResult((), "unavailable", "permission_required", ("semantic",), False)
- if batch.reason == "deadline":
- return LaneResult((), "unavailable", "timeout_budget", ("semantic",), True)
- if batch.reason in {"vector_unavailable", "index_unavailable"}:
- return LaneResult((), "unavailable", batch.reason, ("semantic",), False)
- store, generation = _index_state(index, database_path)
- from one_memory.retrieval.index_store import RetrievalIndexStore
-
- store = cast("RetrievalIndexStore | None", store)
- if store is None or generation is None:
- return LaneResult((), "unavailable", "index_unavailable", ("semantic",), False)
- try:
- materialized = materialize_raw_candidates(
- store,
- batch,
- generation_id=generation,
- source_ids=source_ids,
- deadline=deadline,
- character_budget=query.character_budget,
- )
- except Exception:
- return LaneResult((), "unavailable", "index_unavailable", ("semantic",), False)
- if materialized.reason == "deadline":
- return LaneResult((), "unavailable", "timeout_budget", ("semantic",), True)
- if materialized.reason in {"index_unavailable", "not_authorized"}:
- return LaneResult((), "unavailable", materialized.reason, ("semantic",), False)
- complete = materialized.complete and not unpublished
- if materialized.reason == "evidence_changed":
- status: Literal["ok", "empty", "unavailable", "not_searched"] = "unavailable"
- reason = "evidence_changed"
- elif not materialized.items and not complete:
- status, reason = "unavailable", "index_incomplete"
- elif materialized.reason == "character_budget":
- status, reason = "ok", None
- else:
- status, reason = ("ok" if materialized.items else "empty"), None
- return LaneResult(
- tuple(materialized.items),
- status,
- reason,
- ("semantic",),
- materialized.reason == "character_budget",
- )
-
-
-def _index_state(index: RawIndexLookup, database_path: Path) -> tuple[object | None, str | None]:
- store = getattr(index, "_store", None)
- generation = None
- snapshot = getattr(index, "snapshot", None)
- if snapshot is not None:
- try:
- generation = snapshot().generation_id
- except Exception:
- generation = None
- if store is None:
- try:
- from one_memory.retrieval.index_store import RetrievalIndexStore
-
- store = RetrievalIndexStore(database_path)
- except Exception:
- store = None
- return store, generation
-
-
def read_memory_lane(
database_path: Path,
query: CoordinatorQuery,
@@ -857,7 +739,6 @@ def coordinate_context_v2(
query: CoordinatorQuery,
*,
database_path: Path,
- index: RawIndexLookup | None = None,
deadline: float,
expand_refs: ExpandRefs | None = None,
) -> RetrievalResponse:
@@ -871,62 +752,29 @@ def coordinate_context_v2(
lexical = read_lexical_raw_lane(
database_path, query, source_ids=authority.raw_sources, deadline=deadline
)
- semantic = read_semantic_raw_lane(
- index, database_path, query, source_ids=authority.raw_sources, deadline=deadline
- )
timings = StageTimings(raw_ms=(time.monotonic() - raw_started) * 1000)
combined: dict[tuple[str, str], RawEvidence] = {}
- for item in (*lexical.items, *semantic.items):
- if not isinstance(item, RawEvidence):
- continue
- identity = _identity(item)
- previous = combined.get(identity)
- if previous is None or _semantic_only_raw(previous):
- combined[identity] = item
- # Semantic similarity is a candidate signal, not a verified recall
- # result. Keep semantic-only candidates visible in coverage, but do not
- # release their bodies for host injection unless the item is
- # independently confirmed by the lexical/object lane. A global distance
- # threshold is deliberately not used: the measured positive and
- # negative distributions overlap.
- semantic_only = {_identity(item) for item in combined.values() if _semantic_only_raw(item)}
- confirmed = {_identity(item) for item in lexical.items if isinstance(item, RawEvidence)}
- deferred_raw = semantic_only - confirmed
- if deferred_raw:
- combined = {
- identity: item
- for identity, item in combined.items()
- if identity not in deferred_raw
- }
- channels = tuple(dict.fromkeys((*lexical.channels, *semantic.channels)))
+ for item in lexical.items:
+ if isinstance(item, RawEvidence):
+ combined.setdefault(_identity(item), item)
if lexical.status == "unavailable":
status: Literal["ok", "empty", "unavailable", "not_searched", "degraded"] = (
"unavailable"
)
reason = lexical.reason
- elif semantic.status == "unavailable":
- status = "unavailable"
- reason = semantic.reason
- elif combined:
- status, reason = "ok", semantic.reason if semantic.status != "ok" else None
- elif deferred_raw:
- status, reason = "degraded", "relevance_confirmation"
- elif lexical.status == "empty":
- status, reason = ("empty" if semantic.status != "not_searched" else "ok"), None
+ elif combined or lexical.status == "empty":
+ status, reason = "ok", None
else:
status, reason = "empty", None
- truncated = lexical.truncated or semantic.truncated
raw = LaneResult(
tuple(combined.values()),
status,
reason,
- channels,
- truncated,
+ lexical.channels,
+ lexical.truncated,
lexical.starved,
lexical.coverage_limited,
)
- if deferred_raw:
- object.__setattr__(raw, "coverage_limited", True)
elif "raw" in query.kinds and not authority.raw_allowed:
raw = LaneResult((), "unavailable", "permission_required", (), False)
if "memory" in query.kinds:
@@ -995,9 +843,7 @@ def coordinate_context_v2(
return _path("unavailable", raw.reason, raw.channels, "not_searched", False, 0)
if dropped and not final_raw:
return _path("unavailable", "permission_required", raw.channels, "unavailable", True, 0)
- # Lexical-only is complete when the lexical lane actually answered. An
- # unconfigured semantic lane is a capability absence, not proof that
- # less than the authorized range was searched.
+ # The raw lane is complete when the lexical lane actually answered.
complete = (
raw.status in {"ok", "empty"}
and not dropped
@@ -1022,9 +868,7 @@ def coordinate_context_v2(
"count_budget"
if truncated
else "coverage_budget"
- if raw.coverage_limited and not deferred_raw
- else "relevance_confirmation"
- if deferred_raw
+ if raw.coverage_limited
else "character_budget"
)
return _path(
@@ -1112,7 +956,7 @@ def coordinate_context_v2(
"group_id": f"group-{index}",
"answer_target": query.intent,
"decision_status": "missing_confirmation",
- "missing_requirements": _group_missing_requirements(item),
+ "missing_requirements": [],
"object_refs": _object_refs(item, plan.object_keys),
"items": [group_item.model_dump(mode="json") for group_item in (item,)],
}
@@ -1185,7 +1029,6 @@ async def search_context_v2(
query: CoordinatorQuery,
*,
database_path: Path,
- index: RawIndexLookup | None = None,
deadline: float,
expand_refs: ExpandRefs | None = None,
) -> RetrievalResponse:
@@ -1198,7 +1041,6 @@ async def search_context_v2(
authority,
query,
database_path=database_path,
- index=index,
deadline=deadline,
expand_refs=expand_refs,
),
diff --git a/memhub/apps/core/src/one_memory/local_embedding.py b/memhub/apps/core/src/one_memory/local_embedding.py
index fd9aee7..26114c6 100644
--- a/memhub/apps/core/src/one_memory/local_embedding.py
+++ b/memhub/apps/core/src/one_memory/local_embedding.py
@@ -10,16 +10,12 @@ weights alone.
from __future__ import annotations
import threading
-from dataclasses import dataclass
-from hashlib import sha256
from importlib import import_module
from typing import TYPE_CHECKING, Any
from one_memory.embedding_assets import (
BUILTIN_EMBEDDING_DIMENSIONS,
BUILTIN_EMBEDDING_MAX_TOKENS,
- EMBEDDING_ASSETS,
- AssetVerificationError,
MissingEmbeddingAssetsError,
assets_present,
resolve_asset_dir,
@@ -34,28 +30,6 @@ _FETCH_HINT = (
)
-@dataclass(frozen=True)
-class TokenOffset:
- """Half-open Python Unicode code point offsets; synthetic specials use [0,0)."""
-
- start_cp: int
- end_cp: int
- special: bool = False
-
-
-@dataclass(frozen=True)
-class TokenizedText:
- """Untruncated, unpadded encoding, including tokenizer postprocessor specials."""
-
- token_ids: tuple[int, ...]
- offsets: tuple[TokenOffset, ...]
-
- def __post_init__(self) -> None:
- # Defensive copies also keep runtime callers from retaining mutable lists.
- object.__setattr__(self, "token_ids", tuple(self.token_ids))
- object.__setattr__(self, "offsets", tuple(self.offsets))
-
-
class LocalEmbedder:
"""Encodes text with the built-in model, loading it on first use."""
@@ -68,43 +42,6 @@ class LocalEmbedder:
self._session: Any | None = None
self._tokenizer: Any | None = None
self._input_names: frozenset[str] = frozenset()
- self._offset_lock = threading.Lock()
- self._offset_tokenizer: Any | None = None
-
- def tokenize_with_offsets(self, text: str) -> TokenizedText:
- """Tokenize original text without encode()'s truncation or batch padding.
-
- The Python tokenizers binding returns code point offsets into the input,
- even when its model normalizer changes spelling. This separate lazy
- tokenizer uses the exact locked asset; it never loads an ONNX session or
- mutates the tokenizer shared by encode(). Offsets are not UTF-16 indices.
- """
- if not isinstance(text, str):
- raise TypeError("embedding_offsets_require_text")
- with self._offset_lock:
- if self._offset_tokenizer is None:
- directory = resolve_asset_dir()
- path = directory / "tokenizer.json"
- if not path.is_file():
- raise MissingEmbeddingAssetsError(f"{_FETCH_HINT} Looked in {directory}.")
- data = path.read_bytes()
- expected = next(a.sha256 for a in EMBEDDING_ASSETS if a.filename == path.name)
- if sha256(data).hexdigest() != expected:
- raise AssetVerificationError("tokenizer.json: locked digest mismatch")
- from tokenizers import Tokenizer
-
- tokenizer = Tokenizer.from_str(data.decode("utf-8"))
- tokenizer.no_truncation()
- tokenizer.no_padding()
- self._offset_tokenizer = tokenizer
- encoding = self._offset_tokenizer.encode(text, add_special_tokens=True)
- offsets = tuple(
- TokenOffset(start, end, bool(special))
- for (start, end), special in zip(
- encoding.offsets, encoding.special_tokens_mask, strict=True
- )
- )
- return TokenizedText(tuple(encoding.ids), offsets)
def _load(self) -> tuple[Any, Any]:
# Double-checked under a lock: memory processing encodes from worker
diff --git a/memhub/apps/core/src/one_memory/memory_server/migrations.py b/memhub/apps/core/src/one_memory/memory_server/migrations.py
index e5fa270..ff129ae 100644
--- a/memhub/apps/core/src/one_memory/memory_server/migrations.py
+++ b/memhub/apps/core/src/one_memory/memory_server/migrations.py
@@ -2217,34 +2217,6 @@ CREATE TABLE IF NOT EXISTS retrieval_chunks (
"""
-def retrieval_documents_v2_ddl(dialect: str = "mysql") -> str:
- """The v2 documents table under its temporary `_v2` name.
-
- `index_store.migrate_v2` creates this, copies rows from the v1 table, drops
- the v1 table and renames. The rename is the only reason a `_v2` name exists
- at all, and it is derived from the single definition so the copy-across
- cannot drift from what fresh databases get.
- """
- documents = next(
- statement
- for statement in _split_statements(retrieval_index_ddl(dialect))
- if "CREATE TABLE IF NOT EXISTS retrieval_documents (" in statement
- )
- return documents.replace(
- "CREATE TABLE IF NOT EXISTS retrieval_documents (",
- "CREATE TABLE retrieval_documents_v2 (",
- )
-
-
-def retrieval_build_maps_ddl(dialect: str = "mysql") -> str:
- """The build-maps table added by v2."""
- return next(
- statement
- for statement in _split_statements(retrieval_index_ddl(dialect))
- if "retrieval_build_maps" in statement
- )
-
-
async def _033_retrieval_index(cursor: Any) -> None:
for statement in _split_statements(retrieval_index_ddl(_dialect(cursor))):
await cursor.execute(statement)
diff --git a/memhub/apps/core/src/one_memory/retrieval/chunking.py b/memhub/apps/core/src/one_memory/retrieval/chunking.py
deleted file mode 100644
index 9d59121..0000000
--- a/memhub/apps/core/src/one_memory/retrieval/chunking.py
+++ /dev/null
@@ -1,167 +0,0 @@
-"""Single-Evidence identity chunking; no reads, persistence, normalization or grants.
-
-Callers must first establish eligibility/sanitization/permission. This module
-does not strip wrappers or secrets and must not be used as a content safety gate.
-"""
-
-from dataclasses import dataclass
-from typing import Protocol
-
-from one_memory.embedding_assets import BUILTIN_EMBEDDING_MAX_TOKENS
-from one_memory.local_embedding import TokenizedText, TokenOffset
-
-
-class OffsetTokenizer(Protocol):
- def tokenize_with_offsets(self, text: str) -> TokenizedText: ...
-
-
-class ChunkingError(ValueError):
- """Invalid offsets, uncovered text or insufficient encoding budget; never truncate."""
-
-
-@dataclass(frozen=True)
-class TextChunk:
- ordinal: int
- start_cp: int
- end_cp: int
- token_count: int
- text: str
-
-
-@dataclass(frozen=True)
-class _Span:
- start: int
- end: int
- tokens: int
-
-
-def _spans(text: str, encoded: TokenizedText) -> tuple[_Span, ...]:
- if not isinstance(encoded, TokenizedText) or len(encoded.token_ids) != len(encoded.offsets):
- raise ChunkingError("chunk_token_offset_count_mismatch")
- spans: list[_Span] = []
- previous_start = previous_end = 0
- for token_id, offset in zip(encoded.token_ids, encoded.offsets, strict=True):
- if type(token_id) is not int or token_id < 0 or not isinstance(offset, TokenOffset):
- raise ChunkingError("chunk_invalid_token_offset")
- if (
- type(offset.start_cp) is not int
- or type(offset.end_cp) is not int
- or type(offset.special) is not bool
- ):
- raise ChunkingError("chunk_invalid_token_offset")
- start, end = offset.start_cp, offset.end_cp
- if offset.special:
- if start != 0 or end != 0:
- raise ChunkingError("chunk_invalid_special_offset")
- continue
- if not 0 <= start < end <= len(text) or start < previous_start or end < previous_end:
- raise ChunkingError("chunk_invalid_offset_order_or_bounds")
- if start > previous_end and not text[previous_end:start].isspace():
- raise ChunkingError("chunk_uncovered_nonwhitespace")
- # Overlapping subpieces (including several pieces for one code point)
- # form an indivisible interval; slicing inside it loses token identity.
- if spans and start < spans[-1].end:
- prior = spans[-1]
- spans[-1] = _Span(prior.start, end, prior.tokens + 1)
- else:
- spans.append(_Span(start, end, 1))
- previous_start, previous_end = start, end
- if text[previous_end:] and not text[previous_end:].isspace():
- raise ChunkingError("chunk_uncovered_nonwhitespace")
- return tuple(spans)
-
-
-def _encoding(text: str, tokenizer: OffsetTokenizer) -> tuple[TokenizedText, tuple[_Span, ...]]:
- try:
- result = tokenizer.tokenize_with_offsets(text)
- except Exception as error:
- raise ChunkingError("chunk_tokenization_failed") from error
- return result, _spans(text, result)
-
-
-def chunk_evidence(
- text: str,
- tokenizer: OffsetTokenizer,
- *,
- prefix: str = "",
- target_tokens: int = 384,
- overlap_tokens: int = 64,
- max_tokens: int = BUILTIN_EMBEDDING_MAX_TOKENS,
-) -> tuple[TextChunk, ...]:
- """Chunk one original with code point [start,end), preserving exact slices.
-
- target/overlap count original non-special tokens. Overlap is up to the
- requested token count at indivisible offset boundaries; forward progress is
- mandatory. token_count is re-tokenized prefix + chunk.text INCLUDING specials,
- never an additive estimate. Prefix is caller-provided, unchanged, and omitted
- from chunk.text/offsets. Empty/whitespace-only inputs have no index chunks.
- """
- if not isinstance(text, str) or not isinstance(prefix, str):
- raise ChunkingError("chunk_requires_text")
- if (
- any(type(value) is not int for value in (target_tokens, overlap_tokens, max_tokens))
- or not 0 <= overlap_tokens < target_tokens
- or not 1 <= max_tokens <= BUILTIN_EMBEDDING_MAX_TOKENS
- or target_tokens > max_tokens
- ):
- raise ChunkingError("chunk_invalid_token_budget")
- if not text or text.isspace():
- return ()
- _, spans = _encoding(text, tokenizer)
- if not spans:
- raise ChunkingError("chunk_uncovered_nonwhitespace")
- chunks: list[TextChunk] = []
- start_index = 0
- previous_end_cp = 0
- while start_index < len(spans):
- end_index = start_index
- body_tokens = 0
- while end_index < len(spans):
- candidate = body_tokens + spans[end_index].tokens
- if end_index > start_index and candidate > target_tokens:
- break
- body_tokens = candidate
- end_index += 1
- # Prefer the last paragraph boundary in the latter half of the target.
- # Never trade away an entire overlap window or split a token interval.
- if end_index < len(spans):
- count = 0
- preferred = None
- for boundary in range(start_index + 1, end_index + 1):
- count += spans[boundary - 1].tokens
- if count >= max(target_tokens // 2, overlap_tokens + 1):
- left, right = spans[boundary - 1].end, spans[boundary].start
- if "\n" in text[max(spans[boundary - 1].start, left - 1) : right]:
- preferred = boundary
- if preferred is not None:
- end_index = preferred
- start_cp = 0 if start_index == 0 else spans[start_index].start
- while end_index > start_index:
- # Include gaps verbatim, including whitespace and combining marks;
- # source coverage is never reconstructed by joining decoded tokens.
- end_cp = spans[end_index].start if end_index < len(spans) else len(text)
- encoded, _ = _encoding(prefix + text[start_cp:end_cp], tokenizer)
- token_count = len(encoded.token_ids)
- if token_count <= max_tokens:
- break
- end_index -= 1
- else:
- raise ChunkingError("chunk_indivisible_span_exceeds_token_budget")
- if end_cp <= previous_end_cp or start_cp > previous_end_cp:
- raise ChunkingError("chunk_coverage_cannot_progress")
- chunks.append(TextChunk(len(chunks), start_cp, end_cp, token_count, text[start_cp:end_cp]))
- previous_end_cp = end_cp
- if end_index == len(spans):
- break
- next_start = end_index
- overlap = 0
- while next_start > start_index + 1:
- candidate = overlap + spans[next_start - 1].tokens
- if candidate > overlap_tokens:
- break
- next_start -= 1
- overlap = candidate
- start_index = next_start
- if previous_end_cp != len(text):
- raise ChunkingError("chunk_incomplete_coverage")
- return tuple(chunks)
diff --git a/memhub/apps/core/src/one_memory/retrieval/content.py b/memhub/apps/core/src/one_memory/retrieval/content.py
deleted file mode 100644
index 2c1aa68..0000000
--- a/memhub/apps/core/src/one_memory/retrieval/content.py
+++ /dev/null
@@ -1,92 +0,0 @@
-"""Shared qualification for identity-mapped raw indexing and candidate reads."""
-
-from collections.abc import Sequence
-
-from one_memory.conversation_processing import _is_hy_input_turn
-from one_memory.imports.models import EvidenceContentKind, EvidenceRecord
-from one_memory.imports.sources.parsing import is_host_generated_user_turn, strip_host_wrappers
-from one_memory.input_sanitization import sanitize_model_input
-from one_memory.retrieval.chunking import TextChunk
-from one_memory.retrieval.mapped_content import PreparedMappedContent, prepare_mapped_content
-
-
-def identity_content_ready(record: EvidenceRecord) -> bool:
- # Reuse pure existing hygiene checks, without invoking HY. Nonidentity
- # cleaning requires a source map and remains explicitly uncovered for now.
- return (
- record.content_kind is EvidenceContentKind.RAW_CONVERSATION
- and record.role in {"user", "assistant"}
- and _is_hy_input_turn(record)
- and not is_host_generated_user_turn(record.verbatim)
- and strip_host_wrappers(record.verbatim) == record.verbatim.strip()
- and sanitize_model_input(record.verbatim) == record.verbatim
- )
-
-
-def index_content(
- record: EvidenceRecord, *, normalizer_version: int, chunker_version: int
-) -> PreparedMappedContent | None:
- """Generation-owned content semantics, never silently upgrade identity generations."""
- if (normalizer_version, chunker_version) == (1, 1):
- if not identity_content_ready(record):
- raise ValueError("content_mapping_required")
- return None
- if (normalizer_version, chunker_version) == (2, 2):
- return prepare_mapped_content(record)
- raise ValueError("unsupported_content_generation")
-
-
-def validate_chunk_coverage(
- record: EvidenceRecord, chunks: Sequence[TextChunk], prepared: PreparedMappedContent | None
-) -> None:
- """Verify exact slices and complete citation-safe coverage without a tokenizer call."""
- if len(chunks) > 8192 or (prepared is None and not chunks):
- raise ValueError("invalid_chunk_manifest")
- if prepared is None:
- ranges = [(0, len(record.verbatim))]
- else:
- ranges = []
- for span in prepared.metadata.segments:
- if span.kind == "copy":
- assert span.source_start_cp is not None and span.source_end_cp is not None
- ranges.append((span.source_start_cp, span.source_end_cp))
- range_index = 0
- covered_end = ranges[0][0] if ranges else 0
- previous_end = -1
- for index, chunk in enumerate(chunks):
- if not isinstance(chunk, TextChunk) or any(
- type(value) is not int
- for value in (chunk.ordinal, chunk.start_cp, chunk.end_cp, chunk.token_count)
- ):
- raise ValueError("invalid_chunk_manifest")
- while range_index < len(ranges) and chunk.start_cp >= ranges[range_index][1]:
- if record.verbatim[covered_end : ranges[range_index][1]].strip():
- raise ValueError("incomplete_chunk_manifest")
- range_index += 1
- covered_end = ranges[range_index][0] if range_index < len(ranges) else 0
- if range_index >= len(ranges):
- raise ValueError("invalid_chunk_manifest")
- left, right = ranges[range_index]
- if (
- chunk.ordinal != index
- or not left <= chunk.start_cp < chunk.end_cp <= right
- or chunk.end_cp <= previous_end
- or not 1 <= chunk.token_count <= 512
- or record.verbatim[chunk.start_cp : chunk.end_cp] != chunk.text
- or (
- chunk.start_cp > covered_end
- and (prepared is None or record.verbatim[covered_end : chunk.start_cp].strip())
- )
- ):
- raise ValueError("invalid_chunk_manifest")
- covered_end = previous_end = chunk.end_cp
- while range_index < len(ranges):
- if prepared is None:
- if covered_end != ranges[range_index][1]:
- raise ValueError("incomplete_chunk_manifest")
- elif record.verbatim[covered_end : ranges[range_index][1]].strip():
- raise ValueError("incomplete_chunk_manifest")
- range_index += 1
- covered_end = ranges[range_index][0] if range_index < len(ranges) else 0
- if prepared is not None and (not chunks) != (prepared.state == "no_indexable_content"):
- raise ValueError("invalid_chunk_manifest")
diff --git a/memhub/apps/core/src/one_memory/retrieval/index_provisioning.py b/memhub/apps/core/src/one_memory/retrieval/index_provisioning.py
deleted file mode 100644
index 2073949..0000000
--- a/memhub/apps/core/src/one_memory/retrieval/index_provisioning.py
+++ /dev/null
@@ -1,96 +0,0 @@
-"""Publish one active generation for the built-in encoder, when asked to.
-
-The runtime deliberately only resumes an index somebody already initialized, so
-without this the feature could be switched on and still never index anything.
-Provisioning stays separate from `start()` for the same reason it exists: it is
-the one place allowed to create, and it creates only for a first enable.
-"""
-
-from __future__ import annotations
-
-import logging
-import sqlite3
-from pathlib import Path
-
-from one_memory.database_access import main_database
-from one_memory.embedding_assets import assets_present
-from one_memory.retrieval.index_store import RetrievalIndexStore
-from one_memory.retrieval.index_worker import LocalRetrievalEncoder
-from one_memory.retrieval.vector_store import RawVectorStore
-
-DATABASE_NAME = "one-memory.sqlite3"
-VECTOR_DIR_NAME = "retrieval-vectors"
-
-
-def provision_retrieval_index(data_dir: Path) -> str | None:
- """The active generation id for the built-in encoder, creating it once.
-
- Returns None whenever the right answer is to leave the data alone: no
- built-in encoder, no database yet, an index from an older schema, or one
- already published for a different encoder. None is not an error; the
- lexical lane works without an index and says so through `degraded`.
- """
- database = data_dir / DATABASE_NAME
- if not assets_present() or not database.is_file():
- return None
- identity = LocalRetrievalEncoder().identity
- store = RetrievalIndexStore(database)
- try:
- with main_database(database).write() as db, db:
- db.execute("BEGIN IMMEDIATE")
- if (existing := _published(db)) is not None:
- return existing if _matches(store, db, existing, identity) else None
- version = _schema_version(db)
- if version == 1:
- # Someone's v1 index. Migrating it is maintenance, not startup.
- return None
- if version is None:
- store.initialize_v2(db)
- generation = store.create_generation(db, identity)
- store.activate_generation(generation)
- _ensure_vectors(data_dir / VECTOR_DIR_NAME, generation, identity.dimensions)
- except Exception:
- logging.getLogger(__name__).warning("retrieval_index_provision_failed", exc_info=True)
- return None
- return generation
-
-
-def _schema_version(db: sqlite3.Connection) -> int | None:
- if not db.execute(
- "SELECT 1 FROM sqlite_master WHERE type='table' AND name='retrieval_index_state'"
- ).fetchone():
- return None
- row = db.execute(
- "SELECT schema_version FROM retrieval_index_state WHERE singleton=1"
- ).fetchone()
- return None if row is None else int(row[0])
-
-
-def _published(db: sqlite3.Connection) -> str | None:
- if not db.execute(
- "SELECT 1 FROM sqlite_master WHERE type='table' AND name='retrieval_index_generations'"
- ).fetchone():
- return None
- rows = db.execute(
- "SELECT generation_id FROM retrieval_index_generations WHERE state='active' LIMIT 2"
- ).fetchall()
- return str(rows[0][0]) if len(rows) == 1 else None
-
-
-def _matches(
- store: RetrievalIndexStore,
- db: sqlite3.Connection,
- generation: str,
- identity: object,
-) -> bool:
- try:
- return store._identity(db, generation) == identity
- except Exception:
- return False
-
-
-def _ensure_vectors(directory: Path, generation: str, dimensions: int) -> None:
- if directory.exists():
- return
- vectors = RawVectorStore.create(directory, generation_id=generation, dimensions=dimensions)
- vectors.close()
diff --git a/memhub/apps/core/src/one_memory/retrieval/index_store.py b/memhub/apps/core/src/one_memory/retrieval/index_store.py
deleted file mode 100644
index 388f434..0000000
--- a/memhub/apps/core/src/one_memory/retrieval/index_store.py
+++ /dev/null
@@ -1,856 +0,0 @@
-"""Explicit derived-index transactions. No constructor migration or model invocation."""
-
-import hashlib
-import json
-import math
-import sqlite3
-import time
-from collections.abc import Callable, Iterator, Sequence
-from contextlib import contextmanager
-from dataclasses import asdict, dataclass, replace
-from datetime import UTC, datetime
-from pathlib import Path
-from uuid import uuid4
-
-from one_memory import ingest_journal as journal
-from one_memory.context_raw import _check, _connection
-from one_memory.database import take_the_write_lock_first
-from one_memory.database_access import main_database
-from one_memory.imports.models import EvidenceRecord
-from one_memory.imports.storage import EvidenceStore
-from one_memory.memory_server.migrations import (
- _split_statements,
- retrieval_build_maps_ddl,
- retrieval_documents_v2_ddl,
- retrieval_index_ddl,
-)
-from one_memory.retrieval.chunking import TextChunk
-from one_memory.retrieval.content import index_content, validate_chunk_coverage
-from one_memory.retrieval.mapped_content import ContentMap, PreparedMappedContent
-from one_memory.retrieval.models import TimeRange
-
-
-def digest(value: object) -> str:
- return hashlib.sha256(
- json.dumps(
- value, sort_keys=True, ensure_ascii=False, separators=(",", ":"), allow_nan=False
- ).encode()
- ).hexdigest()
-
-
-@dataclass(frozen=True)
-class EncoderIdentity:
- encoder_id: str
- encoder_digest: str
- tokenizer_digest: str
- dimensions: int
- chunker_version: int = 1
- normalizer_version: int = 1
- query_template_version: int = 1
-
- def __post_init__(self) -> None:
- if not self.encoder_id or len(self.encoder_id) > 128:
- raise ValueError("invalid_encoder")
- for value in (self.encoder_digest, self.tokenizer_digest):
- if len(value) != 64 or any(c not in "0123456789abcdef" for c in value):
- raise ValueError("invalid_encoder_digest")
- for number in (
- self.dimensions,
- self.chunker_version,
- self.normalizer_version,
- self.query_template_version,
- ):
- if type(number) is not int or not 1 <= number <= 1000000:
- raise ValueError("invalid_encoder_number")
-
-
-@dataclass(frozen=True)
-class IndexLease:
- generation_id: str
- evidence_id: str
- build_id: str
- event_seq: int
- owner: str
- token: str
- expires_at: float
- grant_fingerprint: str
-
-
-@dataclass(frozen=True)
-class PublishedChunk:
- chunk_id: str
- build_id: str
- evidence_id: str
- revision: int
- start_cp: int
- end_cp: int
- ordinal: int
- source_id: str
- event_time: datetime | None
-
-
-def chunk_identity(lease: IndexLease, chunk: TextChunk, map_digest: str | None = None) -> str:
- values = [lease.build_id, chunk.ordinal, chunk.start_cp, chunk.end_cp, digest(chunk.text)]
- return digest(values if map_digest is None else [*values, map_digest])
-
-
-@contextmanager
-def transaction(db: sqlite3.Connection) -> Iterator[None]:
- if not db.in_transaction:
- raise RuntimeError("retrieval_requires_caller_transaction")
- name = "retrieval_" + uuid4().hex
- previous = db.row_factory
- db.row_factory = sqlite3.Row
- db.execute(f"SAVEPOINT {name}")
- try:
- yield
- except BaseException:
- db.execute(f"ROLLBACK TO {name}")
- db.execute(f"RELEASE {name}")
- raise
- else:
- db.execute(f"RELEASE {name}")
- finally:
- db.row_factory = previous
-
-
-class RetrievalIndexStore:
- def __init__(self, database_path: Path):
- self.database_path = database_path
-
- @contextmanager
- def _db(
- self, *, write: bool = False, deadline: float | None = None
- ) -> Iterator[sqlite3.Connection]:
- if deadline is not None:
- if write:
- raise ValueError("deadline_connection_is_read_only")
- with _connection(self.database_path, deadline) as db:
- db.execute("PRAGMA query_only=ON")
- yield db
- return
- database = main_database(self.database_path)
- with (
- database.write(create=False, foreign_keys=True, row_factory=sqlite3.Row)
- if write
- else database.read(foreign_keys=True, row_factory=sqlite3.Row)
- ) as db:
- try:
- if write:
- take_the_write_lock_first(db)
- else:
- db.execute("PRAGMA query_only=ON")
- db.execute("BEGIN")
- yield db
- if write:
- db.commit()
- except BaseException:
- db.rollback()
- raise
-
- def initialize(self, db: sqlite3.Connection) -> None:
- """Caller owns maintenance/exclusive writer admission; rollback includes baseline."""
- self._initialize(db, version=1)
-
- def initialize_v2(self, db: sqlite3.Connection) -> None:
- """Initialize a fresh store directly at the folded v2 schema.
-
- Startup provisioning uses this path so a brand-new database does not
- create a v1 schema and immediately migrate it. Explicit maintenance
- callers keep `initialize()`'s v1 baseline contract instead.
- """
- self._initialize(db, version=2)
-
- def _initialize(self, db: sqlite3.Connection, *, version: int) -> None:
- with transaction(db):
- exists = db.execute(
- "SELECT 1 FROM sqlite_master WHERE name='retrieval_index_state'"
- ).fetchone()
- if exists:
- self._store_id(db)
- elif db.execute(
- "SELECT 1 FROM sqlite_master WHERE name IN ('retrieval_documents',"
- "'retrieval_chunks','retrieval_index_generations','retrieval_consumers')"
- ).fetchone():
- raise ValueError("retrieval_partial_initialization")
- journal.initialize(db)
- # Migration 33 owns the schema. Explicit initialization keeps the
- # maintenance-facing v1 baseline so `migrate_v2` remains observable;
- # startup provisioning may instead ask for the folded v2 directly.
- for sql in _split_statements(retrieval_index_ddl("sqlite", version=version)):
- db.execute(sql)
- row = db.execute(
- "SELECT schema_version FROM retrieval_index_state WHERE singleton=1"
- ).fetchone()
- if row is None:
- db.execute(
- "INSERT INTO retrieval_index_state VALUES (1,?,?)",
- (version, uuid4().hex),
- )
- elif row[0] not in (1, 2):
- raise ValueError("retrieval_schema_mismatch")
-
- def migrate_v2(self, db: sqlite3.Connection) -> None:
- """Explicit maintenance transaction only; readers/worker never invoke this."""
- with transaction(db):
- self._store_id(db)
- version = db.execute("SELECT schema_version FROM retrieval_index_state").fetchone()[0]
- if version == 2:
- db.execute(
- "SELECT build_id,source_map_json,map_digest,coverage_state "
- "FROM retrieval_build_maps LIMIT 0"
- )
- return
- if (
- db.execute(
- "SELECT 1 FROM sqlite_master WHERE type='table' AND name='retrieval_build_maps'"
- ).fetchone()
- is not None
- ):
- # A v1 index must not already own a v2-only table. Treat this
- # as a foreign/partial schema instead of letting `IF NOT
- # EXISTS` silently accept a table with the wrong columns.
- raise sqlite3.OperationalError("retrieval_build_maps already exists")
- # Both tables come from migration 33, named/derived there so this
- # module holds no DDL text of its own.
- db.execute(retrieval_documents_v2_ddl("sqlite"))
- db.execute("INSERT INTO retrieval_documents_v2 SELECT * FROM retrieval_documents")
- db.execute("DROP TABLE retrieval_documents")
- db.execute("ALTER TABLE retrieval_documents_v2 RENAME TO retrieval_documents")
- db.execute(
- "CREATE INDEX retrieval_jobs ON retrieval_documents(generation_id,state,event_seq)"
- )
- db.execute(retrieval_build_maps_ddl("sqlite"))
- db.execute("UPDATE retrieval_index_state SET schema_version=2 WHERE singleton=1")
-
- @staticmethod
- def _store_id(db: sqlite3.Connection) -> str:
- row = db.execute(
- "SELECT schema_version,data_store_id FROM retrieval_index_state WHERE singleton=1"
- ).fetchone()
- if row is None or row[0] not in (1, 2) or not isinstance(row[1], str) or len(row[1]) != 32:
- raise ValueError("retrieval_not_initialized")
- return str(row[1])
-
- def create_generation(self, db: sqlite3.Connection, identity: EncoderIdentity) -> str:
- with transaction(db):
- self._check_content_version(db, identity)
- key = digest([self._store_id(db), asdict(identity)])
- encoded = json.dumps(asdict(identity), sort_keys=True)
- row = db.execute(
- "SELECT identity_json FROM retrieval_index_generations WHERE generation_id=?",
- (key,),
- ).fetchone()
- if row is not None and row[0] != encoded:
- raise ValueError("generation_identity_changed")
- if row is None:
- db.execute(
- "INSERT INTO retrieval_index_generations VALUES (?,?,'building',?)",
- (key, encoded, time.time()),
- )
- db.execute("INSERT INTO retrieval_consumers VALUES (?,0,0)", (key,))
- return key
-
- def consume(self, db: sqlite3.Connection, generation_id: str, *, limit: int = 256) -> int:
- if type(limit) is not int or not 1 <= limit <= 256:
- raise ValueError("invalid_consume_limit")
- with transaction(db):
- store_id = self._store_id(db)
- generation = db.execute(
- "SELECT state FROM retrieval_index_generations WHERE generation_id=?",
- (generation_id,),
- ).fetchone()
- if generation is None or generation[0] == "retired":
- raise ValueError("generation_not_writable")
- cursor = db.execute(
- "SELECT after_seq FROM retrieval_consumers WHERE generation_id=?", (generation_id,)
- ).fetchone()
- if cursor is None:
- raise ValueError("consumer_missing")
- rows = db.execute(
- "SELECT seq FROM evidence_ingest_events WHERE seq>? ORDER BY seq LIMIT ?",
- (cursor[0], limit),
- ).fetchall()
- for row in rows:
- event = journal.event(db, row[0])
- if event is None:
- raise ValueError("journal_event_missing")
- if event.user_id != "local":
- continue
- build = digest(
- [
- store_id,
- generation_id,
- event.evidence_id,
- event.incarnation,
- event.evidence_revision,
- event.fingerprint,
- ]
- )
- state = "deleted" if event.kind == "delete" else "pending"
- db.execute(
- """INSERT INTO retrieval_documents
- (generation_id,evidence_id,source_id,event_seq,build_id,state)
- VALUES (?,?,?,?,?,?) ON CONFLICT(generation_id,evidence_id) DO UPDATE SET
- source_id=excluded.source_id,event_seq=excluded.event_seq,build_id=excluded.build_id,
- state=excluded.state,active_build_id=NULL,lease_owner=NULL,lease_token=NULL,
- lease_until=NULL,grant_fingerprint=NULL
- WHERE excluded.event_seq>retrieval_documents.event_seq""",
- (generation_id, event.evidence_id, event.source_id, event.seq, build, state),
- )
- if rows:
- db.execute(
- "UPDATE retrieval_consumers SET after_seq=? WHERE generation_id=?",
- (rows[-1][0], generation_id),
- )
- return len(rows)
-
- def consume_pending(self, generation_id: str, *, limit: int = 256) -> int:
- with self._db(write=True) as db:
- return self.consume(db, generation_id, limit=limit)
-
- def encoder_identity(self, generation_id: str) -> EncoderIdentity:
- with self._db() as db:
- return self._identity(db, generation_id)
-
- @staticmethod
- def _check_content_version(db: sqlite3.Connection, identity: EncoderIdentity) -> None:
- versions = (identity.normalizer_version, identity.chunker_version)
- if versions not in ((1, 1), (2, 2)):
- raise ValueError("unsupported_content_generation")
- if versions == (2, 2):
- row = db.execute(
- "SELECT schema_version FROM retrieval_index_state WHERE singleton=1"
- ).fetchone()
- if row is None or row[0] != 2:
- raise ValueError("retrieval_schema2_required")
-
- def _identity(self, db: sqlite3.Connection, generation_id: str) -> EncoderIdentity:
- row = db.execute(
- "SELECT identity_json FROM retrieval_index_generations WHERE generation_id=?",
- (generation_id,),
- ).fetchone()
- if row is None:
- raise ValueError("generation_missing")
- identity = EncoderIdentity(**json.loads(row[0]))
- if digest([self._store_id(db), asdict(identity)]) != generation_id:
- raise ValueError("generation_identity_changed")
- self._check_content_version(db, identity)
- return identity
-
- def _prepare(
- self, db: sqlite3.Connection, generation_id: str, record: EvidenceRecord
- ) -> PreparedMappedContent | None:
- identity = self._identity(db, generation_id)
- return index_content(
- record,
- normalizer_version=identity.normalizer_version,
- chunker_version=identity.chunker_version,
- )
-
- @staticmethod
- def _grant(db: sqlite3.Connection, source_id: str) -> str | None:
- row = db.execute(
- """SELECT root_path,candidate_kind,granted_at,root_identity,revision
- FROM source_grants
- WHERE user_id='local' AND source_id=? AND revoked_at IS NULL""",
- (source_id,),
- ).fetchone()
- return digest(tuple(row)) if row and row[3] and row[4] > 0 else None
-
- def claim(
- self, generation_id: str, owner: str, *, now: float | None = None, lease_seconds: int = 60
- ) -> IndexLease | None:
- now = time.time() if now is None else now
- if not owner or len(owner) > 128 or not math.isfinite(now) or not 1 <= lease_seconds <= 300:
- raise ValueError("invalid_lease")
- with self._db(write=True) as db:
- generation = db.execute(
- "SELECT state FROM retrieval_index_generations WHERE generation_id=?",
- (generation_id,),
- ).fetchone()
- if generation is None or generation[0] == "retired":
- raise ValueError("generation_not_writable")
- rows = db.execute(
- """SELECT * FROM retrieval_documents WHERE generation_id=? AND
- (state='pending' OR (state='leased' AND lease_until<=?))
- ORDER BY event_seq LIMIT 256""",
- (generation_id, now),
- ).fetchall()
- for row in rows:
- event = journal.event(db, row["event_seq"])
- grant = self._grant(db, row["source_id"])
- if event is None or grant is None or not journal.matches_current(db, event):
- db.execute(
- "UPDATE retrieval_documents SET state='blocked',active_build_id=NULL "
- "WHERE generation_id=? AND evidence_id=?",
- (generation_id, row["evidence_id"]),
- )
- continue
- token = uuid4().hex
- until = now + lease_seconds
- db.execute(
- """UPDATE retrieval_documents SET state='leased',lease_owner=?,lease_token=?,
- lease_until=?,grant_fingerprint=?,attempts=attempts+1
- WHERE generation_id=? AND evidence_id=?""",
- (owner, token, until, grant, generation_id, row["evidence_id"]),
- )
- return IndexLease(
- generation_id,
- row["evidence_id"],
- row["build_id"],
- row["event_seq"],
- owner,
- token,
- until,
- grant,
- )
- return None
-
- def _validate_lease(
- self, db: sqlite3.Connection, lease: IndexLease, now: float
- ) -> journal.IngestEvent:
- if not math.isfinite(now):
- raise ValueError("invalid_lease_time")
- generation = db.execute(
- "SELECT state FROM retrieval_index_generations WHERE generation_id=?",
- (lease.generation_id,),
- ).fetchone()
- if generation is None or generation[0] == "retired":
- raise ValueError("lease_generation_retired")
- row = db.execute(
- "SELECT * FROM retrieval_documents WHERE generation_id=? AND evidence_id=?",
- (lease.generation_id, lease.evidence_id),
- ).fetchone()
- if (
- row is None
- or row["state"] != "leased"
- or (
- row["build_id"],
- row["event_seq"],
- row["lease_owner"],
- row["lease_token"],
- row["lease_until"],
- row["grant_fingerprint"],
- )
- != (
- lease.build_id,
- lease.event_seq,
- lease.owner,
- lease.token,
- lease.expires_at,
- lease.grant_fingerprint,
- )
- or lease.expires_at <= now
- ):
- raise ValueError("lease_changed_or_expired")
- event = journal.event(db, lease.event_seq)
- if (
- event is None
- or event.user_id != "local"
- or event.evidence_id != lease.evidence_id
- or not journal.matches_current(db, event)
- ):
- raise ValueError("lease_evidence_changed")
- if self._grant(db, event.source_id) != lease.grant_fingerprint:
- raise ValueError("lease_source_authority_changed")
- return event
-
- def read_lease(self, lease: IndexLease, *, now: float | None = None) -> EvidenceRecord:
- with self._db() as db:
- self._validate_lease(db, lease, time.time() if now is None else now)
- record = EvidenceStore._get_with_connection(db, lease.evidence_id)
- if record is None or len(record.verbatim) > 2_000_000:
- raise ValueError("lease_input_unavailable_or_too_large")
- return record
-
- def renew(
- self, lease: IndexLease, *, now: float | None = None, lease_seconds: int = 60
- ) -> IndexLease:
- now = time.time() if now is None else now
- if type(lease_seconds) is not int or not 1 <= lease_seconds <= 300:
- raise ValueError("invalid_lease")
- with self._db(write=True) as db:
- self._validate_lease(db, lease, now)
- expires = max(lease.expires_at, now + lease_seconds)
- db.execute(
- "UPDATE retrieval_documents SET lease_until=? "
- "WHERE generation_id=? AND evidence_id=?",
- (expires, lease.generation_id, lease.evidence_id),
- )
- return replace(lease, expires_at=expires)
-
- def block(self, lease: IndexLease) -> None:
- """Release only this claim; preserve uncertain derived vectors for later cleanup."""
- with self._db(write=True) as db:
- db.execute(
- """UPDATE retrieval_documents SET state='blocked',active_build_id=NULL,
- lease_owner=NULL,lease_token=NULL,lease_until=NULL
- WHERE generation_id=? AND evidence_id=? AND state='leased'
- AND build_id=? AND lease_token=? AND lease_owner=?""",
- (lease.generation_id, lease.evidence_id, lease.build_id, lease.token, lease.owner),
- )
-
- def retry_blocked(self, generation_id: str) -> int:
- """Bounded, durable round-robin retry/regrant pass; never creates grants."""
- changed = 0
- with self._db(write=True) as db:
- generation = db.execute(
- "SELECT g.state,c.retry_after_seq FROM retrieval_index_generations g "
- "JOIN retrieval_consumers c USING(generation_id) WHERE generation_id=?",
- (generation_id,),
- ).fetchone()
- if generation is None or generation[0] == "retired":
- raise ValueError("generation_not_writable")
- rows = db.execute(
- "SELECT * FROM retrieval_documents WHERE generation_id=? "
- "AND state IN ('blocked','ready','excluded') AND event_seq>? "
- "ORDER BY event_seq LIMIT 256",
- (generation_id, generation[1]),
- ).fetchall()
- for row in rows:
- event = journal.event(db, row["event_seq"])
- grant = self._grant(db, row["source_id"])
- if row["state"] in ("ready", "excluded") and grant == row["grant_fingerprint"]:
- continue
- if (
- event
- and event.source_id == row["source_id"]
- and grant
- and journal.matches_current(db, event)
- ):
- db.execute(
- "UPDATE retrieval_documents SET state='pending',active_build_id=NULL,"
- "lease_owner=NULL,lease_token=NULL,lease_until=NULL,grant_fingerprint=NULL "
- "WHERE generation_id=? AND evidence_id=?",
- (generation_id, row["evidence_id"]),
- )
- changed += 1
- db.execute(
- "UPDATE retrieval_consumers SET retry_after_seq=? WHERE generation_id=?",
- (rows[-1]["event_seq"] if len(rows) == 256 else 0, generation_id),
- )
- return changed
-
- def publish(
- self,
- lease: IndexLease,
- chunks: Sequence[TextChunk],
- *,
- vectors_verified: bool,
- now: float | None = None,
- source_map: ContentMap | None = None,
- checkpoint: Callable[[], None] | None = None,
- ) -> tuple[str, ...]:
- """Caller verifies actual immutable vectors before entering this short transaction."""
- if vectors_verified is not True:
- raise ValueError("vectors_not_verified")
- now = time.time() if now is None else now
- if not math.isfinite(now):
- raise ValueError("invalid_lease_time")
- if len(chunks) > 8192:
- raise ValueError("invalid_chunk_manifest")
- with self._db(write=True) as db:
- if checkpoint is not None:
- checkpoint()
- event = self._validate_lease(db, lease, now)
- record = EvidenceStore._get_with_connection(db, lease.evidence_id)
- if record is None or len(record.verbatim) > 2_000_000:
- raise ValueError("lease_evidence_missing")
- prepared = self._prepare(db, lease.generation_id, record)
- if source_map != (prepared.metadata if prepared is not None else None):
- raise ValueError("source_map_mismatch")
- validate_chunk_coverage(record, chunks, prepared)
- map_digest = source_map.digest if source_map is not None else None
- keys = []
- manifest = []
- for index, chunk in enumerate(chunks):
- text_hash = digest(chunk.text)
- key = chunk_identity(lease, chunk, map_digest)
- keys.append(key)
- manifest.append(
- (
- key,
- lease.generation_id,
- lease.evidence_id,
- lease.build_id,
- index,
- chunk.start_cp,
- chunk.end_cp,
- chunk.token_count,
- text_hash,
- )
- )
- existing = db.execute(
- "SELECT * FROM retrieval_chunks WHERE build_id=? ORDER BY ordinal",
- (lease.build_id,),
- ).fetchall()
- if existing:
- if [tuple(row) for row in existing] != manifest:
- raise ValueError("immutable_chunk_manifest_conflict")
- else:
- db.executemany("INSERT INTO retrieval_chunks VALUES (?,?,?,?,?,?,?,?,?)", manifest)
- state = "ready" if chunks else "excluded"
- if source_map is not None:
- values = (
- lease.build_id,
- source_map.canonical_json(),
- map_digest,
- "indexable" if chunks else "excluded",
- )
- prior = db.execute(
- "SELECT * FROM retrieval_build_maps WHERE build_id=?", (lease.build_id,)
- ).fetchone()
- if prior is not None and tuple(prior) != values:
- raise ValueError("immutable_source_map_conflict")
- if prior is None:
- db.execute("INSERT INTO retrieval_build_maps VALUES (?,?,?,?)", values)
- if not journal.matches_current(db, event):
- raise ValueError("lease_evidence_changed")
- db.execute(
- "UPDATE retrieval_documents SET state=?,active_build_id=build_id,"
- "lease_owner=NULL,lease_token=NULL,lease_until=NULL "
- "WHERE generation_id=? AND evidence_id=?",
- (state, lease.generation_id, lease.evidence_id),
- )
- if checkpoint is not None:
- checkpoint()
- return tuple(keys)
-
- def activate_generation(self, generation_id: str) -> None:
- with self._db(write=True) as db:
- row = db.execute(
- "SELECT state FROM retrieval_index_generations WHERE generation_id=?",
- (generation_id,),
- ).fetchone()
- cursor = db.execute(
- "SELECT after_seq FROM retrieval_consumers WHERE generation_id=?", (generation_id,)
- ).fetchone()
- if (
- row is None
- or row[0] == "retired"
- or cursor is None
- or cursor[0] != journal.watermark(db)
- ):
- raise ValueError("generation_not_caught_up")
- for document in db.execute(
- "SELECT * FROM retrieval_documents WHERE generation_id=? "
- "AND state NOT IN ('ready','excluded','deleted')",
- (generation_id,),
- ):
- if self._grant(db, document["source_id"]) is not None:
- raise ValueError("generation_incomplete")
- for document in db.execute(
- "SELECT * FROM retrieval_documents WHERE generation_id=? "
- "AND state IN ('ready','excluded')",
- (generation_id,),
- ):
- if self._grant(db, document["source_id"]) is not None:
- event = journal.event(db, document["event_seq"])
- if (
- not event
- or not journal.matches_current(db, event)
- or self._grant(db, document["source_id"]) != document["grant_fingerprint"]
- ):
- raise ValueError("generation_stale")
- record = EvidenceStore._get_with_connection(db, document["evidence_id"])
- if record is None:
- raise ValueError("generation_stale")
- self.validate_document(db, document, record)
- db.execute(
- "UPDATE retrieval_index_generations SET state='retired' "
- "WHERE state='active' AND generation_id!=?",
- (generation_id,),
- )
- db.execute(
- "UPDATE retrieval_index_generations SET state='active' WHERE generation_id=?",
- (generation_id,),
- )
-
- def active_chunks(
- self,
- generation_id: str,
- *,
- source_ids: frozenset[str] | None = None,
- time_range: TimeRange | None = None,
- deadline: float | None = None,
- require_active: bool = False,
- ) -> tuple[PublishedChunk, ...]:
- """Return current chunks after SQL-level source/time eligibility filtering.
-
- Grant fingerprints, journal freshness, and manifests are still checked
- below in the same snapshot. The SQL predicate keeps unauthorized rows
- from filling an in-memory candidate set before those checks run.
- """
- if source_ids is not None and not source_ids:
- return ()
- with self._db(deadline=deadline) as db:
- generation = db.execute(
- "SELECT state FROM retrieval_index_generations WHERE generation_id=?",
- (generation_id,),
- ).fetchone()
- if generation is None or generation[0] != "active":
- if require_active:
- raise ValueError("active_generation_changed")
- return ()
- clauses = ["d.generation_id=?", "d.state IN ('ready','excluded')"]
- parameters: list[object] = [generation_id]
- join = ""
- if source_ids is not None:
- clauses.append("d.source_id IN (SELECT value FROM json_each(?))")
- parameters.append(json.dumps(sorted(source_ids)))
- if time_range is not None:
- join = "JOIN evidence e ON e.evidence_id=d.evidence_id "
- clauses.extend(
- (
- "e.created_at IS NOT NULL",
- "julianday(e.created_at) >= julianday(?)",
- "julianday(e.created_at) < julianday(?)",
- )
- )
- parameters.extend((time_range.start.isoformat(), time_range.end.isoformat()))
- output = []
- for row in db.execute(
- "SELECT d.* FROM retrieval_documents d "
- + join
- + f"WHERE {' AND '.join(clauses)} ORDER BY d.event_seq,d.evidence_id",
- parameters,
- ):
- if deadline is not None:
- _check(deadline)
- grant = self._grant(db, row["source_id"])
- if grant is None or grant != row["grant_fingerprint"]:
- continue
- event = journal.event(db, row["event_seq"])
- if (
- event is None
- or event.source_id != row["source_id"]
- or not journal.matches_current(db, event)
- ):
- continue
- record = EvidenceStore._get_with_connection(db, row["evidence_id"])
- if (
- record is None
- or record.user_id != "local"
- or record.source_id != row["source_id"]
- ):
- continue
- event_time = record.created_at
- if event_time is not None:
- event_time = event_time.astimezone(UTC) if event_time.tzinfo else None
- if time_range is not None and (
- event_time is None or not time_range.start <= event_time < time_range.end
- ):
- continue
- chunks = self.validate_document(db, row, record, deadline=deadline)
- for chunk in chunks:
- if deadline is not None:
- _check(deadline)
- output.append(
- PublishedChunk(
- chunk["chunk_id"],
- row["build_id"],
- row["evidence_id"],
- event.evidence_revision,
- chunk["start_cp"],
- chunk["end_cp"],
- chunk["ordinal"],
- record.source_id,
- event_time,
- )
- )
- return tuple(output)
-
- def validate_document(
- self,
- db: sqlite3.Connection,
- document: sqlite3.Row,
- record: EvidenceRecord,
- *,
- deadline: float | None = None,
- ) -> tuple[sqlite3.Row, ...]:
- """Same-snapshot complete manifest/map check shared by candidate and final read."""
- if deadline is not None:
- _check(deadline)
- if document["active_build_id"] != document["build_id"]:
- raise ValueError("published_build_binding_invalid")
- prepared = self._prepare(db, document["generation_id"], record)
- map_digest = None
- if prepared is not None:
- source_map = db.execute(
- "SELECT source_map_json,map_digest,coverage_state FROM retrieval_build_maps "
- "WHERE build_id=?",
- (document["build_id"],),
- ).fetchone()
- expected_state = "excluded" if prepared.state == "no_indexable_content" else "ready"
- if (
- source_map is None
- or tuple(source_map)
- != (
- prepared.metadata.canonical_json(),
- prepared.metadata.digest,
- "excluded" if expected_state == "excluded" else "indexable",
- )
- or document["state"] != expected_state
- ):
- raise ValueError("published_source_map_invalid")
- map_digest = prepared.metadata.digest
- elif document["state"] != "ready":
- raise ValueError("published_manifest_invalid")
- rows = tuple(
- db.execute(
- "SELECT * FROM retrieval_chunks WHERE generation_id=? AND evidence_id=? "
- "AND build_id=? ORDER BY ordinal LIMIT 8193",
- (document["generation_id"], record.evidence_id, document["build_id"]),
- ).fetchall()
- )
- chunks = []
- for row in rows:
- if deadline is not None:
- _check(deadline)
- start, end = row["start_cp"], row["end_cp"]
- if (
- type(start) is not int
- or type(end) is not int
- or not 0 <= start < end <= len(record.verbatim)
- ):
- raise ValueError("published_manifest_invalid")
- text = record.verbatim[start:end]
- text_hash = digest(text)
- identity = [document["build_id"], row["ordinal"], start, end, text_hash]
- if map_digest is not None:
- identity.append(map_digest)
- if row["text_hash"] != text_hash or row["chunk_id"] != digest(identity):
- raise ValueError("published_manifest_invalid")
- chunks.append(TextChunk(row["ordinal"], start, end, row["token_count"], text))
- try:
- validate_chunk_coverage(record, chunks, prepared)
- except ValueError as error:
- raise ValueError("published_manifest_incomplete_or_invalid") from error
- if deadline is not None:
- _check(deadline)
- return rows
-
- def status(self, generation_id: str) -> dict[str, int]:
- with self._db() as db:
- row = db.execute(
- "SELECT after_seq FROM retrieval_consumers WHERE generation_id=?", (generation_id,)
- ).fetchone()
- if row is None:
- raise ValueError("generation_missing")
- result = {
- "after_seq": int(row[0]),
- "pending": 0,
- "leased": 0,
- "ready": 0,
- "blocked": 0,
- "deleted": 0,
- "excluded": 0,
- }
- for state, count in db.execute(
- "SELECT state,count(*) FROM retrieval_documents "
- "WHERE generation_id=? GROUP BY state",
- (generation_id,),
- ):
- result[state] = count
- return result
diff --git a/memhub/apps/core/src/one_memory/retrieval/index_supervisor.py b/memhub/apps/core/src/one_memory/retrieval/index_supervisor.py
deleted file mode 100644
index 1b53e2d..0000000
--- a/memhub/apps/core/src/one_memory/retrieval/index_supervisor.py
+++ /dev/null
@@ -1,200 +0,0 @@
-"""Explicit ownership of one admitted index worker; no migration or resource close."""
-
-import asyncio
-import math
-import threading
-import time
-from collections.abc import Callable
-from dataclasses import dataclass
-from typing import Literal, Protocol
-
-from one_memory.retrieval.index_worker import IndexWorkResult
-from one_memory.service_levels import ServiceLevel, service_level_context
-
-
-class IndexWorker(Protocol):
- def run_once(self, *, cancel: threading.Event | None = None) -> IndexWorkResult: ...
-
-
-@dataclass(frozen=True)
-class IndexSupervisorSnapshot:
- state: Literal["new", "running", "stopping", "stopped"]
- last_result: str | None
- error_code: str | None
- slices: int
- failures: int
- thread_alive: bool
-
-
-class RetrievalIndexSupervisor:
- """Single-use supervisor. Caller owns admission and closes vectors only after stop.
-
- Cancellation is cooperative: stop/wait cannot finish until a blocking native
- run_once returns. The lifecycle owner must drain before shutting its event loop.
- """
-
- def __init__(
- self,
- worker: IndexWorker,
- *,
- idle_wait_seconds: float = 5,
- failure_wait_seconds: float = 5,
- yield_wait_seconds: float = 0.05,
- max_consecutive_slices: int = 8,
- background_should_yield: Callable[[], bool] | None = None,
- ):
- for interval in (idle_wait_seconds, failure_wait_seconds, yield_wait_seconds):
- if (
- isinstance(interval, bool)
- or not isinstance(interval, (int, float))
- or not math.isfinite(interval)
- or not 0.001 <= interval <= 300
- ):
- raise ValueError("index_supervisor_invalid_config")
- if type(max_consecutive_slices) is not int or not 1 <= max_consecutive_slices <= 256:
- raise ValueError("index_supervisor_invalid_config")
- self._worker = worker
- self._idle_wait = idle_wait_seconds
- self._failure_wait = failure_wait_seconds
- self._yield_wait = yield_wait_seconds
- self._max_slices = max_consecutive_slices
- self._background_should_yield = background_should_yield or (lambda: False)
- self._condition = threading.Condition()
- self._cancel = threading.Event()
- self._wake = False
- self._thread: threading.Thread | None = None
- self._state: Literal["new", "running", "stopping", "stopped"] = "new"
- self._last_result: str | None = None
- self._error_code: str | None = None
- self._slices = self._failures = 0
-
- def start(self) -> None:
- with self._condition:
- if self._state != "new":
- raise RuntimeError("index_supervisor_already_started")
- thread = threading.Thread(target=self._run, name="retrieval-index", daemon=False)
- self._thread = thread
- self._state = "running"
- try:
- thread.start()
- except Exception:
- self._thread = None
- self._state = "stopped"
- self._error_code = "index_supervisor_start_failed"
- raise RuntimeError("index_supervisor_start_failed") from None
-
- def trigger(self) -> bool:
- with self._condition:
- if self._state != "running":
- return False
- self._wake = True
- self._condition.notify_all()
- return True
-
- def snapshot(self) -> IndexSupervisorSnapshot:
- with self._condition:
- return IndexSupervisorSnapshot(
- self._state,
- self._last_result,
- self._error_code,
- self._slices,
- self._failures,
- self._thread is not None and self._thread.is_alive(),
- )
-
- def _request_stop(self) -> None:
- with self._condition:
- self._cancel.set()
- self._wake = False
- if self._state == "new":
- self._state = "stopped"
- elif self._state == "running":
- self._state = "stopping"
- self._condition.notify_all()
-
- async def stop(self) -> None:
- self._request_stop()
- await self.wait()
-
- async def wait(self) -> None:
- """Cancellation requests shutdown, drains the real thread, then propagates."""
- cancelled = False
- while True:
- with self._condition:
- thread = self._thread
- if thread is None or not thread.is_alive():
- if thread is not None:
- thread.join()
- break
- try:
- # No executor waiter or to_thread future to orphan on cancellation.
- await asyncio.sleep(0.01)
- except asyncio.CancelledError:
- cancelled = True
- self._request_stop()
- if cancelled:
- raise asyncio.CancelledError
-
- def _pause(self, seconds: float, *, wakeable: bool) -> None:
- deadline = time.monotonic() + seconds
- with self._condition:
- while not self._cancel.is_set():
- if wakeable and self._wake:
- break
- remaining = deadline - time.monotonic()
- if remaining <= 0:
- break
- self._condition.wait(remaining)
- self._wake = False
-
- def _run(self) -> None:
- consecutive = 0
- try:
- while not self._cancel.is_set():
- if self._background_should_yield():
- self._pause(self._yield_wait, wakeable=False)
- continue
- result: str | None = None
- error: str | None = None
- try:
- with service_level_context(ServiceLevel.BACKGROUND):
- value = self._worker.run_once(cancel=self._cancel)
- if not isinstance(value, IndexWorkResult) or value.state not in {
- "idle",
- "ready",
- "excluded",
- "blocked",
- "cancelled",
- "deferred",
- }:
- error = "index_worker_invalid_result"
- else:
- result = value.state
- if result == "blocked":
- error = "index_worker_blocked"
- except BaseException:
- # Worker/provider exceptions must never escape via thread stderr.
- error = "index_worker_failed"
- with self._condition:
- self._last_result, self._error_code = result, error
- self._slices += 1
- if error is not None:
- self._failures += 1
- if self._cancel.is_set():
- break
- if error is not None:
- consecutive = 0
- self._pause(self._failure_wait, wakeable=False)
- elif result in {"ready", "excluded"}:
- consecutive += 1
- if consecutive >= self._max_slices:
- consecutive = 0
- self._pause(self._yield_wait, wakeable=False)
- else:
- consecutive = 0
- self._pause(self._idle_wait, wakeable=True)
- finally:
- with self._condition:
- self._state = "stopped"
- self._wake = False
- self._condition.notify_all()
diff --git a/memhub/apps/core/src/one_memory/retrieval/index_worker.py b/memhub/apps/core/src/one_memory/retrieval/index_worker.py
deleted file mode 100644
index d5f0df5..0000000
--- a/memhub/apps/core/src/one_memory/retrieval/index_worker.py
+++ /dev/null
@@ -1,262 +0,0 @@
-"""Bounded raw-index work. No migration, scheduler, HY or remote model invocation.
-
-Runtime must admit a single worker, own its thread/process, and drain that owner
-before closing the vector client. Cancellation here is cooperative between
-native calls, not a claim that ONNX/Chroma can be preempted in a Python thread.
-"""
-
-import math
-import threading
-import time
-from collections.abc import Callable, Sequence
-from dataclasses import dataclass
-from typing import Literal, Protocol
-
-from one_memory.embedding_assets import (
- BUILTIN_EMBEDDING_MODEL,
- EMBEDDING_ASSETS,
- verify_assets,
-)
-from one_memory.local_embedding import LocalEmbedder, TokenizedText
-from one_memory.retrieval.chunking import ChunkingError, chunk_evidence
-from one_memory.retrieval.content import index_content
-from one_memory.retrieval.index_store import (
- EncoderIdentity,
- IndexLease,
- RetrievalIndexStore,
- chunk_identity,
-)
-from one_memory.retrieval.mapped_content import MappedContentError, chunk_mapped_content
-from one_memory.retrieval.vector_store import RawVectorRecord, RawVectorStore, VectorStoreError
-
-
-class RetrievalEncoder(Protocol):
- @property
- def identity(self) -> EncoderIdentity: ...
-
- def tokenize_with_offsets(self, text: str) -> TokenizedText: ...
-
- def encode_documents(self, texts: list[str]) -> list[list[float]]: ...
-
-
-class LocalRetrievalEncoder:
- """Pinned local assets; mapped content v2 by default, explicit v1 for old generations."""
-
- def __init__(self, embedder: LocalEmbedder | None = None, *, content_version: int = 2):
- if type(content_version) is not int or content_version not in (1, 2):
- raise ValueError("unsupported_content_generation")
- self._embedder = embedder if embedder is not None else LocalEmbedder()
- digests = {asset.filename: asset.sha256 for asset in EMBEDDING_ASSETS}
- self.identity = EncoderIdentity(
- BUILTIN_EMBEDDING_MODEL,
- digests["model_int8.onnx"],
- digests["tokenizer.json"],
- self._embedder.dimensions,
- chunker_version=content_version,
- normalizer_version=content_version,
- )
- self._verified = False
- self._lock = threading.Lock()
-
- def _verify(self) -> None:
- with self._lock:
- if not self._verified:
- verify_assets()
- self._verified = True
-
- def tokenize_with_offsets(self, text: str) -> TokenizedText:
- self._verify()
- return self._embedder.tokenize_with_offsets(text)
-
- def encode_documents(self, texts: list[str]) -> list[list[float]]:
- self._verify()
- if len(texts) > 8 or any(
- not 1 <= len(self.tokenize_with_offsets(text).token_ids) <= 512 for text in texts
- ):
- raise ValueError("encoder_batch_or_token_budget")
- with self._lock:
- return self._embedder.encode(texts)
-
-
-@dataclass(frozen=True)
-class IndexWorkResult:
- state: Literal["idle", "ready", "excluded", "blocked", "cancelled", "deferred"]
- reason: str | None = None
- chunk_count: int = 0
-
-
-class _Interrupted(Exception):
- def __init__(self, state: Literal["cancelled", "deferred"]):
- self.state = state
-
-
-class RetrievalIndexWorker:
- def __init__(
- self,
- store: RetrievalIndexStore,
- vectors: RawVectorStore,
- encoder: RetrievalEncoder,
- *,
- generation_id: str,
- owner: str,
- clock: Callable[[], float] = time.monotonic,
- wall_clock: Callable[[], float] = time.time,
- max_job_seconds: float = 120,
- ):
- if (
- not owner
- or len(owner) > 128
- or isinstance(max_job_seconds, bool)
- or not math.isfinite(max_job_seconds)
- or not 0 < max_job_seconds <= 300
- ):
- raise ValueError("invalid_worker_budget")
- if store.encoder_identity(generation_id) != encoder.identity:
- raise ValueError("worker_encoder_identity_mismatch")
- if (
- vectors.generation_id != generation_id
- or vectors.dimensions != encoder.identity.dimensions
- ):
- raise ValueError("worker_vector_identity_mismatch")
- self.store, self.vectors, self.encoder = store, vectors, encoder
- self.generation_id, self.owner = generation_id, owner
- self.clock, self.wall_clock = clock, wall_clock
- self.max_job_seconds = max_job_seconds
- self._run_lock = threading.Lock()
-
- def run_once(self, *, cancel: threading.Event | None = None) -> IndexWorkResult:
- """Consume <=256 events and handle <=1 document; never activate a generation.
-
- Interrupted jobs retain their finite lease for restart recovery. Failed
- jobs are blocked for explicit retry; arbitrary provider exceptions and
- original text never enter the public result or a log.
- """
- if not self._run_lock.acquire(blocking=False):
- raise RuntimeError("index_worker_already_running")
- lease: IndexLease | None = None
- deadline = self.clock() + self.max_job_seconds
-
- def check() -> None:
- if cancel is not None and cancel.is_set():
- raise _Interrupted("cancelled")
- now = self.clock()
- if not math.isfinite(now) or now >= deadline:
- raise _Interrupted("deferred")
-
- def boundary() -> None:
- nonlocal lease
- check()
- assert lease is not None
- # This revalidates the original's current fingerprint and authority,
- # not just a heartbeat on an otherwise stale claim.
- lease = self.store.renew(lease, now=self.wall_clock())
- check()
-
- def publish_checkpoint() -> None:
- check()
- assert lease is not None
- now = self.wall_clock()
- if not math.isfinite(now) or now >= lease.expires_at:
- raise ValueError("lease_changed_or_expired")
-
- try:
- check()
- self.store.consume_pending(self.generation_id)
- check()
- lease = self.store.claim(self.generation_id, self.owner, now=self.wall_clock())
- if lease is None:
- return IndexWorkResult("idle")
- record = self.store.read_lease(lease, now=self.wall_clock())
- check()
- identity = self.encoder.identity
- try:
- prepared = index_content(
- record,
- normalizer_version=identity.normalizer_version,
- chunker_version=identity.chunker_version,
- )
- except ValueError:
- self.store.block(lease)
- return IndexWorkResult("blocked", "content_mapping_required")
-
- worker_encoder = self.encoder
-
- class CheckedTokenizer:
- def tokenize_with_offsets(self, text: str) -> TokenizedText:
- check()
- result = worker_encoder.tokenize_with_offsets(text)
- check()
- return result
-
- chunks = (
- chunk_mapped_content(prepared, CheckedTokenizer())
- if prepared is not None
- else chunk_evidence(record.verbatim, CheckedTokenizer())
- )
- if (not chunks and prepared is None) or len(chunks) > 8192:
- raise ChunkingError("invalid_chunk_count")
- source_map = prepared.metadata if prepared is not None else None
- map_digest = source_map.digest if source_map is not None else None
- boundary()
- expected: list[RawVectorRecord] = []
- for start in range(0, len(chunks), 8):
- batch = chunks[start : start + 8]
- boundary()
- encoded = self.encoder.encode_documents([chunk.text for chunk in batch])
- boundary()
- if len(encoded) != len(batch):
- raise ValueError("encoder_count_mismatch")
- records = [
- RawVectorRecord(
- chunk_identity(lease, chunk, map_digest),
- lease.build_id,
- lease.generation_id,
- tuple(vector),
- )
- for chunk, vector in zip(batch, encoded, strict=True)
- ]
- self.vectors.upsert(records)
- boundary()
- expected.extend(records)
- self._verify(expected, boundary)
- assert lease is not None
- check()
- self.store.publish(
- lease,
- chunks,
- vectors_verified=True,
- now=self.wall_clock(),
- source_map=source_map,
- checkpoint=publish_checkpoint,
- )
- return IndexWorkResult("ready" if chunks else "excluded", chunk_count=len(chunks))
- except _Interrupted as error:
- return IndexWorkResult(
- error.state, "cancelled" if error.state == "cancelled" else "deadline"
- )
- except Exception as error:
- cause: BaseException | None = error
- while cause is not None:
- if isinstance(cause, _Interrupted):
- return IndexWorkResult(
- cause.state, "cancelled" if cause.state == "cancelled" else "deadline"
- )
- cause = cause.__cause__
- if lease is None:
- raise
- self.store.block(lease)
- reason = (
- "chunking_unavailable"
- if isinstance(error, (ChunkingError, MappedContentError))
- else "vector_unavailable"
- if isinstance(error, VectorStoreError)
- else "index_build_unavailable"
- )
- return IndexWorkResult("blocked", reason)
- finally:
- self._run_lock.release()
-
- def _verify(self, records: Sequence[RawVectorRecord], boundary: Callable[[], None]) -> None:
- for start in range(0, len(records), 8):
- self.vectors.verify(records[start : start + 8])
- boundary()
diff --git a/memhub/apps/core/src/one_memory/retrieval/mapped_content.py b/memhub/apps/core/src/one_memory/retrieval/mapped_content.py
deleted file mode 100644
index 1f85efd..0000000
--- a/memhub/apps/core/src/one_memory/retrieval/mapped_content.py
+++ /dev/null
@@ -1,349 +0,0 @@
-"""Pure raw-content admission and exact source maps; no grants, writes or HY calls."""
-
-import hashlib
-import json
-from dataclasses import asdict, dataclass, field
-from typing import Literal
-
-from one_memory.conversation_processing import _is_hy_input_turn
-from one_memory.embedding_assets import BUILTIN_EMBEDDING_MAX_TOKENS
-from one_memory.imports.models import (
- MAX_VERBATIM_CHARACTERS,
- EvidenceContentKind,
- EvidenceRecord,
-)
-from one_memory.imports.sources.parsing import (
- is_host_generated_user_turn,
- unwrap_host_envelope_with_positions,
-)
-from one_memory.input_sanitization import sanitize_model_input_with_spans
-from one_memory.local_embedding import TokenizedText
-from one_memory.retrieval.chunking import ChunkingError, OffsetTokenizer, TextChunk, chunk_evidence
-
-MAP_VERSION = "raw-content-map-v1"
-PARSER_VERSION = "host-envelope-positions-v1"
-SANITIZER_VERSION = "model-input-spans-v1"
-MAX_MAPPED_CHUNKS = 8192
-
-
-class MappedContentError(ValueError):
- """Fixed reason only; never include original or sanitized content."""
-
-
-@dataclass(frozen=True)
-class MappedSegment:
- kind: Literal["copy", "synthetic"]
- start_cp: int
- end_cp: int
- source_start_cp: int | None
- source_end_cp: int | None
-
-
-@dataclass(frozen=True)
-class ExcludedRange:
- start_cp: int
- end_cp: int
- reason: Literal["envelope_or_trim", "sanitizer"]
-
-
-@dataclass(frozen=True)
-class ContentMap:
- map_version: str
- parser_version: str
- sanitizer_version: str
- source_length_cp: int
- body_start_cp: int | None
- body_end_cp: int | None
- sanitized_length_cp: int
- sanitized_sha256: str
- segments: tuple[MappedSegment, ...]
- excluded: tuple[ExcludedRange, ...]
-
- def canonical_json(self) -> str:
- return json.dumps(asdict(self), sort_keys=True, separators=(",", ":"), allow_nan=False)
-
- @property
- def digest(self) -> str:
- return hashlib.sha256(self.canonical_json().encode()).hexdigest()
-
-
-@dataclass(frozen=True)
-class PreparedMappedContent:
- actual_text: str = field(repr=False)
- metadata: ContentMap
- state: Literal["ready", "no_indexable_content"]
-
-
-def _validate(content: PreparedMappedContent, original: str | None = None) -> None:
- meta, text = content.metadata, content.actual_text
- invalid = "mapped_invalid_source_map"
- if (
- (meta.map_version, meta.parser_version, meta.sanitizer_version)
- != (MAP_VERSION, PARSER_VERSION, SANITIZER_VERSION)
- or type(meta.source_length_cp) is not int
- or not 0 <= meta.source_length_cp <= MAX_VERBATIM_CHARACTERS
- or type(meta.sanitized_length_cp) is not int
- or meta.sanitized_length_cp != len(text)
- or meta.sanitized_sha256 != hashlib.sha256(text.encode()).hexdigest()
- or type(meta.segments) is not tuple
- or type(meta.excluded) is not tuple
- ):
- raise MappedContentError(invalid)
- body_start, body_end = meta.body_start_cp, meta.body_end_cp
- if body_start is None or body_end is None:
- if body_start is not None or body_end is not None or text or meta.segments:
- raise MappedContentError(invalid)
- elif (
- type(body_start) is not int
- or type(body_end) is not int
- or not 0 <= body_start < body_end <= meta.source_length_cp
- ):
- raise MappedContentError(invalid)
- cursor = source_cursor = 0
- has_content = False
- partition: list[tuple[int, int]] = []
- for span in meta.segments:
- if (
- not isinstance(span, MappedSegment)
- or type(span.start_cp) is not int
- or type(span.end_cp) is not int
- or not cursor == span.start_cp < span.end_cp <= len(text)
- ):
- raise MappedContentError(invalid)
- cursor = span.end_cp
- if span.kind == "synthetic":
- if span.source_start_cp is not None or span.source_end_cp is not None:
- raise MappedContentError(invalid)
- continue
- left, right = span.source_start_cp, span.source_end_cp
- if (
- span.kind != "copy"
- or type(left) is not int
- or type(right) is not int
- or body_start is None
- or body_end is None
- or not body_start <= left < right <= body_end
- or left < source_cursor
- or right - left != span.end_cp - span.start_cp
- ):
- raise MappedContentError(invalid)
- if original is not None and text[span.start_cp : span.end_cp] != original[left:right]:
- raise MappedContentError(invalid)
- has_content |= bool(text[span.start_cp : span.end_cp].strip())
- source_cursor = right
- partition.append((left, right))
- if cursor != len(text):
- raise MappedContentError(invalid)
- excluded_end = 0
- for excluded in meta.excluded:
- if (
- not isinstance(excluded, ExcludedRange)
- or type(excluded.start_cp) is not int
- or type(excluded.end_cp) is not int
- or not excluded_end <= excluded.start_cp < excluded.end_cp <= meta.source_length_cp
- or excluded.reason not in {"envelope_or_trim", "sanitizer"}
- ):
- raise MappedContentError(invalid)
- excluded_end = excluded.end_cp
- partition.append((excluded.start_cp, excluded.end_cp))
- source_cursor = 0
- for left, right in sorted(partition):
- if left != source_cursor:
- raise MappedContentError(invalid)
- source_cursor = right
- if source_cursor != meta.source_length_cp or content.state != (
- "ready" if has_content else "no_indexable_content"
- ):
- raise MappedContentError(invalid)
-
-
-def prepare_mapped_content(record: EvidenceRecord) -> PreparedMappedContent:
- """Map a qualified body to Evidence.verbatim code points, without modifying it.
-
- The returned text is transient; only metadata is suitable for an outer manifest.
- That manifest must independently bind Evidence identity/revision/hash and grants.
- """
- if record.content_kind is not EvidenceContentKind.RAW_CONVERSATION or record.role not in {
- "user",
- "assistant",
- }:
- raise MappedContentError("mapped_content_ineligible")
- original = record.verbatim
- if len(original) > MAX_VERBATIM_CHARACTERS:
- raise MappedContentError("mapped_source_limit_exceeded")
- envelope = unwrap_host_envelope_with_positions(original)
- if envelope.status == "ambiguous":
- raise MappedContentError("mapped_ambiguous_envelope")
- body, start, end = envelope.text, envelope.start_cp, envelope.end_cp
- segments: list[MappedSegment] = []
- excluded: list[ExcludedRange] = []
- text = ""
- if body is None:
- if start is not None or end is not None:
- raise MappedContentError("mapped_invalid_envelope")
- if original:
- excluded.append(ExcludedRange(0, len(original), "envelope_or_trim"))
- else:
- if (
- type(start) is not int
- or type(end) is not int
- or not 0 <= start < end <= len(original)
- or original[start:end] != body
- ):
- raise MappedContentError("mapped_invalid_envelope")
- if is_host_generated_user_turn(original) or is_host_generated_user_turn(body):
- raise MappedContentError("mapped_generated_content")
- if not _is_hy_input_turn(record.model_copy(update={"verbatim": body})):
- raise MappedContentError("mapped_body_hygiene_rejected")
- sanitized = sanitize_model_input_with_spans(body)
- text = sanitized.text
- cursor = 0
- for span in sanitized.spans:
- if span.start_cp > cursor:
- segments.append(MappedSegment("synthetic", cursor, span.start_cp, None, None))
- copied = MappedSegment(
- "copy",
- span.start_cp,
- span.end_cp,
- start + span.source_start_cp,
- start + span.source_end_cp,
- )
- # Only source AND output adjacency can make one citation-safe interval.
- if (
- segments
- and segments[-1].kind == "copy"
- and (
- segments[-1].end_cp == copied.start_cp
- and segments[-1].source_end_cp == copied.source_start_cp
- )
- ):
- previous = segments.pop()
- copied = MappedSegment(
- "copy",
- previous.start_cp,
- copied.end_cp,
- previous.source_start_cp,
- copied.source_end_cp,
- )
- segments.append(copied)
- cursor = span.end_cp
- if cursor < len(text):
- segments.append(MappedSegment("synthetic", cursor, len(text), None, None))
- if start:
- excluded.append(ExcludedRange(0, start, "envelope_or_trim"))
- excluded.extend(
- ExcludedRange(start + left, start + right, "sanitizer")
- for left, right in sanitized.excluded
- )
- if end < len(original):
- excluded.append(ExcludedRange(end, len(original), "envelope_or_trim"))
- metadata = ContentMap(
- MAP_VERSION,
- PARSER_VERSION,
- SANITIZER_VERSION,
- len(original),
- start,
- end,
- len(text),
- hashlib.sha256(text.encode()).hexdigest(),
- tuple(segments),
- tuple(excluded),
- )
- prepared = PreparedMappedContent(
- text,
- metadata,
- "ready"
- if any(
- span.kind == "copy" and text[span.start_cp : span.end_cp].strip() for span in segments
- )
- else "no_indexable_content",
- )
- _validate(prepared, original)
- return prepared
-
-
-class _BoundedTokenizer:
- """Count successful chunk re-encodings without copying the chunking algorithm."""
-
- def __init__(self, tokenizer: OffsetTokenizer, remaining: int, max_tokens: int):
- self.tokenizer = tokenizer
- self.remaining = remaining
- self.max_tokens = max_tokens
- self.initial = True
-
- def tokenize_with_offsets(self, text: str) -> TokenizedText:
- if not self.initial and self.remaining == 0:
- raise MappedContentError("mapped_chunk_limit_exceeded")
- result = self.tokenizer.tokenize_with_offsets(text)
- # chunk_evidence first encodes the whole copy span, then prefix+candidate;
- # an in-budget candidate emits exactly one chunk. Shrink retries do not.
- if (
- not self.initial
- and isinstance(result, TokenizedText)
- and len(result.token_ids) <= self.max_tokens
- ):
- self.remaining -= 1
- self.initial = False
- return result
-
-
-def chunk_mapped_content(
- content: PreparedMappedContent,
- tokenizer: OffsetTokenizer,
- *,
- prefix: str = "",
- target_tokens: int = 384,
- overlap_tokens: int = 64,
- max_tokens: int = BUILTIN_EMBEDDING_MAX_TOKENS,
- max_chunks: int = MAX_MAPPED_CHUNKS,
-) -> tuple[TextChunk, ...]:
- """Chunk each continuous copy independently; never encode synthetic placeholders."""
- if type(max_chunks) is not int or not 1 <= max_chunks <= MAX_MAPPED_CHUNKS:
- raise MappedContentError("mapped_invalid_chunk_limit")
- _validate(content)
- # Reuse budget validation even for an explicitly nonindexable record.
- chunk_evidence(
- "",
- tokenizer,
- prefix=prefix,
- target_tokens=target_tokens,
- overlap_tokens=overlap_tokens,
- max_tokens=max_tokens,
- )
- chunks: list[TextChunk] = []
- for span in content.metadata.segments:
- if span.kind != "copy":
- continue
- text = content.actual_text[span.start_cp : span.end_cp]
- if not text.strip():
- continue
- if len(chunks) == max_chunks:
- raise MappedContentError("mapped_chunk_limit_exceeded")
- try:
- pieces = chunk_evidence(
- text,
- _BoundedTokenizer(tokenizer, max_chunks - len(chunks), max_tokens),
- prefix=prefix,
- target_tokens=target_tokens,
- overlap_tokens=overlap_tokens,
- max_tokens=max_tokens,
- )
- except ChunkingError as error:
- if isinstance(error.__cause__, MappedContentError):
- raise error.__cause__ from None
- raise
- if len(chunks) + len(pieces) > max_chunks:
- raise MappedContentError("mapped_chunk_limit_exceeded")
- assert span.source_start_cp is not None # Validated copy span.
- ordinal = len(chunks)
- chunks.extend(
- TextChunk(
- ordinal + i,
- span.source_start_cp + piece.start_cp,
- span.source_start_cp + piece.end_cp,
- piece.token_count,
- piece.text,
- )
- for i, piece in enumerate(pieces)
- )
- return tuple(chunks)
diff --git a/memhub/apps/core/src/one_memory/retrieval/raw_candidates.py b/memhub/apps/core/src/one_memory/retrieval/raw_candidates.py
deleted file mode 100644
index cebb7e8..0000000
--- a/memhub/apps/core/src/one_memory/retrieval/raw_candidates.py
+++ /dev/null
@@ -1,147 +0,0 @@
-"""Internal raw semantic candidate lookup. No route, raw body release or full coverage claim."""
-
-import math
-import sqlite3
-from collections.abc import Sequence
-from dataclasses import dataclass
-from typing import Literal
-
-from one_memory.context_raw import _check
-from one_memory.retrieval.index_store import PublishedChunk, RetrievalIndexStore
-from one_memory.retrieval.models import RetrievalPath, TimeRange
-from one_memory.retrieval.vector_store import RawVectorStore, VectorStoreError
-
-
-@dataclass(frozen=True)
-class RawSemanticCandidate:
- chunk: PublishedChunk
- distance: float
-
-
-@dataclass(frozen=True)
-class RawCandidateBatch:
- hits: tuple[RawSemanticCandidate, ...] = ()
- complete: bool = False
- reason: str | None = None
- coverage: Literal["indexed_only"] = "indexed_only"
- time_range: TimeRange | None = None
-
-
-def read_raw_semantic_candidates(
- store: RetrievalIndexStore,
- vectors: RawVectorStore,
- query_vector: Sequence[float],
- *,
- generation_id: str,
- source_ids: frozenset[str],
- deadline: float,
- limit: int = 64,
- time_range: TimeRange | None = None,
-) -> RawCandidateBatch:
- """Caller resolves host authority; explicit source IDs can only narrow it.
-
- complete means the eligible published chunks were checked and searched, not
- that every authorized original is indexed. The future coordinator must still
- revalidate host identity and all raw/memory data together before body release.
- """
- if not isinstance(source_ids, frozenset) or any(
- not isinstance(source, str) or not 1 <= len(source) <= 128 for source in source_ids
- ):
- raise ValueError("explicit_resolved_sources_required")
- if type(limit) is not int or not 1 <= limit <= 64:
- raise ValueError("invalid_candidate_limit")
- if isinstance(deadline, bool) or not math.isfinite(deadline):
- raise ValueError("invalid_candidate_deadline")
- if not source_ids:
- return RawCandidateBatch(reason="not_authorized")
- if vectors.generation_id != generation_id:
- raise ValueError("candidate_generation_mismatch")
- try:
- _check(deadline)
- eligible = store.active_chunks(
- generation_id, source_ids=source_ids, time_range=time_range, deadline=deadline
- )
- if not eligible:
- return RawCandidateBatch(
- complete=True, reason="no_eligible_chunks", time_range=time_range
- )
- before = {chunk.chunk_id: chunk for chunk in eligible}
- expected_vectors = [(chunk.chunk_id, chunk.build_id) for chunk in eligible]
- # When one query can return the entire allowlist, that response itself
- # is the integrity check. Avoid a duplicate Chroma get-before-query on
- # the hot small/medium-authority path. Larger allowlists still need the
- # explicit complete verification because search returns only top-K.
- verify_before_search = len(eligible) > limit
- if verify_before_search and not vectors.verify_candidates(
- expected_vectors, deadline=deadline
- ):
- return RawCandidateBatch(reason="deadline")
- result = vectors.search(query_vector, tuple(before), limit, deadline)
- if not verify_before_search and {hit.chunk_id for hit in result.hits} != set(before):
- raise VectorStoreError("vector_record_missing")
- _check(deadline)
- current = {
- chunk.chunk_id: chunk
- for chunk in store.active_chunks(
- generation_id, source_ids=source_ids, time_range=time_range, deadline=deadline
- )
- }
- hits = []
- for hit in result.hits:
- if (
- hit.chunk_id not in before
- or hit.generation_id != generation_id
- or hit.build_id != before[hit.chunk_id].build_id
- ):
- raise VectorStoreError("candidate_identity_mismatch")
- if current.get(hit.chunk_id) == before[hit.chunk_id]:
- hits.append(RawSemanticCandidate(before[hit.chunk_id], hit.distance))
- _check(deadline)
- unchanged = before == current
- return RawCandidateBatch(
- tuple(hits),
- result.complete and unchanged,
- "evidence_changed" if not unchanged else result.reason,
- time_range=time_range,
- )
- except TimeoutError:
- return RawCandidateBatch(reason="deadline")
- except VectorStoreError:
- return RawCandidateBatch(reason="vector_unavailable")
- except (sqlite3.Error, OSError, ValueError):
- return RawCandidateBatch(reason="index_unavailable")
-
-
-def compose_raw_retrieval_path(
- batch: RawCandidateBatch,
- *,
- unpublished_authorized: bool,
- lexical_hit_count: int,
-) -> RetrievalPath:
- """Map channel results to a path. Indexed-only search is never complete empty."""
- if type(lexical_hit_count) is not int or lexical_hit_count < 0:
- raise ValueError("invalid_lexical_hit_count")
- hits = len(batch.hits) + lexical_hit_count
- incomplete = unpublished_authorized or not batch.complete
- channels: tuple[Literal["lexical", "semantic"], ...] = ("lexical", "semantic")
- if hits:
- return RetrievalPath(
- status="ok",
- returned_count=min(hits, 32),
- searched_channels=channels,
- coverage="partial" if incomplete else "complete",
- )
- if incomplete:
- return RetrievalPath(
- status="degraded",
- reason="indexed_only",
- returned_count=0,
- searched_channels=channels,
- coverage="partial",
- )
- return RetrievalPath(
- status="empty",
- returned_count=0,
- searched_channels=channels,
- coverage="complete",
- )
diff --git a/memhub/apps/core/src/one_memory/retrieval/raw_evidence.py b/memhub/apps/core/src/one_memory/retrieval/raw_evidence.py
deleted file mode 100644
index cfb01c6..0000000
--- a/memhub/apps/core/src/one_memory/retrieval/raw_evidence.py
+++ /dev/null
@@ -1,189 +0,0 @@
-"""Materialize internal semantic candidates from current authorized Evidence.
-
-Not an arbitrary-ID endpoint or signed expansion capability. The combined v2
-coordinator must revalidate this lane with memory and host identity at release.
-"""
-
-import json
-import sqlite3
-from dataclasses import dataclass
-from datetime import UTC
-
-from one_memory import ingest_journal as journal
-from one_memory.context_raw import _check, _connection
-from one_memory.imports.models import EvidenceRecord
-from one_memory.imports.storage import EvidenceStore
-from one_memory.retrieval.index_store import RetrievalIndexStore
-from one_memory.retrieval.models import EvidenceSpan, RawEvidence
-from one_memory.retrieval.raw_candidates import RawCandidateBatch
-
-
-@dataclass(frozen=True)
-class RawEvidenceBatch:
- items: tuple[RawEvidence, ...] = ()
- complete: bool = False
- reason: str | None = None
- used_characters: int = 0
-
-
-def _merge(intervals: list[tuple[int, int]]) -> list[tuple[int, int]]:
- output: list[tuple[int, int]] = []
- for start, end in sorted(intervals):
- if output and start <= output[-1][1]:
- output[-1] = (output[-1][0], max(end, output[-1][1]))
- else:
- output.append((start, end))
- return output
-
-
-def materialize_raw_candidates(
- store: RetrievalIndexStore,
- candidates: RawCandidateBatch,
- *,
- generation_id: str,
- source_ids: frozenset[str],
- deadline: float,
- character_budget: int = 6000,
-) -> RawEvidenceBatch:
- if not isinstance(source_ids, frozenset) or any(
- not isinstance(source, str) or not 1 <= len(source) <= 128 for source in source_ids
- ):
- raise ValueError("explicit_resolved_sources_required")
- if type(character_budget) is not int or not 1 <= character_budget <= 20000:
- raise ValueError("invalid_evidence_budget")
- if len(candidates.hits) > 64:
- raise ValueError("candidate_budget_exceeded")
- if not source_ids:
- return RawEvidenceBatch(reason="not_authorized")
- try:
- _check(deadline)
- expected = {hit.chunk.chunk_id: hit.chunk for hit in candidates.hits}
- if len(expected) != len(candidates.hits):
- raise ValueError("duplicate_candidates")
- if not expected:
- # Empty/excluded builds still carry an integrity claim. Validate
- # current authorized completed manifests, never only nonempty hits.
- current_chunks = store.active_chunks(
- generation_id,
- source_ids=source_ids,
- time_range=candidates.time_range,
- deadline=deadline,
- require_active=True,
- )
- _check(deadline)
- if current_chunks and candidates.reason == "no_eligible_chunks":
- return RawEvidenceBatch(reason="evidence_changed")
- return RawEvidenceBatch(complete=candidates.complete, reason=candidates.reason)
- changed = False
- grouped: dict[str, tuple[EvidenceRecord, list[tuple[int, int]]]] = {}
- with _connection(store.database_path, deadline) as db:
- db.execute("PRAGMA query_only=ON")
- rows = {
- row["chunk_id"]: row
- for row in db.execute(
- """SELECT c.*, d.event_seq,d.grant_fingerprint,d.source_id
- FROM retrieval_chunks c JOIN retrieval_documents d
- ON d.generation_id=c.generation_id AND d.evidence_id=c.evidence_id
- AND d.active_build_id=c.build_id AND d.build_id=c.build_id
- JOIN retrieval_index_generations g ON g.generation_id=c.generation_id
- WHERE c.generation_id=? AND g.state='active' AND d.state='ready'
- AND c.chunk_id IN (SELECT value FROM json_each(?))""",
- (generation_id, json.dumps(tuple(expected))),
- )
- }
- for key, chunk in expected.items():
- _check(deadline)
- row = rows.get(key)
- if row is None or row["source_id"] not in source_ids:
- changed = True
- continue
- event = journal.event(db, row["event_seq"])
- grant = store._grant(db, row["source_id"])
- if (
- event is None
- or event.user_id != "local"
- or event.source_id != row["source_id"]
- or event.evidence_id != chunk.evidence_id
- or event.evidence_revision != chunk.revision
- or grant is None
- or grant != row["grant_fingerprint"]
- or not journal.matches_current(db, event)
- ):
- changed = True
- continue
- record = EvidenceStore._get_with_connection(db, chunk.evidence_id)
- if (
- record is None
- or record.user_id != "local"
- or record.source_id != row["source_id"]
- ):
- changed = True
- continue
- event_time = record.created_at
- if event_time is not None:
- event_time = event_time.astimezone(UTC) if event_time.tzinfo else None
- if (
- (row["build_id"], row["ordinal"], row["start_cp"], row["end_cp"])
- != (chunk.build_id, chunk.ordinal, chunk.start_cp, chunk.end_cp)
- or (record.source_id, event_time) != (chunk.source_id, chunk.event_time)
- or not 0 <= chunk.start_cp < chunk.end_cp <= len(record.verbatim)
- ):
- changed = True
- continue
- if record.evidence_id not in grouped:
- document = db.execute(
- "SELECT * FROM retrieval_documents WHERE generation_id=? AND evidence_id=?",
- (generation_id, record.evidence_id),
- ).fetchone()
- if document is None:
- raise ValueError("published_document_missing")
- store.validate_document(db, document, record, deadline=deadline)
- grouped[record.evidence_id] = (record, [])
- grouped[record.evidence_id][1].append((chunk.start_cp, chunk.end_cp))
- items = []
- used = 0
- truncated = False
- for record, intervals in grouped.values():
- _check(deadline)
- spans: list[EvidenceSpan] = []
- for start, end in _merge(intervals):
- if len(spans) >= 4 or end - start > min(20000, character_budget - used):
- truncated = True
- continue
- spans.append(
- EvidenceSpan(start_cp=start, end_cp=end, text=record.verbatim[start:end])
- )
- used += end - start
- if not spans:
- continue
- event_time = record.created_at
- if event_time is not None:
- event_time = event_time.astimezone(UTC) if event_time.tzinfo else None
- items.append(
- RawEvidence(
- evidence_id=record.evidence_id,
- revision=record.revision,
- source_id=record.source_id,
- conversation_id=record.conversation_id,
- role=record.role,
- written_by="person" if record.role == "user" else "agent",
- event_time=event_time,
- spans=tuple(spans),
- matched_by=("semantic",),
- )
- )
- _check(deadline)
- return RawEvidenceBatch(
- tuple(items),
- candidates.complete and not changed and not truncated,
- "evidence_changed"
- if changed
- else "character_budget"
- if truncated
- else candidates.reason,
- used,
- )
- except TimeoutError:
- return RawEvidenceBatch(reason="deadline")
- except (sqlite3.Error, OSError, ValueError):
- return RawEvidenceBatch(reason="index_unavailable")
diff --git a/memhub/apps/core/src/one_memory/retrieval/raw_lane.py b/memhub/apps/core/src/one_memory/retrieval/raw_lane.py
deleted file mode 100644
index 92a92a2..0000000
--- a/memhub/apps/core/src/one_memory/retrieval/raw_lane.py
+++ /dev/null
@@ -1,181 +0,0 @@
-"""Fuse lexical raw context with an optional derived index. No new HTTP schema."""
-
-import sqlite3
-from collections.abc import Callable
-from pathlib import Path
-from typing import Protocol
-
-from pydantic import ValidationError
-
-from one_memory.context_models import (
- ContextLaneResult,
- ContextSearchRequest,
- ContextSemanticStatus,
- RawContextItem,
-)
-from one_memory.context_raw import (
- _METADATA,
- _check,
- _connection,
- _execute,
- _message,
- _readable,
- read_raw_context,
-)
-from one_memory.retrieval.models import TimeRange
-from one_memory.retrieval.raw_candidates import RawCandidateBatch, compose_raw_retrieval_path
-
-
-class RawIndexLookup(Protocol):
- def lookup_raw(
- self,
- query: str,
- *,
- source_ids: frozenset[str] | None,
- deadline: float,
- time_range: TimeRange | None = None,
- ) -> tuple[RawCandidateBatch, bool]: ...
-
-
-def fuse_raw_lane(
- lexical: ContextLaneResult,
- batch: RawCandidateBatch,
- *,
- unpublished_authorized: bool,
- extra_items: tuple[RawContextItem, ...] = (),
-) -> ContextLaneResult:
- """v1 has no coverage field; degraded + lexical_only stands in for partial index."""
- semantic_status: ContextSemanticStatus = (
- "timed_out"
- if batch.reason == "deadline"
- else "failed"
- if batch.reason in {"index_unavailable", "vector_unavailable"}
- else "completed"
- )
- index_complete = batch.complete and not unpublished_authorized
- timed_out = semantic_status == "timed_out"
- if lexical.status not in {"ok", "empty", "degraded"}:
- return lexical.model_copy(
- update={
- "searched_channels": ("lexical", "semantic"),
- "semantic_status": semantic_status,
- "index_complete": index_complete,
- "timed_out": timed_out,
- }
- )
- seen = {item.anchor_evidence_id for item in lexical.items if isinstance(item, RawContextItem)}
- extras = tuple(item for item in extra_items if isinstance(item, RawContextItem))
- # Semantic similarity is a candidate signal, not a verified recall result.
- # Release only semantic items that the lexical lane independently confirmed
- # for the same evidence identity. Unconfirmed semantic hits stay in the
- # coverage receipt as partial/degraded and must not reach host injection.
- unconfirmed_semantic = any(item.anchor_evidence_id not in seen for item in extras)
- items = lexical.items
- path = compose_raw_retrieval_path(
- batch,
- unpublished_authorized=unpublished_authorized,
- lexical_hit_count=len(lexical.items),
- )
- status = path.status
- if unconfirmed_semantic:
- status = "degraded"
- if lexical.status == "degraded" and status == "ok":
- status = "degraded"
- return ContextLaneResult(
- status=status,
- reason=(
- "lexical_only" if unconfirmed_semantic and lexical.reason is None else lexical.reason
- ),
- items=items[:21],
- truncated_by=lexical.truncated_by,
- searched_channels=("lexical", "semantic"),
- semantic_status=semantic_status,
- index_complete=index_complete,
- timed_out=timed_out,
- )
-
-
-def load_raw_items_from_candidates(
- database_path: Path,
- batch: RawCandidateBatch,
- *,
- source_ids: frozenset[str] | None,
- deadline: float,
- character_budget: int = 6000,
-) -> tuple[RawContextItem, ...]:
- """Release authorized Evidence bodies for semantic IDs. Not a vector dump."""
- ids = tuple(dict.fromkeys(hit.chunk.evidence_id for hit in batch.hits))
- if not ids or source_ids == frozenset():
- return ()
- readable, authority = _readable(source_ids)
- body_limit = min(20000, max(1, character_budget))
- items: list[RawContextItem] = []
- try:
- with _connection(database_path, deadline) as db:
- for evidence_id in ids[:21]:
- _check(deadline)
- row = _execute(
- db,
- deadline,
- f"""
- SELECT {_METADATA}, substr(e.verbatim,1,?) AS text,
- length(e.verbatim)>? AS body_truncated
- FROM evidence e
- WHERE {readable} AND e.evidence_id=?
- """,
- [body_limit, body_limit, *authority, evidence_id],
- ).fetchone()
- if row is None:
- continue
- try:
- message = _message(row)
- items.append(
- RawContextItem(
- source_id=row["source_id"],
- conversation_id=row["conversation_id"],
- anchor_evidence_id=message.evidence_id,
- messages=(message,),
- truncated=message.truncated,
- )
- )
- except (ValidationError, ValueError, TypeError):
- continue
- except TimeoutError:
- raise
- except (sqlite3.Error, OSError):
- return ()
- return tuple(items)
-
-
-def read_raw_indexed_context(
- database_path: Path,
- request: ContextSearchRequest,
- *,
- source_ids: frozenset[str] | None,
- deadline: float,
- index: RawIndexLookup | None = None,
- load_semantic_items: Callable[..., tuple[RawContextItem, ...]] | None = None,
-) -> ContextLaneResult:
- lexical = read_raw_context(database_path, request, source_ids=source_ids, deadline=deadline)
- if index is None:
- return lexical
- try:
- batch, unpublished = index.lookup_raw(
- request.query, source_ids=source_ids, deadline=deadline
- )
- except Exception:
- batch, unpublished = RawCandidateBatch(reason="index_unavailable"), True
- extra: tuple[RawContextItem, ...] = ()
- if batch.hits:
- extra = (
- load_semantic_items(database_path, batch, source_ids=source_ids, deadline=deadline)
- if load_semantic_items is not None
- else load_raw_items_from_candidates(
- database_path,
- batch,
- source_ids=source_ids,
- deadline=deadline,
- character_budget=request.character_budget,
- )
- )
- return fuse_raw_lane(lexical, batch, unpublished_authorized=unpublished, extra_items=extra)
diff --git a/memhub/apps/core/src/one_memory/retrieval/runtime.py b/memhub/apps/core/src/one_memory/retrieval/runtime.py
deleted file mode 100644
index 95bc2e9..0000000
--- a/memhub/apps/core/src/one_memory/retrieval/runtime.py
+++ /dev/null
@@ -1,379 +0,0 @@
-"""Resume an explicitly configured local index; never create or migrate one."""
-
-import asyncio
-import json
-import os
-import shutil
-import sqlite3
-import stat
-import sys
-import threading
-from collections.abc import Callable
-from dataclasses import asdict, dataclass
-from pathlib import Path
-from typing import Literal
-
-from one_memory.database_access import main_database
-from one_memory.retrieval.index_store import EncoderIdentity, RetrievalIndexStore
-from one_memory.retrieval.index_supervisor import IndexSupervisorSnapshot, RetrievalIndexSupervisor
-from one_memory.retrieval.index_worker import LocalRetrievalEncoder, RetrievalIndexWorker
-from one_memory.retrieval.models import TimeRange
-from one_memory.retrieval.raw_candidates import RawCandidateBatch, read_raw_semantic_candidates
-from one_memory.retrieval.vector_store import RawVectorStore, VectorStoreError
-from one_memory.service_levels import ServiceLevel, ServiceLevelScheduler, ServiceResource
-
-type RuntimeState = Literal[
- "new", "starting", "running", "not_initialized", "failed", "stopping", "stopped"
-]
-
-
-@dataclass(frozen=True)
-class ExistingRetrievalSnapshot:
- state: RuntimeState
- error_code: str | None
- generation_id: str | None
- supervisor: IndexSupervisorSnapshot | None
-
-
-class _AdmissionError(Exception):
- def __init__(self, code: str, *, missing: bool = False):
- self.code, self.missing = code, missing
-
-
-async def _native[T](operation: Callable[[], T]) -> T:
- """Owned, joined native call. Only retained internal lifecycle tasks invoke this."""
- result: list[T] = []
- errors: list[BaseException] = []
-
- def run() -> None:
- try:
- result.append(operation())
- except BaseException as error:
- errors.append(error)
-
- thread = threading.Thread(target=run, name="retrieval-resource", daemon=False)
- thread.start()
- while thread.is_alive():
- await asyncio.sleep(0.01)
- thread.join()
- if errors:
- raise errors[0]
- return result[0]
-
-
-class ExistingRetrievalRuntime:
- """Caller guarantees managed Core single ownership and explicit configuration.
-
- Construct without I/O; start and stop on the owning event loop. A retained
- lifecycle task shields native work from caller cancellation until cleanup.
- """
-
- def __init__(
- self,
- data_dir: Path,
- *,
- minimum_free_bytes: int = 256 * 1024 * 1024,
- service_levels: ServiceLevelScheduler | None = None,
- ):
- if type(minimum_free_bytes) is not int or minimum_free_bytes < 0:
- raise ValueError("retrieval_runtime_invalid_disk_limit")
- self._directory = Path(data_dir)
- self._minimum_free = minimum_free_bytes
- self._service_levels = service_levels or ServiceLevelScheduler()
- self._state: ExistingRetrievalSnapshot = ExistingRetrievalSnapshot("new", None, None, None)
- self._vectors: RawVectorStore | None = None
- self._store: RetrievalIndexStore | None = None
- self._encoder: LocalRetrievalEncoder | None = None
- self._supervisor: RetrievalIndexSupervisor | None = None
- self._last_supervisor_snapshot: IndexSupervisorSnapshot | None = None
- self._start_task: asyncio.Task[bool] | None = None
- self._stop_task: asyncio.Task[None] | None = None
- self._stop_requested = False
-
- def snapshot(self) -> ExistingRetrievalSnapshot:
- return ExistingRetrievalSnapshot(
- self._state.state,
- self._state.error_code,
- self._state.generation_id,
- self._supervisor.snapshot()
- if self._supervisor is not None
- else self._last_supervisor_snapshot,
- )
-
- def _set(
- self, state: RuntimeState, code: str | None = None, generation: str | None = None
- ) -> None:
- self._state = ExistingRetrievalSnapshot(
- state,
- code,
- generation if generation is not None else self._state.generation_id,
- None,
- )
-
- def trigger(self) -> bool:
- return bool(
- self._state.state == "running" and self._supervisor and self._supervisor.trigger()
- )
-
- def lookup_raw(
- self,
- query: str,
- *,
- source_ids: frozenset[str] | None,
- deadline: float,
- time_range: TimeRange | None = None,
- ) -> tuple[RawCandidateBatch, bool]:
- """Read-only candidate lookup. Does not release original bodies."""
- store, encoder, vectors, generation = (
- self._store,
- self._encoder,
- self._vectors,
- self._state.generation_id,
- )
- if (
- self._state.state != "running"
- or store is None
- or encoder is None
- or vectors is None
- or generation is None
- ):
- return RawCandidateBatch(reason="index_unavailable"), True
- counts = store.status(generation)
- unpublished = counts["pending"] + counts["leased"] + counts["blocked"] > 0
- resolved = source_ids
- if resolved is None:
- resolved = frozenset(
- chunk.source_id for chunk in store.active_chunks(generation, deadline=deadline)
- )
- if not resolved:
- return (
- RawCandidateBatch(complete=not unpublished, reason="no_eligible_chunks"),
- unpublished,
- )
- try:
- vector = encoder.encode_documents([query.strip()])[0]
- except Exception:
- return RawCandidateBatch(reason="index_unavailable"), True
- batch = read_raw_semantic_candidates(
- store,
- vectors,
- vector,
- generation_id=generation,
- source_ids=resolved,
- deadline=deadline,
- time_range=time_range,
- )
- return batch, unpublished
-
- async def start(self) -> bool:
- if self._state.state != "new" or self._stop_requested:
- raise RuntimeError("retrieval_runtime_already_started")
- self._set("starting")
- self._start_task = asyncio.create_task(self._start())
- try:
- return await asyncio.shield(self._start_task)
- except asyncio.CancelledError:
- # Retain and drain opening resources, then release them, before cancellation escapes.
- await self.stop()
- raise
-
- async def stop(self) -> None:
- self._stop_requested = True
- if self._stop_task is None:
- self._stop_task = asyncio.create_task(self._stop())
- cancelled = False
- while not self._stop_task.done():
- try:
- await asyncio.shield(self._stop_task)
- except asyncio.CancelledError:
- cancelled = True
- self._stop_task.result()
- if cancelled:
- raise asyncio.CancelledError
-
- async def _start(self) -> bool:
- try:
- store, encoder, vectors, generation = await _native(self._open)
- self._store = store
- self._encoder = encoder
- self._vectors = vectors
- self._set("starting", generation=generation)
- if self._stop_requested:
- return False
- worker = RetrievalIndexWorker(
- store, vectors, encoder, generation_id=generation, owner="managed-core-retrieval"
- )
- self._supervisor = RetrievalIndexSupervisor(
- worker,
- background_should_yield=lambda: self._service_levels.should_yield(
- ServiceResource.INDEX,
- ServiceLevel.BACKGROUND,
- ),
- )
- self._supervisor.start()
- self._set("running")
- return True
- except _AdmissionError as error:
- self._set("not_initialized" if error.missing else "failed", error.code)
- except Exception:
- self._set("failed", "retrieval_runtime_start_failed")
- await self._release()
- return False
-
- async def _release(self) -> None:
- stop_failed = False
- if self._supervisor is not None:
- try:
- await self._supervisor.stop()
- except Exception:
- stop_failed = True
- supervisor_snapshot = self._supervisor.snapshot()
- if supervisor_snapshot.thread_alive:
- self._set("failed", "retrieval_runtime_stop_failed")
- raise RuntimeError("retrieval_runtime_stop_failed") from None
- # The supervisor owns the worker, which owns the native vector
- # client. Retain only the immutable receipt once its thread drains.
- self._last_supervisor_snapshot = supervisor_snapshot
- self._supervisor = None
- if self._vectors is not None:
- try:
- await _native(self._vectors.close)
- except Exception:
- self._set("failed", "retrieval_runtime_close_failed")
- raise RuntimeError("retrieval_runtime_close_failed") from None
- self._vectors = None
- self._store = None
- self._encoder = None
- if stop_failed:
- self._set("failed", "retrieval_runtime_stop_failed")
- raise RuntimeError("retrieval_runtime_stop_failed") from None
-
- async def _stop(self) -> None:
- if self._start_task is not None:
- try:
- await asyncio.shield(self._start_task)
- except Exception:
- # Startup can itself fail during cleanup. Do not lose resource ownership.
- self._set("failed", "retrieval_runtime_start_failed")
- self._set("stopping", self._state.error_code)
- await self._release()
- if self._state.error_code == "retrieval_runtime_close_failed":
- return
- self._set("stopped", self._state.error_code)
-
- @staticmethod
- def _path(path: Path, *, directory: bool) -> tuple[int, int]:
- if any(part.is_symlink() for part in (path, *path.parents)):
- raise _AdmissionError("retrieval_runtime_unsafe_path")
- try:
- info = path.stat()
- except FileNotFoundError:
- raise _AdmissionError("retrieval_runtime_not_initialized", missing=True) from None
- if not (stat.S_ISDIR(info.st_mode) if directory else stat.S_ISREG(info.st_mode)):
- raise _AdmissionError("retrieval_runtime_unsafe_path")
- if not directory and info.st_nlink != 1:
- raise _AdmissionError("retrieval_runtime_unsafe_path")
- if info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) & 0o022:
- raise _AdmissionError("retrieval_runtime_unsafe_path")
- return info.st_dev, info.st_ino
-
- def _sidecars(self, database: Path) -> None:
- for suffix in ("-wal", "-shm", "-journal"):
- path = database.with_name(database.name + suffix)
- if path.is_symlink() or path.exists():
- self._path(path, directory=False)
-
- def _inspect(self, database: Path) -> tuple[RetrievalIndexStore, EncoderIdentity, str]:
- with main_database(database.absolute()).read(busy_timeout_ms=1_000) as db:
- db.row_factory = sqlite3.Row
- db.execute("PRAGMA query_only=ON")
- db.execute("BEGIN")
- if not db.execute(
- "SELECT 1 FROM sqlite_master WHERE name='retrieval_index_state'"
- ).fetchone():
- raise _AdmissionError("retrieval_runtime_not_initialized", missing=True)
- state = db.execute(
- "SELECT schema_version FROM retrieval_index_state WHERE singleton=1"
- ).fetchone()
- if state is not None and state[0] == 1:
- raise _AdmissionError("retrieval_runtime_not_initialized", missing=True)
- if state is None or state[0] != 2:
- raise _AdmissionError("retrieval_runtime_invalid_index")
- rows = db.execute(
- "SELECT generation_id,identity_json FROM retrieval_index_generations "
- "WHERE state='active' LIMIT 2"
- ).fetchall()
- if not rows:
- raise _AdmissionError("retrieval_runtime_not_initialized", missing=True)
- if len(rows) != 1:
- raise _AdmissionError("retrieval_runtime_invalid_index")
- store = RetrievalIndexStore(database)
- generation = rows[0]["generation_id"]
- identity = store._identity(db, generation)
- raw = json.loads(rows[0]["identity_json"])
- if raw != asdict(identity) or set(raw) != set(asdict(identity)):
- raise _AdmissionError("retrieval_runtime_invalid_index")
- # Check required schema shape without repairing it or reading original bodies.
- db.execute(
- "SELECT build_id,source_map_json,map_digest,coverage_state "
- "FROM retrieval_build_maps LIMIT 0"
- )
- db.execute("SELECT after_seq,retry_after_seq FROM retrieval_consumers LIMIT 0")
- db.execute("SELECT lease_token,active_build_id FROM retrieval_documents LIMIT 0")
- db.execute("SELECT chunk_id,text_hash FROM retrieval_chunks LIMIT 0")
- return store, identity, generation
-
- def _open(self) -> tuple[RetrievalIndexStore, LocalRetrievalEncoder, RawVectorStore, str]:
- if sys.platform == "win32" or not hasattr(os, "getuid"):
- raise _AdmissionError("retrieval_runtime_platform_unsupported")
- directory = self._directory.absolute()
- database, vector_dir = directory / "one-memory.sqlite3", directory / "retrieval-vectors"
- root_pin = self._path(directory, directory=True)
- database_pin = self._path(database, directory=False)
- self._sidecars(database)
- try:
- store, identity, generation = self._inspect(database)
- except (ValueError, TypeError, sqlite3.Error):
- raise _AdmissionError("retrieval_runtime_invalid_index") from None
- vector_pin = self._path(vector_dir, directory=True)
- vector_database = vector_dir / "chroma.sqlite3"
- vector_db_pin = self._path(vector_database, directory=False)
- self._sidecars(vector_database)
-
- def recheck() -> None:
- if (
- self._path(directory, directory=True) != root_pin
- or self._path(database, directory=False) != database_pin
- or self._path(vector_dir, directory=True) != vector_pin
- or self._path(vector_database, directory=False) != vector_db_pin
- ):
- raise _AdmissionError("retrieval_runtime_index_changed")
- self._sidecars(database)
- self._sidecars(vector_database)
-
- if shutil.disk_usage(directory).free < self._minimum_free:
- raise _AdmissionError("retrieval_runtime_low_disk")
- encoder = LocalRetrievalEncoder(content_version=identity.normalizer_version)
- if encoder.identity != identity:
- raise _AdmissionError("retrieval_runtime_encoder_mismatch")
- recheck()
- try:
- vectors = RawVectorStore.open_writer_existing(
- vector_dir, generation_id=generation, dimensions=identity.dimensions
- )
- # Retain ownership immediately so all later failures use the same cleanup path.
- self._vectors = vectors
- # A generation change during the native open is not authorization to resume another.
- recheck()
- if self._inspect(database)[1:] != (identity, generation):
- raise _AdmissionError("retrieval_runtime_index_changed")
- recheck()
- return store, encoder, vectors, generation
- except BaseException as error:
- if isinstance(error, VectorStoreError):
- if str(error) in {"vector_store_not_found", "vector_collection_not_found"}:
- raise _AdmissionError(
- "retrieval_runtime_not_initialized", missing=True
- ) from None
- raise _AdmissionError("retrieval_runtime_vector_unavailable") from None
- raise
diff --git a/memhub/apps/core/src/one_memory/retrieval/vector_store.py b/memhub/apps/core/src/one_memory/retrieval/vector_store.py
deleted file mode 100644
index 5f95d6f..0000000
--- a/memhub/apps/core/src/one_memory/retrieval/vector_store.py
+++ /dev/null
@@ -1,534 +0,0 @@
-"""Independent raw-vector adapter. Eligibility is supplied by Core's SQLite snapshot.
-
-No HY collection, document text, embedding function, migration/repair or authority
-store lives here. Chroma PersistentClient is NOT physically read-only: even the
-existing-collection handle can maintain backend metadata/index files. A caller
-requiring filesystem read-only operation must not use this backend.
-
-Scoring is an exact pass over the eligible vectors, not a global ANN top-K that
-is filtered afterwards. Chroma's filtered query does not score every candidate
-in its `where $in` allowlist: at 10k stored chunks an explicit 64-ID allowlist
-returned 26 of 64 while reporting the query as complete, and 0 of 64 after the
-writer handle was reopened. A post-filtered global search would be neither
-permission-correct nor recall-correct, so the eligible embeddings are read
-back and ranked here.
-"""
-
-from __future__ import annotations
-
-import math
-import re
-import sqlite3
-import struct
-import threading
-from collections.abc import Callable, Sequence
-from contextlib import closing
-from dataclasses import dataclass
-from hashlib import sha256
-from numbers import Real
-from pathlib import Path
-from time import monotonic
-from typing import Any, Literal
-
-ALLOWLIST_BATCH_SIZE = 2048
-
-
-class VectorStoreError(RuntimeError):
- """Bounded failure code; callers must not treat corruption as complete empty search."""
-
-
-@dataclass(frozen=True)
-class RawVectorRecord:
- chunk_id: str
- build_id: str
- generation_id: str
- vector: tuple[float, ...]
-
- def __post_init__(self) -> None:
- object.__setattr__(self, "vector", tuple(self.vector))
-
-
-@dataclass(frozen=True)
-class VectorHit:
- chunk_id: str
- build_id: str
- generation_id: str
- distance: float
-
-
-@dataclass(frozen=True)
-class VectorSearchResult:
- hits: tuple[VectorHit, ...]
- complete: bool
- eligible_count: int
- searched_count: int
- reason: Literal["deadline"] | None
-
-
-class RawVectorStore:
- """Unopened constructor has no I/O. Use create/open_existing explicitly.
-
- Directory must be a dedicated, trusted Runtime retrieval-vectors directory,
- never the HY data directory. Only create/open_writer_existing enable upsert.
- Core must serialize writers across handles/processes; this instance's lock
- is not a persistent writer lease. close() releases only this Chroma client,
- retaining other live clients' references.
- """
-
- def __init__(
- self,
- directory: Path,
- *,
- generation_id: str,
- dimensions: int,
- clock: Callable[[], float] = monotonic,
- ) -> None:
- _identifier(generation_id)
- if type(dimensions) is not int or not 1 <= dimensions <= 65536:
- raise VectorStoreError("vector_invalid_dimensions")
- self.directory = Path(directory)
- self.generation_id = generation_id
- self.dimensions = dimensions
- self.collection_name = "raw_" + sha256(generation_id.encode()).hexdigest()
- self._clock = clock
- self._client: Any = None
- self._collection: Any = None
- self._writable = False
- self._closed = False
- self._lock = threading.RLock()
-
- @classmethod
- def create(cls, directory: Path, *, generation_id: str, dimensions: int) -> RawVectorStore:
- """Explicit writer initialization. Refuses to reuse an existing collection."""
- value = cls(directory, generation_id=generation_id, dimensions=dimensions)
- value._open(create=True)
- return value
-
- @classmethod
- def open_existing(
- cls,
- directory: Path,
- *,
- generation_id: str,
- dimensions: int,
- ) -> RawVectorStore:
- """No logical index writes; persistent Chroma may still write maintenance files."""
- value = cls(directory, generation_id=generation_id, dimensions=dimensions)
- value._open(create=False)
- return value
-
- @classmethod
- def open_writer_existing(
- cls,
- directory: Path,
- *,
- generation_id: str,
- dimensions: int,
- ) -> RawVectorStore:
- """Explicit worker recovery; verifies existing collection, never creates/repairs."""
- value = cls(directory, generation_id=generation_id, dimensions=dimensions)
- value._open(create=False, writable=True)
- return value
-
- def _metadata(self) -> dict[str, str | int]:
- return {
- "adapter": "one-memory-raw-v1",
- "generation_id": self.generation_id,
- "dimensions": self.dimensions,
- "distance": "cosine",
- "normalization": "l2-float32-v1",
- }
-
- def _preflight(self, create: bool) -> None:
- if any(part.is_symlink() for part in (self.directory, *self.directory.parents)):
- raise VectorStoreError("vector_unsafe_directory")
- if not self.directory.exists():
- if not create:
- raise VectorStoreError("vector_store_not_found")
- self.directory.mkdir(parents=True)
- if not self.directory.is_dir():
- raise VectorStoreError("vector_unsafe_directory")
- database = self.directory / "chroma.sqlite3"
- if database.is_symlink():
- raise VectorStoreError("vector_unsafe_database")
- if not database.is_file():
- if not create:
- raise VectorStoreError("vector_store_not_found")
- if any(self.directory.iterdir()):
- raise VectorStoreError("vector_unknown_directory")
- return
- if database.stat().st_nlink != 1:
- raise VectorStoreError("vector_unsafe_database")
- # Refuse missing/corrupt collections BEFORE PersistentClient can perform
- # startup maintenance. This checks the installed Chroma 1.5.9 schema.
- try:
- with closing(sqlite3.connect(database.resolve().as_uri() + "?mode=ro", uri=True)) as db:
- found = db.execute(
- "SELECT 1 FROM collections WHERE name=?", (self.collection_name,)
- ).fetchone()
- if not create and found is None:
- raise VectorStoreError("vector_collection_not_found")
- if create and found is not None:
- raise VectorStoreError("vector_collection_exists")
- except sqlite3.Error:
- raise VectorStoreError("vector_store_unavailable") from None
-
- def _open(self, *, create: bool, writable: bool = False) -> None:
- self._preflight(create)
- try:
- import chromadb
- from chromadb.config import Settings
-
- self._client = chromadb.PersistentClient(
- path=str(self.directory), settings=Settings(anonymized_telemetry=False)
- )
- if create:
- self._collection = self._client.create_collection(
- self.collection_name,
- embedding_function=None,
- metadata=self._metadata(),
- # Unit vectors + inner product give cosine distance without
- # cosine HNSW's lossy normalization/de-normalization on get.
- configuration={"hnsw": {"space": "ip"}},
- )
- else:
- self._collection = self._client.get_collection(
- self.collection_name, embedding_function=None
- )
- if (
- self._collection.metadata != self._metadata()
- or type(self._collection.metadata.get("dimensions")) is not int
- ):
- raise VectorStoreError("vector_collection_identity_mismatch")
- config = self._collection.configuration
- if (
- config.get("hnsw", {}).get("space") != "ip"
- or config.get("embedding_function") is not None
- ):
- raise VectorStoreError("vector_collection_configuration_mismatch")
- self._writable = create or writable
- except Exception as error:
- self.close()
- if isinstance(error, VectorStoreError):
- raise
- raise VectorStoreError("vector_store_unavailable") from None
-
- def close(self) -> None:
- with self._lock:
- if self._closed:
- return
- self._closed = True
- if self._client is not None:
- self._client.close()
- self._client = self._collection = None
-
- def _require_open(self) -> Any:
- if self._closed:
- raise VectorStoreError("vector_store_closed")
- if self._collection is None:
- raise VectorStoreError("vector_store_not_open")
- return self._collection
-
- def _records(self, records: Sequence[RawVectorRecord]) -> tuple[RawVectorRecord, ...]:
- result = []
- seen: set[str] = set()
- for record in records:
- if not isinstance(record, RawVectorRecord):
- raise VectorStoreError("vector_invalid_record")
- for identifier in (record.chunk_id, record.build_id, record.generation_id):
- _identifier(identifier)
- if record.generation_id != self.generation_id or record.chunk_id in seen:
- raise VectorStoreError("vector_invalid_record_identity")
- seen.add(record.chunk_id)
- result.append(
- RawVectorRecord(
- record.chunk_id,
- record.build_id,
- record.generation_id,
- _vector(record.vector, self.dimensions),
- )
- )
- return tuple(result)
-
- def _read_records(self, ids: Sequence[str]) -> dict[str, tuple[RawVectorRecord, str]]:
- if not ids:
- return {}
- collection = self._require_open()
- result: dict[str, tuple[RawVectorRecord, str]] = {}
- for start in range(0, len(ids), ALLOWLIST_BATCH_SIZE):
- batch = ids[start : start + ALLOWLIST_BATCH_SIZE]
- try:
- data = collection.get(ids=list(batch), include=["embeddings", "metadatas"])
- returned, vectors, metadatas = data["ids"], data["embeddings"], data["metadatas"]
- if (
- vectors is None
- or metadatas is None
- or not len(returned) == len(vectors) == len(metadatas)
- ):
- raise VectorStoreError("vector_invalid_record_response")
- for key, vector, metadata in zip(returned, vectors, metadatas, strict=True):
- if key not in batch or key in result:
- raise VectorStoreError("vector_ineligible_record")
- result[key] = self._decode(key, vector, metadata)
- except VectorStoreError:
- raise
- except Exception:
- raise VectorStoreError("vector_read_unavailable") from None
- return result
-
- def _decode(
- self, key: str, vector: Sequence[float], metadata: Any
- ) -> tuple[RawVectorRecord, str]:
- if not isinstance(metadata, dict) or set(metadata) != {
- "chunk_id",
- "build_id",
- "generation_id",
- "vector_sha256",
- "input_sha256",
- }:
- raise VectorStoreError("vector_invalid_metadata")
- if metadata["chunk_id"] != key or metadata["generation_id"] != self.generation_id:
- raise VectorStoreError("vector_metadata_identity_mismatch")
- _identifier(key)
- _identifier(metadata["build_id"])
- if not isinstance(metadata["input_sha256"], str) or not re.fullmatch(
- r"[0-9a-f]{64}", metadata["input_sha256"]
- ):
- raise VectorStoreError("vector_invalid_input_digest")
- normalized = _vector(vector, self.dimensions)
- if metadata["vector_sha256"] != _digest(normalized):
- raise VectorStoreError("vector_content_mismatch")
- return RawVectorRecord(key, metadata["build_id"], self.generation_id, normalized), metadata[
- "input_sha256"
- ]
-
- def upsert(self, records: Sequence[RawVectorRecord]) -> None:
- """Idempotent immutable insertion; add-only at Chroma, never overwrite IDs.
-
- Equality is canonical float32 input identity plus explicit normalized
- storage identity. Conflict is checked before
- any add. Backend failure may leave a partial new batch: caller must verify
- the entire build before publication. Existing IDs are never modified.
- """
- with self._lock:
- collection = self._require_open()
- if not self._writable:
- raise VectorStoreError("vector_store_read_only")
- incoming = self._records(records)
- existing = self._read_records([record.chunk_id for record in incoming])
- for record in incoming:
- if record.chunk_id in existing and not _matches(existing[record.chunk_id], record):
- raise VectorStoreError("vector_immutable_conflict")
- new = [record for record in incoming if record.chunk_id not in existing]
- for start in range(0, len(new), ALLOWLIST_BATCH_SIZE):
- batch = new[start : start + ALLOWLIST_BATCH_SIZE]
- try:
- collection.add(
- ids=[record.chunk_id for record in batch],
- embeddings=[list(_unit_vector(record.vector)) for record in batch],
- metadatas=[
- {
- "chunk_id": record.chunk_id,
- "build_id": record.build_id,
- "generation_id": record.generation_id,
- "vector_sha256": _digest(_unit_vector(record.vector)),
- "input_sha256": _digest(record.vector),
- }
- for record in batch
- ],
- )
- except Exception:
- raise VectorStoreError("vector_write_unavailable") from None
- self.verify(incoming)
-
- def verify(self, records: Sequence[RawVectorRecord]) -> None:
- """Verify every expected ID, metadata, dimensions and exact float32 vector."""
- with self._lock:
- self._require_open()
- expected = self._records(records)
- actual = self._read_records([record.chunk_id for record in expected])
- for record in expected:
- if record.chunk_id not in actual:
- raise VectorStoreError("vector_record_missing")
- if not _matches(actual[record.chunk_id], record):
- raise VectorStoreError("vector_immutable_conflict")
-
- def verify_candidates(
- self,
- candidates: Sequence[tuple[str, str]],
- *,
- deadline: float,
- ) -> bool:
- """Read-only presence/integrity check for (chunk_id, expected_build_id).
-
- Costs O(candidates * dimensions), not a bounded top-K lookup. False means
- deadline exhaustion, never proof of absence. Each read covers at most
- 2048 IDs; synchronous backend calls cannot be interrupted. This is not a
- snapshot across Chroma and SQLite: Core still owns final eligibility
- checks and writer serialization. Chroma may perform maintenance writes.
- """
- with self._lock:
- self._require_open()
- if (
- isinstance(deadline, bool)
- or not isinstance(deadline, Real)
- or not math.isfinite(deadline)
- ):
- raise VectorStoreError("vector_invalid_deadline")
- if not isinstance(candidates, Sequence) or isinstance(candidates, (str, bytes)):
- raise VectorStoreError("vector_invalid_candidates")
- expected: list[tuple[str, str]] = []
- seen: set[str] = set()
- for pair in candidates:
- if (
- not isinstance(pair, Sequence)
- or isinstance(pair, (str, bytes))
- or len(pair) != 2
- ):
- raise VectorStoreError("vector_invalid_candidates")
- key, build = pair
- _identifier(key)
- _identifier(build)
- if key in seen:
- raise VectorStoreError("vector_duplicate_candidates")
- seen.add(key)
- expected.append((key, build))
- for start in range(0, len(expected), ALLOWLIST_BATCH_SIZE):
- if self._clock() >= deadline:
- return False
- batch = expected[start : start + ALLOWLIST_BATCH_SIZE]
- # Bound this call to one batch so it cannot read past the next
- # deadline checkpoint via _read_records' internal loop.
- actual = self._read_records([key for key, _build in batch])
- for key, build in batch:
- if key not in actual:
- raise VectorStoreError("vector_record_missing")
- if actual[key][0].build_id != build:
- raise VectorStoreError("vector_metadata_identity_mismatch")
- if self._clock() >= deadline:
- return False
- return True
-
- def search(
- self,
- query_vector: Sequence[float],
- eligible_ids: Sequence[str],
- limit: int,
- deadline: float,
- ) -> VectorSearchResult:
- """Rank exactly the eligible candidates; synchronous backend cannot be preempted.
-
- deadline is absolute monotonic seconds. A late batch is discarded; prior
- on-time batches may return partial. complete means every eligible
- vector was scored, NOT complete index coverage: only chunks the caller
- already resolved as eligible and published are searched.
-
- The eligible embeddings are read back in bounded batches and ranked
- here, because Chroma's own filtered query does not score every ID in
- its allowlist. A global top-K filtered afterwards would rank
- unauthorized vectors first and let them crowd the eligible ones out.
- """
- with self._lock:
- self._require_open()
- vector = _unit_vector(_vector(query_vector, self.dimensions))
- if type(limit) is not int or not 1 <= limit <= ALLOWLIST_BATCH_SIZE:
- raise VectorStoreError("vector_invalid_limit")
- if (
- isinstance(deadline, bool)
- or not isinstance(deadline, Real)
- or not math.isfinite(deadline)
- ):
- raise VectorStoreError("vector_invalid_deadline")
- if isinstance(eligible_ids, (str, bytes)):
- raise VectorStoreError("vector_invalid_allowlist")
- ids = tuple(eligible_ids)
- for key in ids:
- _identifier(key)
- if len(set(ids)) != len(ids):
- raise VectorStoreError("vector_duplicate_allowlist")
- hits: list[VectorHit] = []
- searched = 0
- for start in range(0, len(ids), ALLOWLIST_BATCH_SIZE):
- if self._clock() >= deadline:
- break
- batch = ids[start : start + ALLOWLIST_BATCH_SIZE]
- try:
- read = self._read_records(batch)
- except VectorStoreError as error:
- # A missing or corrupt eligible vector is not "no hits".
- raise VectorStoreError("vector_query_unavailable") from error
- if self._clock() >= deadline:
- break
- try:
- missing = [key for key in batch if key not in read]
- if missing:
- # The caller resolved these as eligible and published,
- # so an absent vector is a broken build, not a miss.
- raise VectorStoreError("vector_query_unavailable")
- validated = [
- VectorHit(
- key,
- read[key][0].build_id,
- read[key][0].generation_id,
- # Stored vectors are unit length, so the inner
- # product is the cosine similarity.
- 1.0 - _dot(read[key][0].vector, vector),
- )
- for key in batch
- ]
- except VectorStoreError:
- raise
- except Exception:
- raise VectorStoreError("vector_invalid_query_response") from None
- if self._clock() >= deadline:
- break
- searched += len(batch)
- hits.extend(validated)
- hits.sort(key=lambda hit: (hit.distance, hit.chunk_id))
- del hits[limit:]
- complete = searched == len(ids)
- return VectorSearchResult(
- tuple(hits), complete, len(ids), searched, None if complete else "deadline"
- )
-
-
-def _dot(left: Sequence[float], right: Sequence[float]) -> float:
- return math.fsum(a * b for a, b in zip(left, right, strict=True))
-
-
-def _identifier(value: Any) -> None:
- if not isinstance(value, str) or not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_.:-]{0,255}", value):
- raise VectorStoreError("vector_invalid_identifier")
-
-
-def _vector(value: Sequence[float], dimensions: int) -> tuple[float, ...]:
- try:
- if len(value) != dimensions or any(
- isinstance(v, bool) or not isinstance(v, Real) or not math.isfinite(v) for v in value
- ):
- raise VectorStoreError("vector_invalid_values_or_dimensions")
- packed = struct.pack(f"<{dimensions}f", *value)
- normalized = struct.unpack(f"<{dimensions}f", packed)
- if not all(math.isfinite(v) for v in normalized) or not any(normalized):
- raise VectorStoreError("vector_invalid_values_or_dimensions")
- return normalized
- except VectorStoreError:
- raise
- except (ValueError, TypeError, OverflowError, struct.error):
- raise VectorStoreError("vector_invalid_values_or_dimensions") from None
-
-
-def _digest(vector: tuple[float, ...]) -> str:
- return sha256(struct.pack(f"<{len(vector)}f", *vector)).hexdigest()
-
-
-def _unit_vector(vector: tuple[float, ...]) -> tuple[float, ...]:
- length = math.sqrt(math.fsum(component * component for component in vector))
- return _vector(tuple(component / length for component in vector), len(vector))
-
-
-def _matches(actual: tuple[RawVectorRecord, str], expected: RawVectorRecord) -> bool:
- stored, input_digest = actual
- return input_digest == _digest(expected.vector) and stored == RawVectorRecord(
- expected.chunk_id, expected.build_id, expected.generation_id, _unit_vector(expected.vector)
- )
diff --git a/memhub/apps/core/tests/TEST_INDEX.md b/memhub/apps/core/tests/TEST_INDEX.md
index 3d4c0ea..953b4e4 100644
--- a/memhub/apps/core/tests/TEST_INDEX.md
+++ b/memhub/apps/core/tests/TEST_INDEX.md
@@ -139,7 +139,6 @@
| `test_live_watch.py` | `contract` | 覆盖 a new turn lands well inside the ten second target、start prime catches a change that predates the watcher、a run that predates the change is followed by a fresh one 等 15 个场景。 |
| `test_live_watch_grants.py` | `integration` | 覆盖 explicit sync grant starts watch without restart and revocation stops it 场景。 |
| `test_local_embedding.py` | `contract` | Guards for the built-in embedding model. |
-| `test_local_embedding_offsets.py` | `contract` | 覆盖 real locked tokenizer offsets are unicode codepoints、known emoji and combining offsets are not utf8 or utf16 units、offsets are not truncated or padded and do not create onnx session 等 9 个场景。 |
| `test_local_onnx_provider.py` | `contract` | Expect local ONNX embedding to be the default and to feed Hy Memory. |
| `test_log_file.py` | `contract` | 覆盖 what the watchdog says survives a dock launch、the log lives where the data does、a machine left running for months does not fill up 等 4 个场景。 |
| `test_log_privacy.py` | `integration` | The file log is shipped to us in bug reports; it must not carry originals. |
@@ -232,21 +231,8 @@
| `test_recall_scope_control.py` | `integration` | 覆盖 external scope cannot override shared or host workspace boundary、scope only mutation is versioned without changing status or reindexing 场景。 |
| `test_recall_scope_off_loop.py` | `contract` | The workspace lookup behind an external recall must not run on the loop. |
| `test_recent_first_import.py` | `contract` | Recent-file priority survives restart through a selection-bound frozen copy. |
-| `test_retrieval_chunking.py` | `contract` | 覆盖 default 384 body 64 overlap and exact source slices、all prefix and special tokens count toward strict 512 budget、real tokenizer covers long chinese middle and tail object locations 等 20 个场景。 |
-| `test_retrieval_eligibility_benchmark.py` | `contract` | 覆盖 scale probe proves composite authority index changes the plan 场景。 |
-| `test_retrieval_index_store.py` | `contract` | 覆盖 constructor and reads do not create database or schema、explicit initialization rolls back and never grants model policy、consumer and jobs commit together 等 19 个场景。 |
-| `test_retrieval_index_supervisor.py` | `contract` | 覆盖 stop waits for native exit and does not close resources、async cancellation repeatedly waits for native thread、idle no trigger no spin and duplicate start rejected 等 14 个场景。 |
-| `test_retrieval_index_worker.py` | `contract` | 覆盖 tick indexes complete original and reopen can search、change during native encoding never publishes、cancel after encode does not publish and is reclaimable 等 10 个场景。 |
| `test_retrieval_lifespan.py` | `integration` | 覆盖 retrieval defaults off and does not create index、enabling retrieval requires runtime ownership、index enabled serve imports without building an unmanaged app 等 6 个场景。 |
-| `test_retrieval_mapped_content.py` | `contract` | 覆盖 raw authors keep exact body and do not mutate evidence、raw and author qualification rejects without tokenizing、ambiguous generated and body hygiene are not admitted 等 22 个场景。 |
-| `test_retrieval_mapped_integration.py` | `contract` | 覆盖 empty materialization rechecks excluded map、completed excluded missing active binding is not empty success、old empty completion is invalidated when a new indexed record arrives 等 24 个场景。 |
-| `test_retrieval_provisioning.py` | `contract` | The index has to come up through the product, not through a test helper. |
| `test_retrieval_query_plan.py` | `contract` | Local QueryPlan: extract URL/path objects. |
-| `test_retrieval_raw_candidates.py` | `contract` | 覆盖 positive only current authorized published chunks are searched、empty authority never reads database or vector、unapproved source never enters vector query 等 15 个场景。 |
-| `test_retrieval_raw_evidence.py` | `contract` | 覆盖 materialized text is exact current original、stale candidates do not authorize original read、budget does not cut a chunk into fake partial object 等 6 个场景。 |
-| `test_retrieval_raw_lane.py` | `contract` | v1 raw lane must not claim complete empty when the derived index is partial. |
-| `test_retrieval_runtime.py` | `contract` | 覆盖 constructor and missing start create nothing、stop releases supervisor worker but preserves immutable snapshot、not initialized or invalid configuration never creates index 等 21 个场景。 |
-| `test_retrieval_vector_store.py` | `contract` | 覆盖 constructor does no io and unopened calls do not create、open existing missing store or collection never creates、real chroma collection has no embedding function or documents 等 32 个场景。 |
| `test_runtime.py` | `integration` | 覆盖 runtime writes and removes ownership receipt、runtime rejects second owner for same data directory、nonpublishing maintenance owner still excludes a second core 等 4 个场景。 |
| `test_runtime_client.py` | `contract` | 覆盖 runtime inspection authenticates the discovered instance、runtime inspection rejects untrusted receipts without probing、runtime inspection distinguishes stopped and identity mismatch 场景。 |
| `test_sanitization_source_map.py` | `contract` | 覆盖 mapping matches existing sanitizer and exact original slices、synthetic replacements never claim original locations、repeated identical suffix is mapped by positions not string search 等 6 个场景。 |
@@ -328,4 +314,4 @@
| `test_workspace_recall.py` | `contract` | Workspace memories were unreachable from every host; these pin the way in. |
| `test_write_queue_lease_renewal.py` | `contract` | Rule 8: a lease must outlive the work it covers, and be renewed while it runs. |
-共 323 个测试文件。
+共 309 个测试文件。
diff --git a/memhub/apps/core/tests/core_test_classification.py b/memhub/apps/core/tests/core_test_classification.py
index c24f9a4..9fa28b3 100644
--- a/memhub/apps/core/tests/core_test_classification.py
+++ b/memhub/apps/core/tests/core_test_classification.py
@@ -14,8 +14,8 @@ from pathlib import Path
import pytest
PRIMARY_MARKERS = ("unit", "contract", "integration", "performance", "live")
-EXPECTED_TEST_FILE_COUNT = 323
-EXPECTED_TEST_FILES_SHA256 = "10a2327bd047a281511980f5b558c1ea64d857f0fff550e4bd96cb53800205ba"
+EXPECTED_TEST_FILE_COUNT = 309
+EXPECTED_TEST_FILES_SHA256 = "ef6165c249971f5a2ab71a0c49f6d859c8c0f3612e3f749fba0acce0eab513ea"
# Pure transformations, validation models, parity checks and state machines.
# This list is intentionally small: uncertain files default to contract.
diff --git a/memhub/apps/core/tests/test_local_embedding_offsets.py b/memhub/apps/core/tests/test_local_embedding_offsets.py
deleted file mode 100644
index 66da996..0000000
--- a/memhub/apps/core/tests/test_local_embedding_offsets.py
+++ /dev/null
@@ -1,132 +0,0 @@
-import socket
-from concurrent.futures import ThreadPoolExecutor
-from dataclasses import FrozenInstanceError
-
-import pytest
-
-from one_memory import local_embedding
-from one_memory.embedding_assets import AssetVerificationError, resolve_asset_dir
-from one_memory.local_embedding import LocalEmbedder, TokenizedText
-
-
[email protected](autouse=True)
-def no_network(monkeypatch):
- def refused(*args, **kwargs):
- pytest.fail("local tokenizer/encoder attempted network")
-
- monkeypatch.setattr(socket.socket, "connect", refused)
- monkeypatch.setattr(socket, "create_connection", refused)
-
-
- "text", ["中文😀e\u0301末尾", "👩‍💻链接", "e\u0301 cafe\u0301", "URL路径 /Case/末尾"]
-)
-def test_real_locked_tokenizer_offsets_are_unicode_codepoints(text):
- embedder = LocalEmbedder()
- tokenized = embedder.tokenize_with_offsets(text)
- assert isinstance(tokenized, TokenizedText)
- assert isinstance(tokenized.token_ids, tuple) and isinstance(tokenized.offsets, tuple)
- ordinary = [offset for offset in tokenized.offsets if not offset.special]
- assert ordinary
- for offset in ordinary:
- assert 0 <= offset.start_cp < offset.end_cp <= len(text)
- assert text[offset.start_cp : offset.end_cp]
- assert all(
- any(o.start_cp <= i < o.end_cp for o in ordinary)
- for i, character in enumerate(text)
- if not character.isspace()
- )
- assert embedder._session is None and embedder._tokenizer is None
- with pytest.raises(FrozenInstanceError):
- ordinary[0].start_cp = 10
-
-
-def test_known_emoji_and_combining_offsets_are_not_utf8_or_utf16_units():
- value = LocalEmbedder().tokenize_with_offsets("中文😀e\u0301末尾")
- offsets = [(o.start_cp, o.end_cp) for o in value.offsets if not o.special]
- assert offsets == [(0, 1), (1, 2), (2, 5), (5, 6), (6, 7)]
-
-
-def test_offsets_are_not_truncated_or_padded_and_do_not_create_onnx_session(monkeypatch):
- monkeypatch.setattr(local_embedding, "import_module", lambda *args: pytest.fail("ONNX load"))
- embedder = LocalEmbedder()
- long = embedder.tokenize_with_offsets("中" * 1600)
- short = embedder.tokenize_with_offsets("中")
- empty = embedder.tokenize_with_offsets("")
- assert len(long.token_ids) == 1602
- assert len(short.token_ids) == 3
- assert len(empty.token_ids) == 2
- assert all(o.special for o in empty.offsets)
- assert long.offsets[-2].end_cp == 1600
- assert embedder._session is None
-
-
-def test_parallel_offset_calls_share_one_lazy_tokenizer_without_shared_encoder_mutation():
- embedder = LocalEmbedder()
- texts = ["中" * 700, "😀中e\u0301", "short"] * 8
- with ThreadPoolExecutor(max_workers=8) as pool:
- actual = list(pool.map(embedder.tokenize_with_offsets, texts))
- assert actual == [embedder.tokenize_with_offsets(text) for text in texts]
- assert embedder._session is None and embedder._tokenizer is None
-
-
-def test_offset_tokenizer_does_not_modify_encode_state_or_vectors():
- embedder = LocalEmbedder()
- baseline = embedder.encode(["原有编码必须保持", "短句"])
- session, tokenizer = embedder._session, embedder._tokenizer
- truncation, padding = tokenizer.truncation.copy(), tokenizer.padding.copy()
- with ThreadPoolExecutor(max_workers=4) as pool:
- tokenized = pool.submit(embedder.tokenize_with_offsets, "中" * 1100)
- encoded = pool.submit(embedder.encode, ["原有编码必须保持", "短句"])
- assert len(tokenized.result().token_ids) == 1102
- assert encoded.result() == baseline
- assert embedder._session is session and embedder._tokenizer is tokenizer
- assert tokenizer.truncation == truncation and tokenizer.padding == padding
- assert len(tokenizer.encode("中" * 1100).ids) == 512
-
-
-def test_offset_assets_reject_unlocked_tokenizer_without_loading_weights(tmp_path, monkeypatch):
- (tmp_path / "tokenizer.json").write_bytes(b'{"invalid":"not locked"}')
- monkeypatch.setattr(local_embedding, "resolve_asset_dir", lambda: tmp_path)
- with pytest.raises(AssetVerificationError):
- LocalEmbedder().tokenize_with_offsets("中")
-
-
-def test_offset_tokenization_uses_only_tokenizer_asset(tmp_path, monkeypatch):
- # Tokenization must not require another ONNX session or even the weights file.
- (tmp_path / "tokenizer.json").write_bytes((resolve_asset_dir() / "tokenizer.json").read_bytes())
- monkeypatch.setattr(local_embedding, "resolve_asset_dir", lambda: tmp_path)
- assert len(LocalEmbedder().tokenize_with_offsets("中文").token_ids) == 4
-
-
-def test_real_adjacent_wordpieces_and_long_ascii_unknown_keep_original_offsets():
- embedder = LocalEmbedder()
- word = "unaffordable"
- parts = [o for o in embedder.tokenize_with_offsets(word).offsets if not o.special]
- assert len(parts) > 1
- assert parts[0].start_cp == 0 and parts[-1].end_cp == len(word)
- assert all(a.end_cp == b.start_cp for a, b in zip(parts, parts[1:], strict=False))
- long_word = "x" * 6000
- unknown = embedder.tokenize_with_offsets(long_word)
- offsets = [o for o in unknown.offsets if not o.special]
- assert len(offsets) == 1 and unknown.token_ids[1] == 100 # locked BERT [UNK]
- assert (offsets[0].start_cp, offsets[0].end_cp) == (0, 6000)
-
-
-def test_parallel_initial_load_constructs_one_offsets_tokenizer(monkeypatch):
- from tokenizers import Tokenizer
-
- calls = []
- original = Tokenizer.from_str
-
- def counted(value):
- calls.append(1)
- return original(value)
-
- monkeypatch.setattr(Tokenizer, "from_str", counted)
- embedder = LocalEmbedder()
- with ThreadPoolExecutor(max_workers=8) as pool:
- results = list(pool.map(embedder.tokenize_with_offsets, ["中😀e\u0301"] * 16))
- assert len(calls) == 1
- assert all(value == results[0] for value in results)
- assert embedder._session is None
diff --git a/memhub/apps/core/tests/test_main_database_entry.py b/memhub/apps/core/tests/test_main_database_entry.py
index 49b2b5d..6d8b9cf 100644
--- a/memhub/apps/core/tests/test_main_database_entry.py
+++ b/memhub/apps/core/tests/test_main_database_entry.py
@@ -87,10 +87,6 @@ ALLOWED = {
"SQLite's own defaults, read once from a connection that opens no file",
opened_with='":memory:"',
),
- ("retrieval/vector_store.py", "RawVectorStore._preflight"): Allowed(
- "Chroma's own chroma.sqlite3, read-only, checked before Chroma opens it",
- opened_with='"?mode=ro"',
- ),
("sdk_migration_verify.py", "legacy_memory_ids_from_bundle"): Allowed(
"the pre-cutover bundle's copy of the legacy database, read-only, to list "
"the memory ids a migrated home must still read back",
diff --git a/memhub/apps/core/tests/test_model_creation_validation_boundary.py b/memhub/apps/core/tests/test_model_creation_validation_boundary.py
index 45d1dd2..dc1dbf4 100644
--- a/memhub/apps/core/tests/test_model_creation_validation_boundary.py
+++ b/memhub/apps/core/tests/test_model_creation_validation_boundary.py
@@ -85,7 +85,6 @@ async def application(tmp_path: Path):
live_sync_enabled=False,
live_sync_watch_enabled=False,
automatic_memory_processing=False,
- retrieval_index_enabled=False,
)
app = create_app(settings, source_catalog={}, secret_store=secrets)
async with (
diff --git a/memhub/apps/core/tests/test_model_probe_operations.py b/memhub/apps/core/tests/test_model_probe_operations.py
index cee54d9..733b6e6 100644
--- a/memhub/apps/core/tests/test_model_probe_operations.py
+++ b/memhub/apps/core/tests/test_model_probe_operations.py
@@ -592,7 +592,6 @@ def test_probe_api_optional_body_lookup_and_authentication(tmp_path, monkeypatch
automatic_memory_processing=False,
live_sync_enabled=False,
live_sync_watch_enabled=False,
- retrieval_index_enabled=False,
),
source_catalog={},
secret_store=secrets,
diff --git a/memhub/apps/core/tests/test_raw_search_without_model.py b/memhub/apps/core/tests/test_raw_search_without_model.py
index 9fb7896..0d987e7 100644
--- a/memhub/apps/core/tests/test_raw_search_without_model.py
+++ b/memhub/apps/core/tests/test_raw_search_without_model.py
@@ -1,15 +1,13 @@
"""Search imported conversations with no generative model configured at all.
-Every other retrieval test either builds the index by hand or stubs the
-encoder, so none of them answers the question a user would ask: I imported my
-history and I never set up a model, can One Memory find things I said in
-different words? This goes through the product's own entry points only:
-authorize the source, run an import job, search `/api/v1/context/search`.
+This answers the question a user would ask: I imported my history and I
+never set up a model, can One Memory find things I said? This goes through the
+product's own entry points only: authorize the source, run an import job,
+search `/api/v1/context/search`.
"""
import asyncio
import json
-import time
from datetime import UTC, datetime
import pytest
@@ -18,7 +16,6 @@ from pydantic import SecretStr
from one_memory.api import create_app
from one_memory.config import Settings
-from one_memory.embedding_assets import assets_present
from one_memory.imports.sources.registry import build_source_catalog
from one_memory.secrets import InMemorySecretStore
from tests.source_consent import authorized_request, thaw_source
@@ -29,14 +26,7 @@ LICENCE = "我们最后决定 One Memory 用 MIT 协议发布,不再考虑 GPL
BACKUP = "备份先写到本地磁盘,再同步到对象存储。"
SYNONYM = "开源软件的授权条款最后怎么挑的"
-pytestmark = [
- pytest.mark.integration,
- pytest.mark.serial,
- pytest.mark.requires_embedding_assets,
- pytest.mark.skipif(
- not assets_present(), reason="built-in embedding weights required for the semantic lane"
- ),
-]
+pytestmark = [pytest.mark.integration, pytest.mark.serial]
def transcript(root, name, *texts):
@@ -57,7 +47,7 @@ def transcript(root, name, *texts):
(root / f"rollout-{name}.jsonl").write_text("\n".join(lines) + "\n")
-def application(tmp_path, *, index: bool):
+def application(tmp_path):
home = tmp_path / "home"
root = home / ".codex/sessions"
root.mkdir(parents=True)
@@ -73,10 +63,8 @@ def application(tmp_path, *, index: bool):
api_token=SecretStr(TOKEN),
automatic_memory_processing=False,
live_sync_enabled=False,
- retrieval_index_enabled=index,
),
source_catalog={"codex": build_source_catalog(home=home, environment={})["codex"]},
- manage_runtime=index,
secret_store=InMemorySecretStore(),
)
@@ -106,20 +94,6 @@ async def search(client, query, *, attempts=1):
raise AssertionError("unreachable")
-def lexical_only(app, query):
- """What the word-matching reader alone returns, for the same request."""
- from one_memory.context_models import ContextSearchRequest
- from one_memory.context_raw import read_raw_context
-
- result = read_raw_context(
- app.state.evidence_store.database_path,
- ContextSearchRequest(query=query, kinds=("raw",)),
- source_ids=frozenset({"codex"}),
- deadline=time.monotonic() + 5,
- )
- return tuple(result.items)
-
-
def texts(body):
return [
message["text"]
@@ -131,7 +105,7 @@ def texts(body):
def test_shared_words_are_found_without_any_model_or_index(tmp_path):
async def scenario():
- app = application(tmp_path, index=False)
+ app = application(tmp_path)
async with (
app.router.lifespan_context(app),
AsyncClient(
@@ -144,38 +118,8 @@ def test_shared_words_are_found_without_any_model_or_index(tmp_path):
assert any(LICENCE in text for text in texts(body))
missed = await search(client, SYNONYM)
- assert not texts(missed), (
- "the lexical lane already answers this phrasing, so the semantic "
- "test below would prove nothing"
- )
-
- asyncio.run(scenario())
-
-
-def test_a_semantic_only_candidate_is_not_released_without_confirmation(tmp_path):
- async def scenario():
- app = application(tmp_path, index=True)
- async with (
- app.router.lifespan_context(app),
- AsyncClient(
- transport=ASGITransport(app=app), base_url="http://test", headers=AUTH
- ) as client,
- ):
- assert app.state.retrieval_runtime.snapshot().state == "running"
- await imported(app, client)
- deadline = time.monotonic() + 30
- while time.monotonic() < deadline:
- snapshot = app.state.retrieval_runtime.snapshot().supervisor
- if snapshot is not None and snapshot.slices >= 4:
- break
- await asyncio.sleep(1)
- assert snapshot is not None and snapshot.slices >= 4 and snapshot.failures == 0
- body = await search(client, SYNONYM)
- assert body["paths"]["raw"]["reason"] == "lexical_only", body
- assert not texts(body), "a semantic-only candidate must not release original text"
- assert not lexical_only(app, SYNONYM), "lexical alone answers this; nothing is proven"
- assert any(LICENCE in text for text in texts(await search(client, "MIT 协议"))), (
- "the index must not cost the lexical lane its own hits"
- )
+ # Different words are not a lexical match, and there is no private
+ # vector index to pretend otherwise.
+ assert not texts(missed)
asyncio.run(scenario())
diff --git a/memhub/apps/core/tests/test_retrieval_chunking.py b/memhub/apps/core/tests/test_retrieval_chunking.py
deleted file mode 100644
index e62223d..0000000
--- a/memhub/apps/core/tests/test_retrieval_chunking.py
+++ /dev/null
@@ -1,273 +0,0 @@
-from dataclasses import FrozenInstanceError
-
-import pytest
-
-from one_memory.local_embedding import LocalEmbedder, TokenizedText, TokenOffset
-from one_memory.retrieval.chunking import ChunkingError, chunk_evidence
-
-
-class Characters:
- def tokenize_with_offsets(self, text):
- offsets = (
- TokenOffset(0, 0, True),
- *(TokenOffset(i, i + 1) for i in range(len(text))),
- TokenOffset(0, 0, True),
- )
- return TokenizedText(tuple(range(len(offsets))), offsets)
-
-
-def assert_coverage(text, chunks, tokenizer, prefix=""):
- assert chunks
- assert [chunk.ordinal for chunk in chunks] == list(range(len(chunks)))
- covered = bytearray(len(text))
- previous_start = -1
- for chunk in chunks:
- assert 0 <= chunk.start_cp < chunk.end_cp <= len(text)
- assert chunk.start_cp > previous_start
- previous_start = chunk.start_cp
- assert chunk.text == text[chunk.start_cp : chunk.end_cp]
- assert chunk.token_count == len(
- tokenizer.tokenize_with_offsets(prefix + chunk.text).token_ids
- )
- assert chunk.token_count <= 512
- covered[chunk.start_cp : chunk.end_cp] = b"\x01" * (chunk.end_cp - chunk.start_cp)
- assert all(covered[i] for i, c in enumerate(text) if not c.isspace())
- assert chunks[-1].end_cp == len(text)
-
-
-def test_default_384_body_64_overlap_and_exact_source_slices():
- text = "中" * 1000
- chunks = chunk_evidence(text, Characters())
- assert [(c.start_cp, c.end_cp) for c in chunks] == [(0, 384), (320, 704), (640, 1000)]
- assert chunks[0].token_count == 386
- assert_coverage(text, chunks, Characters())
- with pytest.raises(FrozenInstanceError):
- chunks[0].text = "replacement"
-
-
-def test_all_prefix_and_special_tokens_count_toward_strict_512_budget():
- prefix = "P" * 200
- text = "中" * 900
- chunks = chunk_evidence(text, Characters(), prefix=prefix)
- assert chunks[0].end_cp == 310
- assert chunks[0].token_count == 512
- assert_coverage(text, chunks, Characters(), prefix)
- with pytest.raises(ChunkingError, match="budget"):
- chunk_evidence("中", Characters(), prefix="P" * 510)
-
-
-def test_real_tokenizer_covers_long_chinese_middle_and_tail_object_locations():
- middle = "https://example.test/Middle?Key=Value#Anchor"
- tail = "/CaseSensitive/路径/final.txt"
- text = "中文😀e\u0301\n" + "中" * 740 + "\n" + middle + "\n" + "尾" * 800 + "\n" + tail
- tokenizer = LocalEmbedder()
- chunks = chunk_evidence(text, tokenizer)
- assert len(chunks) > 3
- assert_coverage(text, chunks, tokenizer)
- for value in (middle, tail):
- location = text.index(value)
- assert any(c.start_cp <= location and c.end_cp >= location + len(value) for c in chunks)
- assert chunks[0].text.startswith("中文😀e\u0301\n")
-
-
-def test_prefers_paragraph_boundary_near_target_without_losing_text():
- text = "中" * 300 + "\n\n" + "尾" * 500
- chunks = chunk_evidence(text, Characters())
- assert chunks[0].end_cp == 302
- assert_coverage(text, chunks, Characters())
-
-
[email protected]("text", ["", " \n\t\r "])
-def test_empty_or_whitespace_has_no_index_chunks(text):
- assert chunk_evidence(text, Characters()) == ()
-
-
- "text", ["😀" * 300, "e\u0301" * 400, "甲\n乙\t丙", "原样 <user_query>原文</user_query>"]
-)
-def test_unicode_and_wrapper_text_identity_is_preserved(text):
- chunks = chunk_evidence(text, Characters())
- assert_coverage(text, chunks, Characters())
-
-
- "offsets,ids",
- [
- ((TokenOffset(0, 4),), (1,)),
- ((TokenOffset(-1, 1),), (1,)),
- ((TokenOffset(1, 2), TokenOffset(0, 1)), (1, 2)),
- ((TokenOffset(0, 1), TokenOffset(2, 3)), (1, 2)),
- ((TokenOffset(0, 0),), (1,)),
- ((TokenOffset(0, 1, True),), (1,)),
- ((TokenOffset(0, 3),), (1, 2)),
- ((), ()),
- ],
-)
-def test_bad_or_incomplete_offsets_fail_explicitly(offsets, ids):
- class Broken:
- def tokenize_with_offsets(self, text):
- return TokenizedText(ids, offsets)
-
- with pytest.raises(ChunkingError):
- chunk_evidence("abc", Broken())
-
-
- "kwargs",
- [
- {"target_tokens": 64},
- {"overlap_tokens": -1},
- {"max_tokens": 513},
- {"max_tokens": True},
- {"target_tokens": 0},
- {"overlap_tokens": 384},
- ],
-)
-def test_invalid_budgets_fail_before_tokenizing(kwargs):
- with pytest.raises(ChunkingError):
- chunk_evidence("content", Characters(), **kwargs)
-
-
-def test_retokenizes_actual_prefix_plus_slice_instead_of_assuming_additive_counts():
- class ContextSensitive(Characters):
- def tokenize_with_offsets(self, text):
- result = super().tokenize_with_offsets(text)
- if text.startswith("P"):
- # Same valid offsets may carry multiple pieces. Budget must include them all.
- return TokenizedText(
- result.token_ids * 2,
- tuple(o for o in result.offsets if o.special)
- + tuple(o for o in result.offsets if o.special)
- + tuple(o for o in result.offsets if not o.special for _ in range(2)),
- )
- return result
-
- tokenizer = ContextSensitive()
- chunks = chunk_evidence("中" * 600, tokenizer, prefix="P")
- assert_coverage("中" * 600, chunks, tokenizer, "P")
-
-
-def test_overlapping_token_offsets_are_not_split_inside_a_character():
- class Repeated(Characters):
- def tokenize_with_offsets(self, text):
- return TokenizedText(
- tuple(range(2 * len(text))),
- tuple(TokenOffset(i, i + 1) for i in range(len(text)) for _ in range(2)),
- )
-
- tokenizer = Repeated()
- text = "😀" * 500
- chunks = chunk_evidence(text, tokenizer)
- assert_coverage(text, chunks, tokenizer)
-
-
-def test_single_indivisible_offset_over_budget_is_not_truncated():
- class Indivisible:
- def tokenize_with_offsets(self, text):
- return TokenizedText(
- tuple(range(600)), tuple(TokenOffset(0, len(text)) for _ in range(600))
- )
-
- with pytest.raises(ChunkingError, match="budget"):
- chunk_evidence("😀", Indivisible())
-
-
-def test_real_long_continuous_ascii_unknown_preserves_all_6000_characters():
- tokenizer = LocalEmbedder()
- text = " \t" + "x" * 6000 + "\n末尾链接 https://example.test/Case?A=B#C\t"
- chunks = chunk_evidence(text, tokenizer)
- assert len(chunks) == 1
- assert chunks[0].text == text
- assert_coverage(text, chunks, tokenizer)
-
-
-def test_real_subword_and_whitespace_boundaries_have_no_gaps():
- tokenizer = LocalEmbedder()
- text = " \t\n" + ("unaffordable embeddings\t中\u0301文 a\u0301b\n\n" * 180) + "\t\n "
- prefix = "本段用于检索:"
- chunks = chunk_evidence(text, tokenizer, prefix=prefix)
- assert len(chunks) > 3
- assert chunks[0].start_cp == 0 and chunks[-1].end_cp == len(text)
- assert_coverage(text, chunks, tokenizer, prefix)
- for first, second in zip(chunks, chunks[1:], strict=False):
- assert second.start_cp <= first.end_cp
-
-
-def test_partially_overlapping_offsets_for_combining_cluster_are_indivisible():
- class Combining(Characters):
- def tokenize_with_offsets(self, text):
- offsets = []
- i = 0
- while i < len(text):
- if text.startswith("e\u0301", i):
- offsets.extend((TokenOffset(i, i + 2), TokenOffset(i + 1, i + 2)))
- i += 2
- else:
- offsets.append(TokenOffset(i, i + 1))
- i += 1
- return TokenizedText(tuple(range(len(offsets))), tuple(offsets))
-
- text = "中" * 383 + "e\u0301" + "尾" * 500
- tokenizer = Combining()
- chunks = chunk_evidence(text, tokenizer)
- assert_coverage(text, chunks, tokenizer)
- assert all(c.start_cp != 384 and c.end_cp != 384 for c in chunks)
-
-
-def test_sparse_whitespace_gaps_are_retained_not_rebuilt_from_tokens():
- tokenizer = LocalEmbedder()
- text = " \n" + "中" * 380 + "\n \t\n" + "尾" * 380 + " \t\r\n"
- chunks = chunk_evidence(text, tokenizer)
- assert_coverage(text, chunks, tokenizer)
- assert chunks[0].text.startswith(" \n")
- assert chunks[-1].text.endswith(" \t\r\n")
-
-
-def test_real_prefix_retokenization_never_exceeds_encoder_window():
- tokenizer = LocalEmbedder()
- prefix = "检索前缀" * 50
- text = ("中文tokenization😀e\u0301\n" * 120) + "https://example.test/Tail"
- chunks = chunk_evidence(text, tokenizer, prefix=prefix)
- assert len(chunks) > 1
- assert_coverage(text, chunks, tokenizer, prefix)
-
-
-def test_tokenizer_failure_is_explicit_not_partial_output():
- class Broken:
- def tokenize_with_offsets(self, text):
- raise RuntimeError("tokenizer unavailable")
-
- with pytest.raises(ChunkingError, match="tokenization_failed"):
- chunk_evidence("中", Broken())
-
-
- "text", ["e\u0301", "\u0301", "中\u0301文", "\n\r\t中\n尾\n", "👩‍💻", "👩‍👩‍👧‍👦", "✈️"]
-)
-def test_locked_normalizer_supported_unicode_examples_cover_original(text):
- tokenizer = LocalEmbedder()
- assert_coverage(text, chunk_evidence(text, tokenizer), tokenizer)
-
-
[email protected]("text", ["中\u200d文", "\u200d", "中\u200b文", "中\x00文", "\x00"])
-def test_locked_normalizer_dropped_nonwhitespace_is_explicitly_unsupported(text):
- tokenizer = LocalEmbedder()
- offsets = tokenizer.tokenize_with_offsets(text).offsets
- assert any(
- not character.isspace()
- and not any(
- not offset.special and offset.start_cp <= i < offset.end_cp for offset in offsets
- )
- for i, character in enumerate(text)
- )
- with pytest.raises(ChunkingError, match="uncovered_nonwhitespace"):
- chunk_evidence(text, tokenizer)
-
-
-def test_another_normalizer_dropping_trailing_combining_mark_is_not_reported_complete():
- class NormalizesAccent:
- def tokenize_with_offsets(self, text):
- return TokenizedText((1,), (TokenOffset(0, 1),))
-
- with pytest.raises(ChunkingError, match="uncovered_nonwhitespace"):
- chunk_evidence("e\u0301", NormalizesAccent())
diff --git a/memhub/apps/core/tests/test_retrieval_eligibility_benchmark.py b/memhub/apps/core/tests/test_retrieval_eligibility_benchmark.py
deleted file mode 100644
index 595d6b2..0000000
--- a/memhub/apps/core/tests/test_retrieval_eligibility_benchmark.py
+++ /dev/null
@@ -1,38 +0,0 @@
-import importlib.util
-import sqlite3
-import sys
-from pathlib import Path
-
-import pytest
-
-pytestmark = [pytest.mark.performance, pytest.mark.serial]
-
-SCRIPT = Path(__file__).resolve().parents[3] / "scripts" / "bench_retrieval_eligibility_scale.py"
-if not SCRIPT.exists():
- pytest.skip(
- f"benchmark script is not part of this checkout: {SCRIPT.name}",
- allow_module_level=True,
- )
-spec = importlib.util.spec_from_file_location("bench_retrieval_eligibility_scale", SCRIPT)
-benchmark = importlib.util.module_from_spec(spec)
-sys.modules["bench_retrieval_eligibility_scale"] = benchmark
-assert spec.loader is not None
-spec.loader.exec_module(benchmark)
-
-
-def test_scale_probe_proves_composite_authority_index_changes_the_plan(tmp_path):
- db = sqlite3.connect(tmp_path / "probe.sqlite3", cached_statements=0)
- try:
- benchmark.create_schema(db)
- benchmark.fill_to(db, 0, 10_000)
- before = benchmark.measure(db, benchmark.FILTERED_SQL, ("g", "allowed"), repeat=3)
- db.execute(
- "CREATE INDEX retrieval_authority_jobs "
- "ON retrieval_documents(generation_id,state,source_id,event_seq)"
- )
- after = benchmark.measure(db, benchmark.FILTERED_SQL, ("g", "allowed"), repeat=3)
- finally:
- db.close()
- assert before["rows"] == after["rows"] == 100
- assert not any("source_id" in step for step in before["plan"])
- assert any("retrieval_authority_jobs" in step for step in after["plan"])
diff --git a/memhub/apps/core/tests/test_retrieval_index_store.py b/memhub/apps/core/tests/test_retrieval_index_store.py
deleted file mode 100644
index c25ef9c..0000000
--- a/memhub/apps/core/tests/test_retrieval_index_store.py
+++ /dev/null
@@ -1,452 +0,0 @@
-import sqlite3
-from contextlib import closing
-from datetime import UTC, datetime
-
-import pytest
-
-from one_memory import ingest_journal
-from one_memory.imports.grants import SourceGrantStore
-from one_memory.imports.models import EvidenceInput, EvidenceOrigin, OriginMetadata, WrittenBy
-from one_memory.imports.sources.registry import RootCandidateKind
-from one_memory.imports.storage import EvidenceStore
-from one_memory.memory_server.config_state import ConfigStore
-from one_memory.retrieval.chunking import TextChunk
-from one_memory.retrieval.index_store import EncoderIdentity, RetrievalIndexStore
-from one_memory.retrieval.models import TimeRange
-
-
-def fixture(tmp_path):
- path = tmp_path / "data.sqlite3"
- evidence = EvidenceStore(path)
- grants = SourceGrantStore(path, home=tmp_path)
- source = tmp_path / "source"
- source.mkdir()
- grants.grant(
- user_id="local", source_id="codex", root=source, candidate_kind=RootCandidateKind.DEFAULT
- )
- store = RetrievalIndexStore(path)
- identity = EncoderIdentity("test", "a" * 64, "b" * 64, 3)
- return path, evidence, grants, store, identity
-
-
-def put(evidence, text="original text", message="one"):
- return evidence.put(
- EvidenceInput(
- source_id="codex",
- conversation_id="thread",
- message_id=message,
- role="user",
- written_by=WrittenBy.PERSON,
- verbatim=text,
- origin_meta={EvidenceOrigin.FILE: OriginMetadata()},
- )
- ).record
-
-
-def initialize(f):
- path, _, _, store, identity = f
- with closing(sqlite3.connect(path)) as db, db:
- db.execute("BEGIN IMMEDIATE")
- store.initialize(db)
- return store.create_generation(db, identity)
-
-
-def test_constructor_and_reads_do_not_create_database_or_schema(tmp_path):
- store = RetrievalIndexStore(tmp_path / "missing")
- assert not store.database_path.exists()
- with pytest.raises(sqlite3.Error):
- store.status("missing")
- assert not store.database_path.exists()
-
-
-def test_explicit_initialization_rolls_back_and_never_grants_model_policy(fixture):
- path, evidence, _, store, _ = fixture
- row = put(evidence)
- with closing(sqlite3.connect(path)) as db:
- with pytest.raises(RuntimeError):
- store.initialize(db)
- db.execute("BEGIN IMMEDIATE")
- store.initialize(db)
- event = ingest_journal.current(db, row.evidence_id)
- assert not event.generation_eligible and event.kind == "migration_baseline"
- db.rollback()
- assert not db.execute(
- "SELECT 1 FROM sqlite_master WHERE name='retrieval_index_state'"
- ).fetchone()
- assert not db.execute(
- "SELECT 1 FROM sqlite_master WHERE name='evidence_ingest_events'"
- ).fetchone()
- initialize(fixture)
- with closing(sqlite3.connect(path)) as db:
- assert not db.execute(
- "SELECT 1 FROM sqlite_master WHERE name='continuous_policy_history'"
- ).fetchone()
-
-
-def test_consumer_and_jobs_commit_together(fixture):
- path, evidence, _, store, _ = fixture
- generation = initialize(fixture)
- row = put(evidence)
- with closing(sqlite3.connect(path)) as db:
- db.execute("BEGIN IMMEDIATE")
- assert store.consume(db, generation) == 1
- assert db.execute("SELECT COUNT(*) FROM retrieval_documents").fetchone()[0] == 1
- db.rollback()
- assert store.status(generation)["after_seq"] == 0
- assert store.consume_pending(generation) == 1
- assert store.consume_pending(generation) == 0
- lease = store.claim(generation, "worker", now=100)
- assert lease.evidence_id == row.evidence_id
-
-
-def test_lease_expiry_old_owner_and_publish_manifest(fixture):
- _, evidence, _, store, _ = fixture
- generation = initialize(fixture)
- row = put(evidence)
- store.consume_pending(generation)
- lease = store.claim(generation, "one", now=100, lease_seconds=10)
- assert store.claim(generation, "two", now=105) is None
- replacement = store.claim(generation, "two", now=111)
- chunks = (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),)
- with pytest.raises(ValueError, match="lease"):
- store.publish(lease, chunks, vectors_verified=True, now=112)
- with pytest.raises(ValueError, match="vectors"):
- store.publish(replacement, chunks, vectors_verified=False, now=112)
- store.publish(replacement, chunks, vectors_verified=True, now=112)
- assert store.status(generation)["ready"] == 1
- assert store.active_chunks(generation) == ()
- store.activate_generation(generation)
- assert len(store.active_chunks(generation)) == 1
- assert store.claim(generation, "three", now=113) is None
-
-
[email protected]("change", ["revision", "unrecorded", "revoke", "deleted"])
-def test_changed_inputs_cannot_publish_or_be_read(fixture, change):
- path, evidence, grants, store, _ = fixture
- generation = initialize(fixture)
- row = put(evidence)
- store.consume_pending(generation)
- lease = store.claim(generation, "worker", now=100)
- if change == "revision":
- put(evidence, "updated text")
- elif change == "unrecorded":
- with closing(sqlite3.connect(path)) as db, db:
- db.execute(
- "UPDATE evidence SET verbatim='unrecorded' WHERE evidence_id=?", (row.evidence_id,)
- )
- elif change == "revoke":
- with closing(sqlite3.connect(path)) as db, db:
- db.execute("UPDATE source_grants SET revoked_at='2026-09-19' WHERE source_id='codex'")
- else:
- with closing(sqlite3.connect(path)) as db, db:
- db.execute("DELETE FROM evidence_origins WHERE evidence_id=?", (row.evidence_id,))
- db.execute("DELETE FROM evidence WHERE evidence_id=?", (row.evidence_id,))
- ingest_journal.record(db, row.evidence_id, cause="delete")
- with pytest.raises(ValueError):
- store.publish(
- lease,
- (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),),
- vectors_verified=True,
- now=101,
- )
- assert store.active_chunks(generation) == ()
-
-
-def test_current_read_rejects_revised_published_content_before_consumer_catches_up(fixture):
- _, evidence, _, store, _ = fixture
- generation = initialize(fixture)
- row = put(evidence)
- store.consume_pending(generation)
- lease = store.claim(generation, "worker", now=100)
- store.publish(
- lease,
- (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),),
- vectors_verified=True,
- now=101,
- )
- store.activate_generation(generation)
- assert store.active_chunks(generation)
- put(evidence, "new version")
- assert store.active_chunks(generation) == ()
- store.consume_pending(generation)
- updated = store.claim(generation, "worker", now=102)
- assert updated.build_id != lease.build_id
-
-
-def test_pausing_harness_sync_does_not_hide_published_memory(fixture):
- path, evidence, _, store, _ = fixture
- generation = initialize(fixture)
- row = put(evidence)
- store.consume_pending(generation)
- lease = store.claim(generation, "worker", now=100)
- store.publish(
- lease,
- (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),),
- vectors_verified=True,
- now=101,
- )
- store.activate_generation(generation)
- before = store.active_chunks(generation)
- assert before
-
- config = ConfigStore(path.parent)
- config.set_sync("codex", True)
- config.set_sync("codex", False)
-
- assert store.active_chunks(generation) == before
-
-
-def test_partial_or_fabricated_chunk_manifest_never_publishes(fixture):
- _, evidence, _, store, _ = fixture
- generation = initialize(fixture)
- row = put(evidence)
- store.consume_pending(generation)
- lease = store.claim(generation, "worker", now=100)
- for chunks in [
- (),
- (TextChunk(0, 0, 3, 3, "ori"),),
- (TextChunk(0, 0, len(row.verbatim), 4, "not original"),),
- ]:
- with pytest.raises(ValueError, match="chunk"):
- store.publish(lease, chunks, vectors_verified=True, now=101)
- assert store.status(generation)["ready"] == 0
-
-
-def test_incomplete_index_cannot_replace_existing_active_generation(fixture):
- path, evidence, _, store, identity = fixture
- first = initialize(fixture)
- row = put(evidence)
- store.consume_pending(first)
- lease = store.claim(first, "worker", now=100)
- chunks = (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),)
- store.publish(lease, chunks, vectors_verified=True, now=101)
- store.activate_generation(first)
- with closing(sqlite3.connect(path)) as db, db:
- db.execute("BEGIN IMMEDIATE")
- second = store.create_generation(db, EncoderIdentity("second", "c" * 64, "d" * 64, 3))
- with pytest.raises(ValueError, match="caught_up"):
- store.activate_generation(second)
- assert store.active_chunks(first)
- store.consume_pending(second)
- with pytest.raises(ValueError, match="incomplete"):
- store.activate_generation(second)
- claim = store.claim(second, "worker", now=102)
- store.publish(claim, chunks, vectors_verified=True, now=103)
- store.activate_generation(second)
- assert store.active_chunks(first) == () and store.active_chunks(second)
-
-
-def test_block_and_explicit_retry_require_same_lease_and_authority(fixture):
- path, evidence, _, store, _ = fixture
- generation = initialize(fixture)
- put(evidence)
- store.consume_pending(generation)
- first = store.claim(generation, "one", now=100, lease_seconds=1)
- second = store.claim(generation, "two", now=102)
- store.block(first)
- assert store.status(generation)["leased"] == 1
- store.block(second)
- assert store.status(generation)["blocked"] == 1
- with closing(sqlite3.connect(path)) as db, db:
- db.execute("UPDATE source_grants SET revoked_at='revoked'")
- assert store.retry_blocked(generation) == 0
- with closing(sqlite3.connect(path)) as db, db:
- db.execute("UPDATE source_grants SET revoked_at=NULL")
- assert store.retry_blocked(generation) == 1
- assert store.claim(generation, "three", now=103) is not None
-
-
-def test_scope_filter_and_revocation_apply_before_exposing_manifest(fixture):
- path, evidence, _, store, _ = fixture
- generation = initialize(fixture)
- row = put(evidence)
- store.consume_pending(generation)
- lease = store.claim(generation, "worker", now=100)
- assert store.read_lease(lease, now=101).verbatim == row.verbatim
- store.publish(
- lease,
- (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),),
- vectors_verified=True,
- now=101,
- )
- store.activate_generation(generation)
- assert store.active_chunks(generation, source_ids=frozenset()) == ()
- assert store.active_chunks(generation, source_ids=frozenset({"codex"}))
- with closing(sqlite3.connect(path)) as db, db:
- db.execute("UPDATE source_grants SET revision=revision+1")
- assert store.active_chunks(generation) == ()
-
-
-def test_source_and_time_eligibility_are_applied_before_manifest_validation(fixture):
- path, evidence, _, store, _ = fixture
- generation = initialize(fixture)
- row = put(evidence)
- store.consume_pending(generation)
- lease = store.claim(generation, "worker", now=100)
- chunks = (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),)
- store.publish(lease, chunks, vectors_verified=True, now=101)
- store.activate_generation(generation)
-
- assert store.active_chunks(generation, source_ids=frozenset()) == ()
- assert store.active_chunks(generation, source_ids=frozenset({"other"})) == ()
- assert len(store.active_chunks(generation, source_ids=frozenset({"codex"}))) == 1
- assert (
- store.active_chunks(
- generation,
- time_range=TimeRange(
- start=datetime(2030, 1, 1, tzinfo=UTC),
- end=datetime(2030, 1, 2, tzinfo=UTC),
- ),
- )
- == ()
- )
-
-
-def test_generation_and_metadata_survive_reopen_without_body_copy(fixture):
- path, evidence, _, store, identity = fixture
- generation = initialize(fixture)
- put(evidence, "sensitive-body-must-not-copy")
- store.consume_pending(generation)
- reopened = RetrievalIndexStore(path)
- assert reopened.status(generation) == store.status(generation)
- with closing(sqlite3.connect(path)) as db, db:
- db.execute("BEGIN IMMEDIATE")
- reopened.initialize(db)
- assert reopened.create_generation(db, identity) == generation
- rows = db.execute("SELECT * FROM retrieval_documents").fetchall()
- assert "sensitive-body-must-not-copy" not in str(rows)
-
-
-def test_partial_schema_is_not_blessed_as_new_instance(fixture):
- path, _, _, store, _ = fixture
- with closing(sqlite3.connect(path)) as db, db:
- db.execute("CREATE TABLE retrieval_chunks (unexpected TEXT)")
- with closing(sqlite3.connect(path)) as db:
- db.execute("BEGIN IMMEDIATE")
- with pytest.raises(ValueError, match="partial_initialization"):
- store.initialize(db)
- assert not db.execute(
- "SELECT 1 FROM sqlite_master WHERE name='retrieval_index_state'"
- ).fetchone()
-
-
-def test_lease_renewal_rechecks_authority_and_rejects_old_expiry(fixture):
- path, evidence, _, store, _ = fixture
- generation = initialize(fixture)
- put(evidence)
- store.consume_pending(generation)
- lease = store.claim(generation, "worker", now=100)
- renewed = store.renew(lease, now=125)
- assert renewed.expires_at == 185
- with pytest.raises(ValueError, match="lease_changed"):
- store.read_lease(lease, now=126)
- assert store.read_lease(renewed, now=126)
- with closing(sqlite3.connect(path)) as db, db:
- db.execute("UPDATE source_grants SET revision=revision+1")
- with pytest.raises(ValueError, match="authority"):
- store.renew(renewed, now=127)
-
-
-def test_regrant_requeues_and_reuses_immutable_manifest(fixture):
- path, evidence, grants, store, _ = fixture
- generation = initialize(fixture)
- row = put(evidence)
- store.consume_pending(generation)
- lease = store.claim(generation, "worker", now=100)
- chunks = (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),)
- store.publish(lease, chunks, vectors_verified=True, now=101)
- store.activate_generation(generation)
- assert len(store.active_chunks(generation)) == 1
- grants.revoke(user_id="local", source_id="codex")
- assert store.active_chunks(generation) == ()
- grants.grant(
- user_id="local",
- source_id="codex",
- root=path.parent / "source",
- candidate_kind=RootCandidateKind.DEFAULT,
- )
- assert store.active_chunks(generation) == ()
- assert store.retry_blocked(generation) == 1
- renewed = store.claim(generation, "worker", now=102)
- assert renewed.build_id == lease.build_id
- store.publish(renewed, chunks, vectors_verified=True, now=103)
- assert len(store.active_chunks(generation)) == 1
-
-
-def test_retry_cursor_passes_ineligible_prefix_across_reopen(fixture):
- path, evidence, grants, store, _ = fixture
- generation = initialize(fixture)
- for index in range(257):
- put(evidence, message=str(index))
- store.consume_pending(generation)
- store.consume_pending(generation)
- with closing(sqlite3.connect(path)) as db, db:
- db.execute("UPDATE retrieval_documents SET state='blocked',source_id='revoked'")
- last_id = db.execute(
- "SELECT evidence_id FROM retrieval_documents ORDER BY event_seq DESC LIMIT 1"
- ).fetchone()[0]
- db.execute(
- "UPDATE retrieval_documents SET source_id='codex' WHERE evidence_id=?", (last_id,)
- )
- assert store.retry_blocked(generation) == 0
- reopened = RetrievalIndexStore(path)
- assert reopened.retry_blocked(generation) == 1
- assert reopened.claim(generation, "worker", now=100).evidence_id == last_id
-
-
-def test_same_build_replay_requires_exact_complete_manifest(fixture):
- path, evidence, _, store, _ = fixture
- generation = initialize(fixture)
- row = put(evidence)
- store.consume_pending(generation)
- lease = store.claim(generation, "worker", now=100)
- chunks = (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),)
- store.publish(lease, chunks, vectors_verified=True, now=101)
- with closing(sqlite3.connect(path)) as db, db:
- db.execute("UPDATE retrieval_documents SET state='pending',active_build_id=NULL")
- replay = store.claim(generation, "worker", now=102)
- with pytest.raises(ValueError, match="manifest"):
- store.publish(
- replay,
- (TextChunk(0, 0, len(row.verbatim), 5, row.verbatim),),
- vectors_verified=True,
- now=103,
- )
- assert store.status(generation)["ready"] == 0
- store.publish(replay, chunks, vectors_verified=True, now=103)
- assert store.status(generation)["ready"] == 1
-
-
-def test_origin_metadata_roundtrip_reuses_old_build_without_losing_new_event(fixture):
- _, evidence, _, store, _ = fixture
- generation = initialize(fixture)
- builds = []
- seqs = []
- for version in ("a", "b", "a"):
- row = evidence.put(
- EvidenceInput(
- source_id="codex",
- conversation_id="thread",
- message_id="one",
- role="user",
- written_by=WrittenBy.PERSON,
- verbatim="unchanged original",
- origin_meta={EvidenceOrigin.FILE: OriginMetadata(source_version=version)},
- )
- ).record
- assert row.revision == 1
- assert store.consume_pending(generation) == 1
- lease = store.claim(generation, "worker", now=100)
- builds.append(lease.build_id)
- seqs.append(lease.event_seq)
- store.publish(
- lease,
- (TextChunk(0, 0, len(row.verbatim), 5, row.verbatim),),
- vectors_verified=True,
- now=101,
- )
- assert builds[0] == builds[2] and builds[0] != builds[1]
- assert seqs[0] < seqs[1] < seqs[2]
- store.activate_generation(generation)
- assert len(store.active_chunks(generation)) == 1
diff --git a/memhub/apps/core/tests/test_retrieval_index_supervisor.py b/memhub/apps/core/tests/test_retrieval_index_supervisor.py
deleted file mode 100644
index 0bd2f69..0000000
--- a/memhub/apps/core/tests/test_retrieval_index_supervisor.py
+++ /dev/null
@@ -1,357 +0,0 @@
-import asyncio
-import socket
-import threading
-import time
-from dataclasses import FrozenInstanceError
-
-import pytest
-
-from one_memory.retrieval.index_supervisor import RetrievalIndexSupervisor
-from one_memory.retrieval.index_worker import IndexWorkResult
-
-
[email protected](autouse=True)
-def no_network(monkeypatch):
- def reject(*args, **kwargs):
- raise AssertionError("supervisor tests must remain offline")
-
- monkeypatch.setattr(socket.socket, "connect", reject)
- monkeypatch.setattr(socket.socket, "connect_ex", reject)
-
-
-class Worker:
- def __init__(self, state="idle", blocking=False):
- self.state = state
- self.entered = threading.Event()
- self.release = threading.Event()
- self.exited = threading.Event()
- self.cancel = None
- self.blocking = blocking
- self.calls = []
- self.thread_ids = set()
- self.active = 0
- self.maximum = 0
- self.closed = False
-
- def run_once(self, *, cancel=None):
- self.cancel = cancel
- self.active += 1
- self.maximum = max(self.maximum, self.active)
- self.calls.append(time.monotonic())
- self.thread_ids.add(threading.get_ident())
- self.entered.set()
- try:
- if self.blocking:
- assert self.release.wait(3), "test cleanup failed"
- if self.state == "exception":
- raise RuntimeError("SECRET_RAW_ERROR")
- if self.state == "invalid":
- return object()
- return IndexWorkResult(self.state, "SECRET_REASON")
- finally:
- self.active -= 1
- self.exited.set()
-
- def close(self):
- self.closed = True
-
-
-async def until(predicate):
- deadline = time.monotonic() + 2
- while not predicate():
- assert time.monotonic() < deadline
- await asyncio.sleep(0.002)
-
-
-def test_stop_waits_for_native_exit_and_does_not_close_resources():
- async def run():
- worker = Worker(blocking=True)
- supervisor = RetrievalIndexSupervisor(worker)
- supervisor.start()
- await until(worker.entered.is_set)
- stopping = asyncio.create_task(supervisor.stop())
- try:
- await until(lambda: worker.cancel.is_set())
- await asyncio.sleep(0.03)
- assert not stopping.done() and not worker.exited.is_set()
- assert supervisor.snapshot().thread_alive
- finally:
- worker.release.set()
- await stopping
- assert worker.exited.is_set()
- assert not supervisor.snapshot().thread_alive
- assert not worker.closed
- assert supervisor.snapshot().state == "stopped"
-
- asyncio.run(run())
-
-
[email protected]("method", ["stop", "wait"])
-def test_async_cancellation_repeatedly_waits_for_native_thread(method):
- async def run():
- worker = Worker(blocking=True)
- supervisor = RetrievalIndexSupervisor(worker)
- supervisor.start()
- await until(worker.entered.is_set)
- task = asyncio.create_task(getattr(supervisor, method)())
- await asyncio.sleep(0)
- task.cancel()
- try:
- await until(lambda: worker.cancel.is_set())
- task.cancel()
- await asyncio.sleep(0.03)
- assert not task.done()
- assert not worker.exited.is_set()
- finally:
- worker.release.set()
- with pytest.raises(asyncio.CancelledError):
- await task
- await supervisor.stop()
- assert worker.exited.is_set() and not supervisor.snapshot().thread_alive
- assert not worker.closed
-
- asyncio.run(run())
-
-
-def test_idle_no_trigger_no_spin_and_duplicate_start_rejected():
- async def run():
- worker = Worker()
- supervisor = RetrievalIndexSupervisor(worker, idle_wait_seconds=0.3)
- assert not supervisor.trigger()
- assert worker.calls == []
- supervisor.start()
- try:
- with pytest.raises(RuntimeError, match="index_supervisor_already_started"):
- supervisor.start()
- await until(lambda: len(worker.calls) == 1)
- await asyncio.sleep(0.06)
- assert len(worker.calls) == 1
- assert supervisor.snapshot().last_result == "idle"
- finally:
- await supervisor.stop()
- with pytest.raises(RuntimeError, match="index_supervisor_already_started"):
- supervisor.start()
- assert not supervisor.trigger()
-
- asyncio.run(run())
-
-
-def test_trigger_storm_coalesces_during_native_call_without_parallel_workers():
- async def run():
- worker = Worker(blocking=True)
- supervisor = RetrievalIndexSupervisor(worker, idle_wait_seconds=0.5)
- supervisor.start()
- await until(worker.entered.is_set)
- try:
- for _ in range(200):
- assert supervisor.trigger()
- worker.release.set()
- await until(lambda: len(worker.calls) >= 2)
- await asyncio.sleep(0.05)
- assert len(worker.calls) == 2
- assert worker.maximum == 1 and len(worker.thread_ids) == 1
- finally:
- worker.release.set()
- await supervisor.stop()
-
- asyncio.run(run())
-
-
-def test_interactive_activity_pauses_index_slices_then_resumes() -> None:
- async def run() -> None:
- worker = Worker()
- interactive = True
- supervisor = RetrievalIndexSupervisor(
- worker,
- yield_wait_seconds=0.02,
- background_should_yield=lambda: interactive,
- )
- supervisor.start()
- try:
- await asyncio.sleep(0.06)
- assert worker.calls == []
- interactive = False
- supervisor.trigger()
- await until(lambda: len(worker.calls) == 1)
- finally:
- await supervisor.stop()
-
- asyncio.run(run())
-
-
- "state,code",
- [
- ("exception", "index_worker_failed"),
- ("invalid", "index_worker_invalid_result"),
- ("blocked", "index_worker_blocked"),
- ],
-)
-def test_failures_have_fixed_error_and_backoff_even_with_triggers(state, code):
- async def run():
- worker = Worker(state)
- supervisor = RetrievalIndexSupervisor(worker, failure_wait_seconds=0.2)
- supervisor.start()
- try:
- await until(lambda: supervisor.snapshot().error_code is not None)
- for _ in range(100):
- supervisor.trigger()
- await asyncio.sleep(0.05)
- assert len(worker.calls) == 1
- snapshot = supervisor.snapshot()
- assert snapshot.error_code == code
- assert "SECRET" not in repr(snapshot)
- await until(lambda: len(worker.calls) >= 2)
- assert worker.calls[1] - worker.calls[0] >= 0.18
- finally:
- await supervisor.stop()
-
- asyncio.run(run())
-
-
[email protected]("state", ["ready", "excluded"])
-def test_backlog_yields_after_maximum_slices(state):
- async def run():
- worker = Worker(state)
- supervisor = RetrievalIndexSupervisor(
- worker, max_consecutive_slices=3, yield_wait_seconds=0.15
- )
- supervisor.start()
- try:
- await until(lambda: len(worker.calls) >= 3)
- for _ in range(50):
- supervisor.trigger()
- await asyncio.sleep(0.04)
- assert len(worker.calls) == 3
- await until(lambda: len(worker.calls) >= 4)
- assert worker.calls[3] - worker.calls[2] >= 0.13
- finally:
- await supervisor.stop()
- assert worker.maximum == 1
-
- asyncio.run(run())
-
-
-def test_concurrent_stop_is_idempotent_and_stop_before_start_is_terminal():
- async def run():
- worker = Worker()
- supervisor = RetrievalIndexSupervisor(worker)
- supervisor.start()
- await asyncio.gather(supervisor.stop(), supervisor.stop())
- assert not supervisor.snapshot().thread_alive
- unused = RetrievalIndexSupervisor(worker)
- await unused.stop()
- with pytest.raises(RuntimeError, match="index_supervisor_already_started"):
- unused.start()
-
- asyncio.run(run())
-
-
- "kwargs",
- [
- {"idle_wait_seconds": 0},
- {"failure_wait_seconds": float("nan")},
- {"yield_wait_seconds": 0},
- {"max_consecutive_slices": True},
- {"max_consecutive_slices": 257},
- ],
-)
-def test_invalid_configuration_never_starts_worker(kwargs):
- worker = Worker()
- with pytest.raises(ValueError, match="index_supervisor_invalid_config"):
- RetrievalIndexSupervisor(worker, **kwargs)
- assert worker.calls == []
-
-
-def test_idle_timeout_polls_again_and_stop_interrupts_long_wait():
- async def run():
- worker = Worker()
- supervisor = RetrievalIndexSupervisor(worker, idle_wait_seconds=0.04)
- supervisor.start()
- try:
- await until(lambda: len(worker.calls) >= 2)
- assert worker.calls[1] - worker.calls[0] >= 0.035
- finally:
- await supervisor.stop()
- long_idle = RetrievalIndexSupervisor(Worker(), idle_wait_seconds=300)
- long_idle.start()
- await until(lambda: long_idle.snapshot().slices == 1)
- started = time.monotonic()
- await long_idle.stop()
- assert time.monotonic() - started < 0.5
-
- asyncio.run(run())
-
-
-def test_failure_clears_on_recovery_without_leaking_worker_reason():
- async def run():
- worker = Worker("exception")
- supervisor = RetrievalIndexSupervisor(worker, failure_wait_seconds=0.06)
- supervisor.start()
- try:
- await until(lambda: supervisor.snapshot().failures == 1)
- worker.state = "idle"
- await until(lambda: supervisor.snapshot().slices == 2)
- current = supervisor.snapshot()
- assert current.error_code is None and current.last_result == "idle"
- assert current.failures == 1
- assert "SECRET" not in repr(current)
- with pytest.raises(FrozenInstanceError):
- current.slices = 0
- finally:
- await supervisor.stop()
-
- asyncio.run(run())
-
-
-def test_thread_start_failure_is_fixed_and_terminal(monkeypatch):
- def fail(_self):
- raise OSError("SECRET_THREAD_ERROR")
-
- monkeypatch.setattr(threading.Thread, "start", fail)
- supervisor = RetrievalIndexSupervisor(Worker())
- with pytest.raises(RuntimeError, match="^index_supervisor_start_failed$"):
- supervisor.start()
- assert not supervisor.snapshot().thread_alive
- assert supervisor.snapshot().state == "stopped"
- assert "SECRET" not in repr(supervisor.snapshot())
- asyncio.run(supervisor.stop())
-
-
[email protected]("state", ["deferred", "cancelled"])
-def test_nonprogress_results_do_not_drain_in_a_tight_loop(state):
- async def run():
- worker = Worker(state)
- supervisor = RetrievalIndexSupervisor(worker, idle_wait_seconds=0.3)
- supervisor.start()
- try:
- await until(lambda: supervisor.snapshot().slices == 1)
- await asyncio.sleep(0.05)
- assert len(worker.calls) == 1
- finally:
- await supervisor.stop()
-
- asyncio.run(run())
-
-
-def test_thread_is_non_daemon_and_joined_without_default_executor_threads():
- async def run():
- before = set(threading.enumerate())
- worker = Worker(blocking=True)
- supervisor = RetrievalIndexSupervisor(worker)
- supervisor.start()
- await until(worker.entered.is_set)
- own = [t for t in threading.enumerate() if t.ident in worker.thread_ids]
- assert len(own) == 1 and not own[0].daemon
- try:
- task = asyncio.create_task(supervisor.stop())
- await until(lambda: worker.cancel.is_set())
- assert set(threading.enumerate()) - before == set(own)
- finally:
- worker.release.set()
- await task
- assert not own[0].is_alive()
- assert set(threading.enumerate()) == before
-
- asyncio.run(run())
diff --git a/memhub/apps/core/tests/test_retrieval_index_worker.py b/memhub/apps/core/tests/test_retrieval_index_worker.py
deleted file mode 100644
index 0962c45..0000000
--- a/memhub/apps/core/tests/test_retrieval_index_worker.py
+++ /dev/null
@@ -1,222 +0,0 @@
-import socket
-import sqlite3
-import threading
-import time
-from contextlib import closing
-
-import pytest
-
-from one_memory.embedding_assets import assets_present
-from one_memory.local_embedding import TokenizedText, TokenOffset
-from one_memory.retrieval.index_store import RetrievalIndexStore
-from one_memory.retrieval.index_worker import LocalRetrievalEncoder, RetrievalIndexWorker
-from one_memory.retrieval.vector_store import RawVectorStore, VectorStoreError
-from tests.test_retrieval_index_store import fixture, initialize, put # noqa: F401
-
-
[email protected](autouse=True)
-def forbid_network(monkeypatch):
- def reject(*args, **kwargs):
- raise AssertionError("index worker must stay offline")
-
- monkeypatch.setattr(socket.socket, "connect", reject)
- monkeypatch.setattr(socket.socket, "connect_ex", reject)
-
-
-class Encoder:
- def __init__(self, identity):
- self.identity = identity
- self.calls = []
- self.after_encode = lambda: None
-
- def tokenize_with_offsets(self, text):
- return TokenizedText(
- tuple(range(len(text))), tuple(TokenOffset(i, i + 1) for i in range(len(text)))
- )
-
- def encode_documents(self, texts):
- self.calls.append(texts)
- self.after_encode()
- return [[1.0, 0.5, 0.25] for _ in texts]
-
-
-def setup(fixture, tmp_path): # noqa: F811
- generation = initialize(fixture)
- store = fixture[3]
- encoder = Encoder(fixture[4])
- vectors = RawVectorStore.create(tmp_path / "vectors", generation_id=generation, dimensions=3)
- worker = RetrievalIndexWorker(store, vectors, encoder, generation_id=generation, owner="test")
- try:
- yield fixture, generation, encoder, vectors, worker
- finally:
- vectors.close()
-
-
-def test_tick_indexes_complete_original_and_reopen_can_search(setup):
- f, generation, encoder, vectors, worker = setup
- record = put(f[1], "start " + "middle " * 180 + " end")
- assert worker.run_once().state == "ready"
- assert worker.run_once().state == "idle"
- assert all(len(batch) <= 8 for batch in encoder.calls)
- assert "".join(encoder.calls[0]) # Nonempty positive encoding control.
- f[3].activate_generation(generation)
- chunks = RetrievalIndexStore(f[0]).active_chunks(generation)
- assert len(chunks) > 1 and chunks[0].start_cp == 0
- assert chunks[-1].end_cp == len(record.verbatim)
- result = vectors.search(
- [1.0, 0.5, 0.25], [c.chunk_id for c in chunks], 8, time.monotonic() + 10
- )
- assert result.complete and result.hits
- with closing(sqlite3.connect(f[0])) as db:
- assert not db.execute(
- "SELECT 1 FROM sqlite_master WHERE name='continuous_policy_history'"
- ).fetchone()
-
-
[email protected]("change", ["revision", "revoke"])
-def test_change_during_native_encoding_never_publishes(setup, change):
- f, generation, encoder, vectors, worker = setup
- put(f[1])
-
- def mutate():
- if change == "revision":
- put(f[1], "changed original")
- else:
- with closing(sqlite3.connect(f[0])) as db, db:
- db.execute("UPDATE source_grants SET revoked_at='revoked'")
-
- encoder.after_encode = mutate
- result = worker.run_once()
- assert result.state == "blocked"
- assert f[3].status(generation)["ready"] == 0
- assert f[3].active_chunks(generation) == ()
-
-
-def test_cancel_after_encode_does_not_publish_and_is_reclaimable(setup):
- f, generation, encoder, _, worker = setup
- put(f[1])
- cancel = threading.Event()
- encoder.after_encode = cancel.set
- assert worker.run_once(cancel=cancel).state == "cancelled"
- assert f[3].status(generation)["leased"] == 1
- assert f[3].claim(generation, "replacement", now=time.time() + 61)
-
-
-def test_cancel_before_tick_does_not_consume_or_encode(setup):
- f, generation, encoder, _, worker = setup
- put(f[1])
- cancel = threading.Event()
- cancel.set()
- assert worker.run_once(cancel=cancel).state == "cancelled"
- assert f[3].status(generation)["after_seq"] == 0
- assert not encoder.calls
-
-
-def test_partial_vector_write_can_retry_same_build_without_duplicates(setup, monkeypatch):
- f, generation, encoder, vectors, worker = setup
- put(f[1], "long message " * 90)
- original = vectors.upsert
-
- def partial(records):
- original(records[:1])
- raise VectorStoreError("vector_write_unavailable")
-
- monkeypatch.setattr(vectors, "upsert", partial)
- assert worker.run_once().state == "blocked"
- assert f[3].status(generation)["ready"] == 0
- monkeypatch.setattr(vectors, "upsert", original)
- assert f[3].retry_blocked(generation) == 1
- assert worker.run_once().state == "ready"
- f[3].activate_generation(generation)
- assert len(f[3].active_chunks(generation)) > 1
-
-
-def test_bad_vector_count_or_failed_verification_never_publishes(setup, monkeypatch):
- f, generation, encoder, vectors, worker = setup
- put(f[1])
- monkeypatch.setattr(encoder, "encode_documents", lambda _: [])
- assert worker.run_once().state == "blocked"
- assert f[3].status(generation)["ready"] == 0
- assert f[3].retry_blocked(generation) == 1
- monkeypatch.setattr(
- encoder, "encode_documents", lambda texts: [[1.0, 0.5, 0.25] for _ in texts]
- )
- monkeypatch.setattr(
- vectors, "verify", lambda _: (_ for _ in ()).throw(VectorStoreError("failure"))
- )
- assert worker.run_once().state == "blocked"
- assert f[3].status(generation)["ready"] == 0
-
-
- "text",
- ["token=example-secret-value", "<environment_context>system wrapper</environment_context>"],
-)
-def test_content_requiring_source_mapping_is_not_encoded_or_marked_ready(setup, text):
- f, generation, encoder, _, worker = setup
- put(f[1], text)
- result = worker.run_once()
- assert result.state == "blocked" and result.reason == "content_mapping_required"
- assert not encoder.calls
- assert f[3].status(generation)["ready"] == 0
-
-
-def test_deadline_after_native_encode_discards_late_work(setup):
- f, generation, encoder, vectors, _ = setup
- put(f[1])
- clock = [1.0]
- encoder.after_encode = lambda: clock.__setitem__(0, 50.0)
- worker = RetrievalIndexWorker(
- f[3],
- vectors,
- encoder,
- generation_id=generation,
- owner="test",
- clock=lambda: clock[0],
- max_job_seconds=10,
- )
- assert worker.run_once().state == "deferred"
- assert f[3].status(generation)["ready"] == 0
-
-
-def test_generation_encoder_mismatch_rejected_before_claim(setup):
- f, generation, encoder, vectors, _ = setup
- from dataclasses import replace
-
- encoder.identity = replace(encoder.identity, encoder_digest="f" * 64)
- with pytest.raises(ValueError, match="encoder_identity"):
- RetrievalIndexWorker(f[3], vectors, encoder, generation_id=generation, owner="other")
- assert f[3].status(generation)["leased"] == 0
-
-
[email protected](not assets_present(), reason="locked local assets absent")
-def test_real_local_encoder_chroma_full_text_chain(fixture, tmp_path): # noqa: F811
- path, evidence, _, store, _ = fixture
- encoder = LocalRetrievalEncoder(content_version=1)
- with closing(sqlite3.connect(path)) as db, db:
- db.execute("BEGIN IMMEDIATE")
- store.initialize(db)
- generation = store.create_generation(db, encoder.identity)
- text = (
- "A long source message about source control and releases. " * 80
- + " Final artifact at /project/design/final.png"
- )
- record = put(evidence, text)
- vectors = RawVectorStore.create(
- tmp_path / "real-vectors", generation_id=generation, dimensions=encoder.identity.dimensions
- )
- try:
- worker = RetrievalIndexWorker(
- store, vectors, encoder, generation_id=generation, owner="real"
- )
- assert worker.run_once().state == "ready"
- store.activate_generation(generation)
- chunks = store.active_chunks(generation)
- assert len(chunks) > 1 and chunks[-1].end_cp == len(text)
- query = encoder.encode_documents([text[chunks[-1].start_cp : chunks[-1].end_cp]])[0]
- result = vectors.search(query, [c.chunk_id for c in chunks], 8, time.monotonic() + 10)
- assert result.complete and result.hits[0].chunk_id == chunks[-1].chunk_id
- assert record.verbatim.endswith("final.png")
- finally:
- vectors.close()
diff --git a/memhub/apps/core/tests/test_retrieval_lifespan.py b/memhub/apps/core/tests/test_retrieval_lifespan.py
index 543e3ab..3e0066e 100644
--- a/memhub/apps/core/tests/test_retrieval_lifespan.py
+++ b/memhub/apps/core/tests/test_retrieval_lifespan.py
@@ -1,23 +1,14 @@
import asyncio
-import json
-import os
-import socket
import sqlite3
-import subprocess
-import sys
-import time
-import urllib.request
from contextlib import closing
-from pathlib import Path
-
-import pytest
from one_memory.api import create_app
from one_memory.config import Settings
from one_memory.secrets import InMemorySecretStore
-def test_retrieval_defaults_off_and_does_not_create_index(tmp_path, monkeypatch):
+def test_startup_does_not_create_a_private_retrieval_index(tmp_path, monkeypatch):
+ """Raw reads go through the SDK and lexical FTS; MemHub keeps no vector copy."""
monkeypatch.setattr("one_memory.model_presets._preset_data", lambda: None)
async def scenario():
@@ -29,10 +20,10 @@ def test_retrieval_defaults_off_and_does_not_create_index(tmp_path, monkeypatch)
automatic_memory_processing=False,
live_sync_enabled=False,
)
- assert settings.retrieval_index_enabled is False
app = create_app(settings, secret_store=InMemorySecretStore())
async with app.router.lifespan_context(app):
- assert app.state.retrieval_runtime is None
+ assert not hasattr(app.state, "retrieval_runtime")
+ assert not (tmp_path / "retrieval-vectors").exists()
with closing(sqlite3.connect(tmp_path / "one-memory.sqlite3")) as db:
assert (
db.execute(
@@ -42,188 +33,3 @@ def test_retrieval_defaults_off_and_does_not_create_index(tmp_path, monkeypatch)
)
asyncio.run(scenario())
-
-
-def test_enabling_retrieval_requires_runtime_ownership(tmp_path):
- with pytest.raises(ValueError, match="retrieval_runtime_ownership_required"):
- create_app(
- Settings(_env_file=None, data_dir=tmp_path, retrieval_index_enabled=True),
- secret_store=InMemorySecretStore(),
- )
-
-
-def test_index_enabled_serve_imports_without_building_an_unmanaged_app(tmp_path, monkeypatch):
- """The real serve entry must not be killed by an import-time default app.
-
- `cli.serve()` imports `create_app` and then builds the app with
- `manage_runtime=True`. If this module also builds a default app at import
- time, enabling the index makes that throw before `serve()` ever binds.
- Run the public entry in a subprocess, not through `create_app()` directly,
- because the regression exists only in the import/entry-point combination.
- """
- monkeypatch.setenv("ONE_MEMORY_DATA_DIR", str(tmp_path))
- monkeypatch.setenv("ONE_MEMORY_RETRIEVAL_INDEX_ENABLED", "1")
- monkeypatch.setenv("ONE_MEMORY_RETRIEVAL_MINIMUM_FREE_BYTES", "0")
- with socket.socket() as probe:
- probe.bind(("127.0.0.1", 0))
- port = probe.getsockname()[1]
- monkeypatch.setenv("ONE_MEMORY_HOST", "127.0.0.1")
- monkeypatch.setenv("ONE_MEMORY_PORT", str(port))
- monkeypatch.setenv("HOME", str(tmp_path / "home"))
- monkeypatch.setenv("ONE_MEMORY_SOURCE_HOME", str(tmp_path / "source-home"))
- monkeypatch.setenv("ONE_MEMORY_IMPORTS_ENABLED", "0")
- monkeypatch.setenv("ONE_MEMORY_LIVE_SYNC_ENABLED", "0")
- monkeypatch.setenv("ONE_MEMORY_AUTOMATIC_MEMORY_PROCESSING", "0")
-
- process = subprocess.Popen(
- [sys.executable, "-m", "one_memory.cli", "serve"],
- env={
- **os.environ,
- "PYTHONPATH": str(Path(__file__).resolve().parents[1] / "src"),
- },
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- text=True,
- )
- stdout = ""
- stderr = ""
- try:
- deadline = time.monotonic() + 15
- health = None
- while time.monotonic() < deadline and process.poll() is None:
- try:
- with urllib.request.urlopen(
- f"http://127.0.0.1:{port}/api/v1/health", timeout=0.5
- ) as response:
- health = json.loads(response.read())
- break
- except OSError:
- time.sleep(0.1)
- if health is None:
- process.terminate()
- try:
- stdout, stderr = process.communicate(timeout=5)
- except subprocess.TimeoutExpired:
- process.kill()
- stdout, stderr = process.communicate(timeout=5)
- pytest.fail(
- "serve did not become healthy within 15s; "
- f"exit={process.returncode} stdout={stdout!r} stderr={stderr!r}"
- )
- assert health["status"] == "ok"
- assert health["service"] == "one-memory-core"
- finally:
- if process.poll() is None:
- process.terminate()
- try:
- process.communicate(timeout=5)
- except subprocess.TimeoutExpired:
- process.kill()
- process.communicate(timeout=5)
-
-
-def managed(tmp_path):
- return create_app(
- Settings(
- _env_file=None,
- data_dir=tmp_path,
- source_home=tmp_path / "home",
- imports_enabled=False,
- automatic_memory_processing=False,
- live_sync_enabled=False,
- retrieval_index_enabled=True,
- ),
- manage_runtime=True,
- secret_store=InMemorySecretStore(),
- )
-
-
-def test_managed_start_without_an_encoder_preserves_normal_core_startup(tmp_path, monkeypatch):
- """No built-in weights is the real reason an index cannot be admitted.
-
- Core must still come up and the lexical lane must still answer; the runtime
- says not_initialized rather than taking the process down with it.
- """
- monkeypatch.setattr("one_memory.model_presets._preset_data", lambda: None)
- monkeypatch.setattr(
- "one_memory.retrieval.index_provisioning.assets_present", lambda *_args: False
- )
-
- async def scenario():
- app = managed(tmp_path)
- async with app.router.lifespan_context(app):
- assert app.state.retrieval_runtime.snapshot().state == "not_initialized"
- assert (tmp_path / "runtime.json").exists()
- assert not (tmp_path / "retrieval-vectors").exists()
- assert not (tmp_path / "runtime.json").exists()
-
- asyncio.run(scenario())
-
-
-def test_enabling_retrieval_provisions_the_index_instead_of_waiting_to_be_handed_one(
- tmp_path, monkeypatch
-):
- """The product entry point, which until now only a test helper could reach."""
- monkeypatch.setattr("one_memory.model_presets._preset_data", lambda: None)
-
- async def scenario():
- app = managed(tmp_path)
- async with app.router.lifespan_context(app):
- assert app.state.retrieval_runtime.snapshot().state == "running"
- assert (tmp_path / "retrieval-vectors").is_dir()
- with closing(sqlite3.connect(tmp_path / "one-memory.sqlite3")) as db:
- assert [
- row[0]
- for row in db.execute(
- "SELECT state FROM retrieval_index_generations ORDER BY rowid"
- )
- ] == ["active"]
-
- asyncio.run(scenario())
-
-
[email protected]("failure", [RuntimeError, asyncio.CancelledError])
-def test_retrieval_stop_failure_still_drains_other_owned_services(tmp_path, monkeypatch, failure):
- monkeypatch.setattr("one_memory.model_presets._preset_data", lambda: None)
- events = []
-
- class StubRuntime:
- def __init__(self, *args, **kwargs):
- pass
-
- async def start(self):
- events.append("start")
-
- async def stop(self):
- events.append("retrieval_drained")
- raise failure("after drain")
-
- monkeypatch.setattr("one_memory.retrieval.runtime.ExistingRetrievalRuntime", StubRuntime)
-
- async def scenario():
- app = create_app(
- Settings(
- _env_file=None,
- data_dir=tmp_path,
- source_home=tmp_path / "home",
- imports_enabled=False,
- automatic_memory_processing=False,
- live_sync_enabled=False,
- retrieval_index_enabled=True,
- ),
- manage_runtime=True,
- secret_store=InMemorySecretStore(),
- )
- with pytest.raises(failure):
- async with app.router.lifespan_context(app):
- original = app.state.sdk_memory_runtime.shutdown
-
- async def shutdown():
- await original()
- events.append("processing_drained")
-
- monkeypatch.setattr(app.state.sdk_memory_runtime, "shutdown", shutdown)
- assert events == ["start", "retrieval_drained", "processing_drained"]
- assert not (tmp_path / "runtime.json").exists()
-
- asyncio.run(scenario())
diff --git a/memhub/apps/core/tests/test_retrieval_mapped_content.py b/memhub/apps/core/tests/test_retrieval_mapped_content.py
deleted file mode 100644
index b408c94..0000000
--- a/memhub/apps/core/tests/test_retrieval_mapped_content.py
+++ /dev/null
@@ -1,422 +0,0 @@
-import hashlib
-import json
-import socket
-from dataclasses import FrozenInstanceError, replace
-from datetime import UTC, datetime
-
-import pytest
-
-from one_memory.imports.models import EvidenceContentKind, EvidenceRecord, WrittenBy
-from one_memory.input_sanitization import SanitizedCopySpan, SanitizedInput, sanitize_model_input
-from one_memory.local_embedding import LocalEmbedder, TokenizedText, TokenOffset
-from one_memory.retrieval import mapped_content as mapped
-from one_memory.retrieval.chunking import ChunkingError
-from one_memory.retrieval.mapped_content import (
- MappedContentError,
- chunk_mapped_content,
- prepare_mapped_content,
-)
-
-
[email protected](autouse=True)
-def offline(monkeypatch):
- def reject(*args, **kwargs):
- raise AssertionError("mapped content must remain offline")
-
- monkeypatch.setattr(socket.socket, "connect", reject)
- monkeypatch.setattr(socket.socket, "connect_ex", reject)
-
-
-def record(text, **changes):
- values = dict(
- evidence_id="synthetic-evidence",
- user_id="local",
- source_id="synthetic-source",
- workspace=None,
- project=None,
- conversation_id="synthetic-conversation",
- message_id="synthetic-message",
- correlation_id=None,
- source_ordinal=None,
- content_kind=EvidenceContentKind.RAW_CONVERSATION,
- role="user",
- written_by=WrittenBy.PERSON,
- created_at=None,
- verbatim=text,
- origin_meta={},
- content_hash="outer-manifest-owned",
- revision=4,
- imported_at=datetime(2026, 9, 19, tzinfo=UTC),
- updated_at=datetime(2026, 9, 19, tzinfo=UTC),
- )
- return EvidenceRecord(**(values | changes))
-
-
-class Characters:
- def __init__(self):
- self.calls = []
-
- def tokenize_with_offsets(self, text):
- self.calls.append(text)
- offsets = (
- TokenOffset(0, 0, True),
- *(TokenOffset(i, i + 1) for i in range(len(text))),
- TokenOffset(0, 0, True),
- )
- return TokenizedText(tuple(range(len(offsets))), offsets)
-
-
-def assert_map(original, content):
- meta = content.metadata
- assert meta.source_length_cp == len(original)
- assert meta.sanitized_length_cp == len(content.actual_text)
- assert meta.sanitized_sha256 == hashlib.sha256(content.actual_text.encode()).hexdigest()
- output_end = 0
- copied = []
- for span in meta.segments:
- assert span.start_cp == output_end < span.end_cp
- output_end = span.end_cp
- if span.kind == "copy":
- assert (
- content.actual_text[span.start_cp : span.end_cp]
- == original[span.source_start_cp : span.source_end_cp]
- )
- copied.append((span.source_start_cp, span.source_end_cp))
- else:
- assert span.kind == "synthetic"
- assert span.source_start_cp is span.source_end_cp is None
- assert output_end == len(content.actual_text)
- ranges = sorted(copied + [(s.start_cp, s.end_cp) for s in meta.excluded])
- end = 0
- for start, right in ranges:
- assert start == end < right
- end = right
- assert end == len(original)
-
-
-def assert_chunks(original, prepared, chunks):
- assert [c.ordinal for c in chunks] == list(range(len(chunks)))
- coverage = set()
- for c in chunks:
- assert c.text == original[c.start_cp : c.end_cp]
- assert c.token_count <= 512
- assert any(
- s.kind == "copy" and s.source_start_cp <= c.start_cp < c.end_cp <= s.source_end_cp
- for s in prepared.metadata.segments
- )
- coverage.update(range(c.start_cp, c.end_cp))
- for s in prepared.metadata.segments:
- if s.kind == "copy":
- assert all(
- i in coverage
- for i in range(s.source_start_cp, s.source_end_cp)
- if not original[i].isspace()
- )
-
-
- "role,author", [("user", WrittenBy.PERSON), ("assistant", WrittenBy.AGENT)]
-)
-def test_raw_authors_keep_exact_body_and_do_not_mutate_evidence(role, author):
- original = " \t\n\u4e2d\U0001f680e\u0301 same\n same\t "
- evidence = record(original, role=role, written_by=author)
- before = evidence.model_dump()
- content = prepare_mapped_content(evidence)
- assert content.state == "ready"
- assert content.actual_text == original.strip()
- assert evidence.model_dump() == before
- assert_map(original, content)
- assert_chunks(original, content, chunk_mapped_content(content, Characters()))
-
-
- "change",
- [
- {"role": "tool", "written_by": WrittenBy.TOOL},
- {"role": "system"},
- {"content_kind": EvidenceContentKind.MEMORY_CONTENT},
- ],
-)
-def test_raw_role_qualification_rejects_without_tokenizing(change):
- with pytest.raises(MappedContentError, match="mapped_content_ineligible"):
- prepare_mapped_content(record("ordinary body", **change))
-
-
[email protected]("author", list(WrittenBy))
-def test_host_author_guess_does_not_change_protocol_role_qualification(author):
- content = prepare_mapped_content(record("ordinary body", role="user", written_by=author))
- assert content.state == "ready"
-
-
- "text,reason",
- [
- ("<user_query>one</user_query><user_query>two</user_query>", "ambiguous"),
- ("<user_info>unclosed", "ambiguous"),
- ("<rules>context</rules> unexplained trailing body", "ambiguous"),
- ("<dynamic_tool_catalog>generated</dynamic_tool_catalog>", "generated"),
- ('{"request_id":"fixture","type":"write","steps":[]}', "generated"),
- (
- "<user_query><dynamic_tool_catalog>generated</dynamic_tool_catalog></user_query>",
- "generated",
- ),
- ("<user_query><environment_context>harness</environment_context></user_query>", "hygiene"),
- ],
-)
-def test_ambiguous_generated_and_body_hygiene_are_not_admitted(text, reason):
- with pytest.raises(MappedContentError, match=reason):
- prepare_mapped_content(record(text))
-
-
-def test_wrapper_offsets_compose_by_position_despite_repeated_text_and_unicode():
- prefix = " \t<rules>same \U0001f680</rules>\n<user_query> \n"
- body = "same \U0001f680 token=same same e\u0301"
- original = prefix + body + " \n</user_query><image_local_path>same</image_local_path> "
- prepared = prepare_mapped_content(record(original))
- assert prepared.actual_text == sanitize_model_input(body)
- assert prepared.metadata.body_start_cp == len(prefix)
- assert prepared.metadata.body_end_cp == len(prefix) + len(body)
- assert_map(original, prepared)
- tokenizer = Characters()
- chunks = chunk_mapped_content(prepared, tokenizer, target_tokens=10, overlap_tokens=3)
- assert_chunks(original, prepared, chunks)
- assert chunks[0].start_cp == len(prefix)
- assert all("<rules>" not in text and "redacted-secret" not in text for text in tokenizer.calls)
-
-
- "secret",
- [
- "token=private-value",
- "Authorization: Bearer abcdefghijklmnop",
- '--password "two words"',
- "sk-abcdefghijklmnopqrstuvwxyz",
- "https://alice:[email protected]/path",
- "-----BEGIN PRIVATE KEY-----\nprivate\n-----END PRIVATE KEY-----",
- ],
-)
-def test_secrets_and_synthetic_replacements_never_reach_tokenizer(secret):
- original = "left \u4e2d\U0001f680 " + secret + " right e\u0301 /final/file"
- content = prepare_mapped_content(record(original))
- tokenizer = Characters()
- chunks = chunk_mapped_content(content, tokenizer, target_tokens=12, overlap_tokens=4)
- assert_map(original, content)
- assert_chunks(original, content, chunks)
- assert all(secret not in text and "redacted-secret" not in text for text in tokenizer.calls)
- assert any("left" in c.text for c in chunks) and any("right" in c.text for c in chunks)
-
-
- "text", ["", " \n\t ", "<rules>only wrapper</rules>", "token=private", " token=x \t token=y "]
-)
-def test_empty_wrapper_or_all_excluded_is_explicit_nonindexable(text):
- content = prepare_mapped_content(record(text))
- assert content.state == "no_indexable_content"
- assert_map(text, content)
- tokenizer = Characters()
- assert chunk_mapped_content(content, tokenizer) == ()
- assert tokenizer.calls == []
-
-
-def test_metadata_is_immutable_canonical_and_never_contains_text_or_secret_hashes():
- original = "<user_query>prefix token=private-value suffix</user_query>"
- content = prepare_mapped_content(record(original))
- encoded = content.metadata.canonical_json()
- assert encoded == json.dumps(json.loads(encoded), sort_keys=True, separators=(",", ":"))
- assert content.metadata.digest == hashlib.sha256(encoded.encode()).hexdigest()
- for text in [
- "prefix",
- "suffix",
- "private-value",
- "redacted-secret",
- hashlib.sha256(b"private-value").hexdigest(),
- ]:
- assert text not in encoded
- assert content.metadata == prepare_mapped_content(record(original)).metadata
- with pytest.raises(FrozenInstanceError):
- content.metadata.source_length_cp = 1
- with pytest.raises(FrozenInstanceError):
- content.metadata.segments[0].start_cp = 1
- with pytest.raises(FrozenInstanceError):
- content.actual_text = "other"
-
-
-def test_digest_changes_for_safe_text_or_coordinate_version_not_equal_length_excluded_secret():
- first = prepare_mapped_content(record("left token=secret-one right"))
- second = prepare_mapped_content(record("left token=secret-two right"))
- assert first.metadata.digest == second.metadata.digest
- assert (
- first.metadata.digest
- != prepare_mapped_content(record("LEFT token=secret-one right")).metadata.digest
- )
- assert first.metadata.digest != replace(first.metadata, map_version="future-map").digest
-
-
-def test_overlapping_sanitizer_passes_and_literal_placeholder_are_distinct():
- text = "<redacted-secret> token=sk-abcdefghijklmnopqrstuvwxyz /tail"
- content = prepare_mapped_content(record(text))
- assert_map(text, content)
- chunks = chunk_mapped_content(content, Characters())
- assert chunks[0].text == "<redacted-secret> " # Literal source text is copy, not synthetic.
- assert chunks[-1].text == " /tail"
-
-
[email protected]("text", ["\u4e2d\x00\u6587", "\u4e2d\u200b\u6587", "\u4e2d\u200d\u6587"])
-def test_control_characters_are_not_silently_sanitized_or_dropped(text):
- content = prepare_mapped_content(record(text))
- assert content.actual_text == text
- with pytest.raises(ChunkingError, match="uncovered_nonwhitespace"):
- chunk_mapped_content(content, LocalEmbedder())
-
-
-def test_locked_tokenizer_unicode_coverage_and_prefix_budget():
- body = ("\u4e2d\u6587\U0001f680e\u0301\n" * 400) + " token=private " + ("\u5c3e\u6587\n" * 400)
- original = "<user_query>" + body + "</user_query>"
- content = prepare_mapped_content(record(original))
- tokenizer = LocalEmbedder()
- prefix = "\u68c0\u7d22\u524d\u7f00" * 30
- chunks = chunk_mapped_content(content, tokenizer, prefix=prefix)
- assert len(chunks) > 4
- assert_chunks(original, content, chunks)
- assert all(
- c.token_count == len(tokenizer.tokenize_with_offsets(prefix + c.text).token_ids)
- for c in chunks
- )
-
-
-def test_more_than_8192_chunks_aborts_one_copy_span_without_returning_partial_content():
- tokenizer = Characters()
- content = prepare_mapped_content(record("x" * 9000))
- with pytest.raises(MappedContentError, match="mapped_chunk_limit_exceeded"):
- chunk_mapped_content(content, tokenizer, target_tokens=1, overlap_tokens=0)
- assert len(tokenizer.calls) <= 8194
-
-
-def test_global_limit_across_copy_spans_and_exact_limit_succeeds():
- content = prepare_mapped_content(record("a token=private b"))
- with pytest.raises(MappedContentError, match="mapped_chunk_limit_exceeded"):
- chunk_mapped_content(content, Characters(), max_chunks=1)
- chunks = chunk_mapped_content(content, Characters(), max_chunks=2)
- assert [c.ordinal for c in chunks] == [0, 1]
-
-
[email protected]("limit", [0, -1, True, 8193])
-def test_chunk_limit_is_not_raiseable_or_coercible(limit):
- with pytest.raises(MappedContentError, match="mapped_invalid_chunk_limit"):
- chunk_mapped_content(prepare_mapped_content(record("body")), Characters(), max_chunks=limit)
-
-
-def test_forged_text_and_incomplete_sanitizer_mapping_fail_closed(monkeypatch):
- prepared = prepare_mapped_content(record("body"))
- with pytest.raises(MappedContentError, match="mapped_invalid"):
- chunk_mapped_content(replace(prepared, actual_text="BODY"), Characters())
- monkeypatch.setattr(
- mapped,
- "sanitize_model_input_with_spans",
- lambda text: SanitizedInput(text, (SanitizedCopySpan(0, 2, 0, 1),), ()),
- )
- with pytest.raises(MappedContentError, match="mapped_invalid"):
- prepare_mapped_content(record("body"))
-
-
-def test_exact_8192_chunks_and_shrink_retries_do_not_spend_extra_chunk_budget():
- content = prepare_mapped_content(record("x" * 8192))
- chunks = chunk_mapped_content(content, Characters(), target_tokens=1, overlap_tokens=0)
- assert len(chunks) == 8192 and chunks[-1].ordinal == 8191
- content = prepare_mapped_content(record("x" * 10))
- chunks = chunk_mapped_content(
- content,
- Characters(),
- target_tokens=8,
- overlap_tokens=0,
- max_tokens=8,
- prefix="PP",
- max_chunks=3,
- )
- assert len(chunks) == 3
- assert_chunks("x" * 10, content, chunks)
-
-
-def test_fragmented_content_limit_is_global_and_rejects_before_next_copy_tokenization():
- text = " token=private ".join(["x"] * 8193)
- prepared = prepare_mapped_content(record(text))
- tokenizer = Characters()
- with pytest.raises(MappedContentError, match="mapped_chunk_limit_exceeded"):
- chunk_mapped_content(prepared, tokenizer)
- assert len(tokenizer.calls) == 2 * 8192
-
-
-def test_random_repeated_wrapper_and_sanitizer_composition():
- import random
-
- randomizer = random.Random(20260919)
- for _ in range(80):
- body = " ".join(
- randomizer.choices(
- [
- "same",
- "\u4e2d\U0001f680e\u0301",
- "token=same",
- "--password same",
- "Authorization: Bearer abcdefghijklmnop",
- "\t",
- "https://u:p@host/path",
- ],
- k=10,
- )
- )
- original = "<rules>same</rules><user_query> " + body + " </user_query>"
- content = prepare_mapped_content(record(original))
- assert_map(original, content)
- assert_chunks(original, content, chunk_mapped_content(content, Characters()))
-
-
- "case", ["output_gap", "source_overlap", "wrong_copy", "missing_exclusion"]
-)
-def test_bad_upstream_maps_do_not_authorize_original_slices(monkeypatch, case):
- samples = {
- "output_gap": SanitizedInput("body", (SanitizedCopySpan(2, 4, 2, 4),), ()),
- "source_overlap": SanitizedInput(
- "bobo", (SanitizedCopySpan(0, 2, 0, 2), SanitizedCopySpan(2, 4, 0, 2)), ((2, 4),)
- ),
- "wrong_copy": SanitizedInput("BODY", (SanitizedCopySpan(0, 4, 0, 4),), ()),
- "missing_exclusion": SanitizedInput("bo", (SanitizedCopySpan(0, 2, 0, 2),), ()),
- }
- monkeypatch.setattr(mapped, "sanitize_model_input_with_spans", lambda text: samples[case])
- with pytest.raises(MappedContentError, match="mapped_invalid"):
- prepare_mapped_content(record("body"))
-
-
-def test_assistant_context_keeps_existing_hygiene_semantics_but_not_generated_payloads():
- text = "Discuss <environment_context>literal</environment_context> as context"
- content = prepare_mapped_content(record(text, role="assistant", written_by=WrittenBy.AGENT))
- assert content.state == "ready" and content.actual_text == text
- with pytest.raises(MappedContentError, match="generated"):
- prepare_mapped_content(
- record(
- "<dynamic_tool_catalog>generated</dynamic_tool_catalog>",
- role="assistant",
- written_by=WrittenBy.AGENT,
- )
- )
-
-
-def test_copy_boundaries_cannot_reconstruct_a_url_across_secret_gap():
- original = "https://example.test/left token=private /right"
- content = prepare_mapped_content(record(original))
- chunks = chunk_mapped_content(content, Characters())
- assert len(chunks) == 2
- assert chunks[0].text == "https://example.test/left "
- assert chunks[1].text == " /right"
- assert chunks[0].end_cp < chunks[1].start_cp
- assert_chunks(original, content, chunks)
-
-
-def test_source_size_limit_rejects_before_parser(monkeypatch):
- def unexpected(_):
- raise AssertionError("oversize input reached parser")
-
- monkeypatch.setattr(mapped, "unwrap_host_envelope_with_positions", unexpected)
- with pytest.raises(MappedContentError, match="mapped_source_limit_exceeded"):
- prepare_mapped_content(record("x" * 1_000_001))
diff --git a/memhub/apps/core/tests/test_retrieval_mapped_integration.py b/memhub/apps/core/tests/test_retrieval_mapped_integration.py
deleted file mode 100644
index b958e8f..0000000
--- a/memhub/apps/core/tests/test_retrieval_mapped_integration.py
+++ /dev/null
@@ -1,549 +0,0 @@
-import socket
-import sqlite3
-import time
-from contextlib import closing
-from dataclasses import replace
-
-import pytest
-
-from one_memory.retrieval.chunking import TextChunk
-from one_memory.retrieval.index_store import RetrievalIndexStore
-from one_memory.retrieval.index_worker import RetrievalIndexWorker
-from one_memory.retrieval.mapped_content import chunk_mapped_content, prepare_mapped_content
-from one_memory.retrieval.raw_candidates import read_raw_semantic_candidates
-from one_memory.retrieval.raw_evidence import materialize_raw_candidates
-from one_memory.retrieval.vector_store import RawVectorStore
-from tests.test_retrieval_index_store import fixture, initialize, put # noqa: F401
-from tests.test_retrieval_index_worker import Encoder
-
-
[email protected](autouse=True)
-def offline(monkeypatch):
- def fail(*args, **kwargs):
- raise AssertionError("no network permitted")
-
- monkeypatch.setattr(socket.socket, "connect", fail)
- monkeypatch.setattr(socket.socket, "connect_ex", fail)
-
-
-def migrate(f):
- initialize(f)
- identity = replace(f[4], normalizer_version=2, chunker_version=2)
- with closing(sqlite3.connect(f[0])) as db, db:
- db.execute("BEGIN IMMEDIATE")
- f[3].migrate_v2(db)
- generation = f[3].create_generation(db, identity)
- return generation, identity
-
-
-def mapped_setup(fixture, tmp_path): # noqa: F811
- generation, identity = migrate(fixture)
- encoder = Encoder(identity)
- vectors = RawVectorStore.create(
- tmp_path / "mapped-vectors", generation_id=generation, dimensions=3
- )
- worker = RetrievalIndexWorker(
- fixture[3], vectors, encoder, generation_id=generation, owner="mapped"
- )
- try:
- yield fixture, generation, encoder, vectors, worker
- finally:
- vectors.close()
-
-
-def candidates(f, generation, vectors):
- return read_raw_semantic_candidates(
- f[3],
- vectors,
- [1.0, 0.5, 0.25],
- generation_id=generation,
- source_ids=frozenset({"codex"}),
- deadline=time.monotonic() + 10,
- )
-
-
-def materialize(f, generation, hits):
- return materialize_raw_candidates(
- f[3],
- hits,
- generation_id=generation,
- source_ids=frozenset({"codex"}),
- deadline=time.monotonic() + 10,
- )
-
-
-def test_empty_materialization_rechecks_excluded_map(mapped_setup):
- f, generation, _, vectors, worker = mapped_setup
- put(f[1], "token=private-value")
- assert worker.run_once().state == "excluded"
- f[3].activate_generation(generation)
- empty = candidates(f, generation, vectors)
- assert empty.complete and not empty.hits
- assert materialize(f, generation, empty).complete
- with closing(sqlite3.connect(f[0])) as db, db:
- db.execute("DELETE FROM retrieval_build_maps")
- assert not materialize(f, generation, empty).complete
-
-
-def test_completed_excluded_missing_active_binding_is_not_empty_success(mapped_setup):
- f, generation, _, vectors, worker = mapped_setup
- put(f[1], "token=private-value")
- assert worker.run_once().state == "excluded"
- f[3].activate_generation(generation)
- assert candidates(f, generation, vectors).complete
- with closing(sqlite3.connect(f[0])) as db, db:
- db.execute("UPDATE retrieval_documents SET active_build_id=NULL")
- result = candidates(f, generation, vectors)
- assert not result.complete and result.reason == "index_unavailable"
-
-
-def test_old_empty_completion_is_invalidated_when_a_new_indexed_record_arrives(mapped_setup):
- f, generation, _, vectors, worker = mapped_setup
- put(f[1], "token=private-value")
- assert worker.run_once().state == "excluded"
- f[3].activate_generation(generation)
- empty = candidates(f, generation, vectors)
- assert empty.complete and not empty.hits
- put(f[1], "new actual content", message="new")
- assert worker.run_once().state == "ready"
- result = materialize(f, generation, empty)
- assert not result.complete and result.reason == "evidence_changed"
-
-
-def test_schema2_roundtrip_only_encodes_copy_and_separates_original_spans(mapped_setup):
- f, generation, encoder, vectors, worker = mapped_setup
- original = (
- "<rules>private wrapper</rules><user_query>before token=private-value after</user_query>"
- )
- record = put(f[1], original)
- result = worker.run_once()
- assert result.state == "ready" and result.chunk_count == 2
- assert encoder.calls == [["before ", " after"]]
- f[3].activate_generation(generation)
- hits = candidates(f, generation, vectors)
- assert hits.complete and len(hits.hits) == 2
- output = materialize(f, generation, hits)
- assert output.complete and len(output.items) == 1
- assert [s.text for s in output.items[0].spans] == ["before ", " after"]
- assert all(original[s.start_cp : s.end_cp] == s.text for s in output.items[0].spans)
- with closing(sqlite3.connect(f[0])) as db:
- rows = db.execute(
- "SELECT source_map_json,map_digest,coverage_state FROM retrieval_build_maps"
- ).fetchall()
- assert len(rows) == 1 and rows[0][2] == "indexable"
- assert "before" not in rows[0][0] and "private" not in rows[0][0]
- assert rows[0][1] == prepare_mapped_content(record).metadata.digest
- assert RetrievalIndexStore(f[0]).active_chunks(generation)
-
-
-def test_explicit_migration_rolls_back_and_v1_remains_readable(fixture): # noqa: F811
- f = fixture
- generation = initialize(f)
- row = put(f[1], "legacy text")
- f[3].consume_pending(generation)
- lease = f[3].claim(generation, "legacy")
- f[3].publish(
- lease, (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),), vectors_verified=True
- )
- f[3].activate_generation(generation)
- before = f[3].active_chunks(generation)
- with closing(sqlite3.connect(f[0])) as db:
- with pytest.raises(RuntimeError):
- f[3].migrate_v2(db)
- db.execute("BEGIN IMMEDIATE")
- f[3].migrate_v2(db)
- db.rollback()
- assert db.execute("SELECT schema_version FROM retrieval_index_state").fetchone()[0] == 1
- assert not db.execute(
- "SELECT 1 FROM sqlite_master WHERE name='retrieval_build_maps'"
- ).fetchone()
- assert f[3].active_chunks(generation) == before
- with closing(sqlite3.connect(f[0])) as db, db:
- db.execute("BEGIN IMMEDIATE")
- f[3].migrate_v2(db)
- f[3].migrate_v2(db)
- assert f[3].active_chunks(generation) == before
- from one_memory.retrieval.raw_candidates import RawCandidateBatch, RawSemanticCandidate
-
- output = materialize(
- f, generation, RawCandidateBatch(tuple(RawSemanticCandidate(c, 0.0) for c in before), True)
- )
- assert output.complete and output.items[0].spans[0].text == "legacy text"
-
-
-def test_version2_requires_explicit_schema_upgrade(fixture): # noqa: F811
- f = fixture
- initialize(f)
- with closing(sqlite3.connect(f[0])) as db:
- db.execute("BEGIN IMMEDIATE")
- with pytest.raises(ValueError, match="schema"):
- f[3].create_generation(db, replace(f[4], normalizer_version=2, chunker_version=2))
- assert db.execute("SELECT schema_version FROM retrieval_index_state").fetchone()[0] == 1
-
-
[email protected]("text", ["token=private", "<rules>only wrapper</rules>"])
-def test_excluded_completion_has_no_vectors_and_is_not_ready(mapped_setup, text):
- f, generation, encoder, vectors, worker = mapped_setup
- put(f[1], text)
- result = worker.run_once()
- assert result.state == "excluded" and result.chunk_count == 0
- assert encoder.calls == []
- assert f[3].status(generation)["excluded"] == 1
- assert f[3].status(generation)["ready"] == 0
- f[3].activate_generation(generation)
- assert candidates(f, generation, vectors).hits == ()
-
-
[email protected]("tamper", ["json", "digest", "offset", "missing_map", "missing_chunk"])
-def test_persisted_tampering_rejected_at_candidate_and_final_read(mapped_setup, tamper):
- f, generation, _, vectors, worker = mapped_setup
- put(f[1], "before token=private after")
- assert worker.run_once().state == "ready"
- f[3].activate_generation(generation)
- hits = candidates(f, generation, vectors)
- with closing(sqlite3.connect(f[0])) as db, db:
- if tamper == "json":
- db.execute("UPDATE retrieval_build_maps SET source_map_json='{}'")
- elif tamper == "digest":
- db.execute("UPDATE retrieval_build_maps SET map_digest=?", ("0" * 64,))
- elif tamper == "offset":
- db.execute("UPDATE retrieval_chunks SET end_cp=end_cp+1 WHERE ordinal=0")
- elif tamper == "missing_map":
- db.execute("DELETE FROM retrieval_build_maps")
- else:
- db.execute("DELETE FROM retrieval_chunks WHERE ordinal=1")
- assert not candidates(f, generation, vectors).complete
- final = materialize(f, generation, hits)
- assert not final.complete and not final.items
-
-
[email protected]("change", ["secret", "revoke", "policy"])
-def test_current_source_secret_revision_and_grant_changes_invalidate_hits(mapped_setup, change):
- f, generation, _, vectors, worker = mapped_setup
- put(f[1], "before token=private-one after")
- assert worker.run_once().state == "ready"
- f[3].activate_generation(generation)
- hits = candidates(f, generation, vectors)
- if change == "secret":
- put(f[1], "before token=private-two after")
- else:
- with closing(sqlite3.connect(f[0])) as db, db:
- db.execute(
- "UPDATE source_grants SET revoked_at='revoked'"
- if change == "revoke"
- else "UPDATE source_grants SET revision=revision+1"
- )
- assert not materialize(f, generation, hits).items
- assert not f[3].active_chunks(generation)
-
-
-def test_publish_recomputes_map_and_rejects_cross_gap_or_incomplete_manifest(mapped_setup):
- f, generation, encoder, _, _ = mapped_setup
- row = put(f[1], "before token=private after")
- f[3].consume_pending(generation)
- lease = f[3].claim(generation, "publisher")
- prepared = prepare_mapped_content(row)
- chunks = chunk_mapped_content(prepared, encoder)
- for bad in [chunks[:1], (TextChunk(0, 0, len(row.verbatim), 10, row.verbatim),)]:
- with pytest.raises(ValueError, match="manifest"):
- f[3].publish(lease, bad, vectors_verified=True, source_map=prepared.metadata)
- with pytest.raises(ValueError, match="map"):
- f[3].publish(
- lease,
- chunks,
- vectors_verified=True,
- source_map=replace(prepared.metadata, sanitized_sha256="0" * 64),
- )
- assert f[3].status(generation)["ready"] == 0
-
-
-def test_many_copy_regions_remain_partial_not_a_cross_gap_quote(mapped_setup):
- f, generation, _, vectors, worker = mapped_setup
- put(f[1], " token=private ".join([f"part-{i}" for i in range(6)]))
- assert worker.run_once().chunk_count == 6
- f[3].activate_generation(generation)
- final = materialize(f, generation, candidates(f, generation, vectors))
- assert not final.complete and len(final.items[0].spans) == 4
- assert all("token=" not in s.text for s in final.items[0].spans)
-
-
-def test_local_onnx_chroma_mapped_roundtrip_without_download(fixture, tmp_path): # noqa: F811
- from one_memory.retrieval.index_worker import LocalRetrievalEncoder
-
- f = fixture
- encoder = LocalRetrievalEncoder()
- assert encoder.identity.normalizer_version == encoder.identity.chunker_version == 2
- with closing(sqlite3.connect(f[0])) as db, db:
- db.execute("BEGIN IMMEDIATE")
- f[3].initialize(db)
- f[3].migrate_v2(db)
- generation = f[3].create_generation(db, encoder.identity)
- row = put(
- f[1],
- "<user_query>\u4e2d\u6587\U0001f680e\u0301\n"
- + "\u524d\u6587" * 300
- + " token=private-value "
- + "\u540e\u6587" * 300
- + "</user_query>",
- )
- vectors = RawVectorStore.create(
- tmp_path / "native-mapped", generation_id=generation, dimensions=encoder.identity.dimensions
- )
- try:
- worker = RetrievalIndexWorker(
- f[3], vectors, encoder, generation_id=generation, owner="local"
- )
- result = worker.run_once()
- assert result.state == "ready" and result.chunk_count >= 4
- f[3].activate_generation(generation)
- hits = read_raw_semantic_candidates(
- f[3],
- vectors,
- encoder.encode_documents(["\u540e\u6587"])[0],
- generation_id=generation,
- source_ids=frozenset({"codex"}),
- deadline=time.monotonic() + 15,
- )
- final = materialize(f, generation, hits)
- assert final.complete and len(final.items[0].spans) == 2
- for span in final.items[0].spans:
- assert span.text == row.verbatim[span.start_cp : span.end_cp]
- assert "private-value" not in span.text and "redacted-secret" not in span.text
- finally:
- vectors.close()
-
-
- "text",
- [
- "<user_query>one</user_query><user_query>two</user_query>",
- "<dynamic_tool_catalog>generated</dynamic_tool_catalog>",
- ],
-)
-def test_ambiguous_generated_stay_blocked(mapped_setup, text):
- f, generation, encoder, _, worker = mapped_setup
- put(f[1], text)
- assert worker.run_once().state == "blocked"
- assert not encoder.calls
- with pytest.raises(ValueError, match="incomplete"):
- f[3].activate_generation(generation)
-
-
[email protected]("change", ["revoke", "secret"])
-def test_change_during_encoding_never_publishes(mapped_setup, change):
- f, generation, encoder, _, worker = mapped_setup
- put(f[1], "before token=private-one after")
- original = encoder.encode_documents
-
- def changed(texts):
- result = original(texts)
- if change == "secret":
- put(f[1], "before token=private-two after")
- else:
- f[2].revoke(user_id="local", source_id="codex")
- return result
-
- encoder.encode_documents = changed
- assert worker.run_once().state == "blocked"
- assert f[3].status(generation)["ready"] == 0
- with closing(sqlite3.connect(f[0])) as db:
- assert not db.execute("SELECT * FROM retrieval_build_maps").fetchall()
-
-
-def test_cancellation_inside_publish_rolls_back_map_chunks_and_completion(
- mapped_setup, monkeypatch
-):
- import threading
-
- f, generation, _, _, worker = mapped_setup
- put(f[1], "before token=private after")
- cancel = threading.Event()
- original = f[3]._prepare
-
- def cancelled(*args):
- prepared = original(*args)
- cancel.set()
- return prepared
-
- monkeypatch.setattr(f[3], "_prepare", cancelled)
- assert worker.run_once(cancel=cancel).state == "cancelled"
- assert f[3].status(generation)["leased"] == 1
- with closing(sqlite3.connect(f[0])) as db:
- assert not db.execute("SELECT * FROM retrieval_build_maps").fetchall()
- assert not db.execute("SELECT * FROM retrieval_chunks").fetchall()
-
-
-def test_excluded_metadata_is_validated_on_activation_and_candidate_read(mapped_setup):
- f, generation, _, vectors, worker = mapped_setup
- put(f[1], "token=private")
- assert worker.run_once().state == "excluded"
- f[3].activate_generation(generation)
- with closing(sqlite3.connect(f[0])) as db, db:
- db.execute("DELETE FROM retrieval_build_maps")
- assert not candidates(f, generation, vectors).complete
- with pytest.raises(ValueError, match="map"):
- f[3].activate_generation(generation)
-
-
-def test_migration_preserves_an_existing_v1_lease_and_store_identity(fixture): # noqa: F811
- f = fixture
- generation = initialize(f)
- row = put(f[1], "legacy leased")
- f[3].consume_pending(generation)
- lease = f[3].claim(generation, "legacy")
- with closing(sqlite3.connect(f[0])) as db, db:
- db.execute("BEGIN IMMEDIATE")
- f[3].migrate_v2(db)
- assert f[3].read_lease(lease) == row
- f[3].publish(
- lease, (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),), vectors_verified=True
- )
- f[3].activate_generation(generation)
- assert f[3].active_chunks(generation)
-
-
-def test_bad_token_offsets_leave_no_partial_mapped_manifest(mapped_setup):
- from one_memory.local_embedding import TokenizedText, TokenOffset
-
- f, generation, encoder, _, worker = mapped_setup
- put(f[1], "before token=private after")
- encoder.tokenize_with_offsets = lambda text: TokenizedText((1,), (TokenOffset(0, 1),))
- assert worker.run_once().state == "blocked"
- assert not encoder.calls
- assert f[3].status(generation)["ready"] == 0
-
-
-def test_failed_migration_keeps_original_table_lease_and_schema(fixture): # noqa: F811
- f = fixture
- generation = initialize(f)
- put(f[1], "preserved")
- f[3].consume_pending(generation)
- lease = f[3].claim(generation, "legacy")
- with closing(sqlite3.connect(f[0])) as db, db:
- db.execute("CREATE TABLE retrieval_build_maps (foreign_value TEXT)")
- db.execute("BEGIN IMMEDIATE")
- with pytest.raises(sqlite3.Error):
- f[3].migrate_v2(db)
- assert db.execute("SELECT schema_version FROM retrieval_index_state").fetchone()[0] == 1
- assert not db.execute(
- "SELECT 1 FROM sqlite_master WHERE name='retrieval_documents_v2'"
- ).fetchone()
- assert f[3].read_lease(lease).verbatim == "preserved"
-
-
- "text,expected", [("before token=private after", "ready"), ("token=private", "excluded")]
-)
-def test_regrant_reuses_immutable_map_and_manifest_only_after_explicit_retry(
- mapped_setup, text, expected
-):
- from one_memory.imports.sources.registry import RootCandidateKind
-
- f, generation, _, _, worker = mapped_setup
- put(f[1], text)
- assert worker.run_once().state == expected
- f[3].activate_generation(generation)
- with closing(sqlite3.connect(f[0])) as db:
- original = db.execute("SELECT * FROM retrieval_build_maps").fetchall()
- f[2].revoke(user_id="local", source_id="codex")
- assert not f[3].active_chunks(generation)
- f[2].grant(
- user_id="local",
- source_id="codex",
- root=f[0].parent / "source",
- candidate_kind=RootCandidateKind.DEFAULT,
- )
- assert not f[3].active_chunks(generation)
- assert f[3].retry_blocked(generation) == 1
- assert worker.run_once().state == expected
- with closing(sqlite3.connect(f[0])) as db:
- assert db.execute("SELECT * FROM retrieval_build_maps").fetchall() == original
-
-
-def test_map_replay_conflict_rolls_back_new_chunks(mapped_setup):
- f, generation, encoder, _, _ = mapped_setup
- row = put(f[1], "before token=private after")
- f[3].consume_pending(generation)
- lease = f[3].claim(generation, "publisher")
- prepared = prepare_mapped_content(row)
- chunks = chunk_mapped_content(prepared, encoder)
- with closing(sqlite3.connect(f[0])) as db, db:
- db.execute(
- "INSERT INTO retrieval_build_maps VALUES (?,?,?,?)",
- (lease.build_id, "{}", "0" * 64, "indexable"),
- )
- with pytest.raises(ValueError, match="immutable_source_map"):
- f[3].publish(lease, chunks, vectors_verified=True, source_map=prepared.metadata)
- with closing(sqlite3.connect(f[0])) as db:
- assert not db.execute("SELECT * FROM retrieval_chunks").fetchall()
- assert f[3].status(generation)["leased"] == 1
-
-
-def test_deadline_expiring_during_publish_rolls_back_completion(mapped_setup, monkeypatch):
- f, generation, _, _, worker = mapped_setup
- put(f[1], "before token=private after")
- current = [0.0]
- worker.clock = lambda: current[0]
- worker.max_job_seconds = 10
- original = f[3]._prepare
-
- def expires(*args):
- result = original(*args)
- current[0] = 11
- return result
-
- monkeypatch.setattr(f[3], "_prepare", expires)
- result = worker.run_once()
- assert result.state == "deferred" and result.reason == "deadline"
- assert f[3].status(generation)["leased"] == 1
- with closing(sqlite3.connect(f[0])) as db:
- assert not db.execute("SELECT * FROM retrieval_chunks").fetchall()
- assert not db.execute("SELECT * FROM retrieval_build_maps").fetchall()
-
-
-def test_deadline_and_character_budget_never_release_cross_gap_spans(mapped_setup):
- f, generation, _, vectors, worker = mapped_setup
- put(f[1], "before token=private after")
- assert worker.run_once().state == "ready"
- f[3].activate_generation(generation)
- hits = candidates(f, generation, vectors)
- expired = materialize_raw_candidates(
- f[3],
- hits,
- generation_id=generation,
- source_ids=frozenset({"codex"}),
- deadline=time.monotonic() - 1,
- )
- assert not expired.items and expired.reason == "deadline"
- bounded = materialize_raw_candidates(
- f[3],
- hits,
- generation_id=generation,
- source_ids=frozenset({"codex"}),
- deadline=time.monotonic() + 10,
- character_budget=7,
- )
- assert not bounded.complete and bounded.reason == "character_budget"
- assert bounded.used_characters <= 7
- assert sum(len(item.spans) for item in bounded.items) == 1
-
-
-def test_lease_expiring_inside_publish_cannot_commit(mapped_setup, monkeypatch):
- f, generation, _, _, worker = mapped_setup
- put(f[1], "before token=private after")
- current = [100.0]
- worker.wall_clock = lambda: current[0]
- original = f[3]._prepare
-
- def expires(*args):
- result = original(*args)
- current[0] = 1000.0
- return result
-
- monkeypatch.setattr(f[3], "_prepare", expires)
- assert worker.run_once().state == "blocked"
- with closing(sqlite3.connect(f[0])) as db:
- assert not db.execute("SELECT * FROM retrieval_chunks").fetchall()
- assert not db.execute("SELECT * FROM retrieval_build_maps").fetchall()
- assert f[3].status(generation)["ready"] == 0
diff --git a/memhub/apps/core/tests/test_retrieval_provisioning.py b/memhub/apps/core/tests/test_retrieval_provisioning.py
deleted file mode 100644
index b8cf32e..0000000
--- a/memhub/apps/core/tests/test_retrieval_provisioning.py
+++ /dev/null
@@ -1,125 +0,0 @@
-"""The index has to come up through the product, not through a test helper.
-
-Every existing retrieval test builds its generation by hand, so none of them
-can tell whether a user who enables the feature ever gets an index at all.
-"""
-
-import logging
-import sqlite3
-from contextlib import closing
-from pathlib import Path
-
-import pytest
-
-from one_memory.imports.storage import EvidenceStore
-from one_memory.retrieval.index_provisioning import provision_retrieval_index
-from one_memory.retrieval.index_store import RetrievalIndexStore
-from one_memory.retrieval.index_worker import LocalRetrievalEncoder
-
-
-def database(tmp_path: Path) -> Path:
- path = tmp_path / "one-memory.sqlite3"
- EvidenceStore(path)
- return path
-
-
-def active(path: Path) -> list[str]:
- with closing(sqlite3.connect(path)) as db:
- if not db.execute(
- "SELECT 1 FROM sqlite_master WHERE name='retrieval_index_generations'"
- ).fetchone():
- return []
- return [
- row[0]
- for row in db.execute(
- "SELECT generation_id FROM retrieval_index_generations WHERE state='active'"
- )
- ]
-
-
-def test_first_enable_publishes_one_active_generation_and_its_vector_store(tmp_path):
- database(tmp_path)
-
- generation = provision_retrieval_index(tmp_path)
-
- assert generation is not None
- assert active(tmp_path / "one-memory.sqlite3") == [generation]
- assert (tmp_path / "retrieval-vectors").is_dir()
-
-
-def test_second_start_reuses_the_same_generation_rather_than_stacking_one(tmp_path):
- database(tmp_path)
-
- first = provision_retrieval_index(tmp_path)
- second = provision_retrieval_index(tmp_path)
-
- assert first == second
- assert active(tmp_path / "one-memory.sqlite3") == [first]
-
-
-def test_a_v1_index_is_left_alone_instead_of_being_migrated_on_startup(tmp_path, caplog):
- """Falsifiable against the first test, where provisioning does publish one.
-
- Migrating someone's existing index while they were only starting the app is
- the one thing the schema note rules out, so this must stay a no-op. The
- absence of a warning is asserted too: crashing into the same `None` would
- otherwise read as a deliberate decision.
- """
- path = database(tmp_path)
- store = RetrievalIndexStore(path)
- with closing(sqlite3.connect(path)) as db, db:
- db.execute("BEGIN IMMEDIATE")
- store.initialize(db)
-
- with caplog.at_level(logging.WARNING):
- assert provision_retrieval_index(tmp_path) is None
- assert active(path) == []
- assert "retrieval_index_provision_failed" not in caplog.text
- with closing(sqlite3.connect(path)) as db:
- version = db.execute("SELECT schema_version FROM retrieval_index_state").fetchone()[0]
- assert version == 1
-
-
-def test_a_generation_for_another_encoder_is_not_swapped_out_silently(tmp_path):
- path = database(tmp_path)
- store = RetrievalIndexStore(path)
- other = LocalRetrievalEncoder(content_version=1).identity
- with closing(sqlite3.connect(path)) as db, db:
- db.execute("BEGIN IMMEDIATE")
- store.initialize(db)
- store.migrate_v2(db)
- foreign = store.create_generation(db, other)
- store.activate_generation(foreign)
-
- assert provision_retrieval_index(tmp_path) is None
- assert active(path) == [foreign]
-
-
-def test_without_the_built_in_encoder_no_index_is_created(tmp_path, monkeypatch):
- path = database(tmp_path)
- monkeypatch.setattr(
- "one_memory.retrieval.index_provisioning.assets_present", lambda *_args: False
- )
-
- assert provision_retrieval_index(tmp_path) is None
- assert active(path) == []
- assert not (tmp_path / "retrieval-vectors").exists()
-
-
-def test_a_missing_database_is_not_created_by_provisioning(tmp_path):
- assert provision_retrieval_index(tmp_path) is None
- assert not (tmp_path / "one-memory.sqlite3").exists()
- assert not (tmp_path / "retrieval-vectors").exists()
-
-
[email protected]("failure", ["initialize_v2", "create_generation"])
-def test_a_failed_provision_reports_none_instead_of_half_an_index(tmp_path, monkeypatch, failure):
- path = database(tmp_path)
-
- def explode(*_args, **_kwargs):
- raise sqlite3.OperationalError("synthetic")
-
- monkeypatch.setattr(RetrievalIndexStore, failure, explode)
-
- assert provision_retrieval_index(tmp_path) is None
- assert active(path) == []
diff --git a/memhub/apps/core/tests/test_retrieval_raw_candidates.py b/memhub/apps/core/tests/test_retrieval_raw_candidates.py
deleted file mode 100644
index 7b041d4..0000000
--- a/memhub/apps/core/tests/test_retrieval_raw_candidates.py
+++ /dev/null
@@ -1,224 +0,0 @@
-import sqlite3
-import time
-from contextlib import closing
-from datetime import datetime
-
-import pytest
-
-from one_memory.imports.models import EvidenceInput
-from one_memory.retrieval.index_worker import RetrievalIndexWorker
-from one_memory.retrieval.models import TimeRange
-from one_memory.retrieval.raw_candidates import (
- compose_raw_retrieval_path,
- read_raw_semantic_candidates,
-)
-from one_memory.retrieval.vector_store import RawVectorStore
-from tests.test_retrieval_index_store import fixture, initialize, put # noqa: F401
-from tests.test_retrieval_index_worker import Encoder, forbid_network # noqa: F401
-
-
-def indexed(fixture, tmp_path): # noqa: F811
- generation = initialize(fixture)
- record = put(fixture[1])
- encoder = Encoder(fixture[4])
- vectors = RawVectorStore.create(tmp_path / "vectors", generation_id=generation, dimensions=3)
- worker = RetrievalIndexWorker(
- fixture[3], vectors, encoder, generation_id=generation, owner="test"
- )
- assert worker.run_once().state == "ready"
- fixture[3].activate_generation(generation)
- try:
- yield fixture, generation, record, vectors
- finally:
- vectors.close()
-
-
-def search(indexed, **kwargs):
- f, generation, _, vectors = indexed
- options = dict(source_ids=frozenset({"codex"}), deadline=time.monotonic() + 10)
- options.update(kwargs)
- return read_raw_semantic_candidates(
- f[3], vectors, [1.0, 0.5, 0.25], generation_id=generation, **options
- )
-
-
-def test_positive_only_current_authorized_published_chunks_are_searched(indexed):
- result = search(indexed)
- assert result.complete and len(result.hits) == 1
- assert result.hits[0].chunk.evidence_id == indexed[2].evidence_id
- assert result.coverage == "indexed_only"
-
-
-def test_empty_authority_never_reads_database_or_vector(indexed, monkeypatch):
- def forbidden(*args, **kwargs):
- raise AssertionError("unauthorized access")
-
- monkeypatch.setattr(indexed[0][3], "active_chunks", forbidden)
- monkeypatch.setattr(indexed[3], "search", forbidden)
- result = search(indexed, source_ids=frozenset())
- assert not result.hits and result.reason == "not_authorized"
-
-
-def test_unapproved_source_never_enters_vector_query(indexed, monkeypatch):
- monkeypatch.setattr(indexed[3], "search", lambda *a, **k: pytest.fail("not eligible"))
- assert search(indexed, source_ids=frozenset({"cursor"})).hits == ()
-
-
[email protected]("change", ["revoke", "revise", "policy"])
-def test_mutation_during_vector_query_discards_stale_candidate(indexed, monkeypatch, change):
- f, _, _, vectors = indexed
- original = vectors.search
-
- def query(*args, **kwargs):
- result = original(*args, **kwargs)
- if change == "revise":
- put(f[1], "changed text")
- elif change == "revoke":
- f[2].revoke(user_id="local", source_id="codex")
- else:
- with closing(sqlite3.connect(f[0])) as db, db:
- db.execute("UPDATE source_grants SET revision=revision+1")
- return result
-
- monkeypatch.setattr(vectors, "search", query)
- result = search(indexed)
- assert not result.hits and not result.complete and result.reason == "evidence_changed"
-
-
-def test_expired_deadline_does_not_read_or_query(indexed, monkeypatch):
- monkeypatch.setattr(indexed[3], "search", lambda *a, **k: pytest.fail("late query"))
- result = search(indexed, deadline=time.monotonic() - 1)
- assert not result.hits and not result.complete and result.reason == "deadline"
-
-
-def test_missing_published_vector_is_error_not_empty(indexed):
- indexed[3]._collection.delete(ids=[c.chunk_id for c in indexed[0][3].active_chunks(indexed[1])])
- result = search(indexed)
- assert not result.hits and not result.complete and result.reason == "vector_unavailable"
-
-
-def test_small_allowlist_uses_query_as_integrity_check_without_duplicate_get(indexed, monkeypatch):
- monkeypatch.setattr(
- indexed[3],
- "verify_candidates",
- lambda *args, **kwargs: pytest.fail("small allowlist must not be read twice"),
- )
- result = search(indexed)
- assert result.complete and len(result.hits) == 1
-
-
-def test_missing_event_time_never_matches_hard_range(indexed, monkeypatch):
- window = TimeRange(start="2026-09-18T00:00:00+08:00", end="2026-09-19T00:00:00+08:00")
- monkeypatch.setattr(indexed[3], "search", lambda *a, **k: pytest.fail("unknown event time"))
- assert search(indexed, time_range=window).hits == ()
-
-
[email protected]("sources", [None, ["codex"], "codex"])
-def test_caller_must_provide_explicit_resolved_authority(indexed, sources):
- with pytest.raises(ValueError, match="source"):
- search(indexed, source_ids=sources)
-
-
-def test_event_time_uses_utc_half_open_range_not_import_time(indexed):
- f, generation, record, vectors = indexed
- f[1].put(
- EvidenceInput(
- source_id=record.source_id,
- conversation_id=record.conversation_id,
- message_id=record.message_id,
- role=record.role,
- written_by=record.written_by,
- verbatim=record.verbatim,
- created_at=datetime.fromisoformat("2026-09-18T16:00:00+00:00"),
- origin_meta=record.origin_meta,
- )
- )
- worker = RetrievalIndexWorker(
- f[3], vectors, Encoder(f[4]), generation_id=generation, owner="test"
- )
- assert worker.run_once().state == "ready"
- before = TimeRange(start="2026-09-18T00:00:00+08:00", end="2026-09-19T00:00:00+08:00")
- after = TimeRange(start="2026-09-19T00:00:00+08:00", end="2026-09-20T00:00:00+08:00")
- assert not search(indexed, time_range=before).hits
- result = search(indexed, time_range=after)
- assert result.complete and len(result.hits) == 1
- assert result.hits[0].chunk.event_time == after.start
-
-
-def test_final_deadline_failure_discards_previously_found_hits(indexed, monkeypatch):
- from one_memory.retrieval import raw_candidates
-
- original = raw_candidates._check
- calls = []
-
- def check(deadline):
- calls.append(deadline)
- if len(calls) == 3:
- raise TimeoutError("late final revalidation")
- return original(deadline)
-
- monkeypatch.setattr(raw_candidates, "_check", check)
- result = search(indexed)
- assert len(calls) == 3 and not result.hits and result.reason == "deadline"
-
-
[email protected]("corruption", ["all_missing", "tail_missing", "offset", "hash"])
-def test_corrupted_published_manifest_is_unavailable_not_complete_empty(indexed, corruption):
- f, generation, _, vectors = indexed
- put(f[1], "long source text " * 90)
- worker = RetrievalIndexWorker(
- f[3], vectors, Encoder(f[4]), generation_id=generation, owner="test"
- )
- assert worker.run_once().state == "ready"
- assert len(search(indexed).hits) > 1
- with closing(sqlite3.connect(f[0])) as db, db:
- if corruption == "all_missing":
- db.execute("DELETE FROM retrieval_chunks")
- elif corruption == "tail_missing":
- db.execute("DELETE FROM retrieval_chunks WHERE ordinal>0")
- elif corruption == "offset":
- db.execute("UPDATE retrieval_chunks SET start_cp=3 WHERE ordinal=0")
- else:
- db.execute("UPDATE retrieval_chunks SET text_hash='broken'")
- result = search(indexed)
- assert not result.complete and not result.hits and result.reason == "index_unavailable"
-
-
-def test_null_stored_grant_with_revocation_never_queries_vectors(indexed, monkeypatch):
- f, _, _, vectors = indexed
- assert search(indexed).hits
- with closing(sqlite3.connect(f[0])) as db, db:
- db.execute("UPDATE retrieval_documents SET grant_fingerprint=NULL")
- f[2].revoke(user_id="local", source_id="codex")
- monkeypatch.setattr(vectors, "search", lambda *a, **k: pytest.fail("revoked vector query"))
- assert not search(indexed).hits
-
-
-def test_unpublished_authorized_evidence_cannot_claim_complete_empty(indexed):
- """A partial index with no hits is incomplete, not an honest empty library."""
- f, generation, _, _vectors = indexed
- put(f[1], "用户决定 One Memory 采用 MIT 协议发布。", message="unindexed")
- assert f[3].consume_pending(generation) >= 1
- pending = f[3].status(generation)
- assert pending["pending"] >= 1
- # Published chunks may still match a probe vector. The unpublished
- # authorized document is what forbids complete-empty, so the candidate
- # batch is forced empty: that is the lying-empty case, not a miss on
- # already-indexed text.
- empty = type(search(indexed))(hits=(), complete=True, reason="no_eligible_chunks")
- path = compose_raw_retrieval_path(
- empty, unpublished_authorized=pending["pending"] > 0, lexical_hit_count=0
- )
- assert path.coverage == "partial"
- assert path.status != "empty"
- assert "semantic" in path.searched_channels
-
-
-def test_complete_index_with_no_hits_may_be_empty(indexed):
- batch = search(indexed, source_ids=frozenset({"codex"}))
- # Query vector still matches the only indexed chunk; force a no-hit batch.
- empty = type(batch)(hits=(), complete=True, reason="no_eligible_chunks")
- path = compose_raw_retrieval_path(empty, unpublished_authorized=False, lexical_hit_count=0)
- assert path.status == "empty"
- assert path.coverage == "complete"
diff --git a/memhub/apps/core/tests/test_retrieval_raw_evidence.py b/memhub/apps/core/tests/test_retrieval_raw_evidence.py
deleted file mode 100644
index d8f7829..0000000
--- a/memhub/apps/core/tests/test_retrieval_raw_evidence.py
+++ /dev/null
@@ -1,88 +0,0 @@
-import sqlite3
-import time
-from contextlib import closing
-
-import pytest
-
-from one_memory.retrieval.index_worker import RetrievalIndexWorker
-from one_memory.retrieval.raw_evidence import materialize_raw_candidates
-from tests.test_retrieval_index_store import fixture, put # noqa: F401
-from tests.test_retrieval_index_worker import Encoder, forbid_network # noqa: F401
-from tests.test_retrieval_raw_candidates import indexed as indexed_fixture
-from tests.test_retrieval_raw_candidates import search
-
-indexed = indexed_fixture
-
-
-def materialize(indexed, candidates=None, **kwargs):
- options = dict(
- source_ids=frozenset({"codex"}), deadline=time.monotonic() + 10, character_budget=6000
- )
- options.update(kwargs)
- return materialize_raw_candidates(
- indexed[0][3], candidates or search(indexed), generation_id=indexed[1], **options
- )
-
-
-def test_materialized_text_is_exact_current_original(indexed):
- result = materialize(indexed)
- assert result.complete and len(result.items) == 1
- item = result.items[0]
- assert item.evidence_id == indexed[2].evidence_id
- assert item.spans[0].text == indexed[2].verbatim
- assert item.written_by == "person" and item.matched_by == ("semantic",)
-
-
[email protected]("change", ["revision", "revoke", "unrecorded"])
-def test_stale_candidates_do_not_authorize_original_read(indexed, change):
- candidates = search(indexed)
- assert candidates.hits
- f = indexed[0]
- if change == "revision":
- put(f[1], "updated original")
- elif change == "revoke":
- f[2].revoke(user_id="local", source_id="codex")
- else:
- with closing(sqlite3.connect(f[0])) as db, db:
- db.execute(
- "UPDATE evidence SET verbatim='unrecorded' WHERE evidence_id=?",
- (indexed[2].evidence_id,),
- )
- result = materialize(indexed, candidates)
- assert not result.items and not result.complete and result.reason == "evidence_changed"
-
-
-def test_budget_does_not_cut_a_chunk_into_fake_partial_object(indexed):
- result = materialize(indexed, character_budget=5)
- assert not result.items and not result.complete and result.reason == "character_budget"
-
-
-def test_forged_offsets_and_modified_manifest_do_not_release_body(indexed):
- candidates = search(indexed)
- with closing(sqlite3.connect(indexed[0][0])) as db, db:
- db.execute("UPDATE retrieval_chunks SET start_cp=2")
- result = materialize(indexed, candidates)
- assert not result.items and not result.complete
-
-
-def test_empty_authority_and_expired_deadline_return_no_body(indexed):
- candidates = search(indexed)
- assert not materialize(indexed, candidates, source_ids=frozenset()).items
- assert not materialize(indexed, candidates, deadline=time.monotonic() - 1).items
-
-
-def test_overlapping_unicode_chunks_merge_without_duplicate_characters(indexed):
- f, generation, _, vectors = indexed
- text = "\u6c49\u5b57\U0001f680e\u0301 " * 180 + " /project/final-design.png"
- put(f[1], text)
- worker = RetrievalIndexWorker(
- f[3], vectors, Encoder(f[4]), generation_id=generation, owner="test"
- )
- assert worker.run_once().state == "ready"
- candidates = search(indexed)
- assert len(candidates.hits) > 1
- result = materialize(indexed, candidates)
- assert result.complete and len(result.items) == 1
- assert len(result.items[0].spans) == 1
- assert result.items[0].spans[0].text == text
- assert result.used_characters == len(text)
diff --git a/memhub/apps/core/tests/test_retrieval_raw_lane.py b/memhub/apps/core/tests/test_retrieval_raw_lane.py
deleted file mode 100644
index 6f80147..0000000
--- a/memhub/apps/core/tests/test_retrieval_raw_lane.py
+++ /dev/null
@@ -1,356 +0,0 @@
-"""v1 raw lane must not claim complete empty when the derived index is partial."""
-
-import asyncio
-import sqlite3
-import threading
-import time
-from types import SimpleNamespace
-
-import pytest
-
-from one_memory.context_models import ContextLaneResult, ContextSearchRequest
-from one_memory.context_raw import read_raw_context
-from one_memory.database_access import main_database
-from one_memory.embedding_assets import assets_present
-from one_memory.retrieval.raw_candidates import RawCandidateBatch
-from one_memory.retrieval.raw_lane import fuse_raw_lane, read_raw_indexed_context
-from tests.test_context_raw import database as database # noqa: F401
-from tests.test_context_raw import put
-
-
-def _require_assets() -> None:
- """Make an absent local fixture explicit instead of a lease-style hang."""
- if not assets_present():
- pytest.skip("_assets_unavailable")
-
-
-def test_unpublished_authorized_evidence_degrades_empty_lexical_lane():
- lexical = ContextLaneResult(status="empty", reason="lexical_only")
- result = fuse_raw_lane(
- lexical,
- RawCandidateBatch(complete=True, reason="no_eligible_chunks"),
- unpublished_authorized=True,
- )
- assert result.status == "degraded"
- assert result.reason == "lexical_only"
- assert result.items == ()
-
-
-def test_complete_index_with_no_hits_stays_empty():
- lexical = ContextLaneResult(status="empty", reason="lexical_only")
- result = fuse_raw_lane(
- lexical,
- RawCandidateBatch(complete=True, reason="no_eligible_chunks"),
- unpublished_authorized=False,
- )
- assert result.status == "empty"
- assert result.items == ()
-
-
-def test_interactive_raw_lane_reads_one_snapshot_while_a_writer_holds(database):
- """The read lane must not wait for an import writer, nor see its uncommitted row.
-
- This is the C4-owned half of the import-contention BLOCK: WAL readers are
- expected to proceed from the last committed snapshot even while the single
- writer holds a transaction. A regression that lets context reads block on
- the writer - or read its uncommitted row - makes this fail.
- """
- with sqlite3.connect(database) as db, db:
- db.execute("PRAGMA journal_mode=WAL")
- put(db, "committed", "needle committed row")
- writer_holds = threading.Event()
- release_writer = threading.Event()
-
- def hold_write_transaction() -> None:
- with main_database(database).write(autocommit=True) as writer:
- writer.execute("BEGIN IMMEDIATE")
- writer.execute(
- """INSERT INTO evidence
- (evidence_id,idempotency_key,user_id,source_id,conversation_id,role,
- written_by,created_at,verbatim,content_hash,revision,imported_at,updated_at)
- VALUES ('uncommitted','uncommitted','local','cursor','c','user',
- 'person','2026-09-20','needle uncommitted row','synthetic',
- 1,'2026-09-20','2026-09-20')"""
- )
- writer_holds.set()
- assert release_writer.wait(5)
- writer.execute("ROLLBACK")
-
- thread = threading.Thread(target=hold_write_transaction)
- thread.start()
- assert writer_holds.wait(5)
- try:
- before = main_database(database).stats()
- started = time.monotonic()
- result = read_raw_context(
- database,
- ContextSearchRequest(query="needle", kinds=("raw",)),
- source_ids=frozenset({"cursor"}),
- deadline=time.monotonic() + 2,
- )
- elapsed = time.monotonic() - started
- after = main_database(database).stats()
- finally:
- release_writer.set()
- thread.join(timeout=5)
- assert not thread.is_alive()
- assert elapsed < 1.0
- assert after.reader_waits == before.reader_waits
- assert [item.anchor_evidence_id for item in result.items] == ["committed"]
- assert all(
- message.evidence_id != "uncommitted" for item in result.items for message in item.messages
- )
-
-
-def test_authorization_and_unavailability_are_not_rewritten():
- denied = ContextLaneResult(status="not_authorized", reason="permission_required")
- assert (
- fuse_raw_lane(denied, RawCandidateBatch(), unpublished_authorized=True).status
- == "not_authorized"
- )
- missing = ContextLaneResult(status="unavailable", reason="read_unavailable")
- assert fuse_raw_lane(missing, RawCandidateBatch(), unpublished_authorized=True).status == (
- "unavailable"
- )
-
-
-def test_indexed_reader_without_runtime_matches_lexical_empty(database):
- with sqlite3.connect(database) as db, db:
- put(db, "e-unrelated", "alpha transcript only")
- request = ContextSearchRequest(query="no-matching-zzzzzz", kinds=("raw",))
- deadline = time.monotonic() + 5
- lexical = read_raw_context(
- database, request, source_ids=frozenset({"cursor"}), deadline=deadline
- )
- indexed = read_raw_indexed_context(
- database, request, source_ids=frozenset({"cursor"}), deadline=deadline, index=None
- )
- assert lexical.status == "empty"
- assert indexed == lexical
- assert indexed.searched_channels == ("lexical",)
- assert indexed.semantic_status == "not_searched"
- assert indexed.index_complete is False
-
-
-def test_indexed_reader_with_unpublished_index_is_not_empty(database):
- with sqlite3.connect(database) as db, db:
- put(db, "e-unrelated", "alpha transcript only")
- request = ContextSearchRequest(query="no-matching-zzzzzz", kinds=("raw",))
- index = SimpleNamespace(
- lookup_raw=lambda query, *, source_ids, deadline: (
- RawCandidateBatch(complete=True, reason="no_eligible_chunks"),
- True,
- )
- )
- result = read_raw_indexed_context(
- database,
- request,
- source_ids=frozenset({"cursor"}),
- deadline=time.monotonic() + 5,
- index=index,
- )
- assert result.status == "degraded"
- assert result.items == ()
- assert result.searched_channels == ("lexical", "semantic")
- assert result.semantic_status == "completed"
- assert result.index_complete is False
-
-
-def test_semantic_hit_without_lexical_overlap_is_not_released(database):
- from datetime import UTC, datetime
-
- from one_memory.retrieval.index_store import PublishedChunk
- from one_memory.retrieval.raw_candidates import RawSemanticCandidate
-
- with sqlite3.connect(database) as db, db:
- put(db, "e-mit", "用户决定 One Memory 采用 MIT 协议发布。")
- chunk = PublishedChunk(
- chunk_id="c1",
- build_id="b1",
- evidence_id="e-mit",
- revision=1,
- start_cp=0,
- end_cp=8,
- ordinal=0,
- source_id="cursor",
- event_time=datetime(2026, 9, 19, tzinfo=UTC),
- )
- index = SimpleNamespace(
- lookup_raw=lambda query, *, source_ids, deadline: (
- RawCandidateBatch(
- hits=(RawSemanticCandidate(chunk, 0.1),),
- complete=True,
- ),
- False,
- )
- )
- result = read_raw_indexed_context(
- database,
- ContextSearchRequest(query="开源许可证怎么选", kinds=("raw",)),
- source_ids=frozenset({"cursor"}),
- deadline=time.monotonic() + 5,
- index=index,
- )
- assert result.status == "degraded"
- assert result.reason == "lexical_only"
- assert result.items == ()
- assert result.searched_channels == ("lexical", "semantic")
- assert result.semantic_status == "completed"
- assert result.index_complete is True
-
-
-def test_semantic_candidate_with_lexical_corroboration_is_released(database):
- from datetime import UTC, datetime
-
- from one_memory.retrieval.index_store import PublishedChunk
- from one_memory.retrieval.raw_candidates import RawSemanticCandidate
-
- with sqlite3.connect(database) as db, db:
- put(db, "e-mit", "用户决定 One Memory 采用 MIT 协议发布。")
- chunk = PublishedChunk(
- chunk_id="c1",
- build_id="b1",
- evidence_id="e-mit",
- revision=1,
- start_cp=0,
- end_cp=8,
- ordinal=0,
- source_id="cursor",
- event_time=datetime(2026, 9, 19, tzinfo=UTC),
- )
- index = SimpleNamespace(
- lookup_raw=lambda query, *, source_ids, deadline: (
- RawCandidateBatch(
- hits=(RawSemanticCandidate(chunk, 0.1),),
- complete=True,
- ),
- False,
- )
- )
- result = read_raw_indexed_context(
- database,
- ContextSearchRequest(query="MIT 协议", kinds=("raw",)),
- source_ids=frozenset({"cursor"}),
- deadline=time.monotonic() + 5,
- index=index,
- )
- assert result.status == "ok"
- assert [item.anchor_evidence_id for item in result.items] == ["e-mit"]
-
-
-def test_raw_semantic_timeout_is_not_reported_as_complete_empty(database):
- with sqlite3.connect(database) as db, db:
- put(db, "e-unrelated", "alpha transcript only")
- index = SimpleNamespace(
- lookup_raw=lambda query, *, source_ids, deadline: (
- RawCandidateBatch(reason="deadline"),
- False,
- )
- )
- result = read_raw_indexed_context(
- database,
- ContextSearchRequest(query="no-matching-zzzzzz", kinds=("raw",)),
- source_ids=frozenset({"cursor"}),
- deadline=time.monotonic() + 5,
- index=index,
- )
- assert result.status == "degraded"
- assert result.semantic_status == "timed_out"
- assert result.index_complete is False
- assert result.timed_out
-
-
-def test_real_built_index_semantic_only_hit_is_not_released_without_confirmation(tmp_path):
- """The whole chain for real: ONNX encode, Chroma, published chunks, authority.
-
- Every other test here hands the lane a hand-written candidate, so none of
- them can tell whether a real build would ever produce one. The query is
- chosen to share no term with the stored sentence, and the lexical-only read
- is asserted empty first, so an `ok` here can only have come from the index.
- """
- _require_assets()
- from one_memory.imports.grants import SourceGrantStore
- from one_memory.imports.sources.registry import RootCandidateKind
- from one_memory.retrieval.runtime import ExistingRetrievalRuntime
- from tests.test_retrieval_index_store import put as put_evidence
- from tests.test_retrieval_runtime import configured, until
-
- store, evidence, generation = configured(tmp_path)
- source = tmp_path / "source"
- source.mkdir()
- SourceGrantStore(store.database_path, home=tmp_path).grant(
- user_id="local",
- source_id="codex",
- root=source,
- candidate_kind=RootCandidateKind.DEFAULT,
- )
- put_evidence(evidence, "我们最后决定 One Memory 用 MIT 协议发布。")
- request = ContextSearchRequest(query="开源软件的授权条款如何挑选", kinds=("raw",))
- sources = frozenset({"codex"})
-
- lexical = read_raw_context(
- store.database_path, request, source_ids=sources, deadline=time.monotonic() + 5
- )
- assert lexical.status == "empty", "词法已经命中,这条测试证明不了语义"
-
- async def scenario():
- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
- assert await handle.start()
- try:
- await until(lambda: store.status(generation)["ready"] >= 1)
- return await asyncio.to_thread(
- read_raw_indexed_context,
- store.database_path,
- request,
- source_ids=sources,
- deadline=time.monotonic() + 10,
- index=handle,
- )
- finally:
- await handle.stop()
-
- result = asyncio.run(scenario())
- # The semantic lane found the row, but lexical-only evidence is not enough
- # confirmation to release it to a host. Keep the candidate visible as a
- # degraded coverage receipt until a relevance/corroboration gate exists.
- assert result.status == "degraded"
- assert result.reason == "lexical_only"
- assert result.items == ()
-
-
-def test_real_built_index_skips_with_a_named_asset_reason(tmp_path, monkeypatch):
- """A missing untracked fixture must not look like a lease or worker hang."""
- monkeypatch.setattr("test_retrieval_raw_lane.assets_present", lambda: False)
- with pytest.raises(pytest.skip.Exception, match="_assets_unavailable"):
- test_real_built_index_semantic_only_hit_is_not_released_without_confirmation(tmp_path)
-
-
-def test_revoked_source_semantic_hit_does_not_release_body(database):
- from datetime import UTC, datetime
-
- from one_memory.retrieval.index_store import PublishedChunk
- from one_memory.retrieval.raw_candidates import RawSemanticCandidate
- from one_memory.retrieval.raw_lane import load_raw_items_from_candidates
- from tests.test_context_raw import revoke
-
- with sqlite3.connect(database) as db, db:
- put(db, "e-revoked", "用户决定 One Memory 采用 MIT 协议发布。")
- revoke(db, "cursor")
- chunk = PublishedChunk(
- chunk_id="c1",
- build_id="b1",
- evidence_id="e-revoked",
- revision=1,
- start_cp=0,
- end_cp=8,
- ordinal=0,
- source_id="cursor",
- event_time=datetime(2026, 9, 19, tzinfo=UTC),
- )
- items = load_raw_items_from_candidates(
- database,
- RawCandidateBatch(hits=(RawSemanticCandidate(chunk, 0.1),), complete=True),
- source_ids=frozenset({"cursor"}),
- deadline=time.monotonic() + 5,
- )
- assert items == ()
diff --git a/memhub/apps/core/tests/test_retrieval_runtime.py b/memhub/apps/core/tests/test_retrieval_runtime.py
deleted file mode 100644
index 102d9c2..0000000
--- a/memhub/apps/core/tests/test_retrieval_runtime.py
+++ /dev/null
@@ -1,522 +0,0 @@
-import asyncio
-import json
-import os
-import shutil
-import socket
-import sqlite3
-import threading
-import time
-from contextlib import closing
-
-import pytest
-
-from one_memory.imports.grants import SourceGrantStore
-from one_memory.imports.sources.registry import RootCandidateKind
-from one_memory.imports.storage import EvidenceStore
-from one_memory.retrieval import runtime
-from one_memory.retrieval.index_store import RetrievalIndexStore
-from one_memory.retrieval.index_worker import IndexWorkResult, LocalRetrievalEncoder
-from one_memory.retrieval.runtime import ExistingRetrievalRuntime
-from one_memory.retrieval.vector_store import RawVectorStore
-from tests.test_retrieval_index_store import put
-
-
[email protected](autouse=True)
-def offline(monkeypatch):
- def reject(*args, **kwargs):
- raise AssertionError("offline runtime test")
-
- monkeypatch.setattr(socket.socket, "connect", reject)
- monkeypatch.setattr(socket.socket, "connect_ex", reject)
-
-
-def configured(tmp_path, *, version=2, collection=True):
- path = tmp_path / "one-memory.sqlite3"
- evidence = EvidenceStore(path)
- identity = LocalRetrievalEncoder(content_version=version).identity
- store = RetrievalIndexStore(path)
- with closing(sqlite3.connect(path)) as db, db:
- db.execute("BEGIN IMMEDIATE")
- store.initialize(db)
- store.migrate_v2(db)
- generation = store.create_generation(db, identity)
- store.activate_generation(generation)
- if collection:
- vectors = RawVectorStore.create(
- tmp_path / "retrieval-vectors", generation_id=generation, dimensions=identity.dimensions
- )
- vectors.close()
- return store, evidence, generation
-
-
-async def until(predicate):
- end = time.monotonic() + 10
- while not predicate():
- assert time.monotonic() < end
- await asyncio.sleep(0.01)
-
-
-def test_constructor_and_missing_start_create_nothing(tmp_path):
- target = tmp_path / "missing"
- handle = ExistingRetrievalRuntime(target)
- assert not target.exists()
- assert not asyncio.run(handle.start())
- assert handle.snapshot().state == "not_initialized"
- assert not target.exists()
- asyncio.run(handle.stop())
-
-
-def test_stop_releases_supervisor_worker_but_preserves_immutable_snapshot(tmp_path):
- configured(tmp_path)
- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
-
- async def scenario():
- assert await handle.start()
- assert handle._supervisor is not None
- await handle.stop()
-
- asyncio.run(scenario())
- assert handle._supervisor is None
- assert handle.snapshot().supervisor is not None
- assert not handle.snapshot().supervisor.thread_alive
-
-
- "kind",
- ["schema1", "no_active", "missing_collection", "bad_identity", "extra_identity", "two_active"],
-)
-def test_not_initialized_or_invalid_configuration_never_creates_index(tmp_path, kind):
- store, _, gen = configured(tmp_path, collection=False)
- with closing(sqlite3.connect(store.database_path)) as db, db:
- if kind == "schema1":
- db.execute("UPDATE retrieval_index_state SET schema_version=1")
- elif kind == "no_active":
- db.execute("UPDATE retrieval_index_generations SET state='building'")
- elif kind in ("bad_identity", "extra_identity"):
- raw = json.loads(
- db.execute("SELECT identity_json FROM retrieval_index_generations").fetchone()[0]
- )
- if kind == "bad_identity":
- raw["encoder_digest"] = "0" * 64
- else:
- raw["unexpected"] = "nope"
- db.execute("UPDATE retrieval_index_generations SET identity_json=?", (json.dumps(raw),))
- elif kind == "two_active":
- db.execute("DROP INDEX retrieval_one_active")
- db.execute(
- "INSERT INTO retrieval_index_generations SELECT 'other',identity_json,state,"
- "created_at FROM retrieval_index_generations"
- )
- before = db.execute("SELECT * FROM retrieval_index_generations").fetchall()
- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
- assert not asyncio.run(handle.start())
- assert handle.snapshot().state in {"not_initialized", "failed"}
- assert not (tmp_path / "retrieval-vectors").exists()
- with closing(sqlite3.connect(store.database_path)) as db:
- assert db.execute("SELECT * FROM retrieval_index_generations").fetchall() == before
- assert str(tmp_path) not in repr(handle.snapshot())
-
-
-def test_low_disk_rejects_before_open_writer(tmp_path, monkeypatch):
- configured(tmp_path)
- monkeypatch.setattr(
- runtime.shutil, "disk_usage", lambda _: shutil._ntuple_diskusage(1000, 999, 1)
- )
- monkeypatch.setattr(
- runtime.RawVectorStore,
- "open_writer_existing",
- lambda *a, **k: pytest.fail("opened on low disk"),
- )
- handle = ExistingRetrievalRuntime(tmp_path)
- assert not asyncio.run(handle.start())
- assert handle.snapshot().error_code == "retrieval_runtime_low_disk"
-
-
[email protected]("kind", ["root", "database", "vectors"])
-def test_symlink_rejected(tmp_path, kind):
- actual = tmp_path / "actual"
- actual.mkdir()
- configured(actual)
- selected = actual
- if kind == "root":
- selected = tmp_path / "link"
- selected.symlink_to(actual, target_is_directory=True)
- else:
- path = actual / ("one-memory.sqlite3" if kind == "database" else "retrieval-vectors")
- moved = tmp_path / "moved"
- path.rename(moved)
- path.symlink_to(moved, target_is_directory=kind == "vectors")
- handle = ExistingRetrievalRuntime(selected, minimum_free_bytes=0)
- assert not asyncio.run(handle.start())
- assert handle.snapshot().error_code == "retrieval_runtime_unsafe_path"
-
-
-def test_local_existing_integration_indexes_only_after_explicit_start(tmp_path):
- store, evidence, gen = configured(tmp_path)
- source = tmp_path / "source"
- source.mkdir()
- SourceGrantStore(store.database_path, home=tmp_path).grant(
- user_id="local", source_id="codex", root=source, candidate_kind=RootCandidateKind.DEFAULT
- )
- row = put(evidence, "before token=private after")
-
- async def run():
- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
- assert store.status(gen)["ready"] == 0
- assert await handle.start()
- try:
- await until(lambda: store.status(gen)["ready"] == 1)
- assert len(store.active_chunks(gen)) == 2
- assert handle.snapshot().generation_id == gen
- with pytest.raises(RuntimeError, match="already_started"):
- await handle.start()
- finally:
- await handle.stop()
- assert handle.snapshot().state == "stopped"
- assert not handle.snapshot().supervisor.thread_alive
- assert evidence.get(row.evidence_id).verbatim == row.verbatim
-
- asyncio.run(run())
-
-
-def fake_resources(tmp_path, monkeypatch, *, fail_worker=False, block_open=False, version=2):
- configured(tmp_path, version=version)
- entered, release, exited, opened = (threading.Event() for _ in range(4))
- events = []
-
- class Vectors:
- def close(self):
- events.append("close")
-
- def open_vectors(*args, **kwargs):
- opened.set()
- if block_open:
- assert release.wait(5)
- return Vectors()
-
- class Worker:
- def __init__(self, *args, **kwargs):
- if fail_worker:
- raise RuntimeError("SECRET failure")
-
- def run_once(self, *, cancel):
- entered.set()
- while not cancel.is_set():
- time.sleep(0.001)
- events.append("cancel")
- assert release.wait(5)
- events.append("exit")
- exited.set()
- return IndexWorkResult("cancelled")
-
- monkeypatch.setattr(runtime.RawVectorStore, "open_writer_existing", open_vectors)
- monkeypatch.setattr(runtime, "RetrievalIndexWorker", Worker)
- return entered, release, exited, opened, events
-
-
-def test_partial_start_failure_closes_owned_vectors(tmp_path, monkeypatch):
- *_, events = fake_resources(tmp_path, monkeypatch, fail_worker=True)
- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
- assert not asyncio.run(handle.start())
- assert events == ["close"]
- assert handle.snapshot().error_code == "retrieval_runtime_start_failed"
- assert "SECRET" not in repr(handle.snapshot())
-
-
-def test_cancelled_stop_drains_before_close(tmp_path, monkeypatch):
- entered, release, exited, _, events = fake_resources(tmp_path, monkeypatch)
-
- async def run():
- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
- assert await handle.start()
- await until(entered.is_set)
- task = asyncio.create_task(handle.stop())
- await asyncio.sleep(0)
- task.cancel()
- try:
- await until(lambda: "cancel" in events)
- task.cancel()
- await asyncio.sleep(0.03)
- assert not task.done() and "close" not in events
- finally:
- release.set()
- with pytest.raises(asyncio.CancelledError):
- await task
- assert exited.is_set() and events == ["cancel", "exit", "close"]
- await handle.stop()
- assert events.count("close") == 1
-
- asyncio.run(run())
-
-
-def test_cancelled_start_drains_open_and_cleans_resources(tmp_path, monkeypatch):
- _, release, _, opened, events = fake_resources(
- tmp_path, monkeypatch, fail_worker=True, block_open=True
- )
-
- async def run():
- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
- task = asyncio.create_task(handle.start())
- await until(opened.is_set)
- task.cancel()
- try:
- await asyncio.sleep(0.03)
- assert not task.done() and not events
- finally:
- release.set()
- with pytest.raises(asyncio.CancelledError):
- await task
- assert events == ["close"]
-
- asyncio.run(run())
-
-
-def test_cancelled_successful_open_does_not_start_worker_and_closes_once(tmp_path, monkeypatch):
- entered, release, _, opened, events = fake_resources(tmp_path, monkeypatch, block_open=True)
-
- async def run():
- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
- starting = asyncio.create_task(handle.start())
- await until(opened.is_set)
- starting.cancel()
- await asyncio.sleep(0.02)
- starting.cancel()
- try:
- await asyncio.sleep(0.02)
- assert not starting.done() and not events and not entered.is_set()
- finally:
- release.set()
- with pytest.raises(asyncio.CancelledError):
- await starting
- assert events == ["close"] and not entered.is_set()
- assert handle.snapshot().state == "stopped"
- await handle.stop()
- assert events == ["close"]
-
- asyncio.run(run())
-
-
-def test_old_identity_one_recovers_without_generation_upgrade(tmp_path, monkeypatch):
- entered, release, _, _, events = fake_resources(tmp_path, monkeypatch, version=1)
- path = tmp_path / "one-memory.sqlite3"
- with closing(sqlite3.connect(path)) as db:
- before = db.execute("SELECT * FROM retrieval_index_generations").fetchall()
-
- async def run():
- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
- assert await handle.start()
- await until(entered.is_set)
- release.set()
- await handle.stop()
-
- asyncio.run(run())
- assert events == ["cancel", "exit", "close"]
- with closing(sqlite3.connect(path)) as db:
- assert db.execute("SELECT * FROM retrieval_index_generations").fetchall() == before
-
-
-def test_missing_identity_fields_fail_even_when_dataclass_defaults_match(tmp_path):
- store, _, _ = configured(tmp_path, version=1, collection=False)
- with closing(sqlite3.connect(store.database_path)) as db, db:
- raw = json.loads(
- db.execute("SELECT identity_json FROM retrieval_index_generations").fetchone()[0]
- )
- raw.pop("normalizer_version")
- db.execute("UPDATE retrieval_index_generations SET identity_json=?", (json.dumps(raw),))
- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
- assert not asyncio.run(handle.start())
- assert handle.snapshot().error_code == "retrieval_runtime_invalid_index"
-
-
-def test_valid_hash_but_foreign_encoder_never_opens_vectors(tmp_path, monkeypatch):
- from dataclasses import asdict
-
- from one_memory.retrieval.index_store import EncoderIdentity, digest
-
- store, _, _ = configured(tmp_path)
- with closing(sqlite3.connect(store.database_path)) as db, db:
- raw = json.loads(
- db.execute("SELECT identity_json FROM retrieval_index_generations").fetchone()[0]
- )
- raw["encoder_digest"] = "0" * 64
- identity = EncoderIdentity(**raw)
- store_id = db.execute("SELECT data_store_id FROM retrieval_index_state").fetchone()[0]
- db.execute(
- "UPDATE retrieval_index_generations SET generation_id=?,identity_json=?",
- (digest([store_id, asdict(identity)]), json.dumps(raw)),
- )
- monkeypatch.setattr(
- runtime.RawVectorStore,
- "open_writer_existing",
- lambda *a, **k: pytest.fail("foreign encoder opened"),
- )
- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
- assert not asyncio.run(handle.start())
- assert handle.snapshot().error_code == "retrieval_runtime_encoder_mismatch"
-
-
-def test_concurrent_stop_waits_start_and_runs_cleanup_once(tmp_path, monkeypatch):
- entered, release, _, opened, events = fake_resources(tmp_path, monkeypatch, block_open=True)
-
- async def run():
- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
- starting = asyncio.create_task(handle.start())
- await until(opened.is_set)
- stops = [asyncio.create_task(handle.stop()), asyncio.create_task(handle.stop())]
- try:
- await asyncio.sleep(0.03)
- assert all(not task.done() for task in stops)
- finally:
- release.set()
- await asyncio.gather(*stops)
- assert not await starting
- assert events == ["close"] and not entered.is_set()
-
- asyncio.run(run())
-
-
-def test_stop_before_start_never_opens_resources(tmp_path):
- handle = ExistingRetrievalRuntime(tmp_path)
- asyncio.run(handle.stop())
- with pytest.raises(RuntimeError, match="already_started"):
- asyncio.run(handle.start())
- assert not list(tmp_path.iterdir())
-
-
-def test_missing_collection_in_existing_chroma_does_not_create_collection(tmp_path):
- store, _, gen = configured(tmp_path)
- with closing(sqlite3.connect(tmp_path / "retrieval-vectors" / "chroma.sqlite3")) as db, db:
- db.execute("UPDATE collections SET name='foreign_collection'")
- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
- assert not asyncio.run(handle.start())
- assert handle.snapshot().state == "not_initialized"
- with closing(sqlite3.connect(tmp_path / "retrieval-vectors" / "chroma.sqlite3")) as db:
- assert db.execute("SELECT name FROM collections").fetchall() == [("foreign_collection",)]
-
-
[email protected]("target", ["root", "database", "vectors", "vector_database"])
-def test_group_writable_paths_rejected_without_open(tmp_path, monkeypatch, target):
- configured(tmp_path)
- paths = {
- "root": tmp_path,
- "database": tmp_path / "one-memory.sqlite3",
- "vectors": tmp_path / "retrieval-vectors",
- "vector_database": tmp_path / "retrieval-vectors/chroma.sqlite3",
- }
- path = paths[target]
- original_mode = path.stat().st_mode & 0o777
- path.chmod(original_mode | 0o020)
- monkeypatch.setattr(
- runtime.RawVectorStore,
- "open_writer_existing",
- lambda *a, **k: pytest.fail("unsafe resource opened"),
- )
- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
- try:
- assert not asyncio.run(handle.start())
- assert handle.snapshot().error_code == "retrieval_runtime_unsafe_path"
- finally:
- path.chmod(original_mode)
-
-
-def test_foreign_uid_and_windows_are_failclosed(tmp_path, monkeypatch):
- configured(tmp_path)
- uid = os.getuid()
- monkeypatch.setattr(runtime.os, "getuid", lambda: uid + 1)
- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
- assert not asyncio.run(handle.start())
- assert handle.snapshot().error_code == "retrieval_runtime_unsafe_path"
- monkeypatch.setattr(runtime.os, "getuid", lambda: uid)
- monkeypatch.setattr(runtime.sys, "platform", "win32")
- windows = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
- assert not asyncio.run(windows.start())
- assert windows.snapshot().error_code == "retrieval_runtime_platform_unsupported"
-
-
[email protected]("target", ["root", "database"])
-def test_identical_store_id_copy_replacement_during_open_is_rejected(tmp_path, monkeypatch, target):
- root = tmp_path / "profile"
- root.mkdir()
- configured(root)
- closed = []
-
- class Vectors:
- def close(self):
- closed.append(True)
-
- def replace_path(*args, **kwargs):
- if target == "database":
- path = root / "one-memory.sqlite3"
- copy = root / "replacement.sqlite3"
- shutil.copy2(path, copy)
- os.replace(copy, path)
- else:
- old = tmp_path / "old-profile"
- root.rename(old)
- shutil.copytree(old, root)
- return Vectors()
-
- monkeypatch.setattr(runtime.RawVectorStore, "open_writer_existing", replace_path)
- handle = ExistingRetrievalRuntime(root, minimum_free_bytes=0)
- assert not asyncio.run(handle.start())
- assert handle.snapshot().error_code == "retrieval_runtime_index_changed"
- assert closed == [True]
-
-
-def test_failed_drain_keeps_live_worker_vectors_open_and_raises_fixed_code(tmp_path, monkeypatch):
- entered, release, _, _, events = fake_resources(tmp_path, monkeypatch)
- actual = runtime.RetrievalIndexSupervisor
- instances = []
-
- class BadStop(actual):
- def __init__(self, worker, **kwargs):
- super().__init__(worker, **kwargs)
- instances.append(self)
-
- async def stop(self):
- raise RuntimeError("SECRET_STOP_ERROR")
-
- monkeypatch.setattr(runtime, "RetrievalIndexSupervisor", BadStop)
-
- async def run():
- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
- assert await handle.start()
- await until(entered.is_set)
- try:
- with pytest.raises(RuntimeError, match="^retrieval_runtime_stop_failed$"):
- await handle.stop()
- assert "close" not in events
- assert handle.snapshot().supervisor.thread_alive
- assert handle.snapshot().state == "failed"
- assert "SECRET" not in repr(handle.snapshot())
- finally:
- release.set()
- await actual.stop(instances[0])
- # Test alone owns cleanup of the intentionally broken supervisor.
- handle._vectors.close()
-
- asyncio.run(run())
-
-
-def test_stop_error_after_confirmed_exit_still_closes_vectors(tmp_path, monkeypatch):
- entered, release, _, _, events = fake_resources(tmp_path, monkeypatch)
- actual = runtime.RetrievalIndexSupervisor
-
- class BadStop(actual):
- async def stop(self):
- await super().stop()
- raise RuntimeError("SECRET_AFTER_EXIT")
-
- monkeypatch.setattr(runtime, "RetrievalIndexSupervisor", BadStop)
-
- async def run():
- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
- assert await handle.start()
- await until(entered.is_set)
- release.set()
- with pytest.raises(RuntimeError, match="^retrieval_runtime_stop_failed$"):
- await handle.stop()
- assert events == ["cancel", "exit", "close"]
- assert not handle.snapshot().supervisor.thread_alive
-
- asyncio.run(run())
diff --git a/memhub/apps/core/tests/test_retrieval_vector_store.py b/memhub/apps/core/tests/test_retrieval_vector_store.py
deleted file mode 100644
index c78c4bb..0000000
--- a/memhub/apps/core/tests/test_retrieval_vector_store.py
+++ /dev/null
@@ -1,528 +0,0 @@
-import math
-import socket
-from time import monotonic
-
-import numpy as np
-import pytest
-
-from one_memory.retrieval.vector_store import (
- RawVectorRecord,
- RawVectorStore,
- VectorStoreError,
- _unit_vector,
-)
-
-
[email protected](autouse=True)
-def no_network(monkeypatch):
- def refused(*args, **kwargs):
- pytest.fail("vector adapter attempted network")
-
- monkeypatch.setattr(socket.socket, "connect", refused)
- monkeypatch.setattr(socket, "create_connection", refused)
-
-
-def store(tmp_path):
- value = RawVectorStore.create(
- tmp_path / "retrieval-vectors", generation_id="generation-a", dimensions=3
- )
- try:
- yield value
- finally:
- value.close()
-
-
-def record(chunk="chunk-a", *, build="build-a", generation="generation-a", vector=(1.0, 0.0, 0.0)):
- return RawVectorRecord(chunk, build, generation, vector)
-
-
-def test_constructor_does_no_io_and_unopened_calls_do_not_create(tmp_path):
- directory = tmp_path / "never-created"
- value = RawVectorStore(directory, generation_id="generation-a", dimensions=3)
- assert not directory.exists()
- with pytest.raises(VectorStoreError, match="not_open"):
- value.search((1.0, 0.0, 0.0), ["chunk-a"], 1, monotonic() + 1)
- assert not directory.exists()
- value.close()
-
-
-def test_open_existing_missing_store_or_collection_never_creates(tmp_path, store):
- missing = tmp_path / "missing"
- with pytest.raises(VectorStoreError, match="not_found"):
- RawVectorStore.open_existing(missing, generation_id="generation-a", dimensions=3)
- assert not missing.exists()
- before = store._client.list_collections()
- with pytest.raises(VectorStoreError, match="not_found"):
- RawVectorStore.open_existing(store.directory, generation_id="other", dimensions=3)
- assert store._client.list_collections() == before
-
-
-def test_real_chroma_collection_has_no_embedding_function_or_documents(store):
- store.upsert([record()])
- collection = store._collection
- assert collection._embedding_function is None
- result = collection.get(ids=["chunk-a"], include=["embeddings", "metadatas", "documents"])
- assert result["documents"] == [None]
- assert result["metadatas"][0]["chunk_id"] == "chunk-a"
- assert result["metadatas"][0]["build_id"] == "build-a"
- assert result["metadatas"][0]["generation_id"] == "generation-a"
- assert len(result["embeddings"][0]) == 3
- store.verify([record()])
-
-
-def test_existing_read_handle_cannot_write_or_repair_and_releases_shared_client(store):
- store.upsert([record()])
- opened = RawVectorStore.open_existing(
- store.directory, generation_id="generation-a", dimensions=3
- )
- try:
- result = opened.search((1.0, 0.0, 0.0), ["chunk-a"], 1, monotonic() + 10)
- assert result.complete and result.hits[0].chunk_id == "chunk-a"
- with pytest.raises(VectorStoreError, match="read_only"):
- opened.upsert([record("new")])
- opened.verify([record()])
- finally:
- opened.close()
- store.verify([record()])
- with pytest.raises(VectorStoreError, match="closed"):
- opened.search((1.0, 0.0, 0.0), ["chunk-a"], 1, monotonic() + 1)
-
-
[email protected]("changed", [record(build="different"), record(vector=(0.0, 1.0, 0.0))])
-def test_immutable_id_rejects_different_metadata_or_vector_without_overwriting(store, changed):
- store.upsert([record()])
- store.upsert([record()])
- with pytest.raises(VectorStoreError, match="immutable_conflict"):
- store.upsert([record("new"), changed])
- store.verify([record()])
- assert store._collection.get(ids=["new"])["ids"] == []
-
-
-def test_verify_detects_missing_and_tampered_records(store):
- with pytest.raises(VectorStoreError, match="missing"):
- store.verify([record()])
- store.upsert([record()])
- store._collection.update(ids=["chunk-a"], embeddings=[[0.0, 1.0, 0.0]])
- with pytest.raises(VectorStoreError):
- store.verify([record()])
- with pytest.raises(VectorStoreError):
- store.search((1.0, 0.0, 0.0), ["chunk-a"], 1, monotonic() + 10)
-
-
- "vector",
- [
- (1.0, 0.0),
- (1.0, 0.0, math.nan),
- (math.inf, 0.0, 1.0),
- (0.0, 0.0, 0.0),
- (True, 0.0, 1.0),
- (1e100, 0.0, 1.0),
- ],
-)
-def test_invalid_vectors_refused_before_writes_or_search(store, vector):
- with pytest.raises(VectorStoreError):
- store.upsert([record(vector=vector)])
- with pytest.raises(VectorStoreError):
- store.search(vector, [], 1, monotonic() + 1)
- assert store._collection.count() == 0
-
-
-def test_wrong_generation_and_duplicate_ids_rejected(store):
- with pytest.raises(VectorStoreError):
- store.upsert([record(generation="other")])
- with pytest.raises(VectorStoreError):
- store.upsert([record(), record()])
- assert store._collection.count() == 0
-
-
-def test_empty_eligible_ids_never_queries_all_collection(store, monkeypatch):
- store.upsert([record()])
- monkeypatch.setattr(
- store._collection, "query", lambda **kwargs: pytest.fail("unfiltered query")
- )
- result = store.search((1.0, 0.0, 0.0), [], 3, monotonic() - 1)
- assert result.complete and result.hits == ()
- assert result.eligible_count == result.searched_count == 0
-
-
-def test_real_in_filter_excludes_closer_unapproved_id_and_matches_small_cosine_oracle(store):
- rows = [
- record("denied", vector=(1.0, 0.0, 0.0)),
- record("b", vector=(0.9, 0.1, 0.0)),
- record("c", vector=(0.0, 1.0, 0.0)),
- record("d", vector=(-1.0, 0.0, 0.0)),
- ]
- store.upsert(rows)
- result = store.search((1.0, 0.0, 0.0), ["b", "c", "d"], 3, monotonic() + 10)
- assert result.complete and result.searched_count == 3
- assert [hit.chunk_id for hit in result.hits] == ["b", "c", "d"]
- for hit, row in zip(result.hits, rows[1:], strict=True):
- vector = np.array(row.vector, dtype=np.float32)
- expected = 1 - float(vector[0] / np.linalg.norm(vector))
- assert hit.distance == pytest.approx(expected, abs=1e-5)
-
-
-def test_eligible_batches_cover_more_than_2048_and_merge_topk(store, monkeypatch):
- ids = [f"chunk-{i:04d}" for i in range(2053)]
- rows = [record(chunk, vector=(float(i + 1), 1.0, 0.0)) for i, chunk in enumerate(ids)]
- store.upsert(rows)
- calls = []
- get = store._collection.get
-
- def tracked(**kwargs):
- calls.append(kwargs)
- return get(**kwargs)
-
- monkeypatch.setattr(store._collection, "get", tracked)
- result = store.search((1.0, 0.0, 0.0), ids, 3, monotonic() + 30)
- assert result.complete and result.searched_count == 2053
- assert len(calls) == 2
- assert [len(call["ids"]) for call in calls] == [2048, 5]
- assert set().union(*(set(call["ids"]) for call in calls)) == set(ids)
- assert all(call["include"] == ["embeddings", "metadatas"] for call in calls)
- assert len(result.hits) == 3
- # The exact ranking over every eligible chunk, not one batch's local top-K:
- # a per-batch top-K would have kept the second batch's steepest vectors out.
- # Distances come from the stored unit vectors, so float32 rounding decides
- # the top of this ranking just as it does for the adapter.
- stored = {row.chunk_id: np.array(_unit_vector(row.vector), dtype=np.float32) for row in rows}
- expected = {
- chunk_id: float(1 - np.dot(unit, np.array([1.0, 0.0, 0.0], dtype=np.float32)))
- for chunk_id, unit in stored.items()
- }
- ranked = sorted(expected.items(), key=lambda pair: (pair[1], pair[0]))[:3]
- assert [hit.chunk_id for hit in result.hits] == [chunk for chunk, _ in ranked]
- for hit in result.hits:
- assert hit.distance == pytest.approx(expected[hit.chunk_id], abs=1e-6)
-
-
-def test_deadline_before_query_and_late_batch_return_are_partial(store, monkeypatch):
- store.upsert([record()])
- calls = []
- now = [0.0]
- store._clock = lambda: now[0]
- get = store._collection.get
-
- def late(**kwargs):
- calls.append(kwargs)
- result = get(**kwargs)
- now[0] = 2.0
- return result
-
- monkeypatch.setattr(store._collection, "get", late)
- first = store.search((1.0, 0.0, 0.0), ["chunk-a"], 1, 0.0)
- assert not first.complete and first.reason == "deadline" and first.hits == ()
- assert calls == []
- second = store.search((1.0, 0.0, 0.0), ["chunk-a"], 1, 1.0)
- assert not second.complete and second.searched_count == 0 and second.hits == ()
- assert len(calls) == 1
-
-
-def test_search_never_scores_a_vector_the_caller_did_not_declare_eligible(store, monkeypatch):
- """Chroma's own filtered query is no longer used at all.
-
- It does not score every ID in its `where $in` allowlist, so a query could
- come back short while claiming completion, and a global query filtered
- afterwards would let unauthorized vectors take the eligible ones' places.
- """
- store.upsert([record(), record("denied", vector=(1.0, 0.0, 0.0))])
- monkeypatch.setattr(
- store._collection,
- "query",
- lambda **kwargs: pytest.fail("global or filtered ANN query is not used"),
- )
- result = store.search((1.0, 0.0, 0.0), ["chunk-a"], 1, monotonic() + 10)
- assert result.complete and [hit.chunk_id for hit in result.hits] == ["chunk-a"]
-
-
-def test_an_eligible_vector_missing_from_the_backend_is_unavailable_not_a_miss(store):
- """The caller published this chunk, so an absent vector is a broken build."""
- store.upsert([record("present")])
- with pytest.raises(VectorStoreError, match="^vector_query_unavailable$"):
- store.search((1.0, 0.0, 0.0), ["present", "absent"], 2, monotonic() + 10)
-
-
-def test_open_existing_validates_generation_dimensions_and_collection_config(store):
- with pytest.raises(VectorStoreError):
- RawVectorStore.open_existing(store.directory, generation_id="generation-a", dimensions=4)
- store._collection.modify(metadata={"generation_id": "other"})
- with pytest.raises(VectorStoreError):
- RawVectorStore.open_existing(store.directory, generation_id="generation-a", dimensions=3)
-
-
-def test_open_existing_refuses_symlink_database_before_chroma(tmp_path):
- directory = tmp_path / "retrieval-vectors"
- directory.mkdir()
- target = tmp_path / "not-a-chroma-db"
- target.write_bytes(b"do not open")
- (directory / "chroma.sqlite3").symlink_to(target)
- with pytest.raises(VectorStoreError):
- RawVectorStore.open_existing(directory, generation_id="generation-a", dimensions=3)
- assert target.read_bytes() == b"do not open"
-
-
-def test_writer_reopen_resumes_idempotently_but_query_handle_still_cannot_write(tmp_path):
- directory = tmp_path / "retrieval-vectors"
- original = RawVectorStore.create(directory, generation_id="generation-a", dimensions=3)
- try:
- original.upsert([record(vector=(1.0, 1.0, 0.0))])
- finally:
- original.close()
- writer = RawVectorStore.open_writer_existing(
- directory, generation_id="generation-a", dimensions=3
- )
- try:
- writer.upsert([record(vector=(1.0, 1.0, 0.0))])
- writer.verify([record(vector=(1.0, 1.0, 0.0))])
- with pytest.raises(VectorStoreError, match="immutable_conflict"):
- writer.upsert([record(build="other", vector=(1.0, 1.0, 0.0))])
- with pytest.raises(VectorStoreError, match="immutable_conflict"):
- writer.upsert([record(vector=(2.0, 2.0, 0.0))])
- assert writer._collection.count() == 1
- finally:
- writer.close()
- reader = RawVectorStore.open_existing(directory, generation_id="generation-a", dimensions=3)
- try:
- with pytest.raises(VectorStoreError, match="read_only"):
- reader.upsert([record("new")])
- result = reader.search((1.0, 1.0, 0.0), ["chunk-a"], 1, monotonic() + 10)
- assert result.complete and result.hits[0].distance == pytest.approx(0, abs=1e-5)
- finally:
- reader.close()
-
-
-def test_writer_existing_does_not_create_missing_store_or_collection(tmp_path, store):
- missing = tmp_path / "writer-missing"
- with pytest.raises(VectorStoreError, match="not_found"):
- RawVectorStore.open_writer_existing(missing, generation_id="generation-a", dimensions=3)
- assert not missing.exists()
- with pytest.raises(VectorStoreError, match="not_found"):
- RawVectorStore.open_writer_existing(store.directory, generation_id="other", dimensions=3)
-
-
-def test_partial_keeps_only_completed_eligible_batches(store, monkeypatch):
- ids = [f"present-{i}" for i in range(2048)] + ["last"]
- store.upsert([*(record(chunk, vector=(1.0, float(i + 1), 0.0)) for i, chunk in enumerate(ids))])
- now = [0.0]
- store._clock = lambda: now[0]
- get = store._collection.get
- calls = []
-
- def timed(**kwargs):
- calls.append(kwargs)
- result = get(**kwargs)
- if len(calls) == 2:
- now[0] = 2.0
- return result
-
- monkeypatch.setattr(store._collection, "get", timed)
- result = store.search((1.0, 0.0, 0.0), ids, 2, 1.0)
- assert not result.complete and result.reason == "deadline"
- assert result.searched_count == 2048 and result.eligible_count == 2049
- assert [hit.chunk_id for hit in result.hits] == ["present-0", "present-1"]
-
-
[email protected]("mutation", ["metadata", "dimension", "hash", "extra_id"])
-def test_malformed_search_response_is_rejected_not_partial_success(store, monkeypatch, mutation):
- store.upsert([record()])
- get = store._collection.get
-
- def malformed(**kwargs):
- result = get(**kwargs)
- if mutation == "metadata":
- result["metadatas"][0][0]["generation_id"] = "wrong"
- elif mutation == "dimension":
- result["embeddings"] = [[1.0, 0.0]]
- elif mutation == "hash":
- result["metadatas"][0]["vector_sha256"] = "0" * 64
- else:
- result["ids"] = result["ids"] * 2
- result["metadatas"] = result["metadatas"] * 2
- result["embeddings"] = [result["embeddings"][0], result["embeddings"][0]]
- return result
-
- monkeypatch.setattr(store._collection, "get", malformed)
- with pytest.raises(VectorStoreError):
- store.search((1.0, 0.0, 0.0), ["chunk-a"], 1, monotonic() + 10)
-
-
- "ids,limit,deadline",
- [
- (["chunk-a", "chunk-a"], 1, 100.0),
- ("chunk-a", 1, 100.0),
- (["chunk-a"], 0, 100.0),
- (["chunk-a"], True, 100.0),
- (["chunk-a"], 1, math.inf),
- (["chunk-a"], 1, math.nan),
- ],
-)
-def test_invalid_query_parameters_do_not_reach_chroma(store, monkeypatch, ids, limit, deadline):
- monkeypatch.setattr(store._collection, "query", lambda **kwargs: pytest.fail("invalid query"))
- with pytest.raises(VectorStoreError):
- store.search((1.0, 0.0, 0.0), ids, limit, deadline)
-
-
-def test_query_backend_error_is_unavailable_not_complete_empty(store, monkeypatch):
- def broken(**kwargs):
- raise RuntimeError("backend request payload")
-
- monkeypatch.setattr(store._collection, "query", broken)
- with pytest.raises(VectorStoreError, match="^vector_query_unavailable$"):
- store.search((1.0, 0.0, 0.0), ["chunk-a"], 1, monotonic() + 10)
-
-
-def test_last_owned_client_close_releases_chroma_system(tmp_path):
- from chromadb.api.shared_system_client import SharedSystemClient
-
- directory = tmp_path / "retrieval-vectors"
- writer = RawVectorStore.create(directory, generation_id="close-test", dimensions=3)
- reader = RawVectorStore.open_existing(directory, generation_id="close-test", dimensions=3)
- identifier = writer._client._identifier
- try:
- assert identifier in SharedSystemClient._identifier_to_system
- writer.close()
- assert identifier in SharedSystemClient._identifier_to_system
- reader.close()
- assert identifier not in SharedSystemClient._identifier_to_system
- reader.close()
- finally:
- reader.close()
- writer.close()
-
-
-def test_open_existing_missing_collection_does_not_construct_a_client(store, monkeypatch):
- import chromadb
-
- monkeypatch.setattr(
- chromadb, "PersistentClient", lambda *args, **kwargs: pytest.fail("must preflight first")
- )
- with pytest.raises(VectorStoreError, match="collection_not_found"):
- RawVectorStore.open_existing(store.directory, generation_id="absent", dimensions=3)
-
-
-def test_existing_dimension_mismatch_does_not_modify_collection(store):
- before = store._collection.metadata.copy()
- with pytest.raises(VectorStoreError):
- RawVectorStore.open_writer_existing(
- store.directory, generation_id="generation-a", dimensions=4
- )
- assert store._collection.metadata == before
- store.upsert([record()])
- store.verify([record()])
-
-
-def test_verify_candidates_reader_checks_every_batch_without_writing(store, monkeypatch):
- rows = [record(f"candidate-{i}") for i in range(2050)]
- store.upsert(rows)
- reader = RawVectorStore.open_existing(
- store.directory, generation_id="generation-a", dimensions=3
- )
- try:
- calls = []
- get = reader._collection.get
-
- def tracked(**kwargs):
- calls.append(kwargs)
- return get(**kwargs)
-
- monkeypatch.setattr(reader._collection, "get", tracked)
- for name in ("add", "upsert", "update", "delete", "query"):
- monkeypatch.setattr(
- reader._collection, name, lambda **kwargs: pytest.fail("unexpected backend call")
- )
- assert (
- reader.verify_candidates(
- [(row.chunk_id, row.build_id) for row in rows], deadline=monotonic() + 30
- )
- is True
- )
- assert [len(call["ids"]) for call in calls] == [2048, 2]
- assert all(call["include"] == ["embeddings", "metadatas"] for call in calls)
- finally:
- reader.close()
-
-
-def test_verify_candidates_empty_and_expired_do_not_read(store, monkeypatch):
- monkeypatch.setattr(store._collection, "get", lambda **kwargs: pytest.fail("unexpected read"))
- store._clock = lambda: 10.0
- assert store.verify_candidates([], deadline=0.0) is True
- assert store.verify_candidates([("chunk-a", "build-a")], deadline=10.0) is False
-
-
[email protected]("expires", ["after_read", "between_batches"])
-def test_verify_candidates_deadline_stops_before_another_batch(store, monkeypatch, expires):
- rows = [record(f"candidate-{i}") for i in range(2049)]
- store.upsert(rows)
- calls = []
- get = store._collection.get
-
- def tracked(**kwargs):
- calls.append(kwargs)
- return get(**kwargs)
-
- ticks = iter([0.0, 2.0] if expires == "after_read" else [0.0, 0.0, 2.0])
- store._clock = lambda: next(ticks)
- monkeypatch.setattr(store._collection, "get", tracked)
- assert (
- store.verify_candidates([(row.chunk_id, row.build_id) for row in rows], deadline=1.0)
- is False
- )
- assert len(calls) == 1 and len(calls[0]["ids"]) == 2048
-
-
[email protected]("pair", [("missing", "build-a"), ("chunk-a", "other-build")])
-def test_verify_candidates_missing_or_wrong_build_refuses(store, pair):
- store.upsert([record()])
- with pytest.raises(VectorStoreError, match="missing|identity_mismatch"):
- store.verify_candidates([pair], deadline=monotonic() + 10)
-
-
[email protected]("mutation", ["generation", "dimension", "hash", "extra_id"])
-def test_verify_candidates_corrupt_backend_record_refuses(store, monkeypatch, mutation):
- store.upsert([record()])
- get = store._collection.get
-
- def corrupt(**kwargs):
- data = get(**kwargs)
- if mutation == "generation":
- data["metadatas"][0]["generation_id"] = "other-generation"
- elif mutation == "dimension":
- data["embeddings"] = [[1.0, 0.0]]
- elif mutation == "hash":
- data["metadatas"][0]["vector_sha256"] = "0" * 64
- else:
- data["ids"][0] = "not-eligible"
- return data
-
- monkeypatch.setattr(store._collection, "get", corrupt)
- with pytest.raises(VectorStoreError):
- store.verify_candidates([("chunk-a", "build-a")], deadline=monotonic() + 10)
-
-
- "candidates",
- [
- [("chunk-a", "build-a"), ("chunk-a", "build-a")],
- [("chunk-a", "build-a"), ("chunk-a", "other")],
- [("../escape", "build-a")],
- [("chunk-a", "")],
- [("chunk-a", 123)],
- [("chunk-a",)],
- ["ab"],
- "ab",
- None,
- ],
-)
-def test_verify_candidates_invalid_input_refuses_before_read(store, monkeypatch, candidates):
- monkeypatch.setattr(store._collection, "get", lambda **kwargs: pytest.fail("unexpected read"))
- with pytest.raises(VectorStoreError):
- store.verify_candidates(candidates, deadline=monotonic() + 10)
-
-
[email protected]("deadline", [math.nan, math.inf, True, "soon"])
-def test_verify_candidates_invalid_deadline_refuses_even_empty(store, deadline):
- with pytest.raises(VectorStoreError, match="invalid_deadline"):
- store.verify_candidates([], deadline=deadline)