Python
refactor(memhub): remove the private retrieval vector index
alvinfeie5ce0cd96445
diff --git a/memhub/apps/cli/src/cli.ts b/memhub/apps/cli/src/cli.tsindex 4cdb1d6..cc30fd6 100644--- a/memhub/apps/cli/src/cli.ts+++ b/memhub/apps/cli/src/cli.ts@@ -4869,10 +4869,10 @@ async function runMaintenanceRestore(options: {const instanceId = randomUUID();const child = spawn(options.coreCommand, ["serve", "--maintenance"], {env: {- // Do not pass the caller's Runtime tuning variables (for example- // retrieval-index flags) into a maintenance process: a restore must- // start the product's minimal maintenance configuration, not inherit- // an unrelated experiment from the shell.+ // Do not pass the caller's Runtime tuning variables into a+ // maintenance process: a restore must start the product's minimal+ // maintenance configuration, not inherit an unrelated experiment from+ // the shell.PATH: process.env.PATH,HOME: process.env.HOME,LANG: process.env.LANG,diff --git a/memhub/apps/core/pyproject.toml b/memhub/apps/core/pyproject.tomlindex dda79b0..3e99cc7 100644--- a/memhub/apps/core/pyproject.toml+++ b/memhub/apps/core/pyproject.toml@@ -13,7 +13,8 @@ dependencies = ["filelock>=3.32.4",# Public SDK version validated against MemHub's permission and recall contracts."hy-memory==2.0.5",- # Retrieval's derived vector index directly consumes these locked versions.+ # chromadb arrives through the HY SDK; locked here so an unrelated upgrade+ # cannot move the SDK's store. Built-in embedding consumes numpy directly."chromadb==1.5.9","numpy==2.5.2","httpx>=0.28.1",diff --git a/memhub/apps/core/src/one_memory/api.py b/memhub/apps/core/src/one_memory/api.pyindex 19d462d..dcd75c5 100644--- a/memhub/apps/core/src/one_memory/api.py+++ b/memhub/apps/core/src/one_memory/api.py@@ -841,8 +841,6 @@ def create_app(secret_store: SecretStore | None = None,) -> FastAPI:runtime_settings = settings or load_settings()- if runtime_settings.retrieval_index_enabled and not manage_runtime:- raise ValueError("retrieval_runtime_ownership_required")@asynccontextmanagerasync def lifespan(_: FastAPI) -> AsyncIterator[None]:@@ -1456,22 +1454,7 @@ def create_app(app.state.resource_diagnostics = watched_task(asyncio.get_running_loop(), _emit_resource_snapshots)- app.state.retrieval_runtime = Nonetry:- if runtime_settings.retrieval_index_enabled:- from one_memory.retrieval.index_provisioning import provision_retrieval_index- from one_memory.retrieval.runtime import ExistingRetrievalRuntime-- # The runtime only resumes an index someone initialized. Turning- # the feature on is that someone; otherwise enabling it would- # leave every search on the lexical lane with nothing to say why.- await asyncio.to_thread(provision_retrieval_index, runtime_settings.data_dir)- app.state.retrieval_runtime = ExistingRetrievalRuntime(- runtime_settings.data_dir,- minimum_free_bytes=runtime_settings.retrieval_minimum_free_bytes,- service_levels=app.state.service_levels,- )- await app.state.retrieval_runtime.start()# Startup writes are done. Only now let the derived-content# rebuild take the single write checkout: it is a long# background job, and on a large library it would otherwise@@ -1498,15 +1481,6 @@ def create_app(outcome="started",instance_id=runtime_settings.instance_id,)- retrieval_stop_error: BaseException | None = None- if app.state.retrieval_runtime is not None:- try:- await app.state.retrieval_runtime.stop()- except (Exception, asyncio.CancelledError) as error:- # Preserve failure while cleaning other owned services.- # Experimental: an exceptional live-thread failure still- # needs an outer ownership fence before production enablement.- retrieval_stop_error = errorapp.state.main_database_report.cancel()app.state.loop_heartbeat.cancel()with suppress(asyncio.CancelledError):@@ -1564,8 +1538,6 @@ def create_app(# Waited for off the loop: this is the task whose whole point# is that the loop never waits on a disk.await asyncio.to_thread(flush_diagnostics)- if retrieval_stop_error is not None:- raise retrieval_stop_errorapp = FastAPI(title="MemHub Core",@@ -4481,9 +4453,8 @@ class _LazyASGIApp:`create_app()` must not run at import time. The managed `serve()` entryimports `create_app` from this module before it can construct its own appwith `manage_runtime=True`; an eager module-level app was therefore built- first with the default `manage_runtime=False`. With retrieval index enabled- that default app hit `retrieval_runtime_ownership_required` and killed the- process before the managed app could bind a port.+ first with the default `manage_runtime=False`, doing startup work for an+ app that never serves."""def __init__(self) -> None:diff --git a/memhub/apps/core/src/one_memory/config.py b/memhub/apps/core/src/one_memory/config.pyindex e68cb34..4f8543e 100644--- a/memhub/apps/core/src/one_memory/config.py+++ b/memhub/apps/core/src/one_memory/config.py@@ -23,10 +23,6 @@ class Settings(BaseSettings):live_sync_enabled: bool = Trueautomatic_memory_processing: bool = Trueimports_enabled: bool = True- # Experimental and unreleased: ownership/error shutdown still requires acceptance.- # Resume only explicitly initialized generations; never migrate at startup.- retrieval_index_enabled: bool = False- retrieval_minimum_free_bytes: int = Field(default=256 * 1024 * 1024, ge=0)# Bounded metadata only: query, original text and answers are never stored.call_event_retention_days: int | None = Field(default=30, ge=1, le=365)call_event_max_events: int | None = Field(default=10_000, ge=1, le=100_000)diff --git a/memhub/apps/core/src/one_memory/context_routes.py b/memhub/apps/core/src/one_memory/context_routes.pyindex 3096d6b..c434939 100644--- a/memhub/apps/core/src/one_memory/context_routes.py+++ b/memhub/apps/core/src/one_memory/context_routes.py@@ -38,7 +38,7 @@ from one_memory.context_models import (SdkMemoryContextItem,SdkRawContextItem,)-from one_memory.context_raw import revalidate_raw_context+from one_memory.context_raw import read_raw_context, revalidate_raw_contextfrom one_memory.context_search import search_contextfrom one_memory.diagnostic_log import diagnostic_event, diagnostic_exceptionfrom one_memory.external_memory_access import external_access@@ -50,7 +50,6 @@ from one_memory.hy_sdk_types import (parse_sdk_search,)from one_memory.imports.storage import EvidenceStore-from one_memory.retrieval.raw_lane import read_raw_indexed_contextfrom one_memory.semantic_deadline import hy_wait_budget, search_withinCALL_EVENT_RESPONSE_BUDGET_MS = 250@@ -99,12 +98,11 @@ def context_router(dependencies: Sequence[Depends]) -> APIRouter:read = asyncio.create_task(in_read_pool(raw_read_pool(),- read_raw_indexed_context,+ read_raw_context,database,value,source_ids=authority.raw_sources,deadline=lane_end,- index=getattr(request.app.state, "retrieval_runtime", None),))done, _ = await asyncio.wait({read}, timeout=max(0, lane_end - time.monotonic()))diff --git a/memhub/apps/core/src/one_memory/context_v2_coordinator.py b/memhub/apps/core/src/one_memory/context_v2_coordinator.pyindex f4ca101..14d9f55 100644--- a/memhub/apps/core/src/one_memory/context_v2_coordinator.py+++ b/memhub/apps/core/src/one_memory/context_v2_coordinator.py@@ -22,7 +22,7 @@ from dataclasses import dataclassfrom datetime import UTC, datetimefrom functools import partialfrom pathlib import Path-from typing import Any, Literal, Protocol, cast+from typing import Any, Literal, castfrom one_memory.background_work import interactive_read_poolfrom one_memory.context_expand import ExpansionAuthority, ExpansionRefSigner@@ -36,23 +36,8 @@ from one_memory.retrieval.models import (RawEvidence,RetrievalPath,RetrievalResponse,- TimeRange,)from one_memory.retrieval.query_plan import plan_local_query-from one_memory.retrieval.raw_candidates import RawCandidateBatch-from one_memory.retrieval.raw_evidence import materialize_raw_candidates---class RawIndexLookup(Protocol):- def lookup_raw(- self,- query: str,- *,- source_ids: frozenset[str],- deadline: float,- time_range: TimeRange | None = None,- ) -> tuple[RawCandidateBatch, bool]: ...-ExpandRefs = tuple[ExpansionRefSigner, ExpansionAuthority]@@ -106,11 +91,6 @@ class LaneResult:coverage_limited: bool = False-def _semantic_only_raw(item: RawEvidence) -> bool:- """Semantic overlap alone is a candidate, not a confirmed recall item."""- return set(item.matched_by) == {"semantic"}--@dataclass(frozen=True)class StageTimings:raw_ms: float = 0.0@@ -191,12 +171,6 @@ def _managed(return selected, truncated-def _group_missing_requirements(item: RawEvidence | GovernedEvidence) -> list[str]:- if isinstance(item, RawEvidence) and _semantic_only_raw(item):- return ["relevance_confirmation"]- return []--def _path(status: str,reason: str | None,@@ -296,8 +270,7 @@ def read_lexical_raw_lane(("lexical", "object") if plan.object_keys else ("lexical",),truncated,# Starvation means: this lane matched, then dropped the item whole. An- # ordinary no-match must stay empty, and an unconfigured semantic lane- # is not starvation.+ # ordinary no-match must stay empty.starved and not items,probe_saturated,)@@ -564,97 +537,6 @@ def warm_raw_read_path(return False-def read_semantic_raw_lane(- index: RawIndexLookup | None,- database_path: Path,- query: CoordinatorQuery,- *,- source_ids: frozenset[str],- deadline: float,-) -> LaneResult:- if not source_ids:- return LaneResult((), "unavailable", "permission_required", (), False)- # No derived index is the no-model baseline, not a failure: the lexical lane- # must still answer. Semantic is genuinely unsearched in that state, so it- # must not claim a channel or a coverage result it never produced.- if index is None:- return LaneResult((), "not_searched", None, (), False)- try:- batch, unpublished = index.lookup_raw(- query.query,- source_ids=source_ids,- deadline=deadline,- time_range=plan_local_query(query.query).time_range,- )- except Exception:- return LaneResult((), "unavailable", "index_unavailable", ("semantic",), False)- if not isinstance(batch, RawCandidateBatch):- return LaneResult((), "unavailable", "index_unavailable", ("semantic",), False)- if batch.reason == "not_authorized":- return LaneResult((), "unavailable", "permission_required", ("semantic",), False)- if batch.reason == "deadline":- return LaneResult((), "unavailable", "timeout_budget", ("semantic",), True)- if batch.reason in {"vector_unavailable", "index_unavailable"}:- return LaneResult((), "unavailable", batch.reason, ("semantic",), False)- store, generation = _index_state(index, database_path)- from one_memory.retrieval.index_store import RetrievalIndexStore-- store = cast("RetrievalIndexStore | None", store)- if store is None or generation is None:- return LaneResult((), "unavailable", "index_unavailable", ("semantic",), False)- try:- materialized = materialize_raw_candidates(- store,- batch,- generation_id=generation,- source_ids=source_ids,- deadline=deadline,- character_budget=query.character_budget,- )- except Exception:- return LaneResult((), "unavailable", "index_unavailable", ("semantic",), False)- if materialized.reason == "deadline":- return LaneResult((), "unavailable", "timeout_budget", ("semantic",), True)- if materialized.reason in {"index_unavailable", "not_authorized"}:- return LaneResult((), "unavailable", materialized.reason, ("semantic",), False)- complete = materialized.complete and not unpublished- if materialized.reason == "evidence_changed":- status: Literal["ok", "empty", "unavailable", "not_searched"] = "unavailable"- reason = "evidence_changed"- elif not materialized.items and not complete:- status, reason = "unavailable", "index_incomplete"- elif materialized.reason == "character_budget":- status, reason = "ok", None- else:- status, reason = ("ok" if materialized.items else "empty"), None- return LaneResult(- tuple(materialized.items),- status,- reason,- ("semantic",),- materialized.reason == "character_budget",- )---def _index_state(index: RawIndexLookup, database_path: Path) -> tuple[object | None, str | None]:- store = getattr(index, "_store", None)- generation = None- snapshot = getattr(index, "snapshot", None)- if snapshot is not None:- try:- generation = snapshot().generation_id- except Exception:- generation = None- if store is None:- try:- from one_memory.retrieval.index_store import RetrievalIndexStore-- store = RetrievalIndexStore(database_path)- except Exception:- store = None- return store, generation--def read_memory_lane(database_path: Path,query: CoordinatorQuery,@@ -857,7 +739,6 @@ def coordinate_context_v2(query: CoordinatorQuery,*,database_path: Path,- index: RawIndexLookup | None = None,deadline: float,expand_refs: ExpandRefs | None = None,) -> RetrievalResponse:@@ -871,62 +752,29 @@ def coordinate_context_v2(lexical = read_lexical_raw_lane(database_path, query, source_ids=authority.raw_sources, deadline=deadline)- semantic = read_semantic_raw_lane(- index, database_path, query, source_ids=authority.raw_sources, deadline=deadline- )timings = StageTimings(raw_ms=(time.monotonic() - raw_started) * 1000)combined: dict[tuple[str, str], RawEvidence] = {}- for item in (*lexical.items, *semantic.items):- if not isinstance(item, RawEvidence):- continue- identity = _identity(item)- previous = combined.get(identity)- if previous is None or _semantic_only_raw(previous):- combined[identity] = item- # Semantic similarity is a candidate signal, not a verified recall- # result. Keep semantic-only candidates visible in coverage, but do not- # release their bodies for host injection unless the item is- # independently confirmed by the lexical/object lane. A global distance- # threshold is deliberately not used: the measured positive and- # negative distributions overlap.- semantic_only = {_identity(item) for item in combined.values() if _semantic_only_raw(item)}- confirmed = {_identity(item) for item in lexical.items if isinstance(item, RawEvidence)}- deferred_raw = semantic_only - confirmed- if deferred_raw:- combined = {- identity: item- for identity, item in combined.items()- if identity not in deferred_raw- }- channels = tuple(dict.fromkeys((*lexical.channels, *semantic.channels)))+ for item in lexical.items:+ if isinstance(item, RawEvidence):+ combined.setdefault(_identity(item), item)if lexical.status == "unavailable":status: Literal["ok", "empty", "unavailable", "not_searched", "degraded"] = ("unavailable")reason = lexical.reason- elif semantic.status == "unavailable":- status = "unavailable"- reason = semantic.reason- elif combined:- status, reason = "ok", semantic.reason if semantic.status != "ok" else None- elif deferred_raw:- status, reason = "degraded", "relevance_confirmation"- elif lexical.status == "empty":- status, reason = ("empty" if semantic.status != "not_searched" else "ok"), None+ elif combined or lexical.status == "empty":+ status, reason = "ok", Noneelse:status, reason = "empty", None- truncated = lexical.truncated or semantic.truncatedraw = LaneResult(tuple(combined.values()),status,reason,- channels,- truncated,+ lexical.channels,+ lexical.truncated,lexical.starved,lexical.coverage_limited,)- if deferred_raw:- object.__setattr__(raw, "coverage_limited", True)elif "raw" in query.kinds and not authority.raw_allowed:raw = LaneResult((), "unavailable", "permission_required", (), False)if "memory" in query.kinds:@@ -995,9 +843,7 @@ def coordinate_context_v2(return _path("unavailable", raw.reason, raw.channels, "not_searched", False, 0)if dropped and not final_raw:return _path("unavailable", "permission_required", raw.channels, "unavailable", True, 0)- # Lexical-only is complete when the lexical lane actually answered. An- # unconfigured semantic lane is a capability absence, not proof that- # less than the authorized range was searched.+ # The raw lane is complete when the lexical lane actually answered.complete = (raw.status in {"ok", "empty"}and not dropped@@ -1022,9 +868,7 @@ def coordinate_context_v2("count_budget"if truncatedelse "coverage_budget"- if raw.coverage_limited and not deferred_raw- else "relevance_confirmation"- if deferred_raw+ if raw.coverage_limitedelse "character_budget")return _path(@@ -1112,7 +956,7 @@ def coordinate_context_v2("group_id": f"group-{index}","answer_target": query.intent,"decision_status": "missing_confirmation",- "missing_requirements": _group_missing_requirements(item),+ "missing_requirements": [],"object_refs": _object_refs(item, plan.object_keys),"items": [group_item.model_dump(mode="json") for group_item in (item,)],}@@ -1185,7 +1029,6 @@ async def search_context_v2(query: CoordinatorQuery,*,database_path: Path,- index: RawIndexLookup | None = None,deadline: float,expand_refs: ExpandRefs | None = None,) -> RetrievalResponse:@@ -1198,7 +1041,6 @@ async def search_context_v2(authority,query,database_path=database_path,- index=index,deadline=deadline,expand_refs=expand_refs,),diff --git a/memhub/apps/core/src/one_memory/local_embedding.py b/memhub/apps/core/src/one_memory/local_embedding.pyindex fd9aee7..26114c6 100644--- a/memhub/apps/core/src/one_memory/local_embedding.py+++ b/memhub/apps/core/src/one_memory/local_embedding.py@@ -10,16 +10,12 @@ weights alone.from __future__ import annotationsimport threading-from dataclasses import dataclass-from hashlib import sha256from importlib import import_modulefrom typing import TYPE_CHECKING, Anyfrom one_memory.embedding_assets import (BUILTIN_EMBEDDING_DIMENSIONS,BUILTIN_EMBEDDING_MAX_TOKENS,- EMBEDDING_ASSETS,- AssetVerificationError,MissingEmbeddingAssetsError,assets_present,resolve_asset_dir,@@ -34,28 +30,6 @@ _FETCH_HINT = ()-@dataclass(frozen=True)-class TokenOffset:- """Half-open Python Unicode code point offsets; synthetic specials use [0,0)."""-- start_cp: int- end_cp: int- special: bool = False---@dataclass(frozen=True)-class TokenizedText:- """Untruncated, unpadded encoding, including tokenizer postprocessor specials."""-- token_ids: tuple[int, ...]- offsets: tuple[TokenOffset, ...]-- def __post_init__(self) -> None:- # Defensive copies also keep runtime callers from retaining mutable lists.- object.__setattr__(self, "token_ids", tuple(self.token_ids))- object.__setattr__(self, "offsets", tuple(self.offsets))--class LocalEmbedder:"""Encodes text with the built-in model, loading it on first use."""@@ -68,43 +42,6 @@ class LocalEmbedder:self._session: Any | None = Noneself._tokenizer: Any | None = Noneself._input_names: frozenset[str] = frozenset()- self._offset_lock = threading.Lock()- self._offset_tokenizer: Any | None = None-- def tokenize_with_offsets(self, text: str) -> TokenizedText:- """Tokenize original text without encode()'s truncation or batch padding.-- The Python tokenizers binding returns code point offsets into the input,- even when its model normalizer changes spelling. This separate lazy- tokenizer uses the exact locked asset; it never loads an ONNX session or- mutates the tokenizer shared by encode(). Offsets are not UTF-16 indices.- """- if not isinstance(text, str):- raise TypeError("embedding_offsets_require_text")- with self._offset_lock:- if self._offset_tokenizer is None:- directory = resolve_asset_dir()- path = directory / "tokenizer.json"- if not path.is_file():- raise MissingEmbeddingAssetsError(f"{_FETCH_HINT} Looked in {directory}.")- data = path.read_bytes()- expected = next(a.sha256 for a in EMBEDDING_ASSETS if a.filename == path.name)- if sha256(data).hexdigest() != expected:- raise AssetVerificationError("tokenizer.json: locked digest mismatch")- from tokenizers import Tokenizer-- tokenizer = Tokenizer.from_str(data.decode("utf-8"))- tokenizer.no_truncation()- tokenizer.no_padding()- self._offset_tokenizer = tokenizer- encoding = self._offset_tokenizer.encode(text, add_special_tokens=True)- offsets = tuple(- TokenOffset(start, end, bool(special))- for (start, end), special in zip(- encoding.offsets, encoding.special_tokens_mask, strict=True- )- )- return TokenizedText(tuple(encoding.ids), offsets)def _load(self) -> tuple[Any, Any]:# Double-checked under a lock: memory processing encodes from workerdiff --git a/memhub/apps/core/src/one_memory/memory_server/migrations.py b/memhub/apps/core/src/one_memory/memory_server/migrations.pyindex e5fa270..ff129ae 100644--- a/memhub/apps/core/src/one_memory/memory_server/migrations.py+++ b/memhub/apps/core/src/one_memory/memory_server/migrations.py@@ -2217,34 +2217,6 @@ CREATE TABLE IF NOT EXISTS retrieval_chunks ("""-def retrieval_documents_v2_ddl(dialect: str = "mysql") -> str:- """The v2 documents table under its temporary `_v2` name.-- `index_store.migrate_v2` creates this, copies rows from the v1 table, drops- the v1 table and renames. The rename is the only reason a `_v2` name exists- at all, and it is derived from the single definition so the copy-across- cannot drift from what fresh databases get.- """- documents = next(- statement- for statement in _split_statements(retrieval_index_ddl(dialect))- if "CREATE TABLE IF NOT EXISTS retrieval_documents (" in statement- )- return documents.replace(- "CREATE TABLE IF NOT EXISTS retrieval_documents (",- "CREATE TABLE retrieval_documents_v2 (",- )---def retrieval_build_maps_ddl(dialect: str = "mysql") -> str:- """The build-maps table added by v2."""- return next(- statement- for statement in _split_statements(retrieval_index_ddl(dialect))- if "retrieval_build_maps" in statement- )--async def _033_retrieval_index(cursor: Any) -> None:for statement in _split_statements(retrieval_index_ddl(_dialect(cursor))):await cursor.execute(statement)diff --git a/memhub/apps/core/src/one_memory/retrieval/chunking.py b/memhub/apps/core/src/one_memory/retrieval/chunking.pydeleted file mode 100644index 9d59121..0000000--- a/memhub/apps/core/src/one_memory/retrieval/chunking.py+++ /dev/null@@ -1,167 +0,0 @@-"""Single-Evidence identity chunking; no reads, persistence, normalization or grants.--Callers must first establish eligibility/sanitization/permission. This module-does not strip wrappers or secrets and must not be used as a content safety gate.-"""--from dataclasses import dataclass-from typing import Protocol--from one_memory.embedding_assets import BUILTIN_EMBEDDING_MAX_TOKENS-from one_memory.local_embedding import TokenizedText, TokenOffset---class OffsetTokenizer(Protocol):- def tokenize_with_offsets(self, text: str) -> TokenizedText: ...---class ChunkingError(ValueError):- """Invalid offsets, uncovered text or insufficient encoding budget; never truncate."""---@dataclass(frozen=True)-class TextChunk:- ordinal: int- start_cp: int- end_cp: int- token_count: int- text: str---@dataclass(frozen=True)-class _Span:- start: int- end: int- tokens: int---def _spans(text: str, encoded: TokenizedText) -> tuple[_Span, ...]:- if not isinstance(encoded, TokenizedText) or len(encoded.token_ids) != len(encoded.offsets):- raise ChunkingError("chunk_token_offset_count_mismatch")- spans: list[_Span] = []- previous_start = previous_end = 0- for token_id, offset in zip(encoded.token_ids, encoded.offsets, strict=True):- if type(token_id) is not int or token_id < 0 or not isinstance(offset, TokenOffset):- raise ChunkingError("chunk_invalid_token_offset")- if (- type(offset.start_cp) is not int- or type(offset.end_cp) is not int- or type(offset.special) is not bool- ):- raise ChunkingError("chunk_invalid_token_offset")- start, end = offset.start_cp, offset.end_cp- if offset.special:- if start != 0 or end != 0:- raise ChunkingError("chunk_invalid_special_offset")- continue- if not 0 <= start < end <= len(text) or start < previous_start or end < previous_end:- raise ChunkingError("chunk_invalid_offset_order_or_bounds")- if start > previous_end and not text[previous_end:start].isspace():- raise ChunkingError("chunk_uncovered_nonwhitespace")- # Overlapping subpieces (including several pieces for one code point)- # form an indivisible interval; slicing inside it loses token identity.- if spans and start < spans[-1].end:- prior = spans[-1]- spans[-1] = _Span(prior.start, end, prior.tokens + 1)- else:- spans.append(_Span(start, end, 1))- previous_start, previous_end = start, end- if text[previous_end:] and not text[previous_end:].isspace():- raise ChunkingError("chunk_uncovered_nonwhitespace")- return tuple(spans)---def _encoding(text: str, tokenizer: OffsetTokenizer) -> tuple[TokenizedText, tuple[_Span, ...]]:- try:- result = tokenizer.tokenize_with_offsets(text)- except Exception as error:- raise ChunkingError("chunk_tokenization_failed") from error- return result, _spans(text, result)---def chunk_evidence(- text: str,- tokenizer: OffsetTokenizer,- *,- prefix: str = "",- target_tokens: int = 384,- overlap_tokens: int = 64,- max_tokens: int = BUILTIN_EMBEDDING_MAX_TOKENS,-) -> tuple[TextChunk, ...]:- """Chunk one original with code point [start,end), preserving exact slices.-- target/overlap count original non-special tokens. Overlap is up to the- requested token count at indivisible offset boundaries; forward progress is- mandatory. token_count is re-tokenized prefix + chunk.text INCLUDING specials,- never an additive estimate. Prefix is caller-provided, unchanged, and omitted- from chunk.text/offsets. Empty/whitespace-only inputs have no index chunks.- """- if not isinstance(text, str) or not isinstance(prefix, str):- raise ChunkingError("chunk_requires_text")- if (- any(type(value) is not int for value in (target_tokens, overlap_tokens, max_tokens))- or not 0 <= overlap_tokens < target_tokens- or not 1 <= max_tokens <= BUILTIN_EMBEDDING_MAX_TOKENS- or target_tokens > max_tokens- ):- raise ChunkingError("chunk_invalid_token_budget")- if not text or text.isspace():- return ()- _, spans = _encoding(text, tokenizer)- if not spans:- raise ChunkingError("chunk_uncovered_nonwhitespace")- chunks: list[TextChunk] = []- start_index = 0- previous_end_cp = 0- while start_index < len(spans):- end_index = start_index- body_tokens = 0- while end_index < len(spans):- candidate = body_tokens + spans[end_index].tokens- if end_index > start_index and candidate > target_tokens:- break- body_tokens = candidate- end_index += 1- # Prefer the last paragraph boundary in the latter half of the target.- # Never trade away an entire overlap window or split a token interval.- if end_index < len(spans):- count = 0- preferred = None- for boundary in range(start_index + 1, end_index + 1):- count += spans[boundary - 1].tokens- if count >= max(target_tokens // 2, overlap_tokens + 1):- left, right = spans[boundary - 1].end, spans[boundary].start- if "\n" in text[max(spans[boundary - 1].start, left - 1) : right]:- preferred = boundary- if preferred is not None:- end_index = preferred- start_cp = 0 if start_index == 0 else spans[start_index].start- while end_index > start_index:- # Include gaps verbatim, including whitespace and combining marks;- # source coverage is never reconstructed by joining decoded tokens.- end_cp = spans[end_index].start if end_index < len(spans) else len(text)- encoded, _ = _encoding(prefix + text[start_cp:end_cp], tokenizer)- token_count = len(encoded.token_ids)- if token_count <= max_tokens:- break- end_index -= 1- else:- raise ChunkingError("chunk_indivisible_span_exceeds_token_budget")- if end_cp <= previous_end_cp or start_cp > previous_end_cp:- raise ChunkingError("chunk_coverage_cannot_progress")- chunks.append(TextChunk(len(chunks), start_cp, end_cp, token_count, text[start_cp:end_cp]))- previous_end_cp = end_cp- if end_index == len(spans):- break- next_start = end_index- overlap = 0- while next_start > start_index + 1:- candidate = overlap + spans[next_start - 1].tokens- if candidate > overlap_tokens:- break- next_start -= 1- overlap = candidate- start_index = next_start- if previous_end_cp != len(text):- raise ChunkingError("chunk_incomplete_coverage")- return tuple(chunks)diff --git a/memhub/apps/core/src/one_memory/retrieval/content.py b/memhub/apps/core/src/one_memory/retrieval/content.pydeleted file mode 100644index 2c1aa68..0000000--- a/memhub/apps/core/src/one_memory/retrieval/content.py+++ /dev/null@@ -1,92 +0,0 @@-"""Shared qualification for identity-mapped raw indexing and candidate reads."""--from collections.abc import Sequence--from one_memory.conversation_processing import _is_hy_input_turn-from one_memory.imports.models import EvidenceContentKind, EvidenceRecord-from one_memory.imports.sources.parsing import is_host_generated_user_turn, strip_host_wrappers-from one_memory.input_sanitization import sanitize_model_input-from one_memory.retrieval.chunking import TextChunk-from one_memory.retrieval.mapped_content import PreparedMappedContent, prepare_mapped_content---def identity_content_ready(record: EvidenceRecord) -> bool:- # Reuse pure existing hygiene checks, without invoking HY. Nonidentity- # cleaning requires a source map and remains explicitly uncovered for now.- return (- record.content_kind is EvidenceContentKind.RAW_CONVERSATION- and record.role in {"user", "assistant"}- and _is_hy_input_turn(record)- and not is_host_generated_user_turn(record.verbatim)- and strip_host_wrappers(record.verbatim) == record.verbatim.strip()- and sanitize_model_input(record.verbatim) == record.verbatim- )---def index_content(- record: EvidenceRecord, *, normalizer_version: int, chunker_version: int-) -> PreparedMappedContent | None:- """Generation-owned content semantics, never silently upgrade identity generations."""- if (normalizer_version, chunker_version) == (1, 1):- if not identity_content_ready(record):- raise ValueError("content_mapping_required")- return None- if (normalizer_version, chunker_version) == (2, 2):- return prepare_mapped_content(record)- raise ValueError("unsupported_content_generation")---def validate_chunk_coverage(- record: EvidenceRecord, chunks: Sequence[TextChunk], prepared: PreparedMappedContent | None-) -> None:- """Verify exact slices and complete citation-safe coverage without a tokenizer call."""- if len(chunks) > 8192 or (prepared is None and not chunks):- raise ValueError("invalid_chunk_manifest")- if prepared is None:- ranges = [(0, len(record.verbatim))]- else:- ranges = []- for span in prepared.metadata.segments:- if span.kind == "copy":- assert span.source_start_cp is not None and span.source_end_cp is not None- ranges.append((span.source_start_cp, span.source_end_cp))- range_index = 0- covered_end = ranges[0][0] if ranges else 0- previous_end = -1- for index, chunk in enumerate(chunks):- if not isinstance(chunk, TextChunk) or any(- type(value) is not int- for value in (chunk.ordinal, chunk.start_cp, chunk.end_cp, chunk.token_count)- ):- raise ValueError("invalid_chunk_manifest")- while range_index < len(ranges) and chunk.start_cp >= ranges[range_index][1]:- if record.verbatim[covered_end : ranges[range_index][1]].strip():- raise ValueError("incomplete_chunk_manifest")- range_index += 1- covered_end = ranges[range_index][0] if range_index < len(ranges) else 0- if range_index >= len(ranges):- raise ValueError("invalid_chunk_manifest")- left, right = ranges[range_index]- if (- chunk.ordinal != index- or not left <= chunk.start_cp < chunk.end_cp <= right- or chunk.end_cp <= previous_end- or not 1 <= chunk.token_count <= 512- or record.verbatim[chunk.start_cp : chunk.end_cp] != chunk.text- or (- chunk.start_cp > covered_end- and (prepared is None or record.verbatim[covered_end : chunk.start_cp].strip())- )- ):- raise ValueError("invalid_chunk_manifest")- covered_end = previous_end = chunk.end_cp- while range_index < len(ranges):- if prepared is None:- if covered_end != ranges[range_index][1]:- raise ValueError("incomplete_chunk_manifest")- elif record.verbatim[covered_end : ranges[range_index][1]].strip():- raise ValueError("incomplete_chunk_manifest")- range_index += 1- covered_end = ranges[range_index][0] if range_index < len(ranges) else 0- if prepared is not None and (not chunks) != (prepared.state == "no_indexable_content"):- raise ValueError("invalid_chunk_manifest")diff --git a/memhub/apps/core/src/one_memory/retrieval/index_provisioning.py b/memhub/apps/core/src/one_memory/retrieval/index_provisioning.pydeleted file mode 100644index 2073949..0000000--- a/memhub/apps/core/src/one_memory/retrieval/index_provisioning.py+++ /dev/null@@ -1,96 +0,0 @@-"""Publish one active generation for the built-in encoder, when asked to.--The runtime deliberately only resumes an index somebody already initialized, so-without this the feature could be switched on and still never index anything.-Provisioning stays separate from `start()` for the same reason it exists: it is-the one place allowed to create, and it creates only for a first enable.-"""--from __future__ import annotations--import logging-import sqlite3-from pathlib import Path--from one_memory.database_access import main_database-from one_memory.embedding_assets import assets_present-from one_memory.retrieval.index_store import RetrievalIndexStore-from one_memory.retrieval.index_worker import LocalRetrievalEncoder-from one_memory.retrieval.vector_store import RawVectorStore--DATABASE_NAME = "one-memory.sqlite3"-VECTOR_DIR_NAME = "retrieval-vectors"---def provision_retrieval_index(data_dir: Path) -> str | None:- """The active generation id for the built-in encoder, creating it once.-- Returns None whenever the right answer is to leave the data alone: no- built-in encoder, no database yet, an index from an older schema, or one- already published for a different encoder. None is not an error; the- lexical lane works without an index and says so through `degraded`.- """- database = data_dir / DATABASE_NAME- if not assets_present() or not database.is_file():- return None- identity = LocalRetrievalEncoder().identity- store = RetrievalIndexStore(database)- try:- with main_database(database).write() as db, db:- db.execute("BEGIN IMMEDIATE")- if (existing := _published(db)) is not None:- return existing if _matches(store, db, existing, identity) else None- version = _schema_version(db)- if version == 1:- # Someone's v1 index. Migrating it is maintenance, not startup.- return None- if version is None:- store.initialize_v2(db)- generation = store.create_generation(db, identity)- store.activate_generation(generation)- _ensure_vectors(data_dir / VECTOR_DIR_NAME, generation, identity.dimensions)- except Exception:- logging.getLogger(__name__).warning("retrieval_index_provision_failed", exc_info=True)- return None- return generation---def _schema_version(db: sqlite3.Connection) -> int | None:- if not db.execute(- "SELECT 1 FROM sqlite_master WHERE type='table' AND name='retrieval_index_state'"- ).fetchone():- return None- row = db.execute(- "SELECT schema_version FROM retrieval_index_state WHERE singleton=1"- ).fetchone()- return None if row is None else int(row[0])---def _published(db: sqlite3.Connection) -> str | None:- if not db.execute(- "SELECT 1 FROM sqlite_master WHERE type='table' AND name='retrieval_index_generations'"- ).fetchone():- return None- rows = db.execute(- "SELECT generation_id FROM retrieval_index_generations WHERE state='active' LIMIT 2"- ).fetchall()- return str(rows[0][0]) if len(rows) == 1 else None---def _matches(- store: RetrievalIndexStore,- db: sqlite3.Connection,- generation: str,- identity: object,-) -> bool:- try:- return store._identity(db, generation) == identity- except Exception:- return False---def _ensure_vectors(directory: Path, generation: str, dimensions: int) -> None:- if directory.exists():- return- vectors = RawVectorStore.create(directory, generation_id=generation, dimensions=dimensions)- vectors.close()diff --git a/memhub/apps/core/src/one_memory/retrieval/index_store.py b/memhub/apps/core/src/one_memory/retrieval/index_store.pydeleted file mode 100644index 388f434..0000000--- a/memhub/apps/core/src/one_memory/retrieval/index_store.py+++ /dev/null@@ -1,856 +0,0 @@-"""Explicit derived-index transactions. No constructor migration or model invocation."""--import hashlib-import json-import math-import sqlite3-import time-from collections.abc import Callable, Iterator, Sequence-from contextlib import contextmanager-from dataclasses import asdict, dataclass, replace-from datetime import UTC, datetime-from pathlib import Path-from uuid import uuid4--from one_memory import ingest_journal as journal-from one_memory.context_raw import _check, _connection-from one_memory.database import take_the_write_lock_first-from one_memory.database_access import main_database-from one_memory.imports.models import EvidenceRecord-from one_memory.imports.storage import EvidenceStore-from one_memory.memory_server.migrations import (- _split_statements,- retrieval_build_maps_ddl,- retrieval_documents_v2_ddl,- retrieval_index_ddl,-)-from one_memory.retrieval.chunking import TextChunk-from one_memory.retrieval.content import index_content, validate_chunk_coverage-from one_memory.retrieval.mapped_content import ContentMap, PreparedMappedContent-from one_memory.retrieval.models import TimeRange---def digest(value: object) -> str:- return hashlib.sha256(- json.dumps(- value, sort_keys=True, ensure_ascii=False, separators=(",", ":"), allow_nan=False- ).encode()- ).hexdigest()---@dataclass(frozen=True)-class EncoderIdentity:- encoder_id: str- encoder_digest: str- tokenizer_digest: str- dimensions: int- chunker_version: int = 1- normalizer_version: int = 1- query_template_version: int = 1-- def __post_init__(self) -> None:- if not self.encoder_id or len(self.encoder_id) > 128:- raise ValueError("invalid_encoder")- for value in (self.encoder_digest, self.tokenizer_digest):- if len(value) != 64 or any(c not in "0123456789abcdef" for c in value):- raise ValueError("invalid_encoder_digest")- for number in (- self.dimensions,- self.chunker_version,- self.normalizer_version,- self.query_template_version,- ):- if type(number) is not int or not 1 <= number <= 1000000:- raise ValueError("invalid_encoder_number")---@dataclass(frozen=True)-class IndexLease:- generation_id: str- evidence_id: str- build_id: str- event_seq: int- owner: str- token: str- expires_at: float- grant_fingerprint: str---@dataclass(frozen=True)-class PublishedChunk:- chunk_id: str- build_id: str- evidence_id: str- revision: int- start_cp: int- end_cp: int- ordinal: int- source_id: str- event_time: datetime | None---def chunk_identity(lease: IndexLease, chunk: TextChunk, map_digest: str | None = None) -> str:- values = [lease.build_id, chunk.ordinal, chunk.start_cp, chunk.end_cp, digest(chunk.text)]- return digest(values if map_digest is None else [*values, map_digest])---@contextmanager-def transaction(db: sqlite3.Connection) -> Iterator[None]:- if not db.in_transaction:- raise RuntimeError("retrieval_requires_caller_transaction")- name = "retrieval_" + uuid4().hex- previous = db.row_factory- db.row_factory = sqlite3.Row- db.execute(f"SAVEPOINT {name}")- try:- yield- except BaseException:- db.execute(f"ROLLBACK TO {name}")- db.execute(f"RELEASE {name}")- raise- else:- db.execute(f"RELEASE {name}")- finally:- db.row_factory = previous---class RetrievalIndexStore:- def __init__(self, database_path: Path):- self.database_path = database_path-- @contextmanager- def _db(- self, *, write: bool = False, deadline: float | None = None- ) -> Iterator[sqlite3.Connection]:- if deadline is not None:- if write:- raise ValueError("deadline_connection_is_read_only")- with _connection(self.database_path, deadline) as db:- db.execute("PRAGMA query_only=ON")- yield db- return- database = main_database(self.database_path)- with (- database.write(create=False, foreign_keys=True, row_factory=sqlite3.Row)- if write- else database.read(foreign_keys=True, row_factory=sqlite3.Row)- ) as db:- try:- if write:- take_the_write_lock_first(db)- else:- db.execute("PRAGMA query_only=ON")- db.execute("BEGIN")- yield db- if write:- db.commit()- except BaseException:- db.rollback()- raise-- def initialize(self, db: sqlite3.Connection) -> None:- """Caller owns maintenance/exclusive writer admission; rollback includes baseline."""- self._initialize(db, version=1)-- def initialize_v2(self, db: sqlite3.Connection) -> None:- """Initialize a fresh store directly at the folded v2 schema.-- Startup provisioning uses this path so a brand-new database does not- create a v1 schema and immediately migrate it. Explicit maintenance- callers keep `initialize()`'s v1 baseline contract instead.- """- self._initialize(db, version=2)-- def _initialize(self, db: sqlite3.Connection, *, version: int) -> None:- with transaction(db):- exists = db.execute(- "SELECT 1 FROM sqlite_master WHERE name='retrieval_index_state'"- ).fetchone()- if exists:- self._store_id(db)- elif db.execute(- "SELECT 1 FROM sqlite_master WHERE name IN ('retrieval_documents',"- "'retrieval_chunks','retrieval_index_generations','retrieval_consumers')"- ).fetchone():- raise ValueError("retrieval_partial_initialization")- journal.initialize(db)- # Migration 33 owns the schema. Explicit initialization keeps the- # maintenance-facing v1 baseline so `migrate_v2` remains observable;- # startup provisioning may instead ask for the folded v2 directly.- for sql in _split_statements(retrieval_index_ddl("sqlite", version=version)):- db.execute(sql)- row = db.execute(- "SELECT schema_version FROM retrieval_index_state WHERE singleton=1"- ).fetchone()- if row is None:- db.execute(- "INSERT INTO retrieval_index_state VALUES (1,?,?)",- (version, uuid4().hex),- )- elif row[0] not in (1, 2):- raise ValueError("retrieval_schema_mismatch")-- def migrate_v2(self, db: sqlite3.Connection) -> None:- """Explicit maintenance transaction only; readers/worker never invoke this."""- with transaction(db):- self._store_id(db)- version = db.execute("SELECT schema_version FROM retrieval_index_state").fetchone()[0]- if version == 2:- db.execute(- "SELECT build_id,source_map_json,map_digest,coverage_state "- "FROM retrieval_build_maps LIMIT 0"- )- return- if (- db.execute(- "SELECT 1 FROM sqlite_master WHERE type='table' AND name='retrieval_build_maps'"- ).fetchone()- is not None- ):- # A v1 index must not already own a v2-only table. Treat this- # as a foreign/partial schema instead of letting `IF NOT- # EXISTS` silently accept a table with the wrong columns.- raise sqlite3.OperationalError("retrieval_build_maps already exists")- # Both tables come from migration 33, named/derived there so this- # module holds no DDL text of its own.- db.execute(retrieval_documents_v2_ddl("sqlite"))- db.execute("INSERT INTO retrieval_documents_v2 SELECT * FROM retrieval_documents")- db.execute("DROP TABLE retrieval_documents")- db.execute("ALTER TABLE retrieval_documents_v2 RENAME TO retrieval_documents")- db.execute(- "CREATE INDEX retrieval_jobs ON retrieval_documents(generation_id,state,event_seq)"- )- db.execute(retrieval_build_maps_ddl("sqlite"))- db.execute("UPDATE retrieval_index_state SET schema_version=2 WHERE singleton=1")-- @staticmethod- def _store_id(db: sqlite3.Connection) -> str:- row = db.execute(- "SELECT schema_version,data_store_id FROM retrieval_index_state WHERE singleton=1"- ).fetchone()- if row is None or row[0] not in (1, 2) or not isinstance(row[1], str) or len(row[1]) != 32:- raise ValueError("retrieval_not_initialized")- return str(row[1])-- def create_generation(self, db: sqlite3.Connection, identity: EncoderIdentity) -> str:- with transaction(db):- self._check_content_version(db, identity)- key = digest([self._store_id(db), asdict(identity)])- encoded = json.dumps(asdict(identity), sort_keys=True)- row = db.execute(- "SELECT identity_json FROM retrieval_index_generations WHERE generation_id=?",- (key,),- ).fetchone()- if row is not None and row[0] != encoded:- raise ValueError("generation_identity_changed")- if row is None:- db.execute(- "INSERT INTO retrieval_index_generations VALUES (?,?,'building',?)",- (key, encoded, time.time()),- )- db.execute("INSERT INTO retrieval_consumers VALUES (?,0,0)", (key,))- return key-- def consume(self, db: sqlite3.Connection, generation_id: str, *, limit: int = 256) -> int:- if type(limit) is not int or not 1 <= limit <= 256:- raise ValueError("invalid_consume_limit")- with transaction(db):- store_id = self._store_id(db)- generation = db.execute(- "SELECT state FROM retrieval_index_generations WHERE generation_id=?",- (generation_id,),- ).fetchone()- if generation is None or generation[0] == "retired":- raise ValueError("generation_not_writable")- cursor = db.execute(- "SELECT after_seq FROM retrieval_consumers WHERE generation_id=?", (generation_id,)- ).fetchone()- if cursor is None:- raise ValueError("consumer_missing")- rows = db.execute(- "SELECT seq FROM evidence_ingest_events WHERE seq>? ORDER BY seq LIMIT ?",- (cursor[0], limit),- ).fetchall()- for row in rows:- event = journal.event(db, row[0])- if event is None:- raise ValueError("journal_event_missing")- if event.user_id != "local":- continue- build = digest(- [- store_id,- generation_id,- event.evidence_id,- event.incarnation,- event.evidence_revision,- event.fingerprint,- ]- )- state = "deleted" if event.kind == "delete" else "pending"- db.execute(- """INSERT INTO retrieval_documents- (generation_id,evidence_id,source_id,event_seq,build_id,state)- VALUES (?,?,?,?,?,?) ON CONFLICT(generation_id,evidence_id) DO UPDATE SET- source_id=excluded.source_id,event_seq=excluded.event_seq,build_id=excluded.build_id,- state=excluded.state,active_build_id=NULL,lease_owner=NULL,lease_token=NULL,- lease_until=NULL,grant_fingerprint=NULL- WHERE excluded.event_seq>retrieval_documents.event_seq""",- (generation_id, event.evidence_id, event.source_id, event.seq, build, state),- )- if rows:- db.execute(- "UPDATE retrieval_consumers SET after_seq=? WHERE generation_id=?",- (rows[-1][0], generation_id),- )- return len(rows)-- def consume_pending(self, generation_id: str, *, limit: int = 256) -> int:- with self._db(write=True) as db:- return self.consume(db, generation_id, limit=limit)-- def encoder_identity(self, generation_id: str) -> EncoderIdentity:- with self._db() as db:- return self._identity(db, generation_id)-- @staticmethod- def _check_content_version(db: sqlite3.Connection, identity: EncoderIdentity) -> None:- versions = (identity.normalizer_version, identity.chunker_version)- if versions not in ((1, 1), (2, 2)):- raise ValueError("unsupported_content_generation")- if versions == (2, 2):- row = db.execute(- "SELECT schema_version FROM retrieval_index_state WHERE singleton=1"- ).fetchone()- if row is None or row[0] != 2:- raise ValueError("retrieval_schema2_required")-- def _identity(self, db: sqlite3.Connection, generation_id: str) -> EncoderIdentity:- row = db.execute(- "SELECT identity_json FROM retrieval_index_generations WHERE generation_id=?",- (generation_id,),- ).fetchone()- if row is None:- raise ValueError("generation_missing")- identity = EncoderIdentity(**json.loads(row[0]))- if digest([self._store_id(db), asdict(identity)]) != generation_id:- raise ValueError("generation_identity_changed")- self._check_content_version(db, identity)- return identity-- def _prepare(- self, db: sqlite3.Connection, generation_id: str, record: EvidenceRecord- ) -> PreparedMappedContent | None:- identity = self._identity(db, generation_id)- return index_content(- record,- normalizer_version=identity.normalizer_version,- chunker_version=identity.chunker_version,- )-- @staticmethod- def _grant(db: sqlite3.Connection, source_id: str) -> str | None:- row = db.execute(- """SELECT root_path,candidate_kind,granted_at,root_identity,revision- FROM source_grants- WHERE user_id='local' AND source_id=? AND revoked_at IS NULL""",- (source_id,),- ).fetchone()- return digest(tuple(row)) if row and row[3] and row[4] > 0 else None-- def claim(- self, generation_id: str, owner: str, *, now: float | None = None, lease_seconds: int = 60- ) -> IndexLease | None:- now = time.time() if now is None else now- if not owner or len(owner) > 128 or not math.isfinite(now) or not 1 <= lease_seconds <= 300:- raise ValueError("invalid_lease")- with self._db(write=True) as db:- generation = db.execute(- "SELECT state FROM retrieval_index_generations WHERE generation_id=?",- (generation_id,),- ).fetchone()- if generation is None or generation[0] == "retired":- raise ValueError("generation_not_writable")- rows = db.execute(- """SELECT * FROM retrieval_documents WHERE generation_id=? AND- (state='pending' OR (state='leased' AND lease_until<=?))- ORDER BY event_seq LIMIT 256""",- (generation_id, now),- ).fetchall()- for row in rows:- event = journal.event(db, row["event_seq"])- grant = self._grant(db, row["source_id"])- if event is None or grant is None or not journal.matches_current(db, event):- db.execute(- "UPDATE retrieval_documents SET state='blocked',active_build_id=NULL "- "WHERE generation_id=? AND evidence_id=?",- (generation_id, row["evidence_id"]),- )- continue- token = uuid4().hex- until = now + lease_seconds- db.execute(- """UPDATE retrieval_documents SET state='leased',lease_owner=?,lease_token=?,- lease_until=?,grant_fingerprint=?,attempts=attempts+1- WHERE generation_id=? AND evidence_id=?""",- (owner, token, until, grant, generation_id, row["evidence_id"]),- )- return IndexLease(- generation_id,- row["evidence_id"],- row["build_id"],- row["event_seq"],- owner,- token,- until,- grant,- )- return None-- def _validate_lease(- self, db: sqlite3.Connection, lease: IndexLease, now: float- ) -> journal.IngestEvent:- if not math.isfinite(now):- raise ValueError("invalid_lease_time")- generation = db.execute(- "SELECT state FROM retrieval_index_generations WHERE generation_id=?",- (lease.generation_id,),- ).fetchone()- if generation is None or generation[0] == "retired":- raise ValueError("lease_generation_retired")- row = db.execute(- "SELECT * FROM retrieval_documents WHERE generation_id=? AND evidence_id=?",- (lease.generation_id, lease.evidence_id),- ).fetchone()- if (- row is None- or row["state"] != "leased"- or (- row["build_id"],- row["event_seq"],- row["lease_owner"],- row["lease_token"],- row["lease_until"],- row["grant_fingerprint"],- )- != (- lease.build_id,- lease.event_seq,- lease.owner,- lease.token,- lease.expires_at,- lease.grant_fingerprint,- )- or lease.expires_at <= now- ):- raise ValueError("lease_changed_or_expired")- event = journal.event(db, lease.event_seq)- if (- event is None- or event.user_id != "local"- or event.evidence_id != lease.evidence_id- or not journal.matches_current(db, event)- ):- raise ValueError("lease_evidence_changed")- if self._grant(db, event.source_id) != lease.grant_fingerprint:- raise ValueError("lease_source_authority_changed")- return event-- def read_lease(self, lease: IndexLease, *, now: float | None = None) -> EvidenceRecord:- with self._db() as db:- self._validate_lease(db, lease, time.time() if now is None else now)- record = EvidenceStore._get_with_connection(db, lease.evidence_id)- if record is None or len(record.verbatim) > 2_000_000:- raise ValueError("lease_input_unavailable_or_too_large")- return record-- def renew(- self, lease: IndexLease, *, now: float | None = None, lease_seconds: int = 60- ) -> IndexLease:- now = time.time() if now is None else now- if type(lease_seconds) is not int or not 1 <= lease_seconds <= 300:- raise ValueError("invalid_lease")- with self._db(write=True) as db:- self._validate_lease(db, lease, now)- expires = max(lease.expires_at, now + lease_seconds)- db.execute(- "UPDATE retrieval_documents SET lease_until=? "- "WHERE generation_id=? AND evidence_id=?",- (expires, lease.generation_id, lease.evidence_id),- )- return replace(lease, expires_at=expires)-- def block(self, lease: IndexLease) -> None:- """Release only this claim; preserve uncertain derived vectors for later cleanup."""- with self._db(write=True) as db:- db.execute(- """UPDATE retrieval_documents SET state='blocked',active_build_id=NULL,- lease_owner=NULL,lease_token=NULL,lease_until=NULL- WHERE generation_id=? AND evidence_id=? AND state='leased'- AND build_id=? AND lease_token=? AND lease_owner=?""",- (lease.generation_id, lease.evidence_id, lease.build_id, lease.token, lease.owner),- )-- def retry_blocked(self, generation_id: str) -> int:- """Bounded, durable round-robin retry/regrant pass; never creates grants."""- changed = 0- with self._db(write=True) as db:- generation = db.execute(- "SELECT g.state,c.retry_after_seq FROM retrieval_index_generations g "- "JOIN retrieval_consumers c USING(generation_id) WHERE generation_id=?",- (generation_id,),- ).fetchone()- if generation is None or generation[0] == "retired":- raise ValueError("generation_not_writable")- rows = db.execute(- "SELECT * FROM retrieval_documents WHERE generation_id=? "- "AND state IN ('blocked','ready','excluded') AND event_seq>? "- "ORDER BY event_seq LIMIT 256",- (generation_id, generation[1]),- ).fetchall()- for row in rows:- event = journal.event(db, row["event_seq"])- grant = self._grant(db, row["source_id"])- if row["state"] in ("ready", "excluded") and grant == row["grant_fingerprint"]:- continue- if (- event- and event.source_id == row["source_id"]- and grant- and journal.matches_current(db, event)- ):- db.execute(- "UPDATE retrieval_documents SET state='pending',active_build_id=NULL,"- "lease_owner=NULL,lease_token=NULL,lease_until=NULL,grant_fingerprint=NULL "- "WHERE generation_id=? AND evidence_id=?",- (generation_id, row["evidence_id"]),- )- changed += 1- db.execute(- "UPDATE retrieval_consumers SET retry_after_seq=? WHERE generation_id=?",- (rows[-1]["event_seq"] if len(rows) == 256 else 0, generation_id),- )- return changed-- def publish(- self,- lease: IndexLease,- chunks: Sequence[TextChunk],- *,- vectors_verified: bool,- now: float | None = None,- source_map: ContentMap | None = None,- checkpoint: Callable[[], None] | None = None,- ) -> tuple[str, ...]:- """Caller verifies actual immutable vectors before entering this short transaction."""- if vectors_verified is not True:- raise ValueError("vectors_not_verified")- now = time.time() if now is None else now- if not math.isfinite(now):- raise ValueError("invalid_lease_time")- if len(chunks) > 8192:- raise ValueError("invalid_chunk_manifest")- with self._db(write=True) as db:- if checkpoint is not None:- checkpoint()- event = self._validate_lease(db, lease, now)- record = EvidenceStore._get_with_connection(db, lease.evidence_id)- if record is None or len(record.verbatim) > 2_000_000:- raise ValueError("lease_evidence_missing")- prepared = self._prepare(db, lease.generation_id, record)- if source_map != (prepared.metadata if prepared is not None else None):- raise ValueError("source_map_mismatch")- validate_chunk_coverage(record, chunks, prepared)- map_digest = source_map.digest if source_map is not None else None- keys = []- manifest = []- for index, chunk in enumerate(chunks):- text_hash = digest(chunk.text)- key = chunk_identity(lease, chunk, map_digest)- keys.append(key)- manifest.append(- (- key,- lease.generation_id,- lease.evidence_id,- lease.build_id,- index,- chunk.start_cp,- chunk.end_cp,- chunk.token_count,- text_hash,- )- )- existing = db.execute(- "SELECT * FROM retrieval_chunks WHERE build_id=? ORDER BY ordinal",- (lease.build_id,),- ).fetchall()- if existing:- if [tuple(row) for row in existing] != manifest:- raise ValueError("immutable_chunk_manifest_conflict")- else:- db.executemany("INSERT INTO retrieval_chunks VALUES (?,?,?,?,?,?,?,?,?)", manifest)- state = "ready" if chunks else "excluded"- if source_map is not None:- values = (- lease.build_id,- source_map.canonical_json(),- map_digest,- "indexable" if chunks else "excluded",- )- prior = db.execute(- "SELECT * FROM retrieval_build_maps WHERE build_id=?", (lease.build_id,)- ).fetchone()- if prior is not None and tuple(prior) != values:- raise ValueError("immutable_source_map_conflict")- if prior is None:- db.execute("INSERT INTO retrieval_build_maps VALUES (?,?,?,?)", values)- if not journal.matches_current(db, event):- raise ValueError("lease_evidence_changed")- db.execute(- "UPDATE retrieval_documents SET state=?,active_build_id=build_id,"- "lease_owner=NULL,lease_token=NULL,lease_until=NULL "- "WHERE generation_id=? AND evidence_id=?",- (state, lease.generation_id, lease.evidence_id),- )- if checkpoint is not None:- checkpoint()- return tuple(keys)-- def activate_generation(self, generation_id: str) -> None:- with self._db(write=True) as db:- row = db.execute(- "SELECT state FROM retrieval_index_generations WHERE generation_id=?",- (generation_id,),- ).fetchone()- cursor = db.execute(- "SELECT after_seq FROM retrieval_consumers WHERE generation_id=?", (generation_id,)- ).fetchone()- if (- row is None- or row[0] == "retired"- or cursor is None- or cursor[0] != journal.watermark(db)- ):- raise ValueError("generation_not_caught_up")- for document in db.execute(- "SELECT * FROM retrieval_documents WHERE generation_id=? "- "AND state NOT IN ('ready','excluded','deleted')",- (generation_id,),- ):- if self._grant(db, document["source_id"]) is not None:- raise ValueError("generation_incomplete")- for document in db.execute(- "SELECT * FROM retrieval_documents WHERE generation_id=? "- "AND state IN ('ready','excluded')",- (generation_id,),- ):- if self._grant(db, document["source_id"]) is not None:- event = journal.event(db, document["event_seq"])- if (- not event- or not journal.matches_current(db, event)- or self._grant(db, document["source_id"]) != document["grant_fingerprint"]- ):- raise ValueError("generation_stale")- record = EvidenceStore._get_with_connection(db, document["evidence_id"])- if record is None:- raise ValueError("generation_stale")- self.validate_document(db, document, record)- db.execute(- "UPDATE retrieval_index_generations SET state='retired' "- "WHERE state='active' AND generation_id!=?",- (generation_id,),- )- db.execute(- "UPDATE retrieval_index_generations SET state='active' WHERE generation_id=?",- (generation_id,),- )-- def active_chunks(- self,- generation_id: str,- *,- source_ids: frozenset[str] | None = None,- time_range: TimeRange | None = None,- deadline: float | None = None,- require_active: bool = False,- ) -> tuple[PublishedChunk, ...]:- """Return current chunks after SQL-level source/time eligibility filtering.-- Grant fingerprints, journal freshness, and manifests are still checked- below in the same snapshot. The SQL predicate keeps unauthorized rows- from filling an in-memory candidate set before those checks run.- """- if source_ids is not None and not source_ids:- return ()- with self._db(deadline=deadline) as db:- generation = db.execute(- "SELECT state FROM retrieval_index_generations WHERE generation_id=?",- (generation_id,),- ).fetchone()- if generation is None or generation[0] != "active":- if require_active:- raise ValueError("active_generation_changed")- return ()- clauses = ["d.generation_id=?", "d.state IN ('ready','excluded')"]- parameters: list[object] = [generation_id]- join = ""- if source_ids is not None:- clauses.append("d.source_id IN (SELECT value FROM json_each(?))")- parameters.append(json.dumps(sorted(source_ids)))- if time_range is not None:- join = "JOIN evidence e ON e.evidence_id=d.evidence_id "- clauses.extend(- (- "e.created_at IS NOT NULL",- "julianday(e.created_at) >= julianday(?)",- "julianday(e.created_at) < julianday(?)",- )- )- parameters.extend((time_range.start.isoformat(), time_range.end.isoformat()))- output = []- for row in db.execute(- "SELECT d.* FROM retrieval_documents d "- + join- + f"WHERE {' AND '.join(clauses)} ORDER BY d.event_seq,d.evidence_id",- parameters,- ):- if deadline is not None:- _check(deadline)- grant = self._grant(db, row["source_id"])- if grant is None or grant != row["grant_fingerprint"]:- continue- event = journal.event(db, row["event_seq"])- if (- event is None- or event.source_id != row["source_id"]- or not journal.matches_current(db, event)- ):- continue- record = EvidenceStore._get_with_connection(db, row["evidence_id"])- if (- record is None- or record.user_id != "local"- or record.source_id != row["source_id"]- ):- continue- event_time = record.created_at- if event_time is not None:- event_time = event_time.astimezone(UTC) if event_time.tzinfo else None- if time_range is not None and (- event_time is None or not time_range.start <= event_time < time_range.end- ):- continue- chunks = self.validate_document(db, row, record, deadline=deadline)- for chunk in chunks:- if deadline is not None:- _check(deadline)- output.append(- PublishedChunk(- chunk["chunk_id"],- row["build_id"],- row["evidence_id"],- event.evidence_revision,- chunk["start_cp"],- chunk["end_cp"],- chunk["ordinal"],- record.source_id,- event_time,- )- )- return tuple(output)-- def validate_document(- self,- db: sqlite3.Connection,- document: sqlite3.Row,- record: EvidenceRecord,- *,- deadline: float | None = None,- ) -> tuple[sqlite3.Row, ...]:- """Same-snapshot complete manifest/map check shared by candidate and final read."""- if deadline is not None:- _check(deadline)- if document["active_build_id"] != document["build_id"]:- raise ValueError("published_build_binding_invalid")- prepared = self._prepare(db, document["generation_id"], record)- map_digest = None- if prepared is not None:- source_map = db.execute(- "SELECT source_map_json,map_digest,coverage_state FROM retrieval_build_maps "- "WHERE build_id=?",- (document["build_id"],),- ).fetchone()- expected_state = "excluded" if prepared.state == "no_indexable_content" else "ready"- if (- source_map is None- or tuple(source_map)- != (- prepared.metadata.canonical_json(),- prepared.metadata.digest,- "excluded" if expected_state == "excluded" else "indexable",- )- or document["state"] != expected_state- ):- raise ValueError("published_source_map_invalid")- map_digest = prepared.metadata.digest- elif document["state"] != "ready":- raise ValueError("published_manifest_invalid")- rows = tuple(- db.execute(- "SELECT * FROM retrieval_chunks WHERE generation_id=? AND evidence_id=? "- "AND build_id=? ORDER BY ordinal LIMIT 8193",- (document["generation_id"], record.evidence_id, document["build_id"]),- ).fetchall()- )- chunks = []- for row in rows:- if deadline is not None:- _check(deadline)- start, end = row["start_cp"], row["end_cp"]- if (- type(start) is not int- or type(end) is not int- or not 0 <= start < end <= len(record.verbatim)- ):- raise ValueError("published_manifest_invalid")- text = record.verbatim[start:end]- text_hash = digest(text)- identity = [document["build_id"], row["ordinal"], start, end, text_hash]- if map_digest is not None:- identity.append(map_digest)- if row["text_hash"] != text_hash or row["chunk_id"] != digest(identity):- raise ValueError("published_manifest_invalid")- chunks.append(TextChunk(row["ordinal"], start, end, row["token_count"], text))- try:- validate_chunk_coverage(record, chunks, prepared)- except ValueError as error:- raise ValueError("published_manifest_incomplete_or_invalid") from error- if deadline is not None:- _check(deadline)- return rows-- def status(self, generation_id: str) -> dict[str, int]:- with self._db() as db:- row = db.execute(- "SELECT after_seq FROM retrieval_consumers WHERE generation_id=?", (generation_id,)- ).fetchone()- if row is None:- raise ValueError("generation_missing")- result = {- "after_seq": int(row[0]),- "pending": 0,- "leased": 0,- "ready": 0,- "blocked": 0,- "deleted": 0,- "excluded": 0,- }- for state, count in db.execute(- "SELECT state,count(*) FROM retrieval_documents "- "WHERE generation_id=? GROUP BY state",- (generation_id,),- ):- result[state] = count- return resultdiff --git a/memhub/apps/core/src/one_memory/retrieval/index_supervisor.py b/memhub/apps/core/src/one_memory/retrieval/index_supervisor.pydeleted file mode 100644index 1b53e2d..0000000--- a/memhub/apps/core/src/one_memory/retrieval/index_supervisor.py+++ /dev/null@@ -1,200 +0,0 @@-"""Explicit ownership of one admitted index worker; no migration or resource close."""--import asyncio-import math-import threading-import time-from collections.abc import Callable-from dataclasses import dataclass-from typing import Literal, Protocol--from one_memory.retrieval.index_worker import IndexWorkResult-from one_memory.service_levels import ServiceLevel, service_level_context---class IndexWorker(Protocol):- def run_once(self, *, cancel: threading.Event | None = None) -> IndexWorkResult: ...---@dataclass(frozen=True)-class IndexSupervisorSnapshot:- state: Literal["new", "running", "stopping", "stopped"]- last_result: str | None- error_code: str | None- slices: int- failures: int- thread_alive: bool---class RetrievalIndexSupervisor:- """Single-use supervisor. Caller owns admission and closes vectors only after stop.-- Cancellation is cooperative: stop/wait cannot finish until a blocking native- run_once returns. The lifecycle owner must drain before shutting its event loop.- """-- def __init__(- self,- worker: IndexWorker,- *,- idle_wait_seconds: float = 5,- failure_wait_seconds: float = 5,- yield_wait_seconds: float = 0.05,- max_consecutive_slices: int = 8,- background_should_yield: Callable[[], bool] | None = None,- ):- for interval in (idle_wait_seconds, failure_wait_seconds, yield_wait_seconds):- if (- isinstance(interval, bool)- or not isinstance(interval, (int, float))- or not math.isfinite(interval)- or not 0.001 <= interval <= 300- ):- raise ValueError("index_supervisor_invalid_config")- if type(max_consecutive_slices) is not int or not 1 <= max_consecutive_slices <= 256:- raise ValueError("index_supervisor_invalid_config")- self._worker = worker- self._idle_wait = idle_wait_seconds- self._failure_wait = failure_wait_seconds- self._yield_wait = yield_wait_seconds- self._max_slices = max_consecutive_slices- self._background_should_yield = background_should_yield or (lambda: False)- self._condition = threading.Condition()- self._cancel = threading.Event()- self._wake = False- self._thread: threading.Thread | None = None- self._state: Literal["new", "running", "stopping", "stopped"] = "new"- self._last_result: str | None = None- self._error_code: str | None = None- self._slices = self._failures = 0-- def start(self) -> None:- with self._condition:- if self._state != "new":- raise RuntimeError("index_supervisor_already_started")- thread = threading.Thread(target=self._run, name="retrieval-index", daemon=False)- self._thread = thread- self._state = "running"- try:- thread.start()- except Exception:- self._thread = None- self._state = "stopped"- self._error_code = "index_supervisor_start_failed"- raise RuntimeError("index_supervisor_start_failed") from None-- def trigger(self) -> bool:- with self._condition:- if self._state != "running":- return False- self._wake = True- self._condition.notify_all()- return True-- def snapshot(self) -> IndexSupervisorSnapshot:- with self._condition:- return IndexSupervisorSnapshot(- self._state,- self._last_result,- self._error_code,- self._slices,- self._failures,- self._thread is not None and self._thread.is_alive(),- )-- def _request_stop(self) -> None:- with self._condition:- self._cancel.set()- self._wake = False- if self._state == "new":- self._state = "stopped"- elif self._state == "running":- self._state = "stopping"- self._condition.notify_all()-- async def stop(self) -> None:- self._request_stop()- await self.wait()-- async def wait(self) -> None:- """Cancellation requests shutdown, drains the real thread, then propagates."""- cancelled = False- while True:- with self._condition:- thread = self._thread- if thread is None or not thread.is_alive():- if thread is not None:- thread.join()- break- try:- # No executor waiter or to_thread future to orphan on cancellation.- await asyncio.sleep(0.01)- except asyncio.CancelledError:- cancelled = True- self._request_stop()- if cancelled:- raise asyncio.CancelledError-- def _pause(self, seconds: float, *, wakeable: bool) -> None:- deadline = time.monotonic() + seconds- with self._condition:- while not self._cancel.is_set():- if wakeable and self._wake:- break- remaining = deadline - time.monotonic()- if remaining <= 0:- break- self._condition.wait(remaining)- self._wake = False-- def _run(self) -> None:- consecutive = 0- try:- while not self._cancel.is_set():- if self._background_should_yield():- self._pause(self._yield_wait, wakeable=False)- continue- result: str | None = None- error: str | None = None- try:- with service_level_context(ServiceLevel.BACKGROUND):- value = self._worker.run_once(cancel=self._cancel)- if not isinstance(value, IndexWorkResult) or value.state not in {- "idle",- "ready",- "excluded",- "blocked",- "cancelled",- "deferred",- }:- error = "index_worker_invalid_result"- else:- result = value.state- if result == "blocked":- error = "index_worker_blocked"- except BaseException:- # Worker/provider exceptions must never escape via thread stderr.- error = "index_worker_failed"- with self._condition:- self._last_result, self._error_code = result, error- self._slices += 1- if error is not None:- self._failures += 1- if self._cancel.is_set():- break- if error is not None:- consecutive = 0- self._pause(self._failure_wait, wakeable=False)- elif result in {"ready", "excluded"}:- consecutive += 1- if consecutive >= self._max_slices:- consecutive = 0- self._pause(self._yield_wait, wakeable=False)- else:- consecutive = 0- self._pause(self._idle_wait, wakeable=True)- finally:- with self._condition:- self._state = "stopped"- self._wake = False- self._condition.notify_all()diff --git a/memhub/apps/core/src/one_memory/retrieval/index_worker.py b/memhub/apps/core/src/one_memory/retrieval/index_worker.pydeleted file mode 100644index d5f0df5..0000000--- a/memhub/apps/core/src/one_memory/retrieval/index_worker.py+++ /dev/null@@ -1,262 +0,0 @@-"""Bounded raw-index work. No migration, scheduler, HY or remote model invocation.--Runtime must admit a single worker, own its thread/process, and drain that owner-before closing the vector client. Cancellation here is cooperative between-native calls, not a claim that ONNX/Chroma can be preempted in a Python thread.-"""--import math-import threading-import time-from collections.abc import Callable, Sequence-from dataclasses import dataclass-from typing import Literal, Protocol--from one_memory.embedding_assets import (- BUILTIN_EMBEDDING_MODEL,- EMBEDDING_ASSETS,- verify_assets,-)-from one_memory.local_embedding import LocalEmbedder, TokenizedText-from one_memory.retrieval.chunking import ChunkingError, chunk_evidence-from one_memory.retrieval.content import index_content-from one_memory.retrieval.index_store import (- EncoderIdentity,- IndexLease,- RetrievalIndexStore,- chunk_identity,-)-from one_memory.retrieval.mapped_content import MappedContentError, chunk_mapped_content-from one_memory.retrieval.vector_store import RawVectorRecord, RawVectorStore, VectorStoreError---class RetrievalEncoder(Protocol):- @property- def identity(self) -> EncoderIdentity: ...-- def tokenize_with_offsets(self, text: str) -> TokenizedText: ...-- def encode_documents(self, texts: list[str]) -> list[list[float]]: ...---class LocalRetrievalEncoder:- """Pinned local assets; mapped content v2 by default, explicit v1 for old generations."""-- def __init__(self, embedder: LocalEmbedder | None = None, *, content_version: int = 2):- if type(content_version) is not int or content_version not in (1, 2):- raise ValueError("unsupported_content_generation")- self._embedder = embedder if embedder is not None else LocalEmbedder()- digests = {asset.filename: asset.sha256 for asset in EMBEDDING_ASSETS}- self.identity = EncoderIdentity(- BUILTIN_EMBEDDING_MODEL,- digests["model_int8.onnx"],- digests["tokenizer.json"],- self._embedder.dimensions,- chunker_version=content_version,- normalizer_version=content_version,- )- self._verified = False- self._lock = threading.Lock()-- def _verify(self) -> None:- with self._lock:- if not self._verified:- verify_assets()- self._verified = True-- def tokenize_with_offsets(self, text: str) -> TokenizedText:- self._verify()- return self._embedder.tokenize_with_offsets(text)-- def encode_documents(self, texts: list[str]) -> list[list[float]]:- self._verify()- if len(texts) > 8 or any(- not 1 <= len(self.tokenize_with_offsets(text).token_ids) <= 512 for text in texts- ):- raise ValueError("encoder_batch_or_token_budget")- with self._lock:- return self._embedder.encode(texts)---@dataclass(frozen=True)-class IndexWorkResult:- state: Literal["idle", "ready", "excluded", "blocked", "cancelled", "deferred"]- reason: str | None = None- chunk_count: int = 0---class _Interrupted(Exception):- def __init__(self, state: Literal["cancelled", "deferred"]):- self.state = state---class RetrievalIndexWorker:- def __init__(- self,- store: RetrievalIndexStore,- vectors: RawVectorStore,- encoder: RetrievalEncoder,- *,- generation_id: str,- owner: str,- clock: Callable[[], float] = time.monotonic,- wall_clock: Callable[[], float] = time.time,- max_job_seconds: float = 120,- ):- if (- not owner- or len(owner) > 128- or isinstance(max_job_seconds, bool)- or not math.isfinite(max_job_seconds)- or not 0 < max_job_seconds <= 300- ):- raise ValueError("invalid_worker_budget")- if store.encoder_identity(generation_id) != encoder.identity:- raise ValueError("worker_encoder_identity_mismatch")- if (- vectors.generation_id != generation_id- or vectors.dimensions != encoder.identity.dimensions- ):- raise ValueError("worker_vector_identity_mismatch")- self.store, self.vectors, self.encoder = store, vectors, encoder- self.generation_id, self.owner = generation_id, owner- self.clock, self.wall_clock = clock, wall_clock- self.max_job_seconds = max_job_seconds- self._run_lock = threading.Lock()-- def run_once(self, *, cancel: threading.Event | None = None) -> IndexWorkResult:- """Consume <=256 events and handle <=1 document; never activate a generation.-- Interrupted jobs retain their finite lease for restart recovery. Failed- jobs are blocked for explicit retry; arbitrary provider exceptions and- original text never enter the public result or a log.- """- if not self._run_lock.acquire(blocking=False):- raise RuntimeError("index_worker_already_running")- lease: IndexLease | None = None- deadline = self.clock() + self.max_job_seconds-- def check() -> None:- if cancel is not None and cancel.is_set():- raise _Interrupted("cancelled")- now = self.clock()- if not math.isfinite(now) or now >= deadline:- raise _Interrupted("deferred")-- def boundary() -> None:- nonlocal lease- check()- assert lease is not None- # This revalidates the original's current fingerprint and authority,- # not just a heartbeat on an otherwise stale claim.- lease = self.store.renew(lease, now=self.wall_clock())- check()-- def publish_checkpoint() -> None:- check()- assert lease is not None- now = self.wall_clock()- if not math.isfinite(now) or now >= lease.expires_at:- raise ValueError("lease_changed_or_expired")-- try:- check()- self.store.consume_pending(self.generation_id)- check()- lease = self.store.claim(self.generation_id, self.owner, now=self.wall_clock())- if lease is None:- return IndexWorkResult("idle")- record = self.store.read_lease(lease, now=self.wall_clock())- check()- identity = self.encoder.identity- try:- prepared = index_content(- record,- normalizer_version=identity.normalizer_version,- chunker_version=identity.chunker_version,- )- except ValueError:- self.store.block(lease)- return IndexWorkResult("blocked", "content_mapping_required")-- worker_encoder = self.encoder-- class CheckedTokenizer:- def tokenize_with_offsets(self, text: str) -> TokenizedText:- check()- result = worker_encoder.tokenize_with_offsets(text)- check()- return result-- chunks = (- chunk_mapped_content(prepared, CheckedTokenizer())- if prepared is not None- else chunk_evidence(record.verbatim, CheckedTokenizer())- )- if (not chunks and prepared is None) or len(chunks) > 8192:- raise ChunkingError("invalid_chunk_count")- source_map = prepared.metadata if prepared is not None else None- map_digest = source_map.digest if source_map is not None else None- boundary()- expected: list[RawVectorRecord] = []- for start in range(0, len(chunks), 8):- batch = chunks[start : start + 8]- boundary()- encoded = self.encoder.encode_documents([chunk.text for chunk in batch])- boundary()- if len(encoded) != len(batch):- raise ValueError("encoder_count_mismatch")- records = [- RawVectorRecord(- chunk_identity(lease, chunk, map_digest),- lease.build_id,- lease.generation_id,- tuple(vector),- )- for chunk, vector in zip(batch, encoded, strict=True)- ]- self.vectors.upsert(records)- boundary()- expected.extend(records)- self._verify(expected, boundary)- assert lease is not None- check()- self.store.publish(- lease,- chunks,- vectors_verified=True,- now=self.wall_clock(),- source_map=source_map,- checkpoint=publish_checkpoint,- )- return IndexWorkResult("ready" if chunks else "excluded", chunk_count=len(chunks))- except _Interrupted as error:- return IndexWorkResult(- error.state, "cancelled" if error.state == "cancelled" else "deadline"- )- except Exception as error:- cause: BaseException | None = error- while cause is not None:- if isinstance(cause, _Interrupted):- return IndexWorkResult(- cause.state, "cancelled" if cause.state == "cancelled" else "deadline"- )- cause = cause.__cause__- if lease is None:- raise- self.store.block(lease)- reason = (- "chunking_unavailable"- if isinstance(error, (ChunkingError, MappedContentError))- else "vector_unavailable"- if isinstance(error, VectorStoreError)- else "index_build_unavailable"- )- return IndexWorkResult("blocked", reason)- finally:- self._run_lock.release()-- def _verify(self, records: Sequence[RawVectorRecord], boundary: Callable[[], None]) -> None:- for start in range(0, len(records), 8):- self.vectors.verify(records[start : start + 8])- boundary()diff --git a/memhub/apps/core/src/one_memory/retrieval/mapped_content.py b/memhub/apps/core/src/one_memory/retrieval/mapped_content.pydeleted file mode 100644index 1f85efd..0000000--- a/memhub/apps/core/src/one_memory/retrieval/mapped_content.py+++ /dev/null@@ -1,349 +0,0 @@-"""Pure raw-content admission and exact source maps; no grants, writes or HY calls."""--import hashlib-import json-from dataclasses import asdict, dataclass, field-from typing import Literal--from one_memory.conversation_processing import _is_hy_input_turn-from one_memory.embedding_assets import BUILTIN_EMBEDDING_MAX_TOKENS-from one_memory.imports.models import (- MAX_VERBATIM_CHARACTERS,- EvidenceContentKind,- EvidenceRecord,-)-from one_memory.imports.sources.parsing import (- is_host_generated_user_turn,- unwrap_host_envelope_with_positions,-)-from one_memory.input_sanitization import sanitize_model_input_with_spans-from one_memory.local_embedding import TokenizedText-from one_memory.retrieval.chunking import ChunkingError, OffsetTokenizer, TextChunk, chunk_evidence--MAP_VERSION = "raw-content-map-v1"-PARSER_VERSION = "host-envelope-positions-v1"-SANITIZER_VERSION = "model-input-spans-v1"-MAX_MAPPED_CHUNKS = 8192---class MappedContentError(ValueError):- """Fixed reason only; never include original or sanitized content."""---@dataclass(frozen=True)-class MappedSegment:- kind: Literal["copy", "synthetic"]- start_cp: int- end_cp: int- source_start_cp: int | None- source_end_cp: int | None---@dataclass(frozen=True)-class ExcludedRange:- start_cp: int- end_cp: int- reason: Literal["envelope_or_trim", "sanitizer"]---@dataclass(frozen=True)-class ContentMap:- map_version: str- parser_version: str- sanitizer_version: str- source_length_cp: int- body_start_cp: int | None- body_end_cp: int | None- sanitized_length_cp: int- sanitized_sha256: str- segments: tuple[MappedSegment, ...]- excluded: tuple[ExcludedRange, ...]-- def canonical_json(self) -> str:- return json.dumps(asdict(self), sort_keys=True, separators=(",", ":"), allow_nan=False)-- @property- def digest(self) -> str:- return hashlib.sha256(self.canonical_json().encode()).hexdigest()---@dataclass(frozen=True)-class PreparedMappedContent:- actual_text: str = field(repr=False)- metadata: ContentMap- state: Literal["ready", "no_indexable_content"]---def _validate(content: PreparedMappedContent, original: str | None = None) -> None:- meta, text = content.metadata, content.actual_text- invalid = "mapped_invalid_source_map"- if (- (meta.map_version, meta.parser_version, meta.sanitizer_version)- != (MAP_VERSION, PARSER_VERSION, SANITIZER_VERSION)- or type(meta.source_length_cp) is not int- or not 0 <= meta.source_length_cp <= MAX_VERBATIM_CHARACTERS- or type(meta.sanitized_length_cp) is not int- or meta.sanitized_length_cp != len(text)- or meta.sanitized_sha256 != hashlib.sha256(text.encode()).hexdigest()- or type(meta.segments) is not tuple- or type(meta.excluded) is not tuple- ):- raise MappedContentError(invalid)- body_start, body_end = meta.body_start_cp, meta.body_end_cp- if body_start is None or body_end is None:- if body_start is not None or body_end is not None or text or meta.segments:- raise MappedContentError(invalid)- elif (- type(body_start) is not int- or type(body_end) is not int- or not 0 <= body_start < body_end <= meta.source_length_cp- ):- raise MappedContentError(invalid)- cursor = source_cursor = 0- has_content = False- partition: list[tuple[int, int]] = []- for span in meta.segments:- if (- not isinstance(span, MappedSegment)- or type(span.start_cp) is not int- or type(span.end_cp) is not int- or not cursor == span.start_cp < span.end_cp <= len(text)- ):- raise MappedContentError(invalid)- cursor = span.end_cp- if span.kind == "synthetic":- if span.source_start_cp is not None or span.source_end_cp is not None:- raise MappedContentError(invalid)- continue- left, right = span.source_start_cp, span.source_end_cp- if (- span.kind != "copy"- or type(left) is not int- or type(right) is not int- or body_start is None- or body_end is None- or not body_start <= left < right <= body_end- or left < source_cursor- or right - left != span.end_cp - span.start_cp- ):- raise MappedContentError(invalid)- if original is not None and text[span.start_cp : span.end_cp] != original[left:right]:- raise MappedContentError(invalid)- has_content |= bool(text[span.start_cp : span.end_cp].strip())- source_cursor = right- partition.append((left, right))- if cursor != len(text):- raise MappedContentError(invalid)- excluded_end = 0- for excluded in meta.excluded:- if (- not isinstance(excluded, ExcludedRange)- or type(excluded.start_cp) is not int- or type(excluded.end_cp) is not int- or not excluded_end <= excluded.start_cp < excluded.end_cp <= meta.source_length_cp- or excluded.reason not in {"envelope_or_trim", "sanitizer"}- ):- raise MappedContentError(invalid)- excluded_end = excluded.end_cp- partition.append((excluded.start_cp, excluded.end_cp))- source_cursor = 0- for left, right in sorted(partition):- if left != source_cursor:- raise MappedContentError(invalid)- source_cursor = right- if source_cursor != meta.source_length_cp or content.state != (- "ready" if has_content else "no_indexable_content"- ):- raise MappedContentError(invalid)---def prepare_mapped_content(record: EvidenceRecord) -> PreparedMappedContent:- """Map a qualified body to Evidence.verbatim code points, without modifying it.-- The returned text is transient; only metadata is suitable for an outer manifest.- That manifest must independently bind Evidence identity/revision/hash and grants.- """- if record.content_kind is not EvidenceContentKind.RAW_CONVERSATION or record.role not in {- "user",- "assistant",- }:- raise MappedContentError("mapped_content_ineligible")- original = record.verbatim- if len(original) > MAX_VERBATIM_CHARACTERS:- raise MappedContentError("mapped_source_limit_exceeded")- envelope = unwrap_host_envelope_with_positions(original)- if envelope.status == "ambiguous":- raise MappedContentError("mapped_ambiguous_envelope")- body, start, end = envelope.text, envelope.start_cp, envelope.end_cp- segments: list[MappedSegment] = []- excluded: list[ExcludedRange] = []- text = ""- if body is None:- if start is not None or end is not None:- raise MappedContentError("mapped_invalid_envelope")- if original:- excluded.append(ExcludedRange(0, len(original), "envelope_or_trim"))- else:- if (- type(start) is not int- or type(end) is not int- or not 0 <= start < end <= len(original)- or original[start:end] != body- ):- raise MappedContentError("mapped_invalid_envelope")- if is_host_generated_user_turn(original) or is_host_generated_user_turn(body):- raise MappedContentError("mapped_generated_content")- if not _is_hy_input_turn(record.model_copy(update={"verbatim": body})):- raise MappedContentError("mapped_body_hygiene_rejected")- sanitized = sanitize_model_input_with_spans(body)- text = sanitized.text- cursor = 0- for span in sanitized.spans:- if span.start_cp > cursor:- segments.append(MappedSegment("synthetic", cursor, span.start_cp, None, None))- copied = MappedSegment(- "copy",- span.start_cp,- span.end_cp,- start + span.source_start_cp,- start + span.source_end_cp,- )- # Only source AND output adjacency can make one citation-safe interval.- if (- segments- and segments[-1].kind == "copy"- and (- segments[-1].end_cp == copied.start_cp- and segments[-1].source_end_cp == copied.source_start_cp- )- ):- previous = segments.pop()- copied = MappedSegment(- "copy",- previous.start_cp,- copied.end_cp,- previous.source_start_cp,- copied.source_end_cp,- )- segments.append(copied)- cursor = span.end_cp- if cursor < len(text):- segments.append(MappedSegment("synthetic", cursor, len(text), None, None))- if start:- excluded.append(ExcludedRange(0, start, "envelope_or_trim"))- excluded.extend(- ExcludedRange(start + left, start + right, "sanitizer")- for left, right in sanitized.excluded- )- if end < len(original):- excluded.append(ExcludedRange(end, len(original), "envelope_or_trim"))- metadata = ContentMap(- MAP_VERSION,- PARSER_VERSION,- SANITIZER_VERSION,- len(original),- start,- end,- len(text),- hashlib.sha256(text.encode()).hexdigest(),- tuple(segments),- tuple(excluded),- )- prepared = PreparedMappedContent(- text,- metadata,- "ready"- if any(- span.kind == "copy" and text[span.start_cp : span.end_cp].strip() for span in segments- )- else "no_indexable_content",- )- _validate(prepared, original)- return prepared---class _BoundedTokenizer:- """Count successful chunk re-encodings without copying the chunking algorithm."""-- def __init__(self, tokenizer: OffsetTokenizer, remaining: int, max_tokens: int):- self.tokenizer = tokenizer- self.remaining = remaining- self.max_tokens = max_tokens- self.initial = True-- def tokenize_with_offsets(self, text: str) -> TokenizedText:- if not self.initial and self.remaining == 0:- raise MappedContentError("mapped_chunk_limit_exceeded")- result = self.tokenizer.tokenize_with_offsets(text)- # chunk_evidence first encodes the whole copy span, then prefix+candidate;- # an in-budget candidate emits exactly one chunk. Shrink retries do not.- if (- not self.initial- and isinstance(result, TokenizedText)- and len(result.token_ids) <= self.max_tokens- ):- self.remaining -= 1- self.initial = False- return result---def chunk_mapped_content(- content: PreparedMappedContent,- tokenizer: OffsetTokenizer,- *,- prefix: str = "",- target_tokens: int = 384,- overlap_tokens: int = 64,- max_tokens: int = BUILTIN_EMBEDDING_MAX_TOKENS,- max_chunks: int = MAX_MAPPED_CHUNKS,-) -> tuple[TextChunk, ...]:- """Chunk each continuous copy independently; never encode synthetic placeholders."""- if type(max_chunks) is not int or not 1 <= max_chunks <= MAX_MAPPED_CHUNKS:- raise MappedContentError("mapped_invalid_chunk_limit")- _validate(content)- # Reuse budget validation even for an explicitly nonindexable record.- chunk_evidence(- "",- tokenizer,- prefix=prefix,- target_tokens=target_tokens,- overlap_tokens=overlap_tokens,- max_tokens=max_tokens,- )- chunks: list[TextChunk] = []- for span in content.metadata.segments:- if span.kind != "copy":- continue- text = content.actual_text[span.start_cp : span.end_cp]- if not text.strip():- continue- if len(chunks) == max_chunks:- raise MappedContentError("mapped_chunk_limit_exceeded")- try:- pieces = chunk_evidence(- text,- _BoundedTokenizer(tokenizer, max_chunks - len(chunks), max_tokens),- prefix=prefix,- target_tokens=target_tokens,- overlap_tokens=overlap_tokens,- max_tokens=max_tokens,- )- except ChunkingError as error:- if isinstance(error.__cause__, MappedContentError):- raise error.__cause__ from None- raise- if len(chunks) + len(pieces) > max_chunks:- raise MappedContentError("mapped_chunk_limit_exceeded")- assert span.source_start_cp is not None # Validated copy span.- ordinal = len(chunks)- chunks.extend(- TextChunk(- ordinal + i,- span.source_start_cp + piece.start_cp,- span.source_start_cp + piece.end_cp,- piece.token_count,- piece.text,- )- for i, piece in enumerate(pieces)- )- return tuple(chunks)diff --git a/memhub/apps/core/src/one_memory/retrieval/raw_candidates.py b/memhub/apps/core/src/one_memory/retrieval/raw_candidates.pydeleted file mode 100644index cebb7e8..0000000--- a/memhub/apps/core/src/one_memory/retrieval/raw_candidates.py+++ /dev/null@@ -1,147 +0,0 @@-"""Internal raw semantic candidate lookup. No route, raw body release or full coverage claim."""--import math-import sqlite3-from collections.abc import Sequence-from dataclasses import dataclass-from typing import Literal--from one_memory.context_raw import _check-from one_memory.retrieval.index_store import PublishedChunk, RetrievalIndexStore-from one_memory.retrieval.models import RetrievalPath, TimeRange-from one_memory.retrieval.vector_store import RawVectorStore, VectorStoreError---@dataclass(frozen=True)-class RawSemanticCandidate:- chunk: PublishedChunk- distance: float---@dataclass(frozen=True)-class RawCandidateBatch:- hits: tuple[RawSemanticCandidate, ...] = ()- complete: bool = False- reason: str | None = None- coverage: Literal["indexed_only"] = "indexed_only"- time_range: TimeRange | None = None---def read_raw_semantic_candidates(- store: RetrievalIndexStore,- vectors: RawVectorStore,- query_vector: Sequence[float],- *,- generation_id: str,- source_ids: frozenset[str],- deadline: float,- limit: int = 64,- time_range: TimeRange | None = None,-) -> RawCandidateBatch:- """Caller resolves host authority; explicit source IDs can only narrow it.-- complete means the eligible published chunks were checked and searched, not- that every authorized original is indexed. The future coordinator must still- revalidate host identity and all raw/memory data together before body release.- """- if not isinstance(source_ids, frozenset) or any(- not isinstance(source, str) or not 1 <= len(source) <= 128 for source in source_ids- ):- raise ValueError("explicit_resolved_sources_required")- if type(limit) is not int or not 1 <= limit <= 64:- raise ValueError("invalid_candidate_limit")- if isinstance(deadline, bool) or not math.isfinite(deadline):- raise ValueError("invalid_candidate_deadline")- if not source_ids:- return RawCandidateBatch(reason="not_authorized")- if vectors.generation_id != generation_id:- raise ValueError("candidate_generation_mismatch")- try:- _check(deadline)- eligible = store.active_chunks(- generation_id, source_ids=source_ids, time_range=time_range, deadline=deadline- )- if not eligible:- return RawCandidateBatch(- complete=True, reason="no_eligible_chunks", time_range=time_range- )- before = {chunk.chunk_id: chunk for chunk in eligible}- expected_vectors = [(chunk.chunk_id, chunk.build_id) for chunk in eligible]- # When one query can return the entire allowlist, that response itself- # is the integrity check. Avoid a duplicate Chroma get-before-query on- # the hot small/medium-authority path. Larger allowlists still need the- # explicit complete verification because search returns only top-K.- verify_before_search = len(eligible) > limit- if verify_before_search and not vectors.verify_candidates(- expected_vectors, deadline=deadline- ):- return RawCandidateBatch(reason="deadline")- result = vectors.search(query_vector, tuple(before), limit, deadline)- if not verify_before_search and {hit.chunk_id for hit in result.hits} != set(before):- raise VectorStoreError("vector_record_missing")- _check(deadline)- current = {- chunk.chunk_id: chunk- for chunk in store.active_chunks(- generation_id, source_ids=source_ids, time_range=time_range, deadline=deadline- )- }- hits = []- for hit in result.hits:- if (- hit.chunk_id not in before- or hit.generation_id != generation_id- or hit.build_id != before[hit.chunk_id].build_id- ):- raise VectorStoreError("candidate_identity_mismatch")- if current.get(hit.chunk_id) == before[hit.chunk_id]:- hits.append(RawSemanticCandidate(before[hit.chunk_id], hit.distance))- _check(deadline)- unchanged = before == current- return RawCandidateBatch(- tuple(hits),- result.complete and unchanged,- "evidence_changed" if not unchanged else result.reason,- time_range=time_range,- )- except TimeoutError:- return RawCandidateBatch(reason="deadline")- except VectorStoreError:- return RawCandidateBatch(reason="vector_unavailable")- except (sqlite3.Error, OSError, ValueError):- return RawCandidateBatch(reason="index_unavailable")---def compose_raw_retrieval_path(- batch: RawCandidateBatch,- *,- unpublished_authorized: bool,- lexical_hit_count: int,-) -> RetrievalPath:- """Map channel results to a path. Indexed-only search is never complete empty."""- if type(lexical_hit_count) is not int or lexical_hit_count < 0:- raise ValueError("invalid_lexical_hit_count")- hits = len(batch.hits) + lexical_hit_count- incomplete = unpublished_authorized or not batch.complete- channels: tuple[Literal["lexical", "semantic"], ...] = ("lexical", "semantic")- if hits:- return RetrievalPath(- status="ok",- returned_count=min(hits, 32),- searched_channels=channels,- coverage="partial" if incomplete else "complete",- )- if incomplete:- return RetrievalPath(- status="degraded",- reason="indexed_only",- returned_count=0,- searched_channels=channels,- coverage="partial",- )- return RetrievalPath(- status="empty",- returned_count=0,- searched_channels=channels,- coverage="complete",- )diff --git a/memhub/apps/core/src/one_memory/retrieval/raw_evidence.py b/memhub/apps/core/src/one_memory/retrieval/raw_evidence.pydeleted file mode 100644index cfb01c6..0000000--- a/memhub/apps/core/src/one_memory/retrieval/raw_evidence.py+++ /dev/null@@ -1,189 +0,0 @@-"""Materialize internal semantic candidates from current authorized Evidence.--Not an arbitrary-ID endpoint or signed expansion capability. The combined v2-coordinator must revalidate this lane with memory and host identity at release.-"""--import json-import sqlite3-from dataclasses import dataclass-from datetime import UTC--from one_memory import ingest_journal as journal-from one_memory.context_raw import _check, _connection-from one_memory.imports.models import EvidenceRecord-from one_memory.imports.storage import EvidenceStore-from one_memory.retrieval.index_store import RetrievalIndexStore-from one_memory.retrieval.models import EvidenceSpan, RawEvidence-from one_memory.retrieval.raw_candidates import RawCandidateBatch---@dataclass(frozen=True)-class RawEvidenceBatch:- items: tuple[RawEvidence, ...] = ()- complete: bool = False- reason: str | None = None- used_characters: int = 0---def _merge(intervals: list[tuple[int, int]]) -> list[tuple[int, int]]:- output: list[tuple[int, int]] = []- for start, end in sorted(intervals):- if output and start <= output[-1][1]:- output[-1] = (output[-1][0], max(end, output[-1][1]))- else:- output.append((start, end))- return output---def materialize_raw_candidates(- store: RetrievalIndexStore,- candidates: RawCandidateBatch,- *,- generation_id: str,- source_ids: frozenset[str],- deadline: float,- character_budget: int = 6000,-) -> RawEvidenceBatch:- if not isinstance(source_ids, frozenset) or any(- not isinstance(source, str) or not 1 <= len(source) <= 128 for source in source_ids- ):- raise ValueError("explicit_resolved_sources_required")- if type(character_budget) is not int or not 1 <= character_budget <= 20000:- raise ValueError("invalid_evidence_budget")- if len(candidates.hits) > 64:- raise ValueError("candidate_budget_exceeded")- if not source_ids:- return RawEvidenceBatch(reason="not_authorized")- try:- _check(deadline)- expected = {hit.chunk.chunk_id: hit.chunk for hit in candidates.hits}- if len(expected) != len(candidates.hits):- raise ValueError("duplicate_candidates")- if not expected:- # Empty/excluded builds still carry an integrity claim. Validate- # current authorized completed manifests, never only nonempty hits.- current_chunks = store.active_chunks(- generation_id,- source_ids=source_ids,- time_range=candidates.time_range,- deadline=deadline,- require_active=True,- )- _check(deadline)- if current_chunks and candidates.reason == "no_eligible_chunks":- return RawEvidenceBatch(reason="evidence_changed")- return RawEvidenceBatch(complete=candidates.complete, reason=candidates.reason)- changed = False- grouped: dict[str, tuple[EvidenceRecord, list[tuple[int, int]]]] = {}- with _connection(store.database_path, deadline) as db:- db.execute("PRAGMA query_only=ON")- rows = {- row["chunk_id"]: row- for row in db.execute(- """SELECT c.*, d.event_seq,d.grant_fingerprint,d.source_id- FROM retrieval_chunks c JOIN retrieval_documents d- ON d.generation_id=c.generation_id AND d.evidence_id=c.evidence_id- AND d.active_build_id=c.build_id AND d.build_id=c.build_id- JOIN retrieval_index_generations g ON g.generation_id=c.generation_id- WHERE c.generation_id=? AND g.state='active' AND d.state='ready'- AND c.chunk_id IN (SELECT value FROM json_each(?))""",- (generation_id, json.dumps(tuple(expected))),- )- }- for key, chunk in expected.items():- _check(deadline)- row = rows.get(key)- if row is None or row["source_id"] not in source_ids:- changed = True- continue- event = journal.event(db, row["event_seq"])- grant = store._grant(db, row["source_id"])- if (- event is None- or event.user_id != "local"- or event.source_id != row["source_id"]- or event.evidence_id != chunk.evidence_id- or event.evidence_revision != chunk.revision- or grant is None- or grant != row["grant_fingerprint"]- or not journal.matches_current(db, event)- ):- changed = True- continue- record = EvidenceStore._get_with_connection(db, chunk.evidence_id)- if (- record is None- or record.user_id != "local"- or record.source_id != row["source_id"]- ):- changed = True- continue- event_time = record.created_at- if event_time is not None:- event_time = event_time.astimezone(UTC) if event_time.tzinfo else None- if (- (row["build_id"], row["ordinal"], row["start_cp"], row["end_cp"])- != (chunk.build_id, chunk.ordinal, chunk.start_cp, chunk.end_cp)- or (record.source_id, event_time) != (chunk.source_id, chunk.event_time)- or not 0 <= chunk.start_cp < chunk.end_cp <= len(record.verbatim)- ):- changed = True- continue- if record.evidence_id not in grouped:- document = db.execute(- "SELECT * FROM retrieval_documents WHERE generation_id=? AND evidence_id=?",- (generation_id, record.evidence_id),- ).fetchone()- if document is None:- raise ValueError("published_document_missing")- store.validate_document(db, document, record, deadline=deadline)- grouped[record.evidence_id] = (record, [])- grouped[record.evidence_id][1].append((chunk.start_cp, chunk.end_cp))- items = []- used = 0- truncated = False- for record, intervals in grouped.values():- _check(deadline)- spans: list[EvidenceSpan] = []- for start, end in _merge(intervals):- if len(spans) >= 4 or end - start > min(20000, character_budget - used):- truncated = True- continue- spans.append(- EvidenceSpan(start_cp=start, end_cp=end, text=record.verbatim[start:end])- )- used += end - start- if not spans:- continue- event_time = record.created_at- if event_time is not None:- event_time = event_time.astimezone(UTC) if event_time.tzinfo else None- items.append(- RawEvidence(- evidence_id=record.evidence_id,- revision=record.revision,- source_id=record.source_id,- conversation_id=record.conversation_id,- role=record.role,- written_by="person" if record.role == "user" else "agent",- event_time=event_time,- spans=tuple(spans),- matched_by=("semantic",),- )- )- _check(deadline)- return RawEvidenceBatch(- tuple(items),- candidates.complete and not changed and not truncated,- "evidence_changed"- if changed- else "character_budget"- if truncated- else candidates.reason,- used,- )- except TimeoutError:- return RawEvidenceBatch(reason="deadline")- except (sqlite3.Error, OSError, ValueError):- return RawEvidenceBatch(reason="index_unavailable")diff --git a/memhub/apps/core/src/one_memory/retrieval/raw_lane.py b/memhub/apps/core/src/one_memory/retrieval/raw_lane.pydeleted file mode 100644index 92a92a2..0000000--- a/memhub/apps/core/src/one_memory/retrieval/raw_lane.py+++ /dev/null@@ -1,181 +0,0 @@-"""Fuse lexical raw context with an optional derived index. No new HTTP schema."""--import sqlite3-from collections.abc import Callable-from pathlib import Path-from typing import Protocol--from pydantic import ValidationError--from one_memory.context_models import (- ContextLaneResult,- ContextSearchRequest,- ContextSemanticStatus,- RawContextItem,-)-from one_memory.context_raw import (- _METADATA,- _check,- _connection,- _execute,- _message,- _readable,- read_raw_context,-)-from one_memory.retrieval.models import TimeRange-from one_memory.retrieval.raw_candidates import RawCandidateBatch, compose_raw_retrieval_path---class RawIndexLookup(Protocol):- def lookup_raw(- self,- query: str,- *,- source_ids: frozenset[str] | None,- deadline: float,- time_range: TimeRange | None = None,- ) -> tuple[RawCandidateBatch, bool]: ...---def fuse_raw_lane(- lexical: ContextLaneResult,- batch: RawCandidateBatch,- *,- unpublished_authorized: bool,- extra_items: tuple[RawContextItem, ...] = (),-) -> ContextLaneResult:- """v1 has no coverage field; degraded + lexical_only stands in for partial index."""- semantic_status: ContextSemanticStatus = (- "timed_out"- if batch.reason == "deadline"- else "failed"- if batch.reason in {"index_unavailable", "vector_unavailable"}- else "completed"- )- index_complete = batch.complete and not unpublished_authorized- timed_out = semantic_status == "timed_out"- if lexical.status not in {"ok", "empty", "degraded"}:- return lexical.model_copy(- update={- "searched_channels": ("lexical", "semantic"),- "semantic_status": semantic_status,- "index_complete": index_complete,- "timed_out": timed_out,- }- )- seen = {item.anchor_evidence_id for item in lexical.items if isinstance(item, RawContextItem)}- extras = tuple(item for item in extra_items if isinstance(item, RawContextItem))- # Semantic similarity is a candidate signal, not a verified recall result.- # Release only semantic items that the lexical lane independently confirmed- # for the same evidence identity. Unconfirmed semantic hits stay in the- # coverage receipt as partial/degraded and must not reach host injection.- unconfirmed_semantic = any(item.anchor_evidence_id not in seen for item in extras)- items = lexical.items- path = compose_raw_retrieval_path(- batch,- unpublished_authorized=unpublished_authorized,- lexical_hit_count=len(lexical.items),- )- status = path.status- if unconfirmed_semantic:- status = "degraded"- if lexical.status == "degraded" and status == "ok":- status = "degraded"- return ContextLaneResult(- status=status,- reason=(- "lexical_only" if unconfirmed_semantic and lexical.reason is None else lexical.reason- ),- items=items[:21],- truncated_by=lexical.truncated_by,- searched_channels=("lexical", "semantic"),- semantic_status=semantic_status,- index_complete=index_complete,- timed_out=timed_out,- )---def load_raw_items_from_candidates(- database_path: Path,- batch: RawCandidateBatch,- *,- source_ids: frozenset[str] | None,- deadline: float,- character_budget: int = 6000,-) -> tuple[RawContextItem, ...]:- """Release authorized Evidence bodies for semantic IDs. Not a vector dump."""- ids = tuple(dict.fromkeys(hit.chunk.evidence_id for hit in batch.hits))- if not ids or source_ids == frozenset():- return ()- readable, authority = _readable(source_ids)- body_limit = min(20000, max(1, character_budget))- items: list[RawContextItem] = []- try:- with _connection(database_path, deadline) as db:- for evidence_id in ids[:21]:- _check(deadline)- row = _execute(- db,- deadline,- f"""- SELECT {_METADATA}, substr(e.verbatim,1,?) AS text,- length(e.verbatim)>? AS body_truncated- FROM evidence e- WHERE {readable} AND e.evidence_id=?- """,- [body_limit, body_limit, *authority, evidence_id],- ).fetchone()- if row is None:- continue- try:- message = _message(row)- items.append(- RawContextItem(- source_id=row["source_id"],- conversation_id=row["conversation_id"],- anchor_evidence_id=message.evidence_id,- messages=(message,),- truncated=message.truncated,- )- )- except (ValidationError, ValueError, TypeError):- continue- except TimeoutError:- raise- except (sqlite3.Error, OSError):- return ()- return tuple(items)---def read_raw_indexed_context(- database_path: Path,- request: ContextSearchRequest,- *,- source_ids: frozenset[str] | None,- deadline: float,- index: RawIndexLookup | None = None,- load_semantic_items: Callable[..., tuple[RawContextItem, ...]] | None = None,-) -> ContextLaneResult:- lexical = read_raw_context(database_path, request, source_ids=source_ids, deadline=deadline)- if index is None:- return lexical- try:- batch, unpublished = index.lookup_raw(- request.query, source_ids=source_ids, deadline=deadline- )- except Exception:- batch, unpublished = RawCandidateBatch(reason="index_unavailable"), True- extra: tuple[RawContextItem, ...] = ()- if batch.hits:- extra = (- load_semantic_items(database_path, batch, source_ids=source_ids, deadline=deadline)- if load_semantic_items is not None- else load_raw_items_from_candidates(- database_path,- batch,- source_ids=source_ids,- deadline=deadline,- character_budget=request.character_budget,- )- )- return fuse_raw_lane(lexical, batch, unpublished_authorized=unpublished, extra_items=extra)diff --git a/memhub/apps/core/src/one_memory/retrieval/runtime.py b/memhub/apps/core/src/one_memory/retrieval/runtime.pydeleted file mode 100644index 95bc2e9..0000000--- a/memhub/apps/core/src/one_memory/retrieval/runtime.py+++ /dev/null@@ -1,379 +0,0 @@-"""Resume an explicitly configured local index; never create or migrate one."""--import asyncio-import json-import os-import shutil-import sqlite3-import stat-import sys-import threading-from collections.abc import Callable-from dataclasses import asdict, dataclass-from pathlib import Path-from typing import Literal--from one_memory.database_access import main_database-from one_memory.retrieval.index_store import EncoderIdentity, RetrievalIndexStore-from one_memory.retrieval.index_supervisor import IndexSupervisorSnapshot, RetrievalIndexSupervisor-from one_memory.retrieval.index_worker import LocalRetrievalEncoder, RetrievalIndexWorker-from one_memory.retrieval.models import TimeRange-from one_memory.retrieval.raw_candidates import RawCandidateBatch, read_raw_semantic_candidates-from one_memory.retrieval.vector_store import RawVectorStore, VectorStoreError-from one_memory.service_levels import ServiceLevel, ServiceLevelScheduler, ServiceResource--type RuntimeState = Literal[- "new", "starting", "running", "not_initialized", "failed", "stopping", "stopped"-]---@dataclass(frozen=True)-class ExistingRetrievalSnapshot:- state: RuntimeState- error_code: str | None- generation_id: str | None- supervisor: IndexSupervisorSnapshot | None---class _AdmissionError(Exception):- def __init__(self, code: str, *, missing: bool = False):- self.code, self.missing = code, missing---async def _native[T](operation: Callable[[], T]) -> T:- """Owned, joined native call. Only retained internal lifecycle tasks invoke this."""- result: list[T] = []- errors: list[BaseException] = []-- def run() -> None:- try:- result.append(operation())- except BaseException as error:- errors.append(error)-- thread = threading.Thread(target=run, name="retrieval-resource", daemon=False)- thread.start()- while thread.is_alive():- await asyncio.sleep(0.01)- thread.join()- if errors:- raise errors[0]- return result[0]---class ExistingRetrievalRuntime:- """Caller guarantees managed Core single ownership and explicit configuration.-- Construct without I/O; start and stop on the owning event loop. A retained- lifecycle task shields native work from caller cancellation until cleanup.- """-- def __init__(- self,- data_dir: Path,- *,- minimum_free_bytes: int = 256 * 1024 * 1024,- service_levels: ServiceLevelScheduler | None = None,- ):- if type(minimum_free_bytes) is not int or minimum_free_bytes < 0:- raise ValueError("retrieval_runtime_invalid_disk_limit")- self._directory = Path(data_dir)- self._minimum_free = minimum_free_bytes- self._service_levels = service_levels or ServiceLevelScheduler()- self._state: ExistingRetrievalSnapshot = ExistingRetrievalSnapshot("new", None, None, None)- self._vectors: RawVectorStore | None = None- self._store: RetrievalIndexStore | None = None- self._encoder: LocalRetrievalEncoder | None = None- self._supervisor: RetrievalIndexSupervisor | None = None- self._last_supervisor_snapshot: IndexSupervisorSnapshot | None = None- self._start_task: asyncio.Task[bool] | None = None- self._stop_task: asyncio.Task[None] | None = None- self._stop_requested = False-- def snapshot(self) -> ExistingRetrievalSnapshot:- return ExistingRetrievalSnapshot(- self._state.state,- self._state.error_code,- self._state.generation_id,- self._supervisor.snapshot()- if self._supervisor is not None- else self._last_supervisor_snapshot,- )-- def _set(- self, state: RuntimeState, code: str | None = None, generation: str | None = None- ) -> None:- self._state = ExistingRetrievalSnapshot(- state,- code,- generation if generation is not None else self._state.generation_id,- None,- )-- def trigger(self) -> bool:- return bool(- self._state.state == "running" and self._supervisor and self._supervisor.trigger()- )-- def lookup_raw(- self,- query: str,- *,- source_ids: frozenset[str] | None,- deadline: float,- time_range: TimeRange | None = None,- ) -> tuple[RawCandidateBatch, bool]:- """Read-only candidate lookup. Does not release original bodies."""- store, encoder, vectors, generation = (- self._store,- self._encoder,- self._vectors,- self._state.generation_id,- )- if (- self._state.state != "running"- or store is None- or encoder is None- or vectors is None- or generation is None- ):- return RawCandidateBatch(reason="index_unavailable"), True- counts = store.status(generation)- unpublished = counts["pending"] + counts["leased"] + counts["blocked"] > 0- resolved = source_ids- if resolved is None:- resolved = frozenset(- chunk.source_id for chunk in store.active_chunks(generation, deadline=deadline)- )- if not resolved:- return (- RawCandidateBatch(complete=not unpublished, reason="no_eligible_chunks"),- unpublished,- )- try:- vector = encoder.encode_documents([query.strip()])[0]- except Exception:- return RawCandidateBatch(reason="index_unavailable"), True- batch = read_raw_semantic_candidates(- store,- vectors,- vector,- generation_id=generation,- source_ids=resolved,- deadline=deadline,- time_range=time_range,- )- return batch, unpublished-- async def start(self) -> bool:- if self._state.state != "new" or self._stop_requested:- raise RuntimeError("retrieval_runtime_already_started")- self._set("starting")- self._start_task = asyncio.create_task(self._start())- try:- return await asyncio.shield(self._start_task)- except asyncio.CancelledError:- # Retain and drain opening resources, then release them, before cancellation escapes.- await self.stop()- raise-- async def stop(self) -> None:- self._stop_requested = True- if self._stop_task is None:- self._stop_task = asyncio.create_task(self._stop())- cancelled = False- while not self._stop_task.done():- try:- await asyncio.shield(self._stop_task)- except asyncio.CancelledError:- cancelled = True- self._stop_task.result()- if cancelled:- raise asyncio.CancelledError-- async def _start(self) -> bool:- try:- store, encoder, vectors, generation = await _native(self._open)- self._store = store- self._encoder = encoder- self._vectors = vectors- self._set("starting", generation=generation)- if self._stop_requested:- return False- worker = RetrievalIndexWorker(- store, vectors, encoder, generation_id=generation, owner="managed-core-retrieval"- )- self._supervisor = RetrievalIndexSupervisor(- worker,- background_should_yield=lambda: self._service_levels.should_yield(- ServiceResource.INDEX,- ServiceLevel.BACKGROUND,- ),- )- self._supervisor.start()- self._set("running")- return True- except _AdmissionError as error:- self._set("not_initialized" if error.missing else "failed", error.code)- except Exception:- self._set("failed", "retrieval_runtime_start_failed")- await self._release()- return False-- async def _release(self) -> None:- stop_failed = False- if self._supervisor is not None:- try:- await self._supervisor.stop()- except Exception:- stop_failed = True- supervisor_snapshot = self._supervisor.snapshot()- if supervisor_snapshot.thread_alive:- self._set("failed", "retrieval_runtime_stop_failed")- raise RuntimeError("retrieval_runtime_stop_failed") from None- # The supervisor owns the worker, which owns the native vector- # client. Retain only the immutable receipt once its thread drains.- self._last_supervisor_snapshot = supervisor_snapshot- self._supervisor = None- if self._vectors is not None:- try:- await _native(self._vectors.close)- except Exception:- self._set("failed", "retrieval_runtime_close_failed")- raise RuntimeError("retrieval_runtime_close_failed") from None- self._vectors = None- self._store = None- self._encoder = None- if stop_failed:- self._set("failed", "retrieval_runtime_stop_failed")- raise RuntimeError("retrieval_runtime_stop_failed") from None-- async def _stop(self) -> None:- if self._start_task is not None:- try:- await asyncio.shield(self._start_task)- except Exception:- # Startup can itself fail during cleanup. Do not lose resource ownership.- self._set("failed", "retrieval_runtime_start_failed")- self._set("stopping", self._state.error_code)- await self._release()- if self._state.error_code == "retrieval_runtime_close_failed":- return- self._set("stopped", self._state.error_code)-- @staticmethod- def _path(path: Path, *, directory: bool) -> tuple[int, int]:- if any(part.is_symlink() for part in (path, *path.parents)):- raise _AdmissionError("retrieval_runtime_unsafe_path")- try:- info = path.stat()- except FileNotFoundError:- raise _AdmissionError("retrieval_runtime_not_initialized", missing=True) from None- if not (stat.S_ISDIR(info.st_mode) if directory else stat.S_ISREG(info.st_mode)):- raise _AdmissionError("retrieval_runtime_unsafe_path")- if not directory and info.st_nlink != 1:- raise _AdmissionError("retrieval_runtime_unsafe_path")- if info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) & 0o022:- raise _AdmissionError("retrieval_runtime_unsafe_path")- return info.st_dev, info.st_ino-- def _sidecars(self, database: Path) -> None:- for suffix in ("-wal", "-shm", "-journal"):- path = database.with_name(database.name + suffix)- if path.is_symlink() or path.exists():- self._path(path, directory=False)-- def _inspect(self, database: Path) -> tuple[RetrievalIndexStore, EncoderIdentity, str]:- with main_database(database.absolute()).read(busy_timeout_ms=1_000) as db:- db.row_factory = sqlite3.Row- db.execute("PRAGMA query_only=ON")- db.execute("BEGIN")- if not db.execute(- "SELECT 1 FROM sqlite_master WHERE name='retrieval_index_state'"- ).fetchone():- raise _AdmissionError("retrieval_runtime_not_initialized", missing=True)- state = db.execute(- "SELECT schema_version FROM retrieval_index_state WHERE singleton=1"- ).fetchone()- if state is not None and state[0] == 1:- raise _AdmissionError("retrieval_runtime_not_initialized", missing=True)- if state is None or state[0] != 2:- raise _AdmissionError("retrieval_runtime_invalid_index")- rows = db.execute(- "SELECT generation_id,identity_json FROM retrieval_index_generations "- "WHERE state='active' LIMIT 2"- ).fetchall()- if not rows:- raise _AdmissionError("retrieval_runtime_not_initialized", missing=True)- if len(rows) != 1:- raise _AdmissionError("retrieval_runtime_invalid_index")- store = RetrievalIndexStore(database)- generation = rows[0]["generation_id"]- identity = store._identity(db, generation)- raw = json.loads(rows[0]["identity_json"])- if raw != asdict(identity) or set(raw) != set(asdict(identity)):- raise _AdmissionError("retrieval_runtime_invalid_index")- # Check required schema shape without repairing it or reading original bodies.- db.execute(- "SELECT build_id,source_map_json,map_digest,coverage_state "- "FROM retrieval_build_maps LIMIT 0"- )- db.execute("SELECT after_seq,retry_after_seq FROM retrieval_consumers LIMIT 0")- db.execute("SELECT lease_token,active_build_id FROM retrieval_documents LIMIT 0")- db.execute("SELECT chunk_id,text_hash FROM retrieval_chunks LIMIT 0")- return store, identity, generation-- def _open(self) -> tuple[RetrievalIndexStore, LocalRetrievalEncoder, RawVectorStore, str]:- if sys.platform == "win32" or not hasattr(os, "getuid"):- raise _AdmissionError("retrieval_runtime_platform_unsupported")- directory = self._directory.absolute()- database, vector_dir = directory / "one-memory.sqlite3", directory / "retrieval-vectors"- root_pin = self._path(directory, directory=True)- database_pin = self._path(database, directory=False)- self._sidecars(database)- try:- store, identity, generation = self._inspect(database)- except (ValueError, TypeError, sqlite3.Error):- raise _AdmissionError("retrieval_runtime_invalid_index") from None- vector_pin = self._path(vector_dir, directory=True)- vector_database = vector_dir / "chroma.sqlite3"- vector_db_pin = self._path(vector_database, directory=False)- self._sidecars(vector_database)-- def recheck() -> None:- if (- self._path(directory, directory=True) != root_pin- or self._path(database, directory=False) != database_pin- or self._path(vector_dir, directory=True) != vector_pin- or self._path(vector_database, directory=False) != vector_db_pin- ):- raise _AdmissionError("retrieval_runtime_index_changed")- self._sidecars(database)- self._sidecars(vector_database)-- if shutil.disk_usage(directory).free < self._minimum_free:- raise _AdmissionError("retrieval_runtime_low_disk")- encoder = LocalRetrievalEncoder(content_version=identity.normalizer_version)- if encoder.identity != identity:- raise _AdmissionError("retrieval_runtime_encoder_mismatch")- recheck()- try:- vectors = RawVectorStore.open_writer_existing(- vector_dir, generation_id=generation, dimensions=identity.dimensions- )- # Retain ownership immediately so all later failures use the same cleanup path.- self._vectors = vectors- # A generation change during the native open is not authorization to resume another.- recheck()- if self._inspect(database)[1:] != (identity, generation):- raise _AdmissionError("retrieval_runtime_index_changed")- recheck()- return store, encoder, vectors, generation- except BaseException as error:- if isinstance(error, VectorStoreError):- if str(error) in {"vector_store_not_found", "vector_collection_not_found"}:- raise _AdmissionError(- "retrieval_runtime_not_initialized", missing=True- ) from None- raise _AdmissionError("retrieval_runtime_vector_unavailable") from None- raisediff --git a/memhub/apps/core/src/one_memory/retrieval/vector_store.py b/memhub/apps/core/src/one_memory/retrieval/vector_store.pydeleted file mode 100644index 5f95d6f..0000000--- a/memhub/apps/core/src/one_memory/retrieval/vector_store.py+++ /dev/null@@ -1,534 +0,0 @@-"""Independent raw-vector adapter. Eligibility is supplied by Core's SQLite snapshot.--No HY collection, document text, embedding function, migration/repair or authority-store lives here. Chroma PersistentClient is NOT physically read-only: even the-existing-collection handle can maintain backend metadata/index files. A caller-requiring filesystem read-only operation must not use this backend.--Scoring is an exact pass over the eligible vectors, not a global ANN top-K that-is filtered afterwards. Chroma's filtered query does not score every candidate-in its `where $in` allowlist: at 10k stored chunks an explicit 64-ID allowlist-returned 26 of 64 while reporting the query as complete, and 0 of 64 after the-writer handle was reopened. A post-filtered global search would be neither-permission-correct nor recall-correct, so the eligible embeddings are read-back and ranked here.-"""--from __future__ import annotations--import math-import re-import sqlite3-import struct-import threading-from collections.abc import Callable, Sequence-from contextlib import closing-from dataclasses import dataclass-from hashlib import sha256-from numbers import Real-from pathlib import Path-from time import monotonic-from typing import Any, Literal--ALLOWLIST_BATCH_SIZE = 2048---class VectorStoreError(RuntimeError):- """Bounded failure code; callers must not treat corruption as complete empty search."""---@dataclass(frozen=True)-class RawVectorRecord:- chunk_id: str- build_id: str- generation_id: str- vector: tuple[float, ...]-- def __post_init__(self) -> None:- object.__setattr__(self, "vector", tuple(self.vector))---@dataclass(frozen=True)-class VectorHit:- chunk_id: str- build_id: str- generation_id: str- distance: float---@dataclass(frozen=True)-class VectorSearchResult:- hits: tuple[VectorHit, ...]- complete: bool- eligible_count: int- searched_count: int- reason: Literal["deadline"] | None---class RawVectorStore:- """Unopened constructor has no I/O. Use create/open_existing explicitly.-- Directory must be a dedicated, trusted Runtime retrieval-vectors directory,- never the HY data directory. Only create/open_writer_existing enable upsert.- Core must serialize writers across handles/processes; this instance's lock- is not a persistent writer lease. close() releases only this Chroma client,- retaining other live clients' references.- """-- def __init__(- self,- directory: Path,- *,- generation_id: str,- dimensions: int,- clock: Callable[[], float] = monotonic,- ) -> None:- _identifier(generation_id)- if type(dimensions) is not int or not 1 <= dimensions <= 65536:- raise VectorStoreError("vector_invalid_dimensions")- self.directory = Path(directory)- self.generation_id = generation_id- self.dimensions = dimensions- self.collection_name = "raw_" + sha256(generation_id.encode()).hexdigest()- self._clock = clock- self._client: Any = None- self._collection: Any = None- self._writable = False- self._closed = False- self._lock = threading.RLock()-- @classmethod- def create(cls, directory: Path, *, generation_id: str, dimensions: int) -> RawVectorStore:- """Explicit writer initialization. Refuses to reuse an existing collection."""- value = cls(directory, generation_id=generation_id, dimensions=dimensions)- value._open(create=True)- return value-- @classmethod- def open_existing(- cls,- directory: Path,- *,- generation_id: str,- dimensions: int,- ) -> RawVectorStore:- """No logical index writes; persistent Chroma may still write maintenance files."""- value = cls(directory, generation_id=generation_id, dimensions=dimensions)- value._open(create=False)- return value-- @classmethod- def open_writer_existing(- cls,- directory: Path,- *,- generation_id: str,- dimensions: int,- ) -> RawVectorStore:- """Explicit worker recovery; verifies existing collection, never creates/repairs."""- value = cls(directory, generation_id=generation_id, dimensions=dimensions)- value._open(create=False, writable=True)- return value-- def _metadata(self) -> dict[str, str | int]:- return {- "adapter": "one-memory-raw-v1",- "generation_id": self.generation_id,- "dimensions": self.dimensions,- "distance": "cosine",- "normalization": "l2-float32-v1",- }-- def _preflight(self, create: bool) -> None:- if any(part.is_symlink() for part in (self.directory, *self.directory.parents)):- raise VectorStoreError("vector_unsafe_directory")- if not self.directory.exists():- if not create:- raise VectorStoreError("vector_store_not_found")- self.directory.mkdir(parents=True)- if not self.directory.is_dir():- raise VectorStoreError("vector_unsafe_directory")- database = self.directory / "chroma.sqlite3"- if database.is_symlink():- raise VectorStoreError("vector_unsafe_database")- if not database.is_file():- if not create:- raise VectorStoreError("vector_store_not_found")- if any(self.directory.iterdir()):- raise VectorStoreError("vector_unknown_directory")- return- if database.stat().st_nlink != 1:- raise VectorStoreError("vector_unsafe_database")- # Refuse missing/corrupt collections BEFORE PersistentClient can perform- # startup maintenance. This checks the installed Chroma 1.5.9 schema.- try:- with closing(sqlite3.connect(database.resolve().as_uri() + "?mode=ro", uri=True)) as db:- found = db.execute(- "SELECT 1 FROM collections WHERE name=?", (self.collection_name,)- ).fetchone()- if not create and found is None:- raise VectorStoreError("vector_collection_not_found")- if create and found is not None:- raise VectorStoreError("vector_collection_exists")- except sqlite3.Error:- raise VectorStoreError("vector_store_unavailable") from None-- def _open(self, *, create: bool, writable: bool = False) -> None:- self._preflight(create)- try:- import chromadb- from chromadb.config import Settings-- self._client = chromadb.PersistentClient(- path=str(self.directory), settings=Settings(anonymized_telemetry=False)- )- if create:- self._collection = self._client.create_collection(- self.collection_name,- embedding_function=None,- metadata=self._metadata(),- # Unit vectors + inner product give cosine distance without- # cosine HNSW's lossy normalization/de-normalization on get.- configuration={"hnsw": {"space": "ip"}},- )- else:- self._collection = self._client.get_collection(- self.collection_name, embedding_function=None- )- if (- self._collection.metadata != self._metadata()- or type(self._collection.metadata.get("dimensions")) is not int- ):- raise VectorStoreError("vector_collection_identity_mismatch")- config = self._collection.configuration- if (- config.get("hnsw", {}).get("space") != "ip"- or config.get("embedding_function") is not None- ):- raise VectorStoreError("vector_collection_configuration_mismatch")- self._writable = create or writable- except Exception as error:- self.close()- if isinstance(error, VectorStoreError):- raise- raise VectorStoreError("vector_store_unavailable") from None-- def close(self) -> None:- with self._lock:- if self._closed:- return- self._closed = True- if self._client is not None:- self._client.close()- self._client = self._collection = None-- def _require_open(self) -> Any:- if self._closed:- raise VectorStoreError("vector_store_closed")- if self._collection is None:- raise VectorStoreError("vector_store_not_open")- return self._collection-- def _records(self, records: Sequence[RawVectorRecord]) -> tuple[RawVectorRecord, ...]:- result = []- seen: set[str] = set()- for record in records:- if not isinstance(record, RawVectorRecord):- raise VectorStoreError("vector_invalid_record")- for identifier in (record.chunk_id, record.build_id, record.generation_id):- _identifier(identifier)- if record.generation_id != self.generation_id or record.chunk_id in seen:- raise VectorStoreError("vector_invalid_record_identity")- seen.add(record.chunk_id)- result.append(- RawVectorRecord(- record.chunk_id,- record.build_id,- record.generation_id,- _vector(record.vector, self.dimensions),- )- )- return tuple(result)-- def _read_records(self, ids: Sequence[str]) -> dict[str, tuple[RawVectorRecord, str]]:- if not ids:- return {}- collection = self._require_open()- result: dict[str, tuple[RawVectorRecord, str]] = {}- for start in range(0, len(ids), ALLOWLIST_BATCH_SIZE):- batch = ids[start : start + ALLOWLIST_BATCH_SIZE]- try:- data = collection.get(ids=list(batch), include=["embeddings", "metadatas"])- returned, vectors, metadatas = data["ids"], data["embeddings"], data["metadatas"]- if (- vectors is None- or metadatas is None- or not len(returned) == len(vectors) == len(metadatas)- ):- raise VectorStoreError("vector_invalid_record_response")- for key, vector, metadata in zip(returned, vectors, metadatas, strict=True):- if key not in batch or key in result:- raise VectorStoreError("vector_ineligible_record")- result[key] = self._decode(key, vector, metadata)- except VectorStoreError:- raise- except Exception:- raise VectorStoreError("vector_read_unavailable") from None- return result-- def _decode(- self, key: str, vector: Sequence[float], metadata: Any- ) -> tuple[RawVectorRecord, str]:- if not isinstance(metadata, dict) or set(metadata) != {- "chunk_id",- "build_id",- "generation_id",- "vector_sha256",- "input_sha256",- }:- raise VectorStoreError("vector_invalid_metadata")- if metadata["chunk_id"] != key or metadata["generation_id"] != self.generation_id:- raise VectorStoreError("vector_metadata_identity_mismatch")- _identifier(key)- _identifier(metadata["build_id"])- if not isinstance(metadata["input_sha256"], str) or not re.fullmatch(- r"[0-9a-f]{64}", metadata["input_sha256"]- ):- raise VectorStoreError("vector_invalid_input_digest")- normalized = _vector(vector, self.dimensions)- if metadata["vector_sha256"] != _digest(normalized):- raise VectorStoreError("vector_content_mismatch")- return RawVectorRecord(key, metadata["build_id"], self.generation_id, normalized), metadata[- "input_sha256"- ]-- def upsert(self, records: Sequence[RawVectorRecord]) -> None:- """Idempotent immutable insertion; add-only at Chroma, never overwrite IDs.-- Equality is canonical float32 input identity plus explicit normalized- storage identity. Conflict is checked before- any add. Backend failure may leave a partial new batch: caller must verify- the entire build before publication. Existing IDs are never modified.- """- with self._lock:- collection = self._require_open()- if not self._writable:- raise VectorStoreError("vector_store_read_only")- incoming = self._records(records)- existing = self._read_records([record.chunk_id for record in incoming])- for record in incoming:- if record.chunk_id in existing and not _matches(existing[record.chunk_id], record):- raise VectorStoreError("vector_immutable_conflict")- new = [record for record in incoming if record.chunk_id not in existing]- for start in range(0, len(new), ALLOWLIST_BATCH_SIZE):- batch = new[start : start + ALLOWLIST_BATCH_SIZE]- try:- collection.add(- ids=[record.chunk_id for record in batch],- embeddings=[list(_unit_vector(record.vector)) for record in batch],- metadatas=[- {- "chunk_id": record.chunk_id,- "build_id": record.build_id,- "generation_id": record.generation_id,- "vector_sha256": _digest(_unit_vector(record.vector)),- "input_sha256": _digest(record.vector),- }- for record in batch- ],- )- except Exception:- raise VectorStoreError("vector_write_unavailable") from None- self.verify(incoming)-- def verify(self, records: Sequence[RawVectorRecord]) -> None:- """Verify every expected ID, metadata, dimensions and exact float32 vector."""- with self._lock:- self._require_open()- expected = self._records(records)- actual = self._read_records([record.chunk_id for record in expected])- for record in expected:- if record.chunk_id not in actual:- raise VectorStoreError("vector_record_missing")- if not _matches(actual[record.chunk_id], record):- raise VectorStoreError("vector_immutable_conflict")-- def verify_candidates(- self,- candidates: Sequence[tuple[str, str]],- *,- deadline: float,- ) -> bool:- """Read-only presence/integrity check for (chunk_id, expected_build_id).-- Costs O(candidates * dimensions), not a bounded top-K lookup. False means- deadline exhaustion, never proof of absence. Each read covers at most- 2048 IDs; synchronous backend calls cannot be interrupted. This is not a- snapshot across Chroma and SQLite: Core still owns final eligibility- checks and writer serialization. Chroma may perform maintenance writes.- """- with self._lock:- self._require_open()- if (- isinstance(deadline, bool)- or not isinstance(deadline, Real)- or not math.isfinite(deadline)- ):- raise VectorStoreError("vector_invalid_deadline")- if not isinstance(candidates, Sequence) or isinstance(candidates, (str, bytes)):- raise VectorStoreError("vector_invalid_candidates")- expected: list[tuple[str, str]] = []- seen: set[str] = set()- for pair in candidates:- if (- not isinstance(pair, Sequence)- or isinstance(pair, (str, bytes))- or len(pair) != 2- ):- raise VectorStoreError("vector_invalid_candidates")- key, build = pair- _identifier(key)- _identifier(build)- if key in seen:- raise VectorStoreError("vector_duplicate_candidates")- seen.add(key)- expected.append((key, build))- for start in range(0, len(expected), ALLOWLIST_BATCH_SIZE):- if self._clock() >= deadline:- return False- batch = expected[start : start + ALLOWLIST_BATCH_SIZE]- # Bound this call to one batch so it cannot read past the next- # deadline checkpoint via _read_records' internal loop.- actual = self._read_records([key for key, _build in batch])- for key, build in batch:- if key not in actual:- raise VectorStoreError("vector_record_missing")- if actual[key][0].build_id != build:- raise VectorStoreError("vector_metadata_identity_mismatch")- if self._clock() >= deadline:- return False- return True-- def search(- self,- query_vector: Sequence[float],- eligible_ids: Sequence[str],- limit: int,- deadline: float,- ) -> VectorSearchResult:- """Rank exactly the eligible candidates; synchronous backend cannot be preempted.-- deadline is absolute monotonic seconds. A late batch is discarded; prior- on-time batches may return partial. complete means every eligible- vector was scored, NOT complete index coverage: only chunks the caller- already resolved as eligible and published are searched.-- The eligible embeddings are read back in bounded batches and ranked- here, because Chroma's own filtered query does not score every ID in- its allowlist. A global top-K filtered afterwards would rank- unauthorized vectors first and let them crowd the eligible ones out.- """- with self._lock:- self._require_open()- vector = _unit_vector(_vector(query_vector, self.dimensions))- if type(limit) is not int or not 1 <= limit <= ALLOWLIST_BATCH_SIZE:- raise VectorStoreError("vector_invalid_limit")- if (- isinstance(deadline, bool)- or not isinstance(deadline, Real)- or not math.isfinite(deadline)- ):- raise VectorStoreError("vector_invalid_deadline")- if isinstance(eligible_ids, (str, bytes)):- raise VectorStoreError("vector_invalid_allowlist")- ids = tuple(eligible_ids)- for key in ids:- _identifier(key)- if len(set(ids)) != len(ids):- raise VectorStoreError("vector_duplicate_allowlist")- hits: list[VectorHit] = []- searched = 0- for start in range(0, len(ids), ALLOWLIST_BATCH_SIZE):- if self._clock() >= deadline:- break- batch = ids[start : start + ALLOWLIST_BATCH_SIZE]- try:- read = self._read_records(batch)- except VectorStoreError as error:- # A missing or corrupt eligible vector is not "no hits".- raise VectorStoreError("vector_query_unavailable") from error- if self._clock() >= deadline:- break- try:- missing = [key for key in batch if key not in read]- if missing:- # The caller resolved these as eligible and published,- # so an absent vector is a broken build, not a miss.- raise VectorStoreError("vector_query_unavailable")- validated = [- VectorHit(- key,- read[key][0].build_id,- read[key][0].generation_id,- # Stored vectors are unit length, so the inner- # product is the cosine similarity.- 1.0 - _dot(read[key][0].vector, vector),- )- for key in batch- ]- except VectorStoreError:- raise- except Exception:- raise VectorStoreError("vector_invalid_query_response") from None- if self._clock() >= deadline:- break- searched += len(batch)- hits.extend(validated)- hits.sort(key=lambda hit: (hit.distance, hit.chunk_id))- del hits[limit:]- complete = searched == len(ids)- return VectorSearchResult(- tuple(hits), complete, len(ids), searched, None if complete else "deadline"- )---def _dot(left: Sequence[float], right: Sequence[float]) -> float:- return math.fsum(a * b for a, b in zip(left, right, strict=True))---def _identifier(value: Any) -> None:- if not isinstance(value, str) or not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_.:-]{0,255}", value):- raise VectorStoreError("vector_invalid_identifier")---def _vector(value: Sequence[float], dimensions: int) -> tuple[float, ...]:- try:- if len(value) != dimensions or any(- isinstance(v, bool) or not isinstance(v, Real) or not math.isfinite(v) for v in value- ):- raise VectorStoreError("vector_invalid_values_or_dimensions")- packed = struct.pack(f"<{dimensions}f", *value)- normalized = struct.unpack(f"<{dimensions}f", packed)- if not all(math.isfinite(v) for v in normalized) or not any(normalized):- raise VectorStoreError("vector_invalid_values_or_dimensions")- return normalized- except VectorStoreError:- raise- except (ValueError, TypeError, OverflowError, struct.error):- raise VectorStoreError("vector_invalid_values_or_dimensions") from None---def _digest(vector: tuple[float, ...]) -> str:- return sha256(struct.pack(f"<{len(vector)}f", *vector)).hexdigest()---def _unit_vector(vector: tuple[float, ...]) -> tuple[float, ...]:- length = math.sqrt(math.fsum(component * component for component in vector))- return _vector(tuple(component / length for component in vector), len(vector))---def _matches(actual: tuple[RawVectorRecord, str], expected: RawVectorRecord) -> bool:- stored, input_digest = actual- return input_digest == _digest(expected.vector) and stored == RawVectorRecord(- expected.chunk_id, expected.build_id, expected.generation_id, _unit_vector(expected.vector)- )diff --git a/memhub/apps/core/tests/TEST_INDEX.md b/memhub/apps/core/tests/TEST_INDEX.mdindex 3d4c0ea..953b4e4 100644--- a/memhub/apps/core/tests/TEST_INDEX.md+++ b/memhub/apps/core/tests/TEST_INDEX.md@@ -139,7 +139,6 @@| `test_live_watch.py` | `contract` | 覆盖 a new turn lands well inside the ten second target、start prime catches a change that predates the watcher、a run that predates the change is followed by a fresh one 等 15 个场景。 || `test_live_watch_grants.py` | `integration` | 覆盖 explicit sync grant starts watch without restart and revocation stops it 场景。 || `test_local_embedding.py` | `contract` | Guards for the built-in embedding model. |-| `test_local_embedding_offsets.py` | `contract` | 覆盖 real locked tokenizer offsets are unicode codepoints、known emoji and combining offsets are not utf8 or utf16 units、offsets are not truncated or padded and do not create onnx session 等 9 个场景。 || `test_local_onnx_provider.py` | `contract` | Expect local ONNX embedding to be the default and to feed Hy Memory. || `test_log_file.py` | `contract` | 覆盖 what the watchdog says survives a dock launch、the log lives where the data does、a machine left running for months does not fill up 等 4 个场景。 || `test_log_privacy.py` | `integration` | The file log is shipped to us in bug reports; it must not carry originals. |@@ -232,21 +231,8 @@| `test_recall_scope_control.py` | `integration` | 覆盖 external scope cannot override shared or host workspace boundary、scope only mutation is versioned without changing status or reindexing 场景。 || `test_recall_scope_off_loop.py` | `contract` | The workspace lookup behind an external recall must not run on the loop. || `test_recent_first_import.py` | `contract` | Recent-file priority survives restart through a selection-bound frozen copy. |-| `test_retrieval_chunking.py` | `contract` | 覆盖 default 384 body 64 overlap and exact source slices、all prefix and special tokens count toward strict 512 budget、real tokenizer covers long chinese middle and tail object locations 等 20 个场景。 |-| `test_retrieval_eligibility_benchmark.py` | `contract` | 覆盖 scale probe proves composite authority index changes the plan 场景。 |-| `test_retrieval_index_store.py` | `contract` | 覆盖 constructor and reads do not create database or schema、explicit initialization rolls back and never grants model policy、consumer and jobs commit together 等 19 个场景。 |-| `test_retrieval_index_supervisor.py` | `contract` | 覆盖 stop waits for native exit and does not close resources、async cancellation repeatedly waits for native thread、idle no trigger no spin and duplicate start rejected 等 14 个场景。 |-| `test_retrieval_index_worker.py` | `contract` | 覆盖 tick indexes complete original and reopen can search、change during native encoding never publishes、cancel after encode does not publish and is reclaimable 等 10 个场景。 || `test_retrieval_lifespan.py` | `integration` | 覆盖 retrieval defaults off and does not create index、enabling retrieval requires runtime ownership、index enabled serve imports without building an unmanaged app 等 6 个场景。 |-| `test_retrieval_mapped_content.py` | `contract` | 覆盖 raw authors keep exact body and do not mutate evidence、raw and author qualification rejects without tokenizing、ambiguous generated and body hygiene are not admitted 等 22 个场景。 |-| `test_retrieval_mapped_integration.py` | `contract` | 覆盖 empty materialization rechecks excluded map、completed excluded missing active binding is not empty success、old empty completion is invalidated when a new indexed record arrives 等 24 个场景。 |-| `test_retrieval_provisioning.py` | `contract` | The index has to come up through the product, not through a test helper. || `test_retrieval_query_plan.py` | `contract` | Local QueryPlan: extract URL/path objects. |-| `test_retrieval_raw_candidates.py` | `contract` | 覆盖 positive only current authorized published chunks are searched、empty authority never reads database or vector、unapproved source never enters vector query 等 15 个场景。 |-| `test_retrieval_raw_evidence.py` | `contract` | 覆盖 materialized text is exact current original、stale candidates do not authorize original read、budget does not cut a chunk into fake partial object 等 6 个场景。 |-| `test_retrieval_raw_lane.py` | `contract` | v1 raw lane must not claim complete empty when the derived index is partial. |-| `test_retrieval_runtime.py` | `contract` | 覆盖 constructor and missing start create nothing、stop releases supervisor worker but preserves immutable snapshot、not initialized or invalid configuration never creates index 等 21 个场景。 |-| `test_retrieval_vector_store.py` | `contract` | 覆盖 constructor does no io and unopened calls do not create、open existing missing store or collection never creates、real chroma collection has no embedding function or documents 等 32 个场景。 || `test_runtime.py` | `integration` | 覆盖 runtime writes and removes ownership receipt、runtime rejects second owner for same data directory、nonpublishing maintenance owner still excludes a second core 等 4 个场景。 || `test_runtime_client.py` | `contract` | 覆盖 runtime inspection authenticates the discovered instance、runtime inspection rejects untrusted receipts without probing、runtime inspection distinguishes stopped and identity mismatch 场景。 || `test_sanitization_source_map.py` | `contract` | 覆盖 mapping matches existing sanitizer and exact original slices、synthetic replacements never claim original locations、repeated identical suffix is mapped by positions not string search 等 6 个场景。 |@@ -328,4 +314,4 @@| `test_workspace_recall.py` | `contract` | Workspace memories were unreachable from every host; these pin the way in. || `test_write_queue_lease_renewal.py` | `contract` | Rule 8: a lease must outlive the work it covers, and be renewed while it runs. |-共 323 个测试文件。+共 309 个测试文件。diff --git a/memhub/apps/core/tests/core_test_classification.py b/memhub/apps/core/tests/core_test_classification.pyindex c24f9a4..9fa28b3 100644--- a/memhub/apps/core/tests/core_test_classification.py+++ b/memhub/apps/core/tests/core_test_classification.py@@ -14,8 +14,8 @@ from pathlib import Pathimport pytestPRIMARY_MARKERS = ("unit", "contract", "integration", "performance", "live")-EXPECTED_TEST_FILE_COUNT = 323-EXPECTED_TEST_FILES_SHA256 = "10a2327bd047a281511980f5b558c1ea64d857f0fff550e4bd96cb53800205ba"+EXPECTED_TEST_FILE_COUNT = 309+EXPECTED_TEST_FILES_SHA256 = "ef6165c249971f5a2ab71a0c49f6d859c8c0f3612e3f749fba0acce0eab513ea"# Pure transformations, validation models, parity checks and state machines.# This list is intentionally small: uncertain files default to contract.diff --git a/memhub/apps/core/tests/test_local_embedding_offsets.py b/memhub/apps/core/tests/test_local_embedding_offsets.pydeleted file mode 100644index 66da996..0000000--- a/memhub/apps/core/tests/test_local_embedding_offsets.py+++ /dev/null@@ -1,132 +0,0 @@-import socket-from concurrent.futures import ThreadPoolExecutor-from dataclasses import FrozenInstanceError--import pytest--from one_memory import local_embedding-from one_memory.embedding_assets import AssetVerificationError, resolve_asset_dir-from one_memory.local_embedding import LocalEmbedder, TokenizedText--[email protected](autouse=True)-def no_network(monkeypatch):- def refused(*args, **kwargs):- pytest.fail("local tokenizer/encoder attempted network")-- monkeypatch.setattr(socket.socket, "connect", refused)- monkeypatch.setattr(socket, "create_connection", refused)--- "text", ["中文😀e\u0301末尾", "👩💻链接", "e\u0301 cafe\u0301", "URL路径 /Case/末尾"]-)-def test_real_locked_tokenizer_offsets_are_unicode_codepoints(text):- embedder = LocalEmbedder()- tokenized = embedder.tokenize_with_offsets(text)- assert isinstance(tokenized, TokenizedText)- assert isinstance(tokenized.token_ids, tuple) and isinstance(tokenized.offsets, tuple)- ordinary = [offset for offset in tokenized.offsets if not offset.special]- assert ordinary- for offset in ordinary:- assert 0 <= offset.start_cp < offset.end_cp <= len(text)- assert text[offset.start_cp : offset.end_cp]- assert all(- any(o.start_cp <= i < o.end_cp for o in ordinary)- for i, character in enumerate(text)- if not character.isspace()- )- assert embedder._session is None and embedder._tokenizer is None- with pytest.raises(FrozenInstanceError):- ordinary[0].start_cp = 10---def test_known_emoji_and_combining_offsets_are_not_utf8_or_utf16_units():- value = LocalEmbedder().tokenize_with_offsets("中文😀e\u0301末尾")- offsets = [(o.start_cp, o.end_cp) for o in value.offsets if not o.special]- assert offsets == [(0, 1), (1, 2), (2, 5), (5, 6), (6, 7)]---def test_offsets_are_not_truncated_or_padded_and_do_not_create_onnx_session(monkeypatch):- monkeypatch.setattr(local_embedding, "import_module", lambda *args: pytest.fail("ONNX load"))- embedder = LocalEmbedder()- long = embedder.tokenize_with_offsets("中" * 1600)- short = embedder.tokenize_with_offsets("中")- empty = embedder.tokenize_with_offsets("")- assert len(long.token_ids) == 1602- assert len(short.token_ids) == 3- assert len(empty.token_ids) == 2- assert all(o.special for o in empty.offsets)- assert long.offsets[-2].end_cp == 1600- assert embedder._session is None---def test_parallel_offset_calls_share_one_lazy_tokenizer_without_shared_encoder_mutation():- embedder = LocalEmbedder()- texts = ["中" * 700, "😀中e\u0301", "short"] * 8- with ThreadPoolExecutor(max_workers=8) as pool:- actual = list(pool.map(embedder.tokenize_with_offsets, texts))- assert actual == [embedder.tokenize_with_offsets(text) for text in texts]- assert embedder._session is None and embedder._tokenizer is None---def test_offset_tokenizer_does_not_modify_encode_state_or_vectors():- embedder = LocalEmbedder()- baseline = embedder.encode(["原有编码必须保持", "短句"])- session, tokenizer = embedder._session, embedder._tokenizer- truncation, padding = tokenizer.truncation.copy(), tokenizer.padding.copy()- with ThreadPoolExecutor(max_workers=4) as pool:- tokenized = pool.submit(embedder.tokenize_with_offsets, "中" * 1100)- encoded = pool.submit(embedder.encode, ["原有编码必须保持", "短句"])- assert len(tokenized.result().token_ids) == 1102- assert encoded.result() == baseline- assert embedder._session is session and embedder._tokenizer is tokenizer- assert tokenizer.truncation == truncation and tokenizer.padding == padding- assert len(tokenizer.encode("中" * 1100).ids) == 512---def test_offset_assets_reject_unlocked_tokenizer_without_loading_weights(tmp_path, monkeypatch):- (tmp_path / "tokenizer.json").write_bytes(b'{"invalid":"not locked"}')- monkeypatch.setattr(local_embedding, "resolve_asset_dir", lambda: tmp_path)- with pytest.raises(AssetVerificationError):- LocalEmbedder().tokenize_with_offsets("中")---def test_offset_tokenization_uses_only_tokenizer_asset(tmp_path, monkeypatch):- # Tokenization must not require another ONNX session or even the weights file.- (tmp_path / "tokenizer.json").write_bytes((resolve_asset_dir() / "tokenizer.json").read_bytes())- monkeypatch.setattr(local_embedding, "resolve_asset_dir", lambda: tmp_path)- assert len(LocalEmbedder().tokenize_with_offsets("中文").token_ids) == 4---def test_real_adjacent_wordpieces_and_long_ascii_unknown_keep_original_offsets():- embedder = LocalEmbedder()- word = "unaffordable"- parts = [o for o in embedder.tokenize_with_offsets(word).offsets if not o.special]- assert len(parts) > 1- assert parts[0].start_cp == 0 and parts[-1].end_cp == len(word)- assert all(a.end_cp == b.start_cp for a, b in zip(parts, parts[1:], strict=False))- long_word = "x" * 6000- unknown = embedder.tokenize_with_offsets(long_word)- offsets = [o for o in unknown.offsets if not o.special]- assert len(offsets) == 1 and unknown.token_ids[1] == 100 # locked BERT [UNK]- assert (offsets[0].start_cp, offsets[0].end_cp) == (0, 6000)---def test_parallel_initial_load_constructs_one_offsets_tokenizer(monkeypatch):- from tokenizers import Tokenizer-- calls = []- original = Tokenizer.from_str-- def counted(value):- calls.append(1)- return original(value)-- monkeypatch.setattr(Tokenizer, "from_str", counted)- embedder = LocalEmbedder()- with ThreadPoolExecutor(max_workers=8) as pool:- results = list(pool.map(embedder.tokenize_with_offsets, ["中😀e\u0301"] * 16))- assert len(calls) == 1- assert all(value == results[0] for value in results)- assert embedder._session is Nonediff --git a/memhub/apps/core/tests/test_main_database_entry.py b/memhub/apps/core/tests/test_main_database_entry.pyindex 49b2b5d..6d8b9cf 100644--- a/memhub/apps/core/tests/test_main_database_entry.py+++ b/memhub/apps/core/tests/test_main_database_entry.py@@ -87,10 +87,6 @@ ALLOWED = {"SQLite's own defaults, read once from a connection that opens no file",opened_with='":memory:"',),- ("retrieval/vector_store.py", "RawVectorStore._preflight"): Allowed(- "Chroma's own chroma.sqlite3, read-only, checked before Chroma opens it",- opened_with='"?mode=ro"',- ),("sdk_migration_verify.py", "legacy_memory_ids_from_bundle"): Allowed("the pre-cutover bundle's copy of the legacy database, read-only, to list ""the memory ids a migrated home must still read back",diff --git a/memhub/apps/core/tests/test_model_creation_validation_boundary.py b/memhub/apps/core/tests/test_model_creation_validation_boundary.pyindex 45d1dd2..dc1dbf4 100644--- a/memhub/apps/core/tests/test_model_creation_validation_boundary.py+++ b/memhub/apps/core/tests/test_model_creation_validation_boundary.py@@ -85,7 +85,6 @@ async def application(tmp_path: Path):live_sync_enabled=False,live_sync_watch_enabled=False,automatic_memory_processing=False,- retrieval_index_enabled=False,)app = create_app(settings, source_catalog={}, secret_store=secrets)async with (diff --git a/memhub/apps/core/tests/test_model_probe_operations.py b/memhub/apps/core/tests/test_model_probe_operations.pyindex cee54d9..733b6e6 100644--- a/memhub/apps/core/tests/test_model_probe_operations.py+++ b/memhub/apps/core/tests/test_model_probe_operations.py@@ -592,7 +592,6 @@ def test_probe_api_optional_body_lookup_and_authentication(tmp_path, monkeypatchautomatic_memory_processing=False,live_sync_enabled=False,live_sync_watch_enabled=False,- retrieval_index_enabled=False,),source_catalog={},secret_store=secrets,diff --git a/memhub/apps/core/tests/test_raw_search_without_model.py b/memhub/apps/core/tests/test_raw_search_without_model.pyindex 9fb7896..0d987e7 100644--- a/memhub/apps/core/tests/test_raw_search_without_model.py+++ b/memhub/apps/core/tests/test_raw_search_without_model.py@@ -1,15 +1,13 @@"""Search imported conversations with no generative model configured at all.-Every other retrieval test either builds the index by hand or stubs the-encoder, so none of them answers the question a user would ask: I imported my-history and I never set up a model, can One Memory find things I said in-different words? This goes through the product's own entry points only:-authorize the source, run an import job, search `/api/v1/context/search`.+This answers the question a user would ask: I imported my history and I+never set up a model, can One Memory find things I said? This goes through the+product's own entry points only: authorize the source, run an import job,+search `/api/v1/context/search`."""import asyncioimport json-import timefrom datetime import UTC, datetimeimport pytest@@ -18,7 +16,6 @@ from pydantic import SecretStrfrom one_memory.api import create_appfrom one_memory.config import Settings-from one_memory.embedding_assets import assets_presentfrom one_memory.imports.sources.registry import build_source_catalogfrom one_memory.secrets import InMemorySecretStorefrom tests.source_consent import authorized_request, thaw_source@@ -29,14 +26,7 @@ LICENCE = "我们最后决定 One Memory 用 MIT 协议发布,不再考虑 GPLBACKUP = "备份先写到本地磁盘,再同步到对象存储。"SYNONYM = "开源软件的授权条款最后怎么挑的"-pytestmark = [- pytest.mark.integration,- pytest.mark.serial,- pytest.mark.requires_embedding_assets,- pytest.mark.skipif(- not assets_present(), reason="built-in embedding weights required for the semantic lane"- ),-]+pytestmark = [pytest.mark.integration, pytest.mark.serial]def transcript(root, name, *texts):@@ -57,7 +47,7 @@ def transcript(root, name, *texts):(root / f"rollout-{name}.jsonl").write_text("\n".join(lines) + "\n")-def application(tmp_path, *, index: bool):+def application(tmp_path):home = tmp_path / "home"root = home / ".codex/sessions"root.mkdir(parents=True)@@ -73,10 +63,8 @@ def application(tmp_path, *, index: bool):api_token=SecretStr(TOKEN),automatic_memory_processing=False,live_sync_enabled=False,- retrieval_index_enabled=index,),source_catalog={"codex": build_source_catalog(home=home, environment={})["codex"]},- manage_runtime=index,secret_store=InMemorySecretStore(),)@@ -106,20 +94,6 @@ async def search(client, query, *, attempts=1):raise AssertionError("unreachable")-def lexical_only(app, query):- """What the word-matching reader alone returns, for the same request."""- from one_memory.context_models import ContextSearchRequest- from one_memory.context_raw import read_raw_context-- result = read_raw_context(- app.state.evidence_store.database_path,- ContextSearchRequest(query=query, kinds=("raw",)),- source_ids=frozenset({"codex"}),- deadline=time.monotonic() + 5,- )- return tuple(result.items)--def texts(body):return [message["text"]@@ -131,7 +105,7 @@ def texts(body):def test_shared_words_are_found_without_any_model_or_index(tmp_path):async def scenario():- app = application(tmp_path, index=False)+ app = application(tmp_path)async with (app.router.lifespan_context(app),AsyncClient(@@ -144,38 +118,8 @@ def test_shared_words_are_found_without_any_model_or_index(tmp_path):assert any(LICENCE in text for text in texts(body))missed = await search(client, SYNONYM)- assert not texts(missed), (- "the lexical lane already answers this phrasing, so the semantic "- "test below would prove nothing"- )-- asyncio.run(scenario())---def test_a_semantic_only_candidate_is_not_released_without_confirmation(tmp_path):- async def scenario():- app = application(tmp_path, index=True)- async with (- app.router.lifespan_context(app),- AsyncClient(- transport=ASGITransport(app=app), base_url="http://test", headers=AUTH- ) as client,- ):- assert app.state.retrieval_runtime.snapshot().state == "running"- await imported(app, client)- deadline = time.monotonic() + 30- while time.monotonic() < deadline:- snapshot = app.state.retrieval_runtime.snapshot().supervisor- if snapshot is not None and snapshot.slices >= 4:- break- await asyncio.sleep(1)- assert snapshot is not None and snapshot.slices >= 4 and snapshot.failures == 0- body = await search(client, SYNONYM)- assert body["paths"]["raw"]["reason"] == "lexical_only", body- assert not texts(body), "a semantic-only candidate must not release original text"- assert not lexical_only(app, SYNONYM), "lexical alone answers this; nothing is proven"- assert any(LICENCE in text for text in texts(await search(client, "MIT 协议"))), (- "the index must not cost the lexical lane its own hits"- )+ # Different words are not a lexical match, and there is no private+ # vector index to pretend otherwise.+ assert not texts(missed)asyncio.run(scenario())diff --git a/memhub/apps/core/tests/test_retrieval_chunking.py b/memhub/apps/core/tests/test_retrieval_chunking.pydeleted file mode 100644index e62223d..0000000--- a/memhub/apps/core/tests/test_retrieval_chunking.py+++ /dev/null@@ -1,273 +0,0 @@-from dataclasses import FrozenInstanceError--import pytest--from one_memory.local_embedding import LocalEmbedder, TokenizedText, TokenOffset-from one_memory.retrieval.chunking import ChunkingError, chunk_evidence---class Characters:- def tokenize_with_offsets(self, text):- offsets = (- TokenOffset(0, 0, True),- *(TokenOffset(i, i + 1) for i in range(len(text))),- TokenOffset(0, 0, True),- )- return TokenizedText(tuple(range(len(offsets))), offsets)---def assert_coverage(text, chunks, tokenizer, prefix=""):- assert chunks- assert [chunk.ordinal for chunk in chunks] == list(range(len(chunks)))- covered = bytearray(len(text))- previous_start = -1- for chunk in chunks:- assert 0 <= chunk.start_cp < chunk.end_cp <= len(text)- assert chunk.start_cp > previous_start- previous_start = chunk.start_cp- assert chunk.text == text[chunk.start_cp : chunk.end_cp]- assert chunk.token_count == len(- tokenizer.tokenize_with_offsets(prefix + chunk.text).token_ids- )- assert chunk.token_count <= 512- covered[chunk.start_cp : chunk.end_cp] = b"\x01" * (chunk.end_cp - chunk.start_cp)- assert all(covered[i] for i, c in enumerate(text) if not c.isspace())- assert chunks[-1].end_cp == len(text)---def test_default_384_body_64_overlap_and_exact_source_slices():- text = "中" * 1000- chunks = chunk_evidence(text, Characters())- assert [(c.start_cp, c.end_cp) for c in chunks] == [(0, 384), (320, 704), (640, 1000)]- assert chunks[0].token_count == 386- assert_coverage(text, chunks, Characters())- with pytest.raises(FrozenInstanceError):- chunks[0].text = "replacement"---def test_all_prefix_and_special_tokens_count_toward_strict_512_budget():- prefix = "P" * 200- text = "中" * 900- chunks = chunk_evidence(text, Characters(), prefix=prefix)- assert chunks[0].end_cp == 310- assert chunks[0].token_count == 512- assert_coverage(text, chunks, Characters(), prefix)- with pytest.raises(ChunkingError, match="budget"):- chunk_evidence("中", Characters(), prefix="P" * 510)---def test_real_tokenizer_covers_long_chinese_middle_and_tail_object_locations():- middle = "https://example.test/Middle?Key=Value#Anchor"- tail = "/CaseSensitive/路径/final.txt"- text = "中文😀e\u0301\n" + "中" * 740 + "\n" + middle + "\n" + "尾" * 800 + "\n" + tail- tokenizer = LocalEmbedder()- chunks = chunk_evidence(text, tokenizer)- assert len(chunks) > 3- assert_coverage(text, chunks, tokenizer)- for value in (middle, tail):- location = text.index(value)- assert any(c.start_cp <= location and c.end_cp >= location + len(value) for c in chunks)- assert chunks[0].text.startswith("中文😀e\u0301\n")---def test_prefers_paragraph_boundary_near_target_without_losing_text():- text = "中" * 300 + "\n\n" + "尾" * 500- chunks = chunk_evidence(text, Characters())- assert chunks[0].end_cp == 302- assert_coverage(text, chunks, Characters())--[email protected]("text", ["", " \n\t\r "])-def test_empty_or_whitespace_has_no_index_chunks(text):- assert chunk_evidence(text, Characters()) == ()--- "text", ["😀" * 300, "e\u0301" * 400, "甲\n乙\t丙", "原样 <user_query>原文</user_query>"]-)-def test_unicode_and_wrapper_text_identity_is_preserved(text):- chunks = chunk_evidence(text, Characters())- assert_coverage(text, chunks, Characters())--- "offsets,ids",- [- ((TokenOffset(0, 4),), (1,)),- ((TokenOffset(-1, 1),), (1,)),- ((TokenOffset(1, 2), TokenOffset(0, 1)), (1, 2)),- ((TokenOffset(0, 1), TokenOffset(2, 3)), (1, 2)),- ((TokenOffset(0, 0),), (1,)),- ((TokenOffset(0, 1, True),), (1,)),- ((TokenOffset(0, 3),), (1, 2)),- ((), ()),- ],-)-def test_bad_or_incomplete_offsets_fail_explicitly(offsets, ids):- class Broken:- def tokenize_with_offsets(self, text):- return TokenizedText(ids, offsets)-- with pytest.raises(ChunkingError):- chunk_evidence("abc", Broken())--- "kwargs",- [- {"target_tokens": 64},- {"overlap_tokens": -1},- {"max_tokens": 513},- {"max_tokens": True},- {"target_tokens": 0},- {"overlap_tokens": 384},- ],-)-def test_invalid_budgets_fail_before_tokenizing(kwargs):- with pytest.raises(ChunkingError):- chunk_evidence("content", Characters(), **kwargs)---def test_retokenizes_actual_prefix_plus_slice_instead_of_assuming_additive_counts():- class ContextSensitive(Characters):- def tokenize_with_offsets(self, text):- result = super().tokenize_with_offsets(text)- if text.startswith("P"):- # Same valid offsets may carry multiple pieces. Budget must include them all.- return TokenizedText(- result.token_ids * 2,- tuple(o for o in result.offsets if o.special)- + tuple(o for o in result.offsets if o.special)- + tuple(o for o in result.offsets if not o.special for _ in range(2)),- )- return result-- tokenizer = ContextSensitive()- chunks = chunk_evidence("中" * 600, tokenizer, prefix="P")- assert_coverage("中" * 600, chunks, tokenizer, "P")---def test_overlapping_token_offsets_are_not_split_inside_a_character():- class Repeated(Characters):- def tokenize_with_offsets(self, text):- return TokenizedText(- tuple(range(2 * len(text))),- tuple(TokenOffset(i, i + 1) for i in range(len(text)) for _ in range(2)),- )-- tokenizer = Repeated()- text = "😀" * 500- chunks = chunk_evidence(text, tokenizer)- assert_coverage(text, chunks, tokenizer)---def test_single_indivisible_offset_over_budget_is_not_truncated():- class Indivisible:- def tokenize_with_offsets(self, text):- return TokenizedText(- tuple(range(600)), tuple(TokenOffset(0, len(text)) for _ in range(600))- )-- with pytest.raises(ChunkingError, match="budget"):- chunk_evidence("😀", Indivisible())---def test_real_long_continuous_ascii_unknown_preserves_all_6000_characters():- tokenizer = LocalEmbedder()- text = " \t" + "x" * 6000 + "\n末尾链接 https://example.test/Case?A=B#C\t"- chunks = chunk_evidence(text, tokenizer)- assert len(chunks) == 1- assert chunks[0].text == text- assert_coverage(text, chunks, tokenizer)---def test_real_subword_and_whitespace_boundaries_have_no_gaps():- tokenizer = LocalEmbedder()- text = " \t\n" + ("unaffordable embeddings\t中\u0301文 a\u0301b\n\n" * 180) + "\t\n "- prefix = "本段用于检索:"- chunks = chunk_evidence(text, tokenizer, prefix=prefix)- assert len(chunks) > 3- assert chunks[0].start_cp == 0 and chunks[-1].end_cp == len(text)- assert_coverage(text, chunks, tokenizer, prefix)- for first, second in zip(chunks, chunks[1:], strict=False):- assert second.start_cp <= first.end_cp---def test_partially_overlapping_offsets_for_combining_cluster_are_indivisible():- class Combining(Characters):- def tokenize_with_offsets(self, text):- offsets = []- i = 0- while i < len(text):- if text.startswith("e\u0301", i):- offsets.extend((TokenOffset(i, i + 2), TokenOffset(i + 1, i + 2)))- i += 2- else:- offsets.append(TokenOffset(i, i + 1))- i += 1- return TokenizedText(tuple(range(len(offsets))), tuple(offsets))-- text = "中" * 383 + "e\u0301" + "尾" * 500- tokenizer = Combining()- chunks = chunk_evidence(text, tokenizer)- assert_coverage(text, chunks, tokenizer)- assert all(c.start_cp != 384 and c.end_cp != 384 for c in chunks)---def test_sparse_whitespace_gaps_are_retained_not_rebuilt_from_tokens():- tokenizer = LocalEmbedder()- text = " \n" + "中" * 380 + "\n \t\n" + "尾" * 380 + " \t\r\n"- chunks = chunk_evidence(text, tokenizer)- assert_coverage(text, chunks, tokenizer)- assert chunks[0].text.startswith(" \n")- assert chunks[-1].text.endswith(" \t\r\n")---def test_real_prefix_retokenization_never_exceeds_encoder_window():- tokenizer = LocalEmbedder()- prefix = "检索前缀" * 50- text = ("中文tokenization😀e\u0301\n" * 120) + "https://example.test/Tail"- chunks = chunk_evidence(text, tokenizer, prefix=prefix)- assert len(chunks) > 1- assert_coverage(text, chunks, tokenizer, prefix)---def test_tokenizer_failure_is_explicit_not_partial_output():- class Broken:- def tokenize_with_offsets(self, text):- raise RuntimeError("tokenizer unavailable")-- with pytest.raises(ChunkingError, match="tokenization_failed"):- chunk_evidence("中", Broken())--- "text", ["e\u0301", "\u0301", "中\u0301文", "\n\r\t中\n尾\n", "👩💻", "👩👩👧👦", "✈️"]-)-def test_locked_normalizer_supported_unicode_examples_cover_original(text):- tokenizer = LocalEmbedder()- assert_coverage(text, chunk_evidence(text, tokenizer), tokenizer)--[email protected]("text", ["中\u200d文", "\u200d", "中\u200b文", "中\x00文", "\x00"])-def test_locked_normalizer_dropped_nonwhitespace_is_explicitly_unsupported(text):- tokenizer = LocalEmbedder()- offsets = tokenizer.tokenize_with_offsets(text).offsets- assert any(- not character.isspace()- and not any(- not offset.special and offset.start_cp <= i < offset.end_cp for offset in offsets- )- for i, character in enumerate(text)- )- with pytest.raises(ChunkingError, match="uncovered_nonwhitespace"):- chunk_evidence(text, tokenizer)---def test_another_normalizer_dropping_trailing_combining_mark_is_not_reported_complete():- class NormalizesAccent:- def tokenize_with_offsets(self, text):- return TokenizedText((1,), (TokenOffset(0, 1),))-- with pytest.raises(ChunkingError, match="uncovered_nonwhitespace"):- chunk_evidence("e\u0301", NormalizesAccent())diff --git a/memhub/apps/core/tests/test_retrieval_eligibility_benchmark.py b/memhub/apps/core/tests/test_retrieval_eligibility_benchmark.pydeleted file mode 100644index 595d6b2..0000000--- a/memhub/apps/core/tests/test_retrieval_eligibility_benchmark.py+++ /dev/null@@ -1,38 +0,0 @@-import importlib.util-import sqlite3-import sys-from pathlib import Path--import pytest--pytestmark = [pytest.mark.performance, pytest.mark.serial]--SCRIPT = Path(__file__).resolve().parents[3] / "scripts" / "bench_retrieval_eligibility_scale.py"-if not SCRIPT.exists():- pytest.skip(- f"benchmark script is not part of this checkout: {SCRIPT.name}",- allow_module_level=True,- )-spec = importlib.util.spec_from_file_location("bench_retrieval_eligibility_scale", SCRIPT)-benchmark = importlib.util.module_from_spec(spec)-sys.modules["bench_retrieval_eligibility_scale"] = benchmark-assert spec.loader is not None-spec.loader.exec_module(benchmark)---def test_scale_probe_proves_composite_authority_index_changes_the_plan(tmp_path):- db = sqlite3.connect(tmp_path / "probe.sqlite3", cached_statements=0)- try:- benchmark.create_schema(db)- benchmark.fill_to(db, 0, 10_000)- before = benchmark.measure(db, benchmark.FILTERED_SQL, ("g", "allowed"), repeat=3)- db.execute(- "CREATE INDEX retrieval_authority_jobs "- "ON retrieval_documents(generation_id,state,source_id,event_seq)"- )- after = benchmark.measure(db, benchmark.FILTERED_SQL, ("g", "allowed"), repeat=3)- finally:- db.close()- assert before["rows"] == after["rows"] == 100- assert not any("source_id" in step for step in before["plan"])- assert any("retrieval_authority_jobs" in step for step in after["plan"])diff --git a/memhub/apps/core/tests/test_retrieval_index_store.py b/memhub/apps/core/tests/test_retrieval_index_store.pydeleted file mode 100644index c25ef9c..0000000--- a/memhub/apps/core/tests/test_retrieval_index_store.py+++ /dev/null@@ -1,452 +0,0 @@-import sqlite3-from contextlib import closing-from datetime import UTC, datetime--import pytest--from one_memory import ingest_journal-from one_memory.imports.grants import SourceGrantStore-from one_memory.imports.models import EvidenceInput, EvidenceOrigin, OriginMetadata, WrittenBy-from one_memory.imports.sources.registry import RootCandidateKind-from one_memory.imports.storage import EvidenceStore-from one_memory.memory_server.config_state import ConfigStore-from one_memory.retrieval.chunking import TextChunk-from one_memory.retrieval.index_store import EncoderIdentity, RetrievalIndexStore-from one_memory.retrieval.models import TimeRange---def fixture(tmp_path):- path = tmp_path / "data.sqlite3"- evidence = EvidenceStore(path)- grants = SourceGrantStore(path, home=tmp_path)- source = tmp_path / "source"- source.mkdir()- grants.grant(- user_id="local", source_id="codex", root=source, candidate_kind=RootCandidateKind.DEFAULT- )- store = RetrievalIndexStore(path)- identity = EncoderIdentity("test", "a" * 64, "b" * 64, 3)- return path, evidence, grants, store, identity---def put(evidence, text="original text", message="one"):- return evidence.put(- EvidenceInput(- source_id="codex",- conversation_id="thread",- message_id=message,- role="user",- written_by=WrittenBy.PERSON,- verbatim=text,- origin_meta={EvidenceOrigin.FILE: OriginMetadata()},- )- ).record---def initialize(f):- path, _, _, store, identity = f- with closing(sqlite3.connect(path)) as db, db:- db.execute("BEGIN IMMEDIATE")- store.initialize(db)- return store.create_generation(db, identity)---def test_constructor_and_reads_do_not_create_database_or_schema(tmp_path):- store = RetrievalIndexStore(tmp_path / "missing")- assert not store.database_path.exists()- with pytest.raises(sqlite3.Error):- store.status("missing")- assert not store.database_path.exists()---def test_explicit_initialization_rolls_back_and_never_grants_model_policy(fixture):- path, evidence, _, store, _ = fixture- row = put(evidence)- with closing(sqlite3.connect(path)) as db:- with pytest.raises(RuntimeError):- store.initialize(db)- db.execute("BEGIN IMMEDIATE")- store.initialize(db)- event = ingest_journal.current(db, row.evidence_id)- assert not event.generation_eligible and event.kind == "migration_baseline"- db.rollback()- assert not db.execute(- "SELECT 1 FROM sqlite_master WHERE name='retrieval_index_state'"- ).fetchone()- assert not db.execute(- "SELECT 1 FROM sqlite_master WHERE name='evidence_ingest_events'"- ).fetchone()- initialize(fixture)- with closing(sqlite3.connect(path)) as db:- assert not db.execute(- "SELECT 1 FROM sqlite_master WHERE name='continuous_policy_history'"- ).fetchone()---def test_consumer_and_jobs_commit_together(fixture):- path, evidence, _, store, _ = fixture- generation = initialize(fixture)- row = put(evidence)- with closing(sqlite3.connect(path)) as db:- db.execute("BEGIN IMMEDIATE")- assert store.consume(db, generation) == 1- assert db.execute("SELECT COUNT(*) FROM retrieval_documents").fetchone()[0] == 1- db.rollback()- assert store.status(generation)["after_seq"] == 0- assert store.consume_pending(generation) == 1- assert store.consume_pending(generation) == 0- lease = store.claim(generation, "worker", now=100)- assert lease.evidence_id == row.evidence_id---def test_lease_expiry_old_owner_and_publish_manifest(fixture):- _, evidence, _, store, _ = fixture- generation = initialize(fixture)- row = put(evidence)- store.consume_pending(generation)- lease = store.claim(generation, "one", now=100, lease_seconds=10)- assert store.claim(generation, "two", now=105) is None- replacement = store.claim(generation, "two", now=111)- chunks = (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),)- with pytest.raises(ValueError, match="lease"):- store.publish(lease, chunks, vectors_verified=True, now=112)- with pytest.raises(ValueError, match="vectors"):- store.publish(replacement, chunks, vectors_verified=False, now=112)- store.publish(replacement, chunks, vectors_verified=True, now=112)- assert store.status(generation)["ready"] == 1- assert store.active_chunks(generation) == ()- store.activate_generation(generation)- assert len(store.active_chunks(generation)) == 1- assert store.claim(generation, "three", now=113) is None--[email protected]("change", ["revision", "unrecorded", "revoke", "deleted"])-def test_changed_inputs_cannot_publish_or_be_read(fixture, change):- path, evidence, grants, store, _ = fixture- generation = initialize(fixture)- row = put(evidence)- store.consume_pending(generation)- lease = store.claim(generation, "worker", now=100)- if change == "revision":- put(evidence, "updated text")- elif change == "unrecorded":- with closing(sqlite3.connect(path)) as db, db:- db.execute(- "UPDATE evidence SET verbatim='unrecorded' WHERE evidence_id=?", (row.evidence_id,)- )- elif change == "revoke":- with closing(sqlite3.connect(path)) as db, db:- db.execute("UPDATE source_grants SET revoked_at='2026-09-19' WHERE source_id='codex'")- else:- with closing(sqlite3.connect(path)) as db, db:- db.execute("DELETE FROM evidence_origins WHERE evidence_id=?", (row.evidence_id,))- db.execute("DELETE FROM evidence WHERE evidence_id=?", (row.evidence_id,))- ingest_journal.record(db, row.evidence_id, cause="delete")- with pytest.raises(ValueError):- store.publish(- lease,- (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),),- vectors_verified=True,- now=101,- )- assert store.active_chunks(generation) == ()---def test_current_read_rejects_revised_published_content_before_consumer_catches_up(fixture):- _, evidence, _, store, _ = fixture- generation = initialize(fixture)- row = put(evidence)- store.consume_pending(generation)- lease = store.claim(generation, "worker", now=100)- store.publish(- lease,- (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),),- vectors_verified=True,- now=101,- )- store.activate_generation(generation)- assert store.active_chunks(generation)- put(evidence, "new version")- assert store.active_chunks(generation) == ()- store.consume_pending(generation)- updated = store.claim(generation, "worker", now=102)- assert updated.build_id != lease.build_id---def test_pausing_harness_sync_does_not_hide_published_memory(fixture):- path, evidence, _, store, _ = fixture- generation = initialize(fixture)- row = put(evidence)- store.consume_pending(generation)- lease = store.claim(generation, "worker", now=100)- store.publish(- lease,- (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),),- vectors_verified=True,- now=101,- )- store.activate_generation(generation)- before = store.active_chunks(generation)- assert before-- config = ConfigStore(path.parent)- config.set_sync("codex", True)- config.set_sync("codex", False)-- assert store.active_chunks(generation) == before---def test_partial_or_fabricated_chunk_manifest_never_publishes(fixture):- _, evidence, _, store, _ = fixture- generation = initialize(fixture)- row = put(evidence)- store.consume_pending(generation)- lease = store.claim(generation, "worker", now=100)- for chunks in [- (),- (TextChunk(0, 0, 3, 3, "ori"),),- (TextChunk(0, 0, len(row.verbatim), 4, "not original"),),- ]:- with pytest.raises(ValueError, match="chunk"):- store.publish(lease, chunks, vectors_verified=True, now=101)- assert store.status(generation)["ready"] == 0---def test_incomplete_index_cannot_replace_existing_active_generation(fixture):- path, evidence, _, store, identity = fixture- first = initialize(fixture)- row = put(evidence)- store.consume_pending(first)- lease = store.claim(first, "worker", now=100)- chunks = (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),)- store.publish(lease, chunks, vectors_verified=True, now=101)- store.activate_generation(first)- with closing(sqlite3.connect(path)) as db, db:- db.execute("BEGIN IMMEDIATE")- second = store.create_generation(db, EncoderIdentity("second", "c" * 64, "d" * 64, 3))- with pytest.raises(ValueError, match="caught_up"):- store.activate_generation(second)- assert store.active_chunks(first)- store.consume_pending(second)- with pytest.raises(ValueError, match="incomplete"):- store.activate_generation(second)- claim = store.claim(second, "worker", now=102)- store.publish(claim, chunks, vectors_verified=True, now=103)- store.activate_generation(second)- assert store.active_chunks(first) == () and store.active_chunks(second)---def test_block_and_explicit_retry_require_same_lease_and_authority(fixture):- path, evidence, _, store, _ = fixture- generation = initialize(fixture)- put(evidence)- store.consume_pending(generation)- first = store.claim(generation, "one", now=100, lease_seconds=1)- second = store.claim(generation, "two", now=102)- store.block(first)- assert store.status(generation)["leased"] == 1- store.block(second)- assert store.status(generation)["blocked"] == 1- with closing(sqlite3.connect(path)) as db, db:- db.execute("UPDATE source_grants SET revoked_at='revoked'")- assert store.retry_blocked(generation) == 0- with closing(sqlite3.connect(path)) as db, db:- db.execute("UPDATE source_grants SET revoked_at=NULL")- assert store.retry_blocked(generation) == 1- assert store.claim(generation, "three", now=103) is not None---def test_scope_filter_and_revocation_apply_before_exposing_manifest(fixture):- path, evidence, _, store, _ = fixture- generation = initialize(fixture)- row = put(evidence)- store.consume_pending(generation)- lease = store.claim(generation, "worker", now=100)- assert store.read_lease(lease, now=101).verbatim == row.verbatim- store.publish(- lease,- (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),),- vectors_verified=True,- now=101,- )- store.activate_generation(generation)- assert store.active_chunks(generation, source_ids=frozenset()) == ()- assert store.active_chunks(generation, source_ids=frozenset({"codex"}))- with closing(sqlite3.connect(path)) as db, db:- db.execute("UPDATE source_grants SET revision=revision+1")- assert store.active_chunks(generation) == ()---def test_source_and_time_eligibility_are_applied_before_manifest_validation(fixture):- path, evidence, _, store, _ = fixture- generation = initialize(fixture)- row = put(evidence)- store.consume_pending(generation)- lease = store.claim(generation, "worker", now=100)- chunks = (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),)- store.publish(lease, chunks, vectors_verified=True, now=101)- store.activate_generation(generation)-- assert store.active_chunks(generation, source_ids=frozenset()) == ()- assert store.active_chunks(generation, source_ids=frozenset({"other"})) == ()- assert len(store.active_chunks(generation, source_ids=frozenset({"codex"}))) == 1- assert (- store.active_chunks(- generation,- time_range=TimeRange(- start=datetime(2030, 1, 1, tzinfo=UTC),- end=datetime(2030, 1, 2, tzinfo=UTC),- ),- )- == ()- )---def test_generation_and_metadata_survive_reopen_without_body_copy(fixture):- path, evidence, _, store, identity = fixture- generation = initialize(fixture)- put(evidence, "sensitive-body-must-not-copy")- store.consume_pending(generation)- reopened = RetrievalIndexStore(path)- assert reopened.status(generation) == store.status(generation)- with closing(sqlite3.connect(path)) as db, db:- db.execute("BEGIN IMMEDIATE")- reopened.initialize(db)- assert reopened.create_generation(db, identity) == generation- rows = db.execute("SELECT * FROM retrieval_documents").fetchall()- assert "sensitive-body-must-not-copy" not in str(rows)---def test_partial_schema_is_not_blessed_as_new_instance(fixture):- path, _, _, store, _ = fixture- with closing(sqlite3.connect(path)) as db, db:- db.execute("CREATE TABLE retrieval_chunks (unexpected TEXT)")- with closing(sqlite3.connect(path)) as db:- db.execute("BEGIN IMMEDIATE")- with pytest.raises(ValueError, match="partial_initialization"):- store.initialize(db)- assert not db.execute(- "SELECT 1 FROM sqlite_master WHERE name='retrieval_index_state'"- ).fetchone()---def test_lease_renewal_rechecks_authority_and_rejects_old_expiry(fixture):- path, evidence, _, store, _ = fixture- generation = initialize(fixture)- put(evidence)- store.consume_pending(generation)- lease = store.claim(generation, "worker", now=100)- renewed = store.renew(lease, now=125)- assert renewed.expires_at == 185- with pytest.raises(ValueError, match="lease_changed"):- store.read_lease(lease, now=126)- assert store.read_lease(renewed, now=126)- with closing(sqlite3.connect(path)) as db, db:- db.execute("UPDATE source_grants SET revision=revision+1")- with pytest.raises(ValueError, match="authority"):- store.renew(renewed, now=127)---def test_regrant_requeues_and_reuses_immutable_manifest(fixture):- path, evidence, grants, store, _ = fixture- generation = initialize(fixture)- row = put(evidence)- store.consume_pending(generation)- lease = store.claim(generation, "worker", now=100)- chunks = (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),)- store.publish(lease, chunks, vectors_verified=True, now=101)- store.activate_generation(generation)- assert len(store.active_chunks(generation)) == 1- grants.revoke(user_id="local", source_id="codex")- assert store.active_chunks(generation) == ()- grants.grant(- user_id="local",- source_id="codex",- root=path.parent / "source",- candidate_kind=RootCandidateKind.DEFAULT,- )- assert store.active_chunks(generation) == ()- assert store.retry_blocked(generation) == 1- renewed = store.claim(generation, "worker", now=102)- assert renewed.build_id == lease.build_id- store.publish(renewed, chunks, vectors_verified=True, now=103)- assert len(store.active_chunks(generation)) == 1---def test_retry_cursor_passes_ineligible_prefix_across_reopen(fixture):- path, evidence, grants, store, _ = fixture- generation = initialize(fixture)- for index in range(257):- put(evidence, message=str(index))- store.consume_pending(generation)- store.consume_pending(generation)- with closing(sqlite3.connect(path)) as db, db:- db.execute("UPDATE retrieval_documents SET state='blocked',source_id='revoked'")- last_id = db.execute(- "SELECT evidence_id FROM retrieval_documents ORDER BY event_seq DESC LIMIT 1"- ).fetchone()[0]- db.execute(- "UPDATE retrieval_documents SET source_id='codex' WHERE evidence_id=?", (last_id,)- )- assert store.retry_blocked(generation) == 0- reopened = RetrievalIndexStore(path)- assert reopened.retry_blocked(generation) == 1- assert reopened.claim(generation, "worker", now=100).evidence_id == last_id---def test_same_build_replay_requires_exact_complete_manifest(fixture):- path, evidence, _, store, _ = fixture- generation = initialize(fixture)- row = put(evidence)- store.consume_pending(generation)- lease = store.claim(generation, "worker", now=100)- chunks = (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),)- store.publish(lease, chunks, vectors_verified=True, now=101)- with closing(sqlite3.connect(path)) as db, db:- db.execute("UPDATE retrieval_documents SET state='pending',active_build_id=NULL")- replay = store.claim(generation, "worker", now=102)- with pytest.raises(ValueError, match="manifest"):- store.publish(- replay,- (TextChunk(0, 0, len(row.verbatim), 5, row.verbatim),),- vectors_verified=True,- now=103,- )- assert store.status(generation)["ready"] == 0- store.publish(replay, chunks, vectors_verified=True, now=103)- assert store.status(generation)["ready"] == 1---def test_origin_metadata_roundtrip_reuses_old_build_without_losing_new_event(fixture):- _, evidence, _, store, _ = fixture- generation = initialize(fixture)- builds = []- seqs = []- for version in ("a", "b", "a"):- row = evidence.put(- EvidenceInput(- source_id="codex",- conversation_id="thread",- message_id="one",- role="user",- written_by=WrittenBy.PERSON,- verbatim="unchanged original",- origin_meta={EvidenceOrigin.FILE: OriginMetadata(source_version=version)},- )- ).record- assert row.revision == 1- assert store.consume_pending(generation) == 1- lease = store.claim(generation, "worker", now=100)- builds.append(lease.build_id)- seqs.append(lease.event_seq)- store.publish(- lease,- (TextChunk(0, 0, len(row.verbatim), 5, row.verbatim),),- vectors_verified=True,- now=101,- )- assert builds[0] == builds[2] and builds[0] != builds[1]- assert seqs[0] < seqs[1] < seqs[2]- store.activate_generation(generation)- assert len(store.active_chunks(generation)) == 1diff --git a/memhub/apps/core/tests/test_retrieval_index_supervisor.py b/memhub/apps/core/tests/test_retrieval_index_supervisor.pydeleted file mode 100644index 0bd2f69..0000000--- a/memhub/apps/core/tests/test_retrieval_index_supervisor.py+++ /dev/null@@ -1,357 +0,0 @@-import asyncio-import socket-import threading-import time-from dataclasses import FrozenInstanceError--import pytest--from one_memory.retrieval.index_supervisor import RetrievalIndexSupervisor-from one_memory.retrieval.index_worker import IndexWorkResult--[email protected](autouse=True)-def no_network(monkeypatch):- def reject(*args, **kwargs):- raise AssertionError("supervisor tests must remain offline")-- monkeypatch.setattr(socket.socket, "connect", reject)- monkeypatch.setattr(socket.socket, "connect_ex", reject)---class Worker:- def __init__(self, state="idle", blocking=False):- self.state = state- self.entered = threading.Event()- self.release = threading.Event()- self.exited = threading.Event()- self.cancel = None- self.blocking = blocking- self.calls = []- self.thread_ids = set()- self.active = 0- self.maximum = 0- self.closed = False-- def run_once(self, *, cancel=None):- self.cancel = cancel- self.active += 1- self.maximum = max(self.maximum, self.active)- self.calls.append(time.monotonic())- self.thread_ids.add(threading.get_ident())- self.entered.set()- try:- if self.blocking:- assert self.release.wait(3), "test cleanup failed"- if self.state == "exception":- raise RuntimeError("SECRET_RAW_ERROR")- if self.state == "invalid":- return object()- return IndexWorkResult(self.state, "SECRET_REASON")- finally:- self.active -= 1- self.exited.set()-- def close(self):- self.closed = True---async def until(predicate):- deadline = time.monotonic() + 2- while not predicate():- assert time.monotonic() < deadline- await asyncio.sleep(0.002)---def test_stop_waits_for_native_exit_and_does_not_close_resources():- async def run():- worker = Worker(blocking=True)- supervisor = RetrievalIndexSupervisor(worker)- supervisor.start()- await until(worker.entered.is_set)- stopping = asyncio.create_task(supervisor.stop())- try:- await until(lambda: worker.cancel.is_set())- await asyncio.sleep(0.03)- assert not stopping.done() and not worker.exited.is_set()- assert supervisor.snapshot().thread_alive- finally:- worker.release.set()- await stopping- assert worker.exited.is_set()- assert not supervisor.snapshot().thread_alive- assert not worker.closed- assert supervisor.snapshot().state == "stopped"-- asyncio.run(run())--[email protected]("method", ["stop", "wait"])-def test_async_cancellation_repeatedly_waits_for_native_thread(method):- async def run():- worker = Worker(blocking=True)- supervisor = RetrievalIndexSupervisor(worker)- supervisor.start()- await until(worker.entered.is_set)- task = asyncio.create_task(getattr(supervisor, method)())- await asyncio.sleep(0)- task.cancel()- try:- await until(lambda: worker.cancel.is_set())- task.cancel()- await asyncio.sleep(0.03)- assert not task.done()- assert not worker.exited.is_set()- finally:- worker.release.set()- with pytest.raises(asyncio.CancelledError):- await task- await supervisor.stop()- assert worker.exited.is_set() and not supervisor.snapshot().thread_alive- assert not worker.closed-- asyncio.run(run())---def test_idle_no_trigger_no_spin_and_duplicate_start_rejected():- async def run():- worker = Worker()- supervisor = RetrievalIndexSupervisor(worker, idle_wait_seconds=0.3)- assert not supervisor.trigger()- assert worker.calls == []- supervisor.start()- try:- with pytest.raises(RuntimeError, match="index_supervisor_already_started"):- supervisor.start()- await until(lambda: len(worker.calls) == 1)- await asyncio.sleep(0.06)- assert len(worker.calls) == 1- assert supervisor.snapshot().last_result == "idle"- finally:- await supervisor.stop()- with pytest.raises(RuntimeError, match="index_supervisor_already_started"):- supervisor.start()- assert not supervisor.trigger()-- asyncio.run(run())---def test_trigger_storm_coalesces_during_native_call_without_parallel_workers():- async def run():- worker = Worker(blocking=True)- supervisor = RetrievalIndexSupervisor(worker, idle_wait_seconds=0.5)- supervisor.start()- await until(worker.entered.is_set)- try:- for _ in range(200):- assert supervisor.trigger()- worker.release.set()- await until(lambda: len(worker.calls) >= 2)- await asyncio.sleep(0.05)- assert len(worker.calls) == 2- assert worker.maximum == 1 and len(worker.thread_ids) == 1- finally:- worker.release.set()- await supervisor.stop()-- asyncio.run(run())---def test_interactive_activity_pauses_index_slices_then_resumes() -> None:- async def run() -> None:- worker = Worker()- interactive = True- supervisor = RetrievalIndexSupervisor(- worker,- yield_wait_seconds=0.02,- background_should_yield=lambda: interactive,- )- supervisor.start()- try:- await asyncio.sleep(0.06)- assert worker.calls == []- interactive = False- supervisor.trigger()- await until(lambda: len(worker.calls) == 1)- finally:- await supervisor.stop()-- asyncio.run(run())--- "state,code",- [- ("exception", "index_worker_failed"),- ("invalid", "index_worker_invalid_result"),- ("blocked", "index_worker_blocked"),- ],-)-def test_failures_have_fixed_error_and_backoff_even_with_triggers(state, code):- async def run():- worker = Worker(state)- supervisor = RetrievalIndexSupervisor(worker, failure_wait_seconds=0.2)- supervisor.start()- try:- await until(lambda: supervisor.snapshot().error_code is not None)- for _ in range(100):- supervisor.trigger()- await asyncio.sleep(0.05)- assert len(worker.calls) == 1- snapshot = supervisor.snapshot()- assert snapshot.error_code == code- assert "SECRET" not in repr(snapshot)- await until(lambda: len(worker.calls) >= 2)- assert worker.calls[1] - worker.calls[0] >= 0.18- finally:- await supervisor.stop()-- asyncio.run(run())--[email protected]("state", ["ready", "excluded"])-def test_backlog_yields_after_maximum_slices(state):- async def run():- worker = Worker(state)- supervisor = RetrievalIndexSupervisor(- worker, max_consecutive_slices=3, yield_wait_seconds=0.15- )- supervisor.start()- try:- await until(lambda: len(worker.calls) >= 3)- for _ in range(50):- supervisor.trigger()- await asyncio.sleep(0.04)- assert len(worker.calls) == 3- await until(lambda: len(worker.calls) >= 4)- assert worker.calls[3] - worker.calls[2] >= 0.13- finally:- await supervisor.stop()- assert worker.maximum == 1-- asyncio.run(run())---def test_concurrent_stop_is_idempotent_and_stop_before_start_is_terminal():- async def run():- worker = Worker()- supervisor = RetrievalIndexSupervisor(worker)- supervisor.start()- await asyncio.gather(supervisor.stop(), supervisor.stop())- assert not supervisor.snapshot().thread_alive- unused = RetrievalIndexSupervisor(worker)- await unused.stop()- with pytest.raises(RuntimeError, match="index_supervisor_already_started"):- unused.start()-- asyncio.run(run())--- "kwargs",- [- {"idle_wait_seconds": 0},- {"failure_wait_seconds": float("nan")},- {"yield_wait_seconds": 0},- {"max_consecutive_slices": True},- {"max_consecutive_slices": 257},- ],-)-def test_invalid_configuration_never_starts_worker(kwargs):- worker = Worker()- with pytest.raises(ValueError, match="index_supervisor_invalid_config"):- RetrievalIndexSupervisor(worker, **kwargs)- assert worker.calls == []---def test_idle_timeout_polls_again_and_stop_interrupts_long_wait():- async def run():- worker = Worker()- supervisor = RetrievalIndexSupervisor(worker, idle_wait_seconds=0.04)- supervisor.start()- try:- await until(lambda: len(worker.calls) >= 2)- assert worker.calls[1] - worker.calls[0] >= 0.035- finally:- await supervisor.stop()- long_idle = RetrievalIndexSupervisor(Worker(), idle_wait_seconds=300)- long_idle.start()- await until(lambda: long_idle.snapshot().slices == 1)- started = time.monotonic()- await long_idle.stop()- assert time.monotonic() - started < 0.5-- asyncio.run(run())---def test_failure_clears_on_recovery_without_leaking_worker_reason():- async def run():- worker = Worker("exception")- supervisor = RetrievalIndexSupervisor(worker, failure_wait_seconds=0.06)- supervisor.start()- try:- await until(lambda: supervisor.snapshot().failures == 1)- worker.state = "idle"- await until(lambda: supervisor.snapshot().slices == 2)- current = supervisor.snapshot()- assert current.error_code is None and current.last_result == "idle"- assert current.failures == 1- assert "SECRET" not in repr(current)- with pytest.raises(FrozenInstanceError):- current.slices = 0- finally:- await supervisor.stop()-- asyncio.run(run())---def test_thread_start_failure_is_fixed_and_terminal(monkeypatch):- def fail(_self):- raise OSError("SECRET_THREAD_ERROR")-- monkeypatch.setattr(threading.Thread, "start", fail)- supervisor = RetrievalIndexSupervisor(Worker())- with pytest.raises(RuntimeError, match="^index_supervisor_start_failed$"):- supervisor.start()- assert not supervisor.snapshot().thread_alive- assert supervisor.snapshot().state == "stopped"- assert "SECRET" not in repr(supervisor.snapshot())- asyncio.run(supervisor.stop())--[email protected]("state", ["deferred", "cancelled"])-def test_nonprogress_results_do_not_drain_in_a_tight_loop(state):- async def run():- worker = Worker(state)- supervisor = RetrievalIndexSupervisor(worker, idle_wait_seconds=0.3)- supervisor.start()- try:- await until(lambda: supervisor.snapshot().slices == 1)- await asyncio.sleep(0.05)- assert len(worker.calls) == 1- finally:- await supervisor.stop()-- asyncio.run(run())---def test_thread_is_non_daemon_and_joined_without_default_executor_threads():- async def run():- before = set(threading.enumerate())- worker = Worker(blocking=True)- supervisor = RetrievalIndexSupervisor(worker)- supervisor.start()- await until(worker.entered.is_set)- own = [t for t in threading.enumerate() if t.ident in worker.thread_ids]- assert len(own) == 1 and not own[0].daemon- try:- task = asyncio.create_task(supervisor.stop())- await until(lambda: worker.cancel.is_set())- assert set(threading.enumerate()) - before == set(own)- finally:- worker.release.set()- await task- assert not own[0].is_alive()- assert set(threading.enumerate()) == before-- asyncio.run(run())diff --git a/memhub/apps/core/tests/test_retrieval_index_worker.py b/memhub/apps/core/tests/test_retrieval_index_worker.pydeleted file mode 100644index 0962c45..0000000--- a/memhub/apps/core/tests/test_retrieval_index_worker.py+++ /dev/null@@ -1,222 +0,0 @@-import socket-import sqlite3-import threading-import time-from contextlib import closing--import pytest--from one_memory.embedding_assets import assets_present-from one_memory.local_embedding import TokenizedText, TokenOffset-from one_memory.retrieval.index_store import RetrievalIndexStore-from one_memory.retrieval.index_worker import LocalRetrievalEncoder, RetrievalIndexWorker-from one_memory.retrieval.vector_store import RawVectorStore, VectorStoreError-from tests.test_retrieval_index_store import fixture, initialize, put # noqa: F401--[email protected](autouse=True)-def forbid_network(monkeypatch):- def reject(*args, **kwargs):- raise AssertionError("index worker must stay offline")-- monkeypatch.setattr(socket.socket, "connect", reject)- monkeypatch.setattr(socket.socket, "connect_ex", reject)---class Encoder:- def __init__(self, identity):- self.identity = identity- self.calls = []- self.after_encode = lambda: None-- def tokenize_with_offsets(self, text):- return TokenizedText(- tuple(range(len(text))), tuple(TokenOffset(i, i + 1) for i in range(len(text)))- )-- def encode_documents(self, texts):- self.calls.append(texts)- self.after_encode()- return [[1.0, 0.5, 0.25] for _ in texts]---def setup(fixture, tmp_path): # noqa: F811- generation = initialize(fixture)- store = fixture[3]- encoder = Encoder(fixture[4])- vectors = RawVectorStore.create(tmp_path / "vectors", generation_id=generation, dimensions=3)- worker = RetrievalIndexWorker(store, vectors, encoder, generation_id=generation, owner="test")- try:- yield fixture, generation, encoder, vectors, worker- finally:- vectors.close()---def test_tick_indexes_complete_original_and_reopen_can_search(setup):- f, generation, encoder, vectors, worker = setup- record = put(f[1], "start " + "middle " * 180 + " end")- assert worker.run_once().state == "ready"- assert worker.run_once().state == "idle"- assert all(len(batch) <= 8 for batch in encoder.calls)- assert "".join(encoder.calls[0]) # Nonempty positive encoding control.- f[3].activate_generation(generation)- chunks = RetrievalIndexStore(f[0]).active_chunks(generation)- assert len(chunks) > 1 and chunks[0].start_cp == 0- assert chunks[-1].end_cp == len(record.verbatim)- result = vectors.search(- [1.0, 0.5, 0.25], [c.chunk_id for c in chunks], 8, time.monotonic() + 10- )- assert result.complete and result.hits- with closing(sqlite3.connect(f[0])) as db:- assert not db.execute(- "SELECT 1 FROM sqlite_master WHERE name='continuous_policy_history'"- ).fetchone()--[email protected]("change", ["revision", "revoke"])-def test_change_during_native_encoding_never_publishes(setup, change):- f, generation, encoder, vectors, worker = setup- put(f[1])-- def mutate():- if change == "revision":- put(f[1], "changed original")- else:- with closing(sqlite3.connect(f[0])) as db, db:- db.execute("UPDATE source_grants SET revoked_at='revoked'")-- encoder.after_encode = mutate- result = worker.run_once()- assert result.state == "blocked"- assert f[3].status(generation)["ready"] == 0- assert f[3].active_chunks(generation) == ()---def test_cancel_after_encode_does_not_publish_and_is_reclaimable(setup):- f, generation, encoder, _, worker = setup- put(f[1])- cancel = threading.Event()- encoder.after_encode = cancel.set- assert worker.run_once(cancel=cancel).state == "cancelled"- assert f[3].status(generation)["leased"] == 1- assert f[3].claim(generation, "replacement", now=time.time() + 61)---def test_cancel_before_tick_does_not_consume_or_encode(setup):- f, generation, encoder, _, worker = setup- put(f[1])- cancel = threading.Event()- cancel.set()- assert worker.run_once(cancel=cancel).state == "cancelled"- assert f[3].status(generation)["after_seq"] == 0- assert not encoder.calls---def test_partial_vector_write_can_retry_same_build_without_duplicates(setup, monkeypatch):- f, generation, encoder, vectors, worker = setup- put(f[1], "long message " * 90)- original = vectors.upsert-- def partial(records):- original(records[:1])- raise VectorStoreError("vector_write_unavailable")-- monkeypatch.setattr(vectors, "upsert", partial)- assert worker.run_once().state == "blocked"- assert f[3].status(generation)["ready"] == 0- monkeypatch.setattr(vectors, "upsert", original)- assert f[3].retry_blocked(generation) == 1- assert worker.run_once().state == "ready"- f[3].activate_generation(generation)- assert len(f[3].active_chunks(generation)) > 1---def test_bad_vector_count_or_failed_verification_never_publishes(setup, monkeypatch):- f, generation, encoder, vectors, worker = setup- put(f[1])- monkeypatch.setattr(encoder, "encode_documents", lambda _: [])- assert worker.run_once().state == "blocked"- assert f[3].status(generation)["ready"] == 0- assert f[3].retry_blocked(generation) == 1- monkeypatch.setattr(- encoder, "encode_documents", lambda texts: [[1.0, 0.5, 0.25] for _ in texts]- )- monkeypatch.setattr(- vectors, "verify", lambda _: (_ for _ in ()).throw(VectorStoreError("failure"))- )- assert worker.run_once().state == "blocked"- assert f[3].status(generation)["ready"] == 0--- "text",- ["token=example-secret-value", "<environment_context>system wrapper</environment_context>"],-)-def test_content_requiring_source_mapping_is_not_encoded_or_marked_ready(setup, text):- f, generation, encoder, _, worker = setup- put(f[1], text)- result = worker.run_once()- assert result.state == "blocked" and result.reason == "content_mapping_required"- assert not encoder.calls- assert f[3].status(generation)["ready"] == 0---def test_deadline_after_native_encode_discards_late_work(setup):- f, generation, encoder, vectors, _ = setup- put(f[1])- clock = [1.0]- encoder.after_encode = lambda: clock.__setitem__(0, 50.0)- worker = RetrievalIndexWorker(- f[3],- vectors,- encoder,- generation_id=generation,- owner="test",- clock=lambda: clock[0],- max_job_seconds=10,- )- assert worker.run_once().state == "deferred"- assert f[3].status(generation)["ready"] == 0---def test_generation_encoder_mismatch_rejected_before_claim(setup):- f, generation, encoder, vectors, _ = setup- from dataclasses import replace-- encoder.identity = replace(encoder.identity, encoder_digest="f" * 64)- with pytest.raises(ValueError, match="encoder_identity"):- RetrievalIndexWorker(f[3], vectors, encoder, generation_id=generation, owner="other")- assert f[3].status(generation)["leased"] == 0--[email protected](not assets_present(), reason="locked local assets absent")-def test_real_local_encoder_chroma_full_text_chain(fixture, tmp_path): # noqa: F811- path, evidence, _, store, _ = fixture- encoder = LocalRetrievalEncoder(content_version=1)- with closing(sqlite3.connect(path)) as db, db:- db.execute("BEGIN IMMEDIATE")- store.initialize(db)- generation = store.create_generation(db, encoder.identity)- text = (- "A long source message about source control and releases. " * 80- + " Final artifact at /project/design/final.png"- )- record = put(evidence, text)- vectors = RawVectorStore.create(- tmp_path / "real-vectors", generation_id=generation, dimensions=encoder.identity.dimensions- )- try:- worker = RetrievalIndexWorker(- store, vectors, encoder, generation_id=generation, owner="real"- )- assert worker.run_once().state == "ready"- store.activate_generation(generation)- chunks = store.active_chunks(generation)- assert len(chunks) > 1 and chunks[-1].end_cp == len(text)- query = encoder.encode_documents([text[chunks[-1].start_cp : chunks[-1].end_cp]])[0]- result = vectors.search(query, [c.chunk_id for c in chunks], 8, time.monotonic() + 10)- assert result.complete and result.hits[0].chunk_id == chunks[-1].chunk_id- assert record.verbatim.endswith("final.png")- finally:- vectors.close()diff --git a/memhub/apps/core/tests/test_retrieval_lifespan.py b/memhub/apps/core/tests/test_retrieval_lifespan.pyindex 543e3ab..3e0066e 100644--- a/memhub/apps/core/tests/test_retrieval_lifespan.py+++ b/memhub/apps/core/tests/test_retrieval_lifespan.py@@ -1,23 +1,14 @@import asyncio-import json-import os-import socketimport sqlite3-import subprocess-import sys-import time-import urllib.requestfrom contextlib import closing-from pathlib import Path--import pytestfrom one_memory.api import create_appfrom one_memory.config import Settingsfrom one_memory.secrets import InMemorySecretStore-def test_retrieval_defaults_off_and_does_not_create_index(tmp_path, monkeypatch):+def test_startup_does_not_create_a_private_retrieval_index(tmp_path, monkeypatch):+ """Raw reads go through the SDK and lexical FTS; MemHub keeps no vector copy."""monkeypatch.setattr("one_memory.model_presets._preset_data", lambda: None)async def scenario():@@ -29,10 +20,10 @@ def test_retrieval_defaults_off_and_does_not_create_index(tmp_path, monkeypatch)automatic_memory_processing=False,live_sync_enabled=False,)- assert settings.retrieval_index_enabled is Falseapp = create_app(settings, secret_store=InMemorySecretStore())async with app.router.lifespan_context(app):- assert app.state.retrieval_runtime is None+ assert not hasattr(app.state, "retrieval_runtime")+ assert not (tmp_path / "retrieval-vectors").exists()with closing(sqlite3.connect(tmp_path / "one-memory.sqlite3")) as db:assert (db.execute(@@ -42,188 +33,3 @@ def test_retrieval_defaults_off_and_does_not_create_index(tmp_path, monkeypatch))asyncio.run(scenario())---def test_enabling_retrieval_requires_runtime_ownership(tmp_path):- with pytest.raises(ValueError, match="retrieval_runtime_ownership_required"):- create_app(- Settings(_env_file=None, data_dir=tmp_path, retrieval_index_enabled=True),- secret_store=InMemorySecretStore(),- )---def test_index_enabled_serve_imports_without_building_an_unmanaged_app(tmp_path, monkeypatch):- """The real serve entry must not be killed by an import-time default app.-- `cli.serve()` imports `create_app` and then builds the app with- `manage_runtime=True`. If this module also builds a default app at import- time, enabling the index makes that throw before `serve()` ever binds.- Run the public entry in a subprocess, not through `create_app()` directly,- because the regression exists only in the import/entry-point combination.- """- monkeypatch.setenv("ONE_MEMORY_DATA_DIR", str(tmp_path))- monkeypatch.setenv("ONE_MEMORY_RETRIEVAL_INDEX_ENABLED", "1")- monkeypatch.setenv("ONE_MEMORY_RETRIEVAL_MINIMUM_FREE_BYTES", "0")- with socket.socket() as probe:- probe.bind(("127.0.0.1", 0))- port = probe.getsockname()[1]- monkeypatch.setenv("ONE_MEMORY_HOST", "127.0.0.1")- monkeypatch.setenv("ONE_MEMORY_PORT", str(port))- monkeypatch.setenv("HOME", str(tmp_path / "home"))- monkeypatch.setenv("ONE_MEMORY_SOURCE_HOME", str(tmp_path / "source-home"))- monkeypatch.setenv("ONE_MEMORY_IMPORTS_ENABLED", "0")- monkeypatch.setenv("ONE_MEMORY_LIVE_SYNC_ENABLED", "0")- monkeypatch.setenv("ONE_MEMORY_AUTOMATIC_MEMORY_PROCESSING", "0")-- process = subprocess.Popen(- [sys.executable, "-m", "one_memory.cli", "serve"],- env={- **os.environ,- "PYTHONPATH": str(Path(__file__).resolve().parents[1] / "src"),- },- stdout=subprocess.PIPE,- stderr=subprocess.PIPE,- text=True,- )- stdout = ""- stderr = ""- try:- deadline = time.monotonic() + 15- health = None- while time.monotonic() < deadline and process.poll() is None:- try:- with urllib.request.urlopen(- f"http://127.0.0.1:{port}/api/v1/health", timeout=0.5- ) as response:- health = json.loads(response.read())- break- except OSError:- time.sleep(0.1)- if health is None:- process.terminate()- try:- stdout, stderr = process.communicate(timeout=5)- except subprocess.TimeoutExpired:- process.kill()- stdout, stderr = process.communicate(timeout=5)- pytest.fail(- "serve did not become healthy within 15s; "- f"exit={process.returncode} stdout={stdout!r} stderr={stderr!r}"- )- assert health["status"] == "ok"- assert health["service"] == "one-memory-core"- finally:- if process.poll() is None:- process.terminate()- try:- process.communicate(timeout=5)- except subprocess.TimeoutExpired:- process.kill()- process.communicate(timeout=5)---def managed(tmp_path):- return create_app(- Settings(- _env_file=None,- data_dir=tmp_path,- source_home=tmp_path / "home",- imports_enabled=False,- automatic_memory_processing=False,- live_sync_enabled=False,- retrieval_index_enabled=True,- ),- manage_runtime=True,- secret_store=InMemorySecretStore(),- )---def test_managed_start_without_an_encoder_preserves_normal_core_startup(tmp_path, monkeypatch):- """No built-in weights is the real reason an index cannot be admitted.-- Core must still come up and the lexical lane must still answer; the runtime- says not_initialized rather than taking the process down with it.- """- monkeypatch.setattr("one_memory.model_presets._preset_data", lambda: None)- monkeypatch.setattr(- "one_memory.retrieval.index_provisioning.assets_present", lambda *_args: False- )-- async def scenario():- app = managed(tmp_path)- async with app.router.lifespan_context(app):- assert app.state.retrieval_runtime.snapshot().state == "not_initialized"- assert (tmp_path / "runtime.json").exists()- assert not (tmp_path / "retrieval-vectors").exists()- assert not (tmp_path / "runtime.json").exists()-- asyncio.run(scenario())---def test_enabling_retrieval_provisions_the_index_instead_of_waiting_to_be_handed_one(- tmp_path, monkeypatch-):- """The product entry point, which until now only a test helper could reach."""- monkeypatch.setattr("one_memory.model_presets._preset_data", lambda: None)-- async def scenario():- app = managed(tmp_path)- async with app.router.lifespan_context(app):- assert app.state.retrieval_runtime.snapshot().state == "running"- assert (tmp_path / "retrieval-vectors").is_dir()- with closing(sqlite3.connect(tmp_path / "one-memory.sqlite3")) as db:- assert [- row[0]- for row in db.execute(- "SELECT state FROM retrieval_index_generations ORDER BY rowid"- )- ] == ["active"]-- asyncio.run(scenario())--[email protected]("failure", [RuntimeError, asyncio.CancelledError])-def test_retrieval_stop_failure_still_drains_other_owned_services(tmp_path, monkeypatch, failure):- monkeypatch.setattr("one_memory.model_presets._preset_data", lambda: None)- events = []-- class StubRuntime:- def __init__(self, *args, **kwargs):- pass-- async def start(self):- events.append("start")-- async def stop(self):- events.append("retrieval_drained")- raise failure("after drain")-- monkeypatch.setattr("one_memory.retrieval.runtime.ExistingRetrievalRuntime", StubRuntime)-- async def scenario():- app = create_app(- Settings(- _env_file=None,- data_dir=tmp_path,- source_home=tmp_path / "home",- imports_enabled=False,- automatic_memory_processing=False,- live_sync_enabled=False,- retrieval_index_enabled=True,- ),- manage_runtime=True,- secret_store=InMemorySecretStore(),- )- with pytest.raises(failure):- async with app.router.lifespan_context(app):- original = app.state.sdk_memory_runtime.shutdown-- async def shutdown():- await original()- events.append("processing_drained")-- monkeypatch.setattr(app.state.sdk_memory_runtime, "shutdown", shutdown)- assert events == ["start", "retrieval_drained", "processing_drained"]- assert not (tmp_path / "runtime.json").exists()-- asyncio.run(scenario())diff --git a/memhub/apps/core/tests/test_retrieval_mapped_content.py b/memhub/apps/core/tests/test_retrieval_mapped_content.pydeleted file mode 100644index b408c94..0000000--- a/memhub/apps/core/tests/test_retrieval_mapped_content.py+++ /dev/null@@ -1,422 +0,0 @@-import hashlib-import json-import socket-from dataclasses import FrozenInstanceError, replace-from datetime import UTC, datetime--import pytest--from one_memory.imports.models import EvidenceContentKind, EvidenceRecord, WrittenBy-from one_memory.input_sanitization import SanitizedCopySpan, SanitizedInput, sanitize_model_input-from one_memory.local_embedding import LocalEmbedder, TokenizedText, TokenOffset-from one_memory.retrieval import mapped_content as mapped-from one_memory.retrieval.chunking import ChunkingError-from one_memory.retrieval.mapped_content import (- MappedContentError,- chunk_mapped_content,- prepare_mapped_content,-)--[email protected](autouse=True)-def offline(monkeypatch):- def reject(*args, **kwargs):- raise AssertionError("mapped content must remain offline")-- monkeypatch.setattr(socket.socket, "connect", reject)- monkeypatch.setattr(socket.socket, "connect_ex", reject)---def record(text, **changes):- values = dict(- evidence_id="synthetic-evidence",- user_id="local",- source_id="synthetic-source",- workspace=None,- project=None,- conversation_id="synthetic-conversation",- message_id="synthetic-message",- correlation_id=None,- source_ordinal=None,- content_kind=EvidenceContentKind.RAW_CONVERSATION,- role="user",- written_by=WrittenBy.PERSON,- created_at=None,- verbatim=text,- origin_meta={},- content_hash="outer-manifest-owned",- revision=4,- imported_at=datetime(2026, 9, 19, tzinfo=UTC),- updated_at=datetime(2026, 9, 19, tzinfo=UTC),- )- return EvidenceRecord(**(values | changes))---class Characters:- def __init__(self):- self.calls = []-- def tokenize_with_offsets(self, text):- self.calls.append(text)- offsets = (- TokenOffset(0, 0, True),- *(TokenOffset(i, i + 1) for i in range(len(text))),- TokenOffset(0, 0, True),- )- return TokenizedText(tuple(range(len(offsets))), offsets)---def assert_map(original, content):- meta = content.metadata- assert meta.source_length_cp == len(original)- assert meta.sanitized_length_cp == len(content.actual_text)- assert meta.sanitized_sha256 == hashlib.sha256(content.actual_text.encode()).hexdigest()- output_end = 0- copied = []- for span in meta.segments:- assert span.start_cp == output_end < span.end_cp- output_end = span.end_cp- if span.kind == "copy":- assert (- content.actual_text[span.start_cp : span.end_cp]- == original[span.source_start_cp : span.source_end_cp]- )- copied.append((span.source_start_cp, span.source_end_cp))- else:- assert span.kind == "synthetic"- assert span.source_start_cp is span.source_end_cp is None- assert output_end == len(content.actual_text)- ranges = sorted(copied + [(s.start_cp, s.end_cp) for s in meta.excluded])- end = 0- for start, right in ranges:- assert start == end < right- end = right- assert end == len(original)---def assert_chunks(original, prepared, chunks):- assert [c.ordinal for c in chunks] == list(range(len(chunks)))- coverage = set()- for c in chunks:- assert c.text == original[c.start_cp : c.end_cp]- assert c.token_count <= 512- assert any(- s.kind == "copy" and s.source_start_cp <= c.start_cp < c.end_cp <= s.source_end_cp- for s in prepared.metadata.segments- )- coverage.update(range(c.start_cp, c.end_cp))- for s in prepared.metadata.segments:- if s.kind == "copy":- assert all(- i in coverage- for i in range(s.source_start_cp, s.source_end_cp)- if not original[i].isspace()- )--- "role,author", [("user", WrittenBy.PERSON), ("assistant", WrittenBy.AGENT)]-)-def test_raw_authors_keep_exact_body_and_do_not_mutate_evidence(role, author):- original = " \t\n\u4e2d\U0001f680e\u0301 same\n same\t "- evidence = record(original, role=role, written_by=author)- before = evidence.model_dump()- content = prepare_mapped_content(evidence)- assert content.state == "ready"- assert content.actual_text == original.strip()- assert evidence.model_dump() == before- assert_map(original, content)- assert_chunks(original, content, chunk_mapped_content(content, Characters()))--- "change",- [- {"role": "tool", "written_by": WrittenBy.TOOL},- {"role": "system"},- {"content_kind": EvidenceContentKind.MEMORY_CONTENT},- ],-)-def test_raw_role_qualification_rejects_without_tokenizing(change):- with pytest.raises(MappedContentError, match="mapped_content_ineligible"):- prepare_mapped_content(record("ordinary body", **change))--[email protected]("author", list(WrittenBy))-def test_host_author_guess_does_not_change_protocol_role_qualification(author):- content = prepare_mapped_content(record("ordinary body", role="user", written_by=author))- assert content.state == "ready"--- "text,reason",- [- ("<user_query>one</user_query><user_query>two</user_query>", "ambiguous"),- ("<user_info>unclosed", "ambiguous"),- ("<rules>context</rules> unexplained trailing body", "ambiguous"),- ("<dynamic_tool_catalog>generated</dynamic_tool_catalog>", "generated"),- ('{"request_id":"fixture","type":"write","steps":[]}', "generated"),- (- "<user_query><dynamic_tool_catalog>generated</dynamic_tool_catalog></user_query>",- "generated",- ),- ("<user_query><environment_context>harness</environment_context></user_query>", "hygiene"),- ],-)-def test_ambiguous_generated_and_body_hygiene_are_not_admitted(text, reason):- with pytest.raises(MappedContentError, match=reason):- prepare_mapped_content(record(text))---def test_wrapper_offsets_compose_by_position_despite_repeated_text_and_unicode():- prefix = " \t<rules>same \U0001f680</rules>\n<user_query> \n"- body = "same \U0001f680 token=same same e\u0301"- original = prefix + body + " \n</user_query><image_local_path>same</image_local_path> "- prepared = prepare_mapped_content(record(original))- assert prepared.actual_text == sanitize_model_input(body)- assert prepared.metadata.body_start_cp == len(prefix)- assert prepared.metadata.body_end_cp == len(prefix) + len(body)- assert_map(original, prepared)- tokenizer = Characters()- chunks = chunk_mapped_content(prepared, tokenizer, target_tokens=10, overlap_tokens=3)- assert_chunks(original, prepared, chunks)- assert chunks[0].start_cp == len(prefix)- assert all("<rules>" not in text and "redacted-secret" not in text for text in tokenizer.calls)--- "secret",- [- "token=private-value",- "Authorization: Bearer abcdefghijklmnop",- '--password "two words"',- "sk-abcdefghijklmnopqrstuvwxyz",- "https://alice:[email protected]/path",- "-----BEGIN PRIVATE KEY-----\nprivate\n-----END PRIVATE KEY-----",- ],-)-def test_secrets_and_synthetic_replacements_never_reach_tokenizer(secret):- original = "left \u4e2d\U0001f680 " + secret + " right e\u0301 /final/file"- content = prepare_mapped_content(record(original))- tokenizer = Characters()- chunks = chunk_mapped_content(content, tokenizer, target_tokens=12, overlap_tokens=4)- assert_map(original, content)- assert_chunks(original, content, chunks)- assert all(secret not in text and "redacted-secret" not in text for text in tokenizer.calls)- assert any("left" in c.text for c in chunks) and any("right" in c.text for c in chunks)--- "text", ["", " \n\t ", "<rules>only wrapper</rules>", "token=private", " token=x \t token=y "]-)-def test_empty_wrapper_or_all_excluded_is_explicit_nonindexable(text):- content = prepare_mapped_content(record(text))- assert content.state == "no_indexable_content"- assert_map(text, content)- tokenizer = Characters()- assert chunk_mapped_content(content, tokenizer) == ()- assert tokenizer.calls == []---def test_metadata_is_immutable_canonical_and_never_contains_text_or_secret_hashes():- original = "<user_query>prefix token=private-value suffix</user_query>"- content = prepare_mapped_content(record(original))- encoded = content.metadata.canonical_json()- assert encoded == json.dumps(json.loads(encoded), sort_keys=True, separators=(",", ":"))- assert content.metadata.digest == hashlib.sha256(encoded.encode()).hexdigest()- for text in [- "prefix",- "suffix",- "private-value",- "redacted-secret",- hashlib.sha256(b"private-value").hexdigest(),- ]:- assert text not in encoded- assert content.metadata == prepare_mapped_content(record(original)).metadata- with pytest.raises(FrozenInstanceError):- content.metadata.source_length_cp = 1- with pytest.raises(FrozenInstanceError):- content.metadata.segments[0].start_cp = 1- with pytest.raises(FrozenInstanceError):- content.actual_text = "other"---def test_digest_changes_for_safe_text_or_coordinate_version_not_equal_length_excluded_secret():- first = prepare_mapped_content(record("left token=secret-one right"))- second = prepare_mapped_content(record("left token=secret-two right"))- assert first.metadata.digest == second.metadata.digest- assert (- first.metadata.digest- != prepare_mapped_content(record("LEFT token=secret-one right")).metadata.digest- )- assert first.metadata.digest != replace(first.metadata, map_version="future-map").digest---def test_overlapping_sanitizer_passes_and_literal_placeholder_are_distinct():- text = "<redacted-secret> token=sk-abcdefghijklmnopqrstuvwxyz /tail"- content = prepare_mapped_content(record(text))- assert_map(text, content)- chunks = chunk_mapped_content(content, Characters())- assert chunks[0].text == "<redacted-secret> " # Literal source text is copy, not synthetic.- assert chunks[-1].text == " /tail"--[email protected]("text", ["\u4e2d\x00\u6587", "\u4e2d\u200b\u6587", "\u4e2d\u200d\u6587"])-def test_control_characters_are_not_silently_sanitized_or_dropped(text):- content = prepare_mapped_content(record(text))- assert content.actual_text == text- with pytest.raises(ChunkingError, match="uncovered_nonwhitespace"):- chunk_mapped_content(content, LocalEmbedder())---def test_locked_tokenizer_unicode_coverage_and_prefix_budget():- body = ("\u4e2d\u6587\U0001f680e\u0301\n" * 400) + " token=private " + ("\u5c3e\u6587\n" * 400)- original = "<user_query>" + body + "</user_query>"- content = prepare_mapped_content(record(original))- tokenizer = LocalEmbedder()- prefix = "\u68c0\u7d22\u524d\u7f00" * 30- chunks = chunk_mapped_content(content, tokenizer, prefix=prefix)- assert len(chunks) > 4- assert_chunks(original, content, chunks)- assert all(- c.token_count == len(tokenizer.tokenize_with_offsets(prefix + c.text).token_ids)- for c in chunks- )---def test_more_than_8192_chunks_aborts_one_copy_span_without_returning_partial_content():- tokenizer = Characters()- content = prepare_mapped_content(record("x" * 9000))- with pytest.raises(MappedContentError, match="mapped_chunk_limit_exceeded"):- chunk_mapped_content(content, tokenizer, target_tokens=1, overlap_tokens=0)- assert len(tokenizer.calls) <= 8194---def test_global_limit_across_copy_spans_and_exact_limit_succeeds():- content = prepare_mapped_content(record("a token=private b"))- with pytest.raises(MappedContentError, match="mapped_chunk_limit_exceeded"):- chunk_mapped_content(content, Characters(), max_chunks=1)- chunks = chunk_mapped_content(content, Characters(), max_chunks=2)- assert [c.ordinal for c in chunks] == [0, 1]--[email protected]("limit", [0, -1, True, 8193])-def test_chunk_limit_is_not_raiseable_or_coercible(limit):- with pytest.raises(MappedContentError, match="mapped_invalid_chunk_limit"):- chunk_mapped_content(prepare_mapped_content(record("body")), Characters(), max_chunks=limit)---def test_forged_text_and_incomplete_sanitizer_mapping_fail_closed(monkeypatch):- prepared = prepare_mapped_content(record("body"))- with pytest.raises(MappedContentError, match="mapped_invalid"):- chunk_mapped_content(replace(prepared, actual_text="BODY"), Characters())- monkeypatch.setattr(- mapped,- "sanitize_model_input_with_spans",- lambda text: SanitizedInput(text, (SanitizedCopySpan(0, 2, 0, 1),), ()),- )- with pytest.raises(MappedContentError, match="mapped_invalid"):- prepare_mapped_content(record("body"))---def test_exact_8192_chunks_and_shrink_retries_do_not_spend_extra_chunk_budget():- content = prepare_mapped_content(record("x" * 8192))- chunks = chunk_mapped_content(content, Characters(), target_tokens=1, overlap_tokens=0)- assert len(chunks) == 8192 and chunks[-1].ordinal == 8191- content = prepare_mapped_content(record("x" * 10))- chunks = chunk_mapped_content(- content,- Characters(),- target_tokens=8,- overlap_tokens=0,- max_tokens=8,- prefix="PP",- max_chunks=3,- )- assert len(chunks) == 3- assert_chunks("x" * 10, content, chunks)---def test_fragmented_content_limit_is_global_and_rejects_before_next_copy_tokenization():- text = " token=private ".join(["x"] * 8193)- prepared = prepare_mapped_content(record(text))- tokenizer = Characters()- with pytest.raises(MappedContentError, match="mapped_chunk_limit_exceeded"):- chunk_mapped_content(prepared, tokenizer)- assert len(tokenizer.calls) == 2 * 8192---def test_random_repeated_wrapper_and_sanitizer_composition():- import random-- randomizer = random.Random(20260919)- for _ in range(80):- body = " ".join(- randomizer.choices(- [- "same",- "\u4e2d\U0001f680e\u0301",- "token=same",- "--password same",- "Authorization: Bearer abcdefghijklmnop",- "\t",- "https://u:p@host/path",- ],- k=10,- )- )- original = "<rules>same</rules><user_query> " + body + " </user_query>"- content = prepare_mapped_content(record(original))- assert_map(original, content)- assert_chunks(original, content, chunk_mapped_content(content, Characters()))--- "case", ["output_gap", "source_overlap", "wrong_copy", "missing_exclusion"]-)-def test_bad_upstream_maps_do_not_authorize_original_slices(monkeypatch, case):- samples = {- "output_gap": SanitizedInput("body", (SanitizedCopySpan(2, 4, 2, 4),), ()),- "source_overlap": SanitizedInput(- "bobo", (SanitizedCopySpan(0, 2, 0, 2), SanitizedCopySpan(2, 4, 0, 2)), ((2, 4),)- ),- "wrong_copy": SanitizedInput("BODY", (SanitizedCopySpan(0, 4, 0, 4),), ()),- "missing_exclusion": SanitizedInput("bo", (SanitizedCopySpan(0, 2, 0, 2),), ()),- }- monkeypatch.setattr(mapped, "sanitize_model_input_with_spans", lambda text: samples[case])- with pytest.raises(MappedContentError, match="mapped_invalid"):- prepare_mapped_content(record("body"))---def test_assistant_context_keeps_existing_hygiene_semantics_but_not_generated_payloads():- text = "Discuss <environment_context>literal</environment_context> as context"- content = prepare_mapped_content(record(text, role="assistant", written_by=WrittenBy.AGENT))- assert content.state == "ready" and content.actual_text == text- with pytest.raises(MappedContentError, match="generated"):- prepare_mapped_content(- record(- "<dynamic_tool_catalog>generated</dynamic_tool_catalog>",- role="assistant",- written_by=WrittenBy.AGENT,- )- )---def test_copy_boundaries_cannot_reconstruct_a_url_across_secret_gap():- original = "https://example.test/left token=private /right"- content = prepare_mapped_content(record(original))- chunks = chunk_mapped_content(content, Characters())- assert len(chunks) == 2- assert chunks[0].text == "https://example.test/left "- assert chunks[1].text == " /right"- assert chunks[0].end_cp < chunks[1].start_cp- assert_chunks(original, content, chunks)---def test_source_size_limit_rejects_before_parser(monkeypatch):- def unexpected(_):- raise AssertionError("oversize input reached parser")-- monkeypatch.setattr(mapped, "unwrap_host_envelope_with_positions", unexpected)- with pytest.raises(MappedContentError, match="mapped_source_limit_exceeded"):- prepare_mapped_content(record("x" * 1_000_001))diff --git a/memhub/apps/core/tests/test_retrieval_mapped_integration.py b/memhub/apps/core/tests/test_retrieval_mapped_integration.pydeleted file mode 100644index b958e8f..0000000--- a/memhub/apps/core/tests/test_retrieval_mapped_integration.py+++ /dev/null@@ -1,549 +0,0 @@-import socket-import sqlite3-import time-from contextlib import closing-from dataclasses import replace--import pytest--from one_memory.retrieval.chunking import TextChunk-from one_memory.retrieval.index_store import RetrievalIndexStore-from one_memory.retrieval.index_worker import RetrievalIndexWorker-from one_memory.retrieval.mapped_content import chunk_mapped_content, prepare_mapped_content-from one_memory.retrieval.raw_candidates import read_raw_semantic_candidates-from one_memory.retrieval.raw_evidence import materialize_raw_candidates-from one_memory.retrieval.vector_store import RawVectorStore-from tests.test_retrieval_index_store import fixture, initialize, put # noqa: F401-from tests.test_retrieval_index_worker import Encoder--[email protected](autouse=True)-def offline(monkeypatch):- def fail(*args, **kwargs):- raise AssertionError("no network permitted")-- monkeypatch.setattr(socket.socket, "connect", fail)- monkeypatch.setattr(socket.socket, "connect_ex", fail)---def migrate(f):- initialize(f)- identity = replace(f[4], normalizer_version=2, chunker_version=2)- with closing(sqlite3.connect(f[0])) as db, db:- db.execute("BEGIN IMMEDIATE")- f[3].migrate_v2(db)- generation = f[3].create_generation(db, identity)- return generation, identity---def mapped_setup(fixture, tmp_path): # noqa: F811- generation, identity = migrate(fixture)- encoder = Encoder(identity)- vectors = RawVectorStore.create(- tmp_path / "mapped-vectors", generation_id=generation, dimensions=3- )- worker = RetrievalIndexWorker(- fixture[3], vectors, encoder, generation_id=generation, owner="mapped"- )- try:- yield fixture, generation, encoder, vectors, worker- finally:- vectors.close()---def candidates(f, generation, vectors):- return read_raw_semantic_candidates(- f[3],- vectors,- [1.0, 0.5, 0.25],- generation_id=generation,- source_ids=frozenset({"codex"}),- deadline=time.monotonic() + 10,- )---def materialize(f, generation, hits):- return materialize_raw_candidates(- f[3],- hits,- generation_id=generation,- source_ids=frozenset({"codex"}),- deadline=time.monotonic() + 10,- )---def test_empty_materialization_rechecks_excluded_map(mapped_setup):- f, generation, _, vectors, worker = mapped_setup- put(f[1], "token=private-value")- assert worker.run_once().state == "excluded"- f[3].activate_generation(generation)- empty = candidates(f, generation, vectors)- assert empty.complete and not empty.hits- assert materialize(f, generation, empty).complete- with closing(sqlite3.connect(f[0])) as db, db:- db.execute("DELETE FROM retrieval_build_maps")- assert not materialize(f, generation, empty).complete---def test_completed_excluded_missing_active_binding_is_not_empty_success(mapped_setup):- f, generation, _, vectors, worker = mapped_setup- put(f[1], "token=private-value")- assert worker.run_once().state == "excluded"- f[3].activate_generation(generation)- assert candidates(f, generation, vectors).complete- with closing(sqlite3.connect(f[0])) as db, db:- db.execute("UPDATE retrieval_documents SET active_build_id=NULL")- result = candidates(f, generation, vectors)- assert not result.complete and result.reason == "index_unavailable"---def test_old_empty_completion_is_invalidated_when_a_new_indexed_record_arrives(mapped_setup):- f, generation, _, vectors, worker = mapped_setup- put(f[1], "token=private-value")- assert worker.run_once().state == "excluded"- f[3].activate_generation(generation)- empty = candidates(f, generation, vectors)- assert empty.complete and not empty.hits- put(f[1], "new actual content", message="new")- assert worker.run_once().state == "ready"- result = materialize(f, generation, empty)- assert not result.complete and result.reason == "evidence_changed"---def test_schema2_roundtrip_only_encodes_copy_and_separates_original_spans(mapped_setup):- f, generation, encoder, vectors, worker = mapped_setup- original = (- "<rules>private wrapper</rules><user_query>before token=private-value after</user_query>"- )- record = put(f[1], original)- result = worker.run_once()- assert result.state == "ready" and result.chunk_count == 2- assert encoder.calls == [["before ", " after"]]- f[3].activate_generation(generation)- hits = candidates(f, generation, vectors)- assert hits.complete and len(hits.hits) == 2- output = materialize(f, generation, hits)- assert output.complete and len(output.items) == 1- assert [s.text for s in output.items[0].spans] == ["before ", " after"]- assert all(original[s.start_cp : s.end_cp] == s.text for s in output.items[0].spans)- with closing(sqlite3.connect(f[0])) as db:- rows = db.execute(- "SELECT source_map_json,map_digest,coverage_state FROM retrieval_build_maps"- ).fetchall()- assert len(rows) == 1 and rows[0][2] == "indexable"- assert "before" not in rows[0][0] and "private" not in rows[0][0]- assert rows[0][1] == prepare_mapped_content(record).metadata.digest- assert RetrievalIndexStore(f[0]).active_chunks(generation)---def test_explicit_migration_rolls_back_and_v1_remains_readable(fixture): # noqa: F811- f = fixture- generation = initialize(f)- row = put(f[1], "legacy text")- f[3].consume_pending(generation)- lease = f[3].claim(generation, "legacy")- f[3].publish(- lease, (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),), vectors_verified=True- )- f[3].activate_generation(generation)- before = f[3].active_chunks(generation)- with closing(sqlite3.connect(f[0])) as db:- with pytest.raises(RuntimeError):- f[3].migrate_v2(db)- db.execute("BEGIN IMMEDIATE")- f[3].migrate_v2(db)- db.rollback()- assert db.execute("SELECT schema_version FROM retrieval_index_state").fetchone()[0] == 1- assert not db.execute(- "SELECT 1 FROM sqlite_master WHERE name='retrieval_build_maps'"- ).fetchone()- assert f[3].active_chunks(generation) == before- with closing(sqlite3.connect(f[0])) as db, db:- db.execute("BEGIN IMMEDIATE")- f[3].migrate_v2(db)- f[3].migrate_v2(db)- assert f[3].active_chunks(generation) == before- from one_memory.retrieval.raw_candidates import RawCandidateBatch, RawSemanticCandidate-- output = materialize(- f, generation, RawCandidateBatch(tuple(RawSemanticCandidate(c, 0.0) for c in before), True)- )- assert output.complete and output.items[0].spans[0].text == "legacy text"---def test_version2_requires_explicit_schema_upgrade(fixture): # noqa: F811- f = fixture- initialize(f)- with closing(sqlite3.connect(f[0])) as db:- db.execute("BEGIN IMMEDIATE")- with pytest.raises(ValueError, match="schema"):- f[3].create_generation(db, replace(f[4], normalizer_version=2, chunker_version=2))- assert db.execute("SELECT schema_version FROM retrieval_index_state").fetchone()[0] == 1--[email protected]("text", ["token=private", "<rules>only wrapper</rules>"])-def test_excluded_completion_has_no_vectors_and_is_not_ready(mapped_setup, text):- f, generation, encoder, vectors, worker = mapped_setup- put(f[1], text)- result = worker.run_once()- assert result.state == "excluded" and result.chunk_count == 0- assert encoder.calls == []- assert f[3].status(generation)["excluded"] == 1- assert f[3].status(generation)["ready"] == 0- f[3].activate_generation(generation)- assert candidates(f, generation, vectors).hits == ()--[email protected]("tamper", ["json", "digest", "offset", "missing_map", "missing_chunk"])-def test_persisted_tampering_rejected_at_candidate_and_final_read(mapped_setup, tamper):- f, generation, _, vectors, worker = mapped_setup- put(f[1], "before token=private after")- assert worker.run_once().state == "ready"- f[3].activate_generation(generation)- hits = candidates(f, generation, vectors)- with closing(sqlite3.connect(f[0])) as db, db:- if tamper == "json":- db.execute("UPDATE retrieval_build_maps SET source_map_json='{}'")- elif tamper == "digest":- db.execute("UPDATE retrieval_build_maps SET map_digest=?", ("0" * 64,))- elif tamper == "offset":- db.execute("UPDATE retrieval_chunks SET end_cp=end_cp+1 WHERE ordinal=0")- elif tamper == "missing_map":- db.execute("DELETE FROM retrieval_build_maps")- else:- db.execute("DELETE FROM retrieval_chunks WHERE ordinal=1")- assert not candidates(f, generation, vectors).complete- final = materialize(f, generation, hits)- assert not final.complete and not final.items--[email protected]("change", ["secret", "revoke", "policy"])-def test_current_source_secret_revision_and_grant_changes_invalidate_hits(mapped_setup, change):- f, generation, _, vectors, worker = mapped_setup- put(f[1], "before token=private-one after")- assert worker.run_once().state == "ready"- f[3].activate_generation(generation)- hits = candidates(f, generation, vectors)- if change == "secret":- put(f[1], "before token=private-two after")- else:- with closing(sqlite3.connect(f[0])) as db, db:- db.execute(- "UPDATE source_grants SET revoked_at='revoked'"- if change == "revoke"- else "UPDATE source_grants SET revision=revision+1"- )- assert not materialize(f, generation, hits).items- assert not f[3].active_chunks(generation)---def test_publish_recomputes_map_and_rejects_cross_gap_or_incomplete_manifest(mapped_setup):- f, generation, encoder, _, _ = mapped_setup- row = put(f[1], "before token=private after")- f[3].consume_pending(generation)- lease = f[3].claim(generation, "publisher")- prepared = prepare_mapped_content(row)- chunks = chunk_mapped_content(prepared, encoder)- for bad in [chunks[:1], (TextChunk(0, 0, len(row.verbatim), 10, row.verbatim),)]:- with pytest.raises(ValueError, match="manifest"):- f[3].publish(lease, bad, vectors_verified=True, source_map=prepared.metadata)- with pytest.raises(ValueError, match="map"):- f[3].publish(- lease,- chunks,- vectors_verified=True,- source_map=replace(prepared.metadata, sanitized_sha256="0" * 64),- )- assert f[3].status(generation)["ready"] == 0---def test_many_copy_regions_remain_partial_not_a_cross_gap_quote(mapped_setup):- f, generation, _, vectors, worker = mapped_setup- put(f[1], " token=private ".join([f"part-{i}" for i in range(6)]))- assert worker.run_once().chunk_count == 6- f[3].activate_generation(generation)- final = materialize(f, generation, candidates(f, generation, vectors))- assert not final.complete and len(final.items[0].spans) == 4- assert all("token=" not in s.text for s in final.items[0].spans)---def test_local_onnx_chroma_mapped_roundtrip_without_download(fixture, tmp_path): # noqa: F811- from one_memory.retrieval.index_worker import LocalRetrievalEncoder-- f = fixture- encoder = LocalRetrievalEncoder()- assert encoder.identity.normalizer_version == encoder.identity.chunker_version == 2- with closing(sqlite3.connect(f[0])) as db, db:- db.execute("BEGIN IMMEDIATE")- f[3].initialize(db)- f[3].migrate_v2(db)- generation = f[3].create_generation(db, encoder.identity)- row = put(- f[1],- "<user_query>\u4e2d\u6587\U0001f680e\u0301\n"- + "\u524d\u6587" * 300- + " token=private-value "- + "\u540e\u6587" * 300- + "</user_query>",- )- vectors = RawVectorStore.create(- tmp_path / "native-mapped", generation_id=generation, dimensions=encoder.identity.dimensions- )- try:- worker = RetrievalIndexWorker(- f[3], vectors, encoder, generation_id=generation, owner="local"- )- result = worker.run_once()- assert result.state == "ready" and result.chunk_count >= 4- f[3].activate_generation(generation)- hits = read_raw_semantic_candidates(- f[3],- vectors,- encoder.encode_documents(["\u540e\u6587"])[0],- generation_id=generation,- source_ids=frozenset({"codex"}),- deadline=time.monotonic() + 15,- )- final = materialize(f, generation, hits)- assert final.complete and len(final.items[0].spans) == 2- for span in final.items[0].spans:- assert span.text == row.verbatim[span.start_cp : span.end_cp]- assert "private-value" not in span.text and "redacted-secret" not in span.text- finally:- vectors.close()--- "text",- [- "<user_query>one</user_query><user_query>two</user_query>",- "<dynamic_tool_catalog>generated</dynamic_tool_catalog>",- ],-)-def test_ambiguous_generated_stay_blocked(mapped_setup, text):- f, generation, encoder, _, worker = mapped_setup- put(f[1], text)- assert worker.run_once().state == "blocked"- assert not encoder.calls- with pytest.raises(ValueError, match="incomplete"):- f[3].activate_generation(generation)--[email protected]("change", ["revoke", "secret"])-def test_change_during_encoding_never_publishes(mapped_setup, change):- f, generation, encoder, _, worker = mapped_setup- put(f[1], "before token=private-one after")- original = encoder.encode_documents-- def changed(texts):- result = original(texts)- if change == "secret":- put(f[1], "before token=private-two after")- else:- f[2].revoke(user_id="local", source_id="codex")- return result-- encoder.encode_documents = changed- assert worker.run_once().state == "blocked"- assert f[3].status(generation)["ready"] == 0- with closing(sqlite3.connect(f[0])) as db:- assert not db.execute("SELECT * FROM retrieval_build_maps").fetchall()---def test_cancellation_inside_publish_rolls_back_map_chunks_and_completion(- mapped_setup, monkeypatch-):- import threading-- f, generation, _, _, worker = mapped_setup- put(f[1], "before token=private after")- cancel = threading.Event()- original = f[3]._prepare-- def cancelled(*args):- prepared = original(*args)- cancel.set()- return prepared-- monkeypatch.setattr(f[3], "_prepare", cancelled)- assert worker.run_once(cancel=cancel).state == "cancelled"- assert f[3].status(generation)["leased"] == 1- with closing(sqlite3.connect(f[0])) as db:- assert not db.execute("SELECT * FROM retrieval_build_maps").fetchall()- assert not db.execute("SELECT * FROM retrieval_chunks").fetchall()---def test_excluded_metadata_is_validated_on_activation_and_candidate_read(mapped_setup):- f, generation, _, vectors, worker = mapped_setup- put(f[1], "token=private")- assert worker.run_once().state == "excluded"- f[3].activate_generation(generation)- with closing(sqlite3.connect(f[0])) as db, db:- db.execute("DELETE FROM retrieval_build_maps")- assert not candidates(f, generation, vectors).complete- with pytest.raises(ValueError, match="map"):- f[3].activate_generation(generation)---def test_migration_preserves_an_existing_v1_lease_and_store_identity(fixture): # noqa: F811- f = fixture- generation = initialize(f)- row = put(f[1], "legacy leased")- f[3].consume_pending(generation)- lease = f[3].claim(generation, "legacy")- with closing(sqlite3.connect(f[0])) as db, db:- db.execute("BEGIN IMMEDIATE")- f[3].migrate_v2(db)- assert f[3].read_lease(lease) == row- f[3].publish(- lease, (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),), vectors_verified=True- )- f[3].activate_generation(generation)- assert f[3].active_chunks(generation)---def test_bad_token_offsets_leave_no_partial_mapped_manifest(mapped_setup):- from one_memory.local_embedding import TokenizedText, TokenOffset-- f, generation, encoder, _, worker = mapped_setup- put(f[1], "before token=private after")- encoder.tokenize_with_offsets = lambda text: TokenizedText((1,), (TokenOffset(0, 1),))- assert worker.run_once().state == "blocked"- assert not encoder.calls- assert f[3].status(generation)["ready"] == 0---def test_failed_migration_keeps_original_table_lease_and_schema(fixture): # noqa: F811- f = fixture- generation = initialize(f)- put(f[1], "preserved")- f[3].consume_pending(generation)- lease = f[3].claim(generation, "legacy")- with closing(sqlite3.connect(f[0])) as db, db:- db.execute("CREATE TABLE retrieval_build_maps (foreign_value TEXT)")- db.execute("BEGIN IMMEDIATE")- with pytest.raises(sqlite3.Error):- f[3].migrate_v2(db)- assert db.execute("SELECT schema_version FROM retrieval_index_state").fetchone()[0] == 1- assert not db.execute(- "SELECT 1 FROM sqlite_master WHERE name='retrieval_documents_v2'"- ).fetchone()- assert f[3].read_lease(lease).verbatim == "preserved"--- "text,expected", [("before token=private after", "ready"), ("token=private", "excluded")]-)-def test_regrant_reuses_immutable_map_and_manifest_only_after_explicit_retry(- mapped_setup, text, expected-):- from one_memory.imports.sources.registry import RootCandidateKind-- f, generation, _, _, worker = mapped_setup- put(f[1], text)- assert worker.run_once().state == expected- f[3].activate_generation(generation)- with closing(sqlite3.connect(f[0])) as db:- original = db.execute("SELECT * FROM retrieval_build_maps").fetchall()- f[2].revoke(user_id="local", source_id="codex")- assert not f[3].active_chunks(generation)- f[2].grant(- user_id="local",- source_id="codex",- root=f[0].parent / "source",- candidate_kind=RootCandidateKind.DEFAULT,- )- assert not f[3].active_chunks(generation)- assert f[3].retry_blocked(generation) == 1- assert worker.run_once().state == expected- with closing(sqlite3.connect(f[0])) as db:- assert db.execute("SELECT * FROM retrieval_build_maps").fetchall() == original---def test_map_replay_conflict_rolls_back_new_chunks(mapped_setup):- f, generation, encoder, _, _ = mapped_setup- row = put(f[1], "before token=private after")- f[3].consume_pending(generation)- lease = f[3].claim(generation, "publisher")- prepared = prepare_mapped_content(row)- chunks = chunk_mapped_content(prepared, encoder)- with closing(sqlite3.connect(f[0])) as db, db:- db.execute(- "INSERT INTO retrieval_build_maps VALUES (?,?,?,?)",- (lease.build_id, "{}", "0" * 64, "indexable"),- )- with pytest.raises(ValueError, match="immutable_source_map"):- f[3].publish(lease, chunks, vectors_verified=True, source_map=prepared.metadata)- with closing(sqlite3.connect(f[0])) as db:- assert not db.execute("SELECT * FROM retrieval_chunks").fetchall()- assert f[3].status(generation)["leased"] == 1---def test_deadline_expiring_during_publish_rolls_back_completion(mapped_setup, monkeypatch):- f, generation, _, _, worker = mapped_setup- put(f[1], "before token=private after")- current = [0.0]- worker.clock = lambda: current[0]- worker.max_job_seconds = 10- original = f[3]._prepare-- def expires(*args):- result = original(*args)- current[0] = 11- return result-- monkeypatch.setattr(f[3], "_prepare", expires)- result = worker.run_once()- assert result.state == "deferred" and result.reason == "deadline"- assert f[3].status(generation)["leased"] == 1- with closing(sqlite3.connect(f[0])) as db:- assert not db.execute("SELECT * FROM retrieval_chunks").fetchall()- assert not db.execute("SELECT * FROM retrieval_build_maps").fetchall()---def test_deadline_and_character_budget_never_release_cross_gap_spans(mapped_setup):- f, generation, _, vectors, worker = mapped_setup- put(f[1], "before token=private after")- assert worker.run_once().state == "ready"- f[3].activate_generation(generation)- hits = candidates(f, generation, vectors)- expired = materialize_raw_candidates(- f[3],- hits,- generation_id=generation,- source_ids=frozenset({"codex"}),- deadline=time.monotonic() - 1,- )- assert not expired.items and expired.reason == "deadline"- bounded = materialize_raw_candidates(- f[3],- hits,- generation_id=generation,- source_ids=frozenset({"codex"}),- deadline=time.monotonic() + 10,- character_budget=7,- )- assert not bounded.complete and bounded.reason == "character_budget"- assert bounded.used_characters <= 7- assert sum(len(item.spans) for item in bounded.items) == 1---def test_lease_expiring_inside_publish_cannot_commit(mapped_setup, monkeypatch):- f, generation, _, _, worker = mapped_setup- put(f[1], "before token=private after")- current = [100.0]- worker.wall_clock = lambda: current[0]- original = f[3]._prepare-- def expires(*args):- result = original(*args)- current[0] = 1000.0- return result-- monkeypatch.setattr(f[3], "_prepare", expires)- assert worker.run_once().state == "blocked"- with closing(sqlite3.connect(f[0])) as db:- assert not db.execute("SELECT * FROM retrieval_chunks").fetchall()- assert not db.execute("SELECT * FROM retrieval_build_maps").fetchall()- assert f[3].status(generation)["ready"] == 0diff --git a/memhub/apps/core/tests/test_retrieval_provisioning.py b/memhub/apps/core/tests/test_retrieval_provisioning.pydeleted file mode 100644index b8cf32e..0000000--- a/memhub/apps/core/tests/test_retrieval_provisioning.py+++ /dev/null@@ -1,125 +0,0 @@-"""The index has to come up through the product, not through a test helper.--Every existing retrieval test builds its generation by hand, so none of them-can tell whether a user who enables the feature ever gets an index at all.-"""--import logging-import sqlite3-from contextlib import closing-from pathlib import Path--import pytest--from one_memory.imports.storage import EvidenceStore-from one_memory.retrieval.index_provisioning import provision_retrieval_index-from one_memory.retrieval.index_store import RetrievalIndexStore-from one_memory.retrieval.index_worker import LocalRetrievalEncoder---def database(tmp_path: Path) -> Path:- path = tmp_path / "one-memory.sqlite3"- EvidenceStore(path)- return path---def active(path: Path) -> list[str]:- with closing(sqlite3.connect(path)) as db:- if not db.execute(- "SELECT 1 FROM sqlite_master WHERE name='retrieval_index_generations'"- ).fetchone():- return []- return [- row[0]- for row in db.execute(- "SELECT generation_id FROM retrieval_index_generations WHERE state='active'"- )- ]---def test_first_enable_publishes_one_active_generation_and_its_vector_store(tmp_path):- database(tmp_path)-- generation = provision_retrieval_index(tmp_path)-- assert generation is not None- assert active(tmp_path / "one-memory.sqlite3") == [generation]- assert (tmp_path / "retrieval-vectors").is_dir()---def test_second_start_reuses_the_same_generation_rather_than_stacking_one(tmp_path):- database(tmp_path)-- first = provision_retrieval_index(tmp_path)- second = provision_retrieval_index(tmp_path)-- assert first == second- assert active(tmp_path / "one-memory.sqlite3") == [first]---def test_a_v1_index_is_left_alone_instead_of_being_migrated_on_startup(tmp_path, caplog):- """Falsifiable against the first test, where provisioning does publish one.-- Migrating someone's existing index while they were only starting the app is- the one thing the schema note rules out, so this must stay a no-op. The- absence of a warning is asserted too: crashing into the same `None` would- otherwise read as a deliberate decision.- """- path = database(tmp_path)- store = RetrievalIndexStore(path)- with closing(sqlite3.connect(path)) as db, db:- db.execute("BEGIN IMMEDIATE")- store.initialize(db)-- with caplog.at_level(logging.WARNING):- assert provision_retrieval_index(tmp_path) is None- assert active(path) == []- assert "retrieval_index_provision_failed" not in caplog.text- with closing(sqlite3.connect(path)) as db:- version = db.execute("SELECT schema_version FROM retrieval_index_state").fetchone()[0]- assert version == 1---def test_a_generation_for_another_encoder_is_not_swapped_out_silently(tmp_path):- path = database(tmp_path)- store = RetrievalIndexStore(path)- other = LocalRetrievalEncoder(content_version=1).identity- with closing(sqlite3.connect(path)) as db, db:- db.execute("BEGIN IMMEDIATE")- store.initialize(db)- store.migrate_v2(db)- foreign = store.create_generation(db, other)- store.activate_generation(foreign)-- assert provision_retrieval_index(tmp_path) is None- assert active(path) == [foreign]---def test_without_the_built_in_encoder_no_index_is_created(tmp_path, monkeypatch):- path = database(tmp_path)- monkeypatch.setattr(- "one_memory.retrieval.index_provisioning.assets_present", lambda *_args: False- )-- assert provision_retrieval_index(tmp_path) is None- assert active(path) == []- assert not (tmp_path / "retrieval-vectors").exists()---def test_a_missing_database_is_not_created_by_provisioning(tmp_path):- assert provision_retrieval_index(tmp_path) is None- assert not (tmp_path / "one-memory.sqlite3").exists()- assert not (tmp_path / "retrieval-vectors").exists()--[email protected]("failure", ["initialize_v2", "create_generation"])-def test_a_failed_provision_reports_none_instead_of_half_an_index(tmp_path, monkeypatch, failure):- path = database(tmp_path)-- def explode(*_args, **_kwargs):- raise sqlite3.OperationalError("synthetic")-- monkeypatch.setattr(RetrievalIndexStore, failure, explode)-- assert provision_retrieval_index(tmp_path) is None- assert active(path) == []diff --git a/memhub/apps/core/tests/test_retrieval_raw_candidates.py b/memhub/apps/core/tests/test_retrieval_raw_candidates.pydeleted file mode 100644index 7b041d4..0000000--- a/memhub/apps/core/tests/test_retrieval_raw_candidates.py+++ /dev/null@@ -1,224 +0,0 @@-import sqlite3-import time-from contextlib import closing-from datetime import datetime--import pytest--from one_memory.imports.models import EvidenceInput-from one_memory.retrieval.index_worker import RetrievalIndexWorker-from one_memory.retrieval.models import TimeRange-from one_memory.retrieval.raw_candidates import (- compose_raw_retrieval_path,- read_raw_semantic_candidates,-)-from one_memory.retrieval.vector_store import RawVectorStore-from tests.test_retrieval_index_store import fixture, initialize, put # noqa: F401-from tests.test_retrieval_index_worker import Encoder, forbid_network # noqa: F401---def indexed(fixture, tmp_path): # noqa: F811- generation = initialize(fixture)- record = put(fixture[1])- encoder = Encoder(fixture[4])- vectors = RawVectorStore.create(tmp_path / "vectors", generation_id=generation, dimensions=3)- worker = RetrievalIndexWorker(- fixture[3], vectors, encoder, generation_id=generation, owner="test"- )- assert worker.run_once().state == "ready"- fixture[3].activate_generation(generation)- try:- yield fixture, generation, record, vectors- finally:- vectors.close()---def search(indexed, **kwargs):- f, generation, _, vectors = indexed- options = dict(source_ids=frozenset({"codex"}), deadline=time.monotonic() + 10)- options.update(kwargs)- return read_raw_semantic_candidates(- f[3], vectors, [1.0, 0.5, 0.25], generation_id=generation, **options- )---def test_positive_only_current_authorized_published_chunks_are_searched(indexed):- result = search(indexed)- assert result.complete and len(result.hits) == 1- assert result.hits[0].chunk.evidence_id == indexed[2].evidence_id- assert result.coverage == "indexed_only"---def test_empty_authority_never_reads_database_or_vector(indexed, monkeypatch):- def forbidden(*args, **kwargs):- raise AssertionError("unauthorized access")-- monkeypatch.setattr(indexed[0][3], "active_chunks", forbidden)- monkeypatch.setattr(indexed[3], "search", forbidden)- result = search(indexed, source_ids=frozenset())- assert not result.hits and result.reason == "not_authorized"---def test_unapproved_source_never_enters_vector_query(indexed, monkeypatch):- monkeypatch.setattr(indexed[3], "search", lambda *a, **k: pytest.fail("not eligible"))- assert search(indexed, source_ids=frozenset({"cursor"})).hits == ()--[email protected]("change", ["revoke", "revise", "policy"])-def test_mutation_during_vector_query_discards_stale_candidate(indexed, monkeypatch, change):- f, _, _, vectors = indexed- original = vectors.search-- def query(*args, **kwargs):- result = original(*args, **kwargs)- if change == "revise":- put(f[1], "changed text")- elif change == "revoke":- f[2].revoke(user_id="local", source_id="codex")- else:- with closing(sqlite3.connect(f[0])) as db, db:- db.execute("UPDATE source_grants SET revision=revision+1")- return result-- monkeypatch.setattr(vectors, "search", query)- result = search(indexed)- assert not result.hits and not result.complete and result.reason == "evidence_changed"---def test_expired_deadline_does_not_read_or_query(indexed, monkeypatch):- monkeypatch.setattr(indexed[3], "search", lambda *a, **k: pytest.fail("late query"))- result = search(indexed, deadline=time.monotonic() - 1)- assert not result.hits and not result.complete and result.reason == "deadline"---def test_missing_published_vector_is_error_not_empty(indexed):- indexed[3]._collection.delete(ids=[c.chunk_id for c in indexed[0][3].active_chunks(indexed[1])])- result = search(indexed)- assert not result.hits and not result.complete and result.reason == "vector_unavailable"---def test_small_allowlist_uses_query_as_integrity_check_without_duplicate_get(indexed, monkeypatch):- monkeypatch.setattr(- indexed[3],- "verify_candidates",- lambda *args, **kwargs: pytest.fail("small allowlist must not be read twice"),- )- result = search(indexed)- assert result.complete and len(result.hits) == 1---def test_missing_event_time_never_matches_hard_range(indexed, monkeypatch):- window = TimeRange(start="2026-09-18T00:00:00+08:00", end="2026-09-19T00:00:00+08:00")- monkeypatch.setattr(indexed[3], "search", lambda *a, **k: pytest.fail("unknown event time"))- assert search(indexed, time_range=window).hits == ()--[email protected]("sources", [None, ["codex"], "codex"])-def test_caller_must_provide_explicit_resolved_authority(indexed, sources):- with pytest.raises(ValueError, match="source"):- search(indexed, source_ids=sources)---def test_event_time_uses_utc_half_open_range_not_import_time(indexed):- f, generation, record, vectors = indexed- f[1].put(- EvidenceInput(- source_id=record.source_id,- conversation_id=record.conversation_id,- message_id=record.message_id,- role=record.role,- written_by=record.written_by,- verbatim=record.verbatim,- created_at=datetime.fromisoformat("2026-09-18T16:00:00+00:00"),- origin_meta=record.origin_meta,- )- )- worker = RetrievalIndexWorker(- f[3], vectors, Encoder(f[4]), generation_id=generation, owner="test"- )- assert worker.run_once().state == "ready"- before = TimeRange(start="2026-09-18T00:00:00+08:00", end="2026-09-19T00:00:00+08:00")- after = TimeRange(start="2026-09-19T00:00:00+08:00", end="2026-09-20T00:00:00+08:00")- assert not search(indexed, time_range=before).hits- result = search(indexed, time_range=after)- assert result.complete and len(result.hits) == 1- assert result.hits[0].chunk.event_time == after.start---def test_final_deadline_failure_discards_previously_found_hits(indexed, monkeypatch):- from one_memory.retrieval import raw_candidates-- original = raw_candidates._check- calls = []-- def check(deadline):- calls.append(deadline)- if len(calls) == 3:- raise TimeoutError("late final revalidation")- return original(deadline)-- monkeypatch.setattr(raw_candidates, "_check", check)- result = search(indexed)- assert len(calls) == 3 and not result.hits and result.reason == "deadline"--[email protected]("corruption", ["all_missing", "tail_missing", "offset", "hash"])-def test_corrupted_published_manifest_is_unavailable_not_complete_empty(indexed, corruption):- f, generation, _, vectors = indexed- put(f[1], "long source text " * 90)- worker = RetrievalIndexWorker(- f[3], vectors, Encoder(f[4]), generation_id=generation, owner="test"- )- assert worker.run_once().state == "ready"- assert len(search(indexed).hits) > 1- with closing(sqlite3.connect(f[0])) as db, db:- if corruption == "all_missing":- db.execute("DELETE FROM retrieval_chunks")- elif corruption == "tail_missing":- db.execute("DELETE FROM retrieval_chunks WHERE ordinal>0")- elif corruption == "offset":- db.execute("UPDATE retrieval_chunks SET start_cp=3 WHERE ordinal=0")- else:- db.execute("UPDATE retrieval_chunks SET text_hash='broken'")- result = search(indexed)- assert not result.complete and not result.hits and result.reason == "index_unavailable"---def test_null_stored_grant_with_revocation_never_queries_vectors(indexed, monkeypatch):- f, _, _, vectors = indexed- assert search(indexed).hits- with closing(sqlite3.connect(f[0])) as db, db:- db.execute("UPDATE retrieval_documents SET grant_fingerprint=NULL")- f[2].revoke(user_id="local", source_id="codex")- monkeypatch.setattr(vectors, "search", lambda *a, **k: pytest.fail("revoked vector query"))- assert not search(indexed).hits---def test_unpublished_authorized_evidence_cannot_claim_complete_empty(indexed):- """A partial index with no hits is incomplete, not an honest empty library."""- f, generation, _, _vectors = indexed- put(f[1], "用户决定 One Memory 采用 MIT 协议发布。", message="unindexed")- assert f[3].consume_pending(generation) >= 1- pending = f[3].status(generation)- assert pending["pending"] >= 1- # Published chunks may still match a probe vector. The unpublished- # authorized document is what forbids complete-empty, so the candidate- # batch is forced empty: that is the lying-empty case, not a miss on- # already-indexed text.- empty = type(search(indexed))(hits=(), complete=True, reason="no_eligible_chunks")- path = compose_raw_retrieval_path(- empty, unpublished_authorized=pending["pending"] > 0, lexical_hit_count=0- )- assert path.coverage == "partial"- assert path.status != "empty"- assert "semantic" in path.searched_channels---def test_complete_index_with_no_hits_may_be_empty(indexed):- batch = search(indexed, source_ids=frozenset({"codex"}))- # Query vector still matches the only indexed chunk; force a no-hit batch.- empty = type(batch)(hits=(), complete=True, reason="no_eligible_chunks")- path = compose_raw_retrieval_path(empty, unpublished_authorized=False, lexical_hit_count=0)- assert path.status == "empty"- assert path.coverage == "complete"diff --git a/memhub/apps/core/tests/test_retrieval_raw_evidence.py b/memhub/apps/core/tests/test_retrieval_raw_evidence.pydeleted file mode 100644index d8f7829..0000000--- a/memhub/apps/core/tests/test_retrieval_raw_evidence.py+++ /dev/null@@ -1,88 +0,0 @@-import sqlite3-import time-from contextlib import closing--import pytest--from one_memory.retrieval.index_worker import RetrievalIndexWorker-from one_memory.retrieval.raw_evidence import materialize_raw_candidates-from tests.test_retrieval_index_store import fixture, put # noqa: F401-from tests.test_retrieval_index_worker import Encoder, forbid_network # noqa: F401-from tests.test_retrieval_raw_candidates import indexed as indexed_fixture-from tests.test_retrieval_raw_candidates import search--indexed = indexed_fixture---def materialize(indexed, candidates=None, **kwargs):- options = dict(- source_ids=frozenset({"codex"}), deadline=time.monotonic() + 10, character_budget=6000- )- options.update(kwargs)- return materialize_raw_candidates(- indexed[0][3], candidates or search(indexed), generation_id=indexed[1], **options- )---def test_materialized_text_is_exact_current_original(indexed):- result = materialize(indexed)- assert result.complete and len(result.items) == 1- item = result.items[0]- assert item.evidence_id == indexed[2].evidence_id- assert item.spans[0].text == indexed[2].verbatim- assert item.written_by == "person" and item.matched_by == ("semantic",)--[email protected]("change", ["revision", "revoke", "unrecorded"])-def test_stale_candidates_do_not_authorize_original_read(indexed, change):- candidates = search(indexed)- assert candidates.hits- f = indexed[0]- if change == "revision":- put(f[1], "updated original")- elif change == "revoke":- f[2].revoke(user_id="local", source_id="codex")- else:- with closing(sqlite3.connect(f[0])) as db, db:- db.execute(- "UPDATE evidence SET verbatim='unrecorded' WHERE evidence_id=?",- (indexed[2].evidence_id,),- )- result = materialize(indexed, candidates)- assert not result.items and not result.complete and result.reason == "evidence_changed"---def test_budget_does_not_cut_a_chunk_into_fake_partial_object(indexed):- result = materialize(indexed, character_budget=5)- assert not result.items and not result.complete and result.reason == "character_budget"---def test_forged_offsets_and_modified_manifest_do_not_release_body(indexed):- candidates = search(indexed)- with closing(sqlite3.connect(indexed[0][0])) as db, db:- db.execute("UPDATE retrieval_chunks SET start_cp=2")- result = materialize(indexed, candidates)- assert not result.items and not result.complete---def test_empty_authority_and_expired_deadline_return_no_body(indexed):- candidates = search(indexed)- assert not materialize(indexed, candidates, source_ids=frozenset()).items- assert not materialize(indexed, candidates, deadline=time.monotonic() - 1).items---def test_overlapping_unicode_chunks_merge_without_duplicate_characters(indexed):- f, generation, _, vectors = indexed- text = "\u6c49\u5b57\U0001f680e\u0301 " * 180 + " /project/final-design.png"- put(f[1], text)- worker = RetrievalIndexWorker(- f[3], vectors, Encoder(f[4]), generation_id=generation, owner="test"- )- assert worker.run_once().state == "ready"- candidates = search(indexed)- assert len(candidates.hits) > 1- result = materialize(indexed, candidates)- assert result.complete and len(result.items) == 1- assert len(result.items[0].spans) == 1- assert result.items[0].spans[0].text == text- assert result.used_characters == len(text)diff --git a/memhub/apps/core/tests/test_retrieval_raw_lane.py b/memhub/apps/core/tests/test_retrieval_raw_lane.pydeleted file mode 100644index 6f80147..0000000--- a/memhub/apps/core/tests/test_retrieval_raw_lane.py+++ /dev/null@@ -1,356 +0,0 @@-"""v1 raw lane must not claim complete empty when the derived index is partial."""--import asyncio-import sqlite3-import threading-import time-from types import SimpleNamespace--import pytest--from one_memory.context_models import ContextLaneResult, ContextSearchRequest-from one_memory.context_raw import read_raw_context-from one_memory.database_access import main_database-from one_memory.embedding_assets import assets_present-from one_memory.retrieval.raw_candidates import RawCandidateBatch-from one_memory.retrieval.raw_lane import fuse_raw_lane, read_raw_indexed_context-from tests.test_context_raw import database as database # noqa: F401-from tests.test_context_raw import put---def _require_assets() -> None:- """Make an absent local fixture explicit instead of a lease-style hang."""- if not assets_present():- pytest.skip("_assets_unavailable")---def test_unpublished_authorized_evidence_degrades_empty_lexical_lane():- lexical = ContextLaneResult(status="empty", reason="lexical_only")- result = fuse_raw_lane(- lexical,- RawCandidateBatch(complete=True, reason="no_eligible_chunks"),- unpublished_authorized=True,- )- assert result.status == "degraded"- assert result.reason == "lexical_only"- assert result.items == ()---def test_complete_index_with_no_hits_stays_empty():- lexical = ContextLaneResult(status="empty", reason="lexical_only")- result = fuse_raw_lane(- lexical,- RawCandidateBatch(complete=True, reason="no_eligible_chunks"),- unpublished_authorized=False,- )- assert result.status == "empty"- assert result.items == ()---def test_interactive_raw_lane_reads_one_snapshot_while_a_writer_holds(database):- """The read lane must not wait for an import writer, nor see its uncommitted row.-- This is the C4-owned half of the import-contention BLOCK: WAL readers are- expected to proceed from the last committed snapshot even while the single- writer holds a transaction. A regression that lets context reads block on- the writer - or read its uncommitted row - makes this fail.- """- with sqlite3.connect(database) as db, db:- db.execute("PRAGMA journal_mode=WAL")- put(db, "committed", "needle committed row")- writer_holds = threading.Event()- release_writer = threading.Event()-- def hold_write_transaction() -> None:- with main_database(database).write(autocommit=True) as writer:- writer.execute("BEGIN IMMEDIATE")- writer.execute(- """INSERT INTO evidence- (evidence_id,idempotency_key,user_id,source_id,conversation_id,role,- written_by,created_at,verbatim,content_hash,revision,imported_at,updated_at)- VALUES ('uncommitted','uncommitted','local','cursor','c','user',- 'person','2026-09-20','needle uncommitted row','synthetic',- 1,'2026-09-20','2026-09-20')"""- )- writer_holds.set()- assert release_writer.wait(5)- writer.execute("ROLLBACK")-- thread = threading.Thread(target=hold_write_transaction)- thread.start()- assert writer_holds.wait(5)- try:- before = main_database(database).stats()- started = time.monotonic()- result = read_raw_context(- database,- ContextSearchRequest(query="needle", kinds=("raw",)),- source_ids=frozenset({"cursor"}),- deadline=time.monotonic() + 2,- )- elapsed = time.monotonic() - started- after = main_database(database).stats()- finally:- release_writer.set()- thread.join(timeout=5)- assert not thread.is_alive()- assert elapsed < 1.0- assert after.reader_waits == before.reader_waits- assert [item.anchor_evidence_id for item in result.items] == ["committed"]- assert all(- message.evidence_id != "uncommitted" for item in result.items for message in item.messages- )---def test_authorization_and_unavailability_are_not_rewritten():- denied = ContextLaneResult(status="not_authorized", reason="permission_required")- assert (- fuse_raw_lane(denied, RawCandidateBatch(), unpublished_authorized=True).status- == "not_authorized"- )- missing = ContextLaneResult(status="unavailable", reason="read_unavailable")- assert fuse_raw_lane(missing, RawCandidateBatch(), unpublished_authorized=True).status == (- "unavailable"- )---def test_indexed_reader_without_runtime_matches_lexical_empty(database):- with sqlite3.connect(database) as db, db:- put(db, "e-unrelated", "alpha transcript only")- request = ContextSearchRequest(query="no-matching-zzzzzz", kinds=("raw",))- deadline = time.monotonic() + 5- lexical = read_raw_context(- database, request, source_ids=frozenset({"cursor"}), deadline=deadline- )- indexed = read_raw_indexed_context(- database, request, source_ids=frozenset({"cursor"}), deadline=deadline, index=None- )- assert lexical.status == "empty"- assert indexed == lexical- assert indexed.searched_channels == ("lexical",)- assert indexed.semantic_status == "not_searched"- assert indexed.index_complete is False---def test_indexed_reader_with_unpublished_index_is_not_empty(database):- with sqlite3.connect(database) as db, db:- put(db, "e-unrelated", "alpha transcript only")- request = ContextSearchRequest(query="no-matching-zzzzzz", kinds=("raw",))- index = SimpleNamespace(- lookup_raw=lambda query, *, source_ids, deadline: (- RawCandidateBatch(complete=True, reason="no_eligible_chunks"),- True,- )- )- result = read_raw_indexed_context(- database,- request,- source_ids=frozenset({"cursor"}),- deadline=time.monotonic() + 5,- index=index,- )- assert result.status == "degraded"- assert result.items == ()- assert result.searched_channels == ("lexical", "semantic")- assert result.semantic_status == "completed"- assert result.index_complete is False---def test_semantic_hit_without_lexical_overlap_is_not_released(database):- from datetime import UTC, datetime-- from one_memory.retrieval.index_store import PublishedChunk- from one_memory.retrieval.raw_candidates import RawSemanticCandidate-- with sqlite3.connect(database) as db, db:- put(db, "e-mit", "用户决定 One Memory 采用 MIT 协议发布。")- chunk = PublishedChunk(- chunk_id="c1",- build_id="b1",- evidence_id="e-mit",- revision=1,- start_cp=0,- end_cp=8,- ordinal=0,- source_id="cursor",- event_time=datetime(2026, 9, 19, tzinfo=UTC),- )- index = SimpleNamespace(- lookup_raw=lambda query, *, source_ids, deadline: (- RawCandidateBatch(- hits=(RawSemanticCandidate(chunk, 0.1),),- complete=True,- ),- False,- )- )- result = read_raw_indexed_context(- database,- ContextSearchRequest(query="开源许可证怎么选", kinds=("raw",)),- source_ids=frozenset({"cursor"}),- deadline=time.monotonic() + 5,- index=index,- )- assert result.status == "degraded"- assert result.reason == "lexical_only"- assert result.items == ()- assert result.searched_channels == ("lexical", "semantic")- assert result.semantic_status == "completed"- assert result.index_complete is True---def test_semantic_candidate_with_lexical_corroboration_is_released(database):- from datetime import UTC, datetime-- from one_memory.retrieval.index_store import PublishedChunk- from one_memory.retrieval.raw_candidates import RawSemanticCandidate-- with sqlite3.connect(database) as db, db:- put(db, "e-mit", "用户决定 One Memory 采用 MIT 协议发布。")- chunk = PublishedChunk(- chunk_id="c1",- build_id="b1",- evidence_id="e-mit",- revision=1,- start_cp=0,- end_cp=8,- ordinal=0,- source_id="cursor",- event_time=datetime(2026, 9, 19, tzinfo=UTC),- )- index = SimpleNamespace(- lookup_raw=lambda query, *, source_ids, deadline: (- RawCandidateBatch(- hits=(RawSemanticCandidate(chunk, 0.1),),- complete=True,- ),- False,- )- )- result = read_raw_indexed_context(- database,- ContextSearchRequest(query="MIT 协议", kinds=("raw",)),- source_ids=frozenset({"cursor"}),- deadline=time.monotonic() + 5,- index=index,- )- assert result.status == "ok"- assert [item.anchor_evidence_id for item in result.items] == ["e-mit"]---def test_raw_semantic_timeout_is_not_reported_as_complete_empty(database):- with sqlite3.connect(database) as db, db:- put(db, "e-unrelated", "alpha transcript only")- index = SimpleNamespace(- lookup_raw=lambda query, *, source_ids, deadline: (- RawCandidateBatch(reason="deadline"),- False,- )- )- result = read_raw_indexed_context(- database,- ContextSearchRequest(query="no-matching-zzzzzz", kinds=("raw",)),- source_ids=frozenset({"cursor"}),- deadline=time.monotonic() + 5,- index=index,- )- assert result.status == "degraded"- assert result.semantic_status == "timed_out"- assert result.index_complete is False- assert result.timed_out---def test_real_built_index_semantic_only_hit_is_not_released_without_confirmation(tmp_path):- """The whole chain for real: ONNX encode, Chroma, published chunks, authority.-- Every other test here hands the lane a hand-written candidate, so none of- them can tell whether a real build would ever produce one. The query is- chosen to share no term with the stored sentence, and the lexical-only read- is asserted empty first, so an `ok` here can only have come from the index.- """- _require_assets()- from one_memory.imports.grants import SourceGrantStore- from one_memory.imports.sources.registry import RootCandidateKind- from one_memory.retrieval.runtime import ExistingRetrievalRuntime- from tests.test_retrieval_index_store import put as put_evidence- from tests.test_retrieval_runtime import configured, until-- store, evidence, generation = configured(tmp_path)- source = tmp_path / "source"- source.mkdir()- SourceGrantStore(store.database_path, home=tmp_path).grant(- user_id="local",- source_id="codex",- root=source,- candidate_kind=RootCandidateKind.DEFAULT,- )- put_evidence(evidence, "我们最后决定 One Memory 用 MIT 协议发布。")- request = ContextSearchRequest(query="开源软件的授权条款如何挑选", kinds=("raw",))- sources = frozenset({"codex"})-- lexical = read_raw_context(- store.database_path, request, source_ids=sources, deadline=time.monotonic() + 5- )- assert lexical.status == "empty", "词法已经命中,这条测试证明不了语义"-- async def scenario():- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)- assert await handle.start()- try:- await until(lambda: store.status(generation)["ready"] >= 1)- return await asyncio.to_thread(- read_raw_indexed_context,- store.database_path,- request,- source_ids=sources,- deadline=time.monotonic() + 10,- index=handle,- )- finally:- await handle.stop()-- result = asyncio.run(scenario())- # The semantic lane found the row, but lexical-only evidence is not enough- # confirmation to release it to a host. Keep the candidate visible as a- # degraded coverage receipt until a relevance/corroboration gate exists.- assert result.status == "degraded"- assert result.reason == "lexical_only"- assert result.items == ()---def test_real_built_index_skips_with_a_named_asset_reason(tmp_path, monkeypatch):- """A missing untracked fixture must not look like a lease or worker hang."""- monkeypatch.setattr("test_retrieval_raw_lane.assets_present", lambda: False)- with pytest.raises(pytest.skip.Exception, match="_assets_unavailable"):- test_real_built_index_semantic_only_hit_is_not_released_without_confirmation(tmp_path)---def test_revoked_source_semantic_hit_does_not_release_body(database):- from datetime import UTC, datetime-- from one_memory.retrieval.index_store import PublishedChunk- from one_memory.retrieval.raw_candidates import RawSemanticCandidate- from one_memory.retrieval.raw_lane import load_raw_items_from_candidates- from tests.test_context_raw import revoke-- with sqlite3.connect(database) as db, db:- put(db, "e-revoked", "用户决定 One Memory 采用 MIT 协议发布。")- revoke(db, "cursor")- chunk = PublishedChunk(- chunk_id="c1",- build_id="b1",- evidence_id="e-revoked",- revision=1,- start_cp=0,- end_cp=8,- ordinal=0,- source_id="cursor",- event_time=datetime(2026, 9, 19, tzinfo=UTC),- )- items = load_raw_items_from_candidates(- database,- RawCandidateBatch(hits=(RawSemanticCandidate(chunk, 0.1),), complete=True),- source_ids=frozenset({"cursor"}),- deadline=time.monotonic() + 5,- )- assert items == ()diff --git a/memhub/apps/core/tests/test_retrieval_runtime.py b/memhub/apps/core/tests/test_retrieval_runtime.pydeleted file mode 100644index 102d9c2..0000000--- a/memhub/apps/core/tests/test_retrieval_runtime.py+++ /dev/null@@ -1,522 +0,0 @@-import asyncio-import json-import os-import shutil-import socket-import sqlite3-import threading-import time-from contextlib import closing--import pytest--from one_memory.imports.grants import SourceGrantStore-from one_memory.imports.sources.registry import RootCandidateKind-from one_memory.imports.storage import EvidenceStore-from one_memory.retrieval import runtime-from one_memory.retrieval.index_store import RetrievalIndexStore-from one_memory.retrieval.index_worker import IndexWorkResult, LocalRetrievalEncoder-from one_memory.retrieval.runtime import ExistingRetrievalRuntime-from one_memory.retrieval.vector_store import RawVectorStore-from tests.test_retrieval_index_store import put--[email protected](autouse=True)-def offline(monkeypatch):- def reject(*args, **kwargs):- raise AssertionError("offline runtime test")-- monkeypatch.setattr(socket.socket, "connect", reject)- monkeypatch.setattr(socket.socket, "connect_ex", reject)---def configured(tmp_path, *, version=2, collection=True):- path = tmp_path / "one-memory.sqlite3"- evidence = EvidenceStore(path)- identity = LocalRetrievalEncoder(content_version=version).identity- store = RetrievalIndexStore(path)- with closing(sqlite3.connect(path)) as db, db:- db.execute("BEGIN IMMEDIATE")- store.initialize(db)- store.migrate_v2(db)- generation = store.create_generation(db, identity)- store.activate_generation(generation)- if collection:- vectors = RawVectorStore.create(- tmp_path / "retrieval-vectors", generation_id=generation, dimensions=identity.dimensions- )- vectors.close()- return store, evidence, generation---async def until(predicate):- end = time.monotonic() + 10- while not predicate():- assert time.monotonic() < end- await asyncio.sleep(0.01)---def test_constructor_and_missing_start_create_nothing(tmp_path):- target = tmp_path / "missing"- handle = ExistingRetrievalRuntime(target)- assert not target.exists()- assert not asyncio.run(handle.start())- assert handle.snapshot().state == "not_initialized"- assert not target.exists()- asyncio.run(handle.stop())---def test_stop_releases_supervisor_worker_but_preserves_immutable_snapshot(tmp_path):- configured(tmp_path)- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)-- async def scenario():- assert await handle.start()- assert handle._supervisor is not None- await handle.stop()-- asyncio.run(scenario())- assert handle._supervisor is None- assert handle.snapshot().supervisor is not None- assert not handle.snapshot().supervisor.thread_alive--- "kind",- ["schema1", "no_active", "missing_collection", "bad_identity", "extra_identity", "two_active"],-)-def test_not_initialized_or_invalid_configuration_never_creates_index(tmp_path, kind):- store, _, gen = configured(tmp_path, collection=False)- with closing(sqlite3.connect(store.database_path)) as db, db:- if kind == "schema1":- db.execute("UPDATE retrieval_index_state SET schema_version=1")- elif kind == "no_active":- db.execute("UPDATE retrieval_index_generations SET state='building'")- elif kind in ("bad_identity", "extra_identity"):- raw = json.loads(- db.execute("SELECT identity_json FROM retrieval_index_generations").fetchone()[0]- )- if kind == "bad_identity":- raw["encoder_digest"] = "0" * 64- else:- raw["unexpected"] = "nope"- db.execute("UPDATE retrieval_index_generations SET identity_json=?", (json.dumps(raw),))- elif kind == "two_active":- db.execute("DROP INDEX retrieval_one_active")- db.execute(- "INSERT INTO retrieval_index_generations SELECT 'other',identity_json,state,"- "created_at FROM retrieval_index_generations"- )- before = db.execute("SELECT * FROM retrieval_index_generations").fetchall()- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)- assert not asyncio.run(handle.start())- assert handle.snapshot().state in {"not_initialized", "failed"}- assert not (tmp_path / "retrieval-vectors").exists()- with closing(sqlite3.connect(store.database_path)) as db:- assert db.execute("SELECT * FROM retrieval_index_generations").fetchall() == before- assert str(tmp_path) not in repr(handle.snapshot())---def test_low_disk_rejects_before_open_writer(tmp_path, monkeypatch):- configured(tmp_path)- monkeypatch.setattr(- runtime.shutil, "disk_usage", lambda _: shutil._ntuple_diskusage(1000, 999, 1)- )- monkeypatch.setattr(- runtime.RawVectorStore,- "open_writer_existing",- lambda *a, **k: pytest.fail("opened on low disk"),- )- handle = ExistingRetrievalRuntime(tmp_path)- assert not asyncio.run(handle.start())- assert handle.snapshot().error_code == "retrieval_runtime_low_disk"--[email protected]("kind", ["root", "database", "vectors"])-def test_symlink_rejected(tmp_path, kind):- actual = tmp_path / "actual"- actual.mkdir()- configured(actual)- selected = actual- if kind == "root":- selected = tmp_path / "link"- selected.symlink_to(actual, target_is_directory=True)- else:- path = actual / ("one-memory.sqlite3" if kind == "database" else "retrieval-vectors")- moved = tmp_path / "moved"- path.rename(moved)- path.symlink_to(moved, target_is_directory=kind == "vectors")- handle = ExistingRetrievalRuntime(selected, minimum_free_bytes=0)- assert not asyncio.run(handle.start())- assert handle.snapshot().error_code == "retrieval_runtime_unsafe_path"---def test_local_existing_integration_indexes_only_after_explicit_start(tmp_path):- store, evidence, gen = configured(tmp_path)- source = tmp_path / "source"- source.mkdir()- SourceGrantStore(store.database_path, home=tmp_path).grant(- user_id="local", source_id="codex", root=source, candidate_kind=RootCandidateKind.DEFAULT- )- row = put(evidence, "before token=private after")-- async def run():- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)- assert store.status(gen)["ready"] == 0- assert await handle.start()- try:- await until(lambda: store.status(gen)["ready"] == 1)- assert len(store.active_chunks(gen)) == 2- assert handle.snapshot().generation_id == gen- with pytest.raises(RuntimeError, match="already_started"):- await handle.start()- finally:- await handle.stop()- assert handle.snapshot().state == "stopped"- assert not handle.snapshot().supervisor.thread_alive- assert evidence.get(row.evidence_id).verbatim == row.verbatim-- asyncio.run(run())---def fake_resources(tmp_path, monkeypatch, *, fail_worker=False, block_open=False, version=2):- configured(tmp_path, version=version)- entered, release, exited, opened = (threading.Event() for _ in range(4))- events = []-- class Vectors:- def close(self):- events.append("close")-- def open_vectors(*args, **kwargs):- opened.set()- if block_open:- assert release.wait(5)- return Vectors()-- class Worker:- def __init__(self, *args, **kwargs):- if fail_worker:- raise RuntimeError("SECRET failure")-- def run_once(self, *, cancel):- entered.set()- while not cancel.is_set():- time.sleep(0.001)- events.append("cancel")- assert release.wait(5)- events.append("exit")- exited.set()- return IndexWorkResult("cancelled")-- monkeypatch.setattr(runtime.RawVectorStore, "open_writer_existing", open_vectors)- monkeypatch.setattr(runtime, "RetrievalIndexWorker", Worker)- return entered, release, exited, opened, events---def test_partial_start_failure_closes_owned_vectors(tmp_path, monkeypatch):- *_, events = fake_resources(tmp_path, monkeypatch, fail_worker=True)- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)- assert not asyncio.run(handle.start())- assert events == ["close"]- assert handle.snapshot().error_code == "retrieval_runtime_start_failed"- assert "SECRET" not in repr(handle.snapshot())---def test_cancelled_stop_drains_before_close(tmp_path, monkeypatch):- entered, release, exited, _, events = fake_resources(tmp_path, monkeypatch)-- async def run():- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)- assert await handle.start()- await until(entered.is_set)- task = asyncio.create_task(handle.stop())- await asyncio.sleep(0)- task.cancel()- try:- await until(lambda: "cancel" in events)- task.cancel()- await asyncio.sleep(0.03)- assert not task.done() and "close" not in events- finally:- release.set()- with pytest.raises(asyncio.CancelledError):- await task- assert exited.is_set() and events == ["cancel", "exit", "close"]- await handle.stop()- assert events.count("close") == 1-- asyncio.run(run())---def test_cancelled_start_drains_open_and_cleans_resources(tmp_path, monkeypatch):- _, release, _, opened, events = fake_resources(- tmp_path, monkeypatch, fail_worker=True, block_open=True- )-- async def run():- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)- task = asyncio.create_task(handle.start())- await until(opened.is_set)- task.cancel()- try:- await asyncio.sleep(0.03)- assert not task.done() and not events- finally:- release.set()- with pytest.raises(asyncio.CancelledError):- await task- assert events == ["close"]-- asyncio.run(run())---def test_cancelled_successful_open_does_not_start_worker_and_closes_once(tmp_path, monkeypatch):- entered, release, _, opened, events = fake_resources(tmp_path, monkeypatch, block_open=True)-- async def run():- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)- starting = asyncio.create_task(handle.start())- await until(opened.is_set)- starting.cancel()- await asyncio.sleep(0.02)- starting.cancel()- try:- await asyncio.sleep(0.02)- assert not starting.done() and not events and not entered.is_set()- finally:- release.set()- with pytest.raises(asyncio.CancelledError):- await starting- assert events == ["close"] and not entered.is_set()- assert handle.snapshot().state == "stopped"- await handle.stop()- assert events == ["close"]-- asyncio.run(run())---def test_old_identity_one_recovers_without_generation_upgrade(tmp_path, monkeypatch):- entered, release, _, _, events = fake_resources(tmp_path, monkeypatch, version=1)- path = tmp_path / "one-memory.sqlite3"- with closing(sqlite3.connect(path)) as db:- before = db.execute("SELECT * FROM retrieval_index_generations").fetchall()-- async def run():- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)- assert await handle.start()- await until(entered.is_set)- release.set()- await handle.stop()-- asyncio.run(run())- assert events == ["cancel", "exit", "close"]- with closing(sqlite3.connect(path)) as db:- assert db.execute("SELECT * FROM retrieval_index_generations").fetchall() == before---def test_missing_identity_fields_fail_even_when_dataclass_defaults_match(tmp_path):- store, _, _ = configured(tmp_path, version=1, collection=False)- with closing(sqlite3.connect(store.database_path)) as db, db:- raw = json.loads(- db.execute("SELECT identity_json FROM retrieval_index_generations").fetchone()[0]- )- raw.pop("normalizer_version")- db.execute("UPDATE retrieval_index_generations SET identity_json=?", (json.dumps(raw),))- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)- assert not asyncio.run(handle.start())- assert handle.snapshot().error_code == "retrieval_runtime_invalid_index"---def test_valid_hash_but_foreign_encoder_never_opens_vectors(tmp_path, monkeypatch):- from dataclasses import asdict-- from one_memory.retrieval.index_store import EncoderIdentity, digest-- store, _, _ = configured(tmp_path)- with closing(sqlite3.connect(store.database_path)) as db, db:- raw = json.loads(- db.execute("SELECT identity_json FROM retrieval_index_generations").fetchone()[0]- )- raw["encoder_digest"] = "0" * 64- identity = EncoderIdentity(**raw)- store_id = db.execute("SELECT data_store_id FROM retrieval_index_state").fetchone()[0]- db.execute(- "UPDATE retrieval_index_generations SET generation_id=?,identity_json=?",- (digest([store_id, asdict(identity)]), json.dumps(raw)),- )- monkeypatch.setattr(- runtime.RawVectorStore,- "open_writer_existing",- lambda *a, **k: pytest.fail("foreign encoder opened"),- )- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)- assert not asyncio.run(handle.start())- assert handle.snapshot().error_code == "retrieval_runtime_encoder_mismatch"---def test_concurrent_stop_waits_start_and_runs_cleanup_once(tmp_path, monkeypatch):- entered, release, _, opened, events = fake_resources(tmp_path, monkeypatch, block_open=True)-- async def run():- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)- starting = asyncio.create_task(handle.start())- await until(opened.is_set)- stops = [asyncio.create_task(handle.stop()), asyncio.create_task(handle.stop())]- try:- await asyncio.sleep(0.03)- assert all(not task.done() for task in stops)- finally:- release.set()- await asyncio.gather(*stops)- assert not await starting- assert events == ["close"] and not entered.is_set()-- asyncio.run(run())---def test_stop_before_start_never_opens_resources(tmp_path):- handle = ExistingRetrievalRuntime(tmp_path)- asyncio.run(handle.stop())- with pytest.raises(RuntimeError, match="already_started"):- asyncio.run(handle.start())- assert not list(tmp_path.iterdir())---def test_missing_collection_in_existing_chroma_does_not_create_collection(tmp_path):- store, _, gen = configured(tmp_path)- with closing(sqlite3.connect(tmp_path / "retrieval-vectors" / "chroma.sqlite3")) as db, db:- db.execute("UPDATE collections SET name='foreign_collection'")- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)- assert not asyncio.run(handle.start())- assert handle.snapshot().state == "not_initialized"- with closing(sqlite3.connect(tmp_path / "retrieval-vectors" / "chroma.sqlite3")) as db:- assert db.execute("SELECT name FROM collections").fetchall() == [("foreign_collection",)]--[email protected]("target", ["root", "database", "vectors", "vector_database"])-def test_group_writable_paths_rejected_without_open(tmp_path, monkeypatch, target):- configured(tmp_path)- paths = {- "root": tmp_path,- "database": tmp_path / "one-memory.sqlite3",- "vectors": tmp_path / "retrieval-vectors",- "vector_database": tmp_path / "retrieval-vectors/chroma.sqlite3",- }- path = paths[target]- original_mode = path.stat().st_mode & 0o777- path.chmod(original_mode | 0o020)- monkeypatch.setattr(- runtime.RawVectorStore,- "open_writer_existing",- lambda *a, **k: pytest.fail("unsafe resource opened"),- )- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)- try:- assert not asyncio.run(handle.start())- assert handle.snapshot().error_code == "retrieval_runtime_unsafe_path"- finally:- path.chmod(original_mode)---def test_foreign_uid_and_windows_are_failclosed(tmp_path, monkeypatch):- configured(tmp_path)- uid = os.getuid()- monkeypatch.setattr(runtime.os, "getuid", lambda: uid + 1)- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)- assert not asyncio.run(handle.start())- assert handle.snapshot().error_code == "retrieval_runtime_unsafe_path"- monkeypatch.setattr(runtime.os, "getuid", lambda: uid)- monkeypatch.setattr(runtime.sys, "platform", "win32")- windows = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)- assert not asyncio.run(windows.start())- assert windows.snapshot().error_code == "retrieval_runtime_platform_unsupported"--[email protected]("target", ["root", "database"])-def test_identical_store_id_copy_replacement_during_open_is_rejected(tmp_path, monkeypatch, target):- root = tmp_path / "profile"- root.mkdir()- configured(root)- closed = []-- class Vectors:- def close(self):- closed.append(True)-- def replace_path(*args, **kwargs):- if target == "database":- path = root / "one-memory.sqlite3"- copy = root / "replacement.sqlite3"- shutil.copy2(path, copy)- os.replace(copy, path)- else:- old = tmp_path / "old-profile"- root.rename(old)- shutil.copytree(old, root)- return Vectors()-- monkeypatch.setattr(runtime.RawVectorStore, "open_writer_existing", replace_path)- handle = ExistingRetrievalRuntime(root, minimum_free_bytes=0)- assert not asyncio.run(handle.start())- assert handle.snapshot().error_code == "retrieval_runtime_index_changed"- assert closed == [True]---def test_failed_drain_keeps_live_worker_vectors_open_and_raises_fixed_code(tmp_path, monkeypatch):- entered, release, _, _, events = fake_resources(tmp_path, monkeypatch)- actual = runtime.RetrievalIndexSupervisor- instances = []-- class BadStop(actual):- def __init__(self, worker, **kwargs):- super().__init__(worker, **kwargs)- instances.append(self)-- async def stop(self):- raise RuntimeError("SECRET_STOP_ERROR")-- monkeypatch.setattr(runtime, "RetrievalIndexSupervisor", BadStop)-- async def run():- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)- assert await handle.start()- await until(entered.is_set)- try:- with pytest.raises(RuntimeError, match="^retrieval_runtime_stop_failed$"):- await handle.stop()- assert "close" not in events- assert handle.snapshot().supervisor.thread_alive- assert handle.snapshot().state == "failed"- assert "SECRET" not in repr(handle.snapshot())- finally:- release.set()- await actual.stop(instances[0])- # Test alone owns cleanup of the intentionally broken supervisor.- handle._vectors.close()-- asyncio.run(run())---def test_stop_error_after_confirmed_exit_still_closes_vectors(tmp_path, monkeypatch):- entered, release, _, _, events = fake_resources(tmp_path, monkeypatch)- actual = runtime.RetrievalIndexSupervisor-- class BadStop(actual):- async def stop(self):- await super().stop()- raise RuntimeError("SECRET_AFTER_EXIT")-- monkeypatch.setattr(runtime, "RetrievalIndexSupervisor", BadStop)-- async def run():- handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)- assert await handle.start()- await until(entered.is_set)- release.set()- with pytest.raises(RuntimeError, match="^retrieval_runtime_stop_failed$"):- await handle.stop()- assert events == ["cancel", "exit", "close"]- assert not handle.snapshot().supervisor.thread_alive-- asyncio.run(run())diff --git a/memhub/apps/core/tests/test_retrieval_vector_store.py b/memhub/apps/core/tests/test_retrieval_vector_store.pydeleted file mode 100644index c78c4bb..0000000--- a/memhub/apps/core/tests/test_retrieval_vector_store.py+++ /dev/null@@ -1,528 +0,0 @@-import math-import socket-from time import monotonic--import numpy as np-import pytest--from one_memory.retrieval.vector_store import (- RawVectorRecord,- RawVectorStore,- VectorStoreError,- _unit_vector,-)--[email protected](autouse=True)-def no_network(monkeypatch):- def refused(*args, **kwargs):- pytest.fail("vector adapter attempted network")-- monkeypatch.setattr(socket.socket, "connect", refused)- monkeypatch.setattr(socket, "create_connection", refused)---def store(tmp_path):- value = RawVectorStore.create(- tmp_path / "retrieval-vectors", generation_id="generation-a", dimensions=3- )- try:- yield value- finally:- value.close()---def record(chunk="chunk-a", *, build="build-a", generation="generation-a", vector=(1.0, 0.0, 0.0)):- return RawVectorRecord(chunk, build, generation, vector)---def test_constructor_does_no_io_and_unopened_calls_do_not_create(tmp_path):- directory = tmp_path / "never-created"- value = RawVectorStore(directory, generation_id="generation-a", dimensions=3)- assert not directory.exists()- with pytest.raises(VectorStoreError, match="not_open"):- value.search((1.0, 0.0, 0.0), ["chunk-a"], 1, monotonic() + 1)- assert not directory.exists()- value.close()---def test_open_existing_missing_store_or_collection_never_creates(tmp_path, store):- missing = tmp_path / "missing"- with pytest.raises(VectorStoreError, match="not_found"):- RawVectorStore.open_existing(missing, generation_id="generation-a", dimensions=3)- assert not missing.exists()- before = store._client.list_collections()- with pytest.raises(VectorStoreError, match="not_found"):- RawVectorStore.open_existing(store.directory, generation_id="other", dimensions=3)- assert store._client.list_collections() == before---def test_real_chroma_collection_has_no_embedding_function_or_documents(store):- store.upsert([record()])- collection = store._collection- assert collection._embedding_function is None- result = collection.get(ids=["chunk-a"], include=["embeddings", "metadatas", "documents"])- assert result["documents"] == [None]- assert result["metadatas"][0]["chunk_id"] == "chunk-a"- assert result["metadatas"][0]["build_id"] == "build-a"- assert result["metadatas"][0]["generation_id"] == "generation-a"- assert len(result["embeddings"][0]) == 3- store.verify([record()])---def test_existing_read_handle_cannot_write_or_repair_and_releases_shared_client(store):- store.upsert([record()])- opened = RawVectorStore.open_existing(- store.directory, generation_id="generation-a", dimensions=3- )- try:- result = opened.search((1.0, 0.0, 0.0), ["chunk-a"], 1, monotonic() + 10)- assert result.complete and result.hits[0].chunk_id == "chunk-a"- with pytest.raises(VectorStoreError, match="read_only"):- opened.upsert([record("new")])- opened.verify([record()])- finally:- opened.close()- store.verify([record()])- with pytest.raises(VectorStoreError, match="closed"):- opened.search((1.0, 0.0, 0.0), ["chunk-a"], 1, monotonic() + 1)--[email protected]("changed", [record(build="different"), record(vector=(0.0, 1.0, 0.0))])-def test_immutable_id_rejects_different_metadata_or_vector_without_overwriting(store, changed):- store.upsert([record()])- store.upsert([record()])- with pytest.raises(VectorStoreError, match="immutable_conflict"):- store.upsert([record("new"), changed])- store.verify([record()])- assert store._collection.get(ids=["new"])["ids"] == []---def test_verify_detects_missing_and_tampered_records(store):- with pytest.raises(VectorStoreError, match="missing"):- store.verify([record()])- store.upsert([record()])- store._collection.update(ids=["chunk-a"], embeddings=[[0.0, 1.0, 0.0]])- with pytest.raises(VectorStoreError):- store.verify([record()])- with pytest.raises(VectorStoreError):- store.search((1.0, 0.0, 0.0), ["chunk-a"], 1, monotonic() + 10)--- "vector",- [- (1.0, 0.0),- (1.0, 0.0, math.nan),- (math.inf, 0.0, 1.0),- (0.0, 0.0, 0.0),- (True, 0.0, 1.0),- (1e100, 0.0, 1.0),- ],-)-def test_invalid_vectors_refused_before_writes_or_search(store, vector):- with pytest.raises(VectorStoreError):- store.upsert([record(vector=vector)])- with pytest.raises(VectorStoreError):- store.search(vector, [], 1, monotonic() + 1)- assert store._collection.count() == 0---def test_wrong_generation_and_duplicate_ids_rejected(store):- with pytest.raises(VectorStoreError):- store.upsert([record(generation="other")])- with pytest.raises(VectorStoreError):- store.upsert([record(), record()])- assert store._collection.count() == 0---def test_empty_eligible_ids_never_queries_all_collection(store, monkeypatch):- store.upsert([record()])- monkeypatch.setattr(- store._collection, "query", lambda **kwargs: pytest.fail("unfiltered query")- )- result = store.search((1.0, 0.0, 0.0), [], 3, monotonic() - 1)- assert result.complete and result.hits == ()- assert result.eligible_count == result.searched_count == 0---def test_real_in_filter_excludes_closer_unapproved_id_and_matches_small_cosine_oracle(store):- rows = [- record("denied", vector=(1.0, 0.0, 0.0)),- record("b", vector=(0.9, 0.1, 0.0)),- record("c", vector=(0.0, 1.0, 0.0)),- record("d", vector=(-1.0, 0.0, 0.0)),- ]- store.upsert(rows)- result = store.search((1.0, 0.0, 0.0), ["b", "c", "d"], 3, monotonic() + 10)- assert result.complete and result.searched_count == 3- assert [hit.chunk_id for hit in result.hits] == ["b", "c", "d"]- for hit, row in zip(result.hits, rows[1:], strict=True):- vector = np.array(row.vector, dtype=np.float32)- expected = 1 - float(vector[0] / np.linalg.norm(vector))- assert hit.distance == pytest.approx(expected, abs=1e-5)---def test_eligible_batches_cover_more_than_2048_and_merge_topk(store, monkeypatch):- ids = [f"chunk-{i:04d}" for i in range(2053)]- rows = [record(chunk, vector=(float(i + 1), 1.0, 0.0)) for i, chunk in enumerate(ids)]- store.upsert(rows)- calls = []- get = store._collection.get-- def tracked(**kwargs):- calls.append(kwargs)- return get(**kwargs)-- monkeypatch.setattr(store._collection, "get", tracked)- result = store.search((1.0, 0.0, 0.0), ids, 3, monotonic() + 30)- assert result.complete and result.searched_count == 2053- assert len(calls) == 2- assert [len(call["ids"]) for call in calls] == [2048, 5]- assert set().union(*(set(call["ids"]) for call in calls)) == set(ids)- assert all(call["include"] == ["embeddings", "metadatas"] for call in calls)- assert len(result.hits) == 3- # The exact ranking over every eligible chunk, not one batch's local top-K:- # a per-batch top-K would have kept the second batch's steepest vectors out.- # Distances come from the stored unit vectors, so float32 rounding decides- # the top of this ranking just as it does for the adapter.- stored = {row.chunk_id: np.array(_unit_vector(row.vector), dtype=np.float32) for row in rows}- expected = {- chunk_id: float(1 - np.dot(unit, np.array([1.0, 0.0, 0.0], dtype=np.float32)))- for chunk_id, unit in stored.items()- }- ranked = sorted(expected.items(), key=lambda pair: (pair[1], pair[0]))[:3]- assert [hit.chunk_id for hit in result.hits] == [chunk for chunk, _ in ranked]- for hit in result.hits:- assert hit.distance == pytest.approx(expected[hit.chunk_id], abs=1e-6)---def test_deadline_before_query_and_late_batch_return_are_partial(store, monkeypatch):- store.upsert([record()])- calls = []- now = [0.0]- store._clock = lambda: now[0]- get = store._collection.get-- def late(**kwargs):- calls.append(kwargs)- result = get(**kwargs)- now[0] = 2.0- return result-- monkeypatch.setattr(store._collection, "get", late)- first = store.search((1.0, 0.0, 0.0), ["chunk-a"], 1, 0.0)- assert not first.complete and first.reason == "deadline" and first.hits == ()- assert calls == []- second = store.search((1.0, 0.0, 0.0), ["chunk-a"], 1, 1.0)- assert not second.complete and second.searched_count == 0 and second.hits == ()- assert len(calls) == 1---def test_search_never_scores_a_vector_the_caller_did_not_declare_eligible(store, monkeypatch):- """Chroma's own filtered query is no longer used at all.-- It does not score every ID in its `where $in` allowlist, so a query could- come back short while claiming completion, and a global query filtered- afterwards would let unauthorized vectors take the eligible ones' places.- """- store.upsert([record(), record("denied", vector=(1.0, 0.0, 0.0))])- monkeypatch.setattr(- store._collection,- "query",- lambda **kwargs: pytest.fail("global or filtered ANN query is not used"),- )- result = store.search((1.0, 0.0, 0.0), ["chunk-a"], 1, monotonic() + 10)- assert result.complete and [hit.chunk_id for hit in result.hits] == ["chunk-a"]---def test_an_eligible_vector_missing_from_the_backend_is_unavailable_not_a_miss(store):- """The caller published this chunk, so an absent vector is a broken build."""- store.upsert([record("present")])- with pytest.raises(VectorStoreError, match="^vector_query_unavailable$"):- store.search((1.0, 0.0, 0.0), ["present", "absent"], 2, monotonic() + 10)---def test_open_existing_validates_generation_dimensions_and_collection_config(store):- with pytest.raises(VectorStoreError):- RawVectorStore.open_existing(store.directory, generation_id="generation-a", dimensions=4)- store._collection.modify(metadata={"generation_id": "other"})- with pytest.raises(VectorStoreError):- RawVectorStore.open_existing(store.directory, generation_id="generation-a", dimensions=3)---def test_open_existing_refuses_symlink_database_before_chroma(tmp_path):- directory = tmp_path / "retrieval-vectors"- directory.mkdir()- target = tmp_path / "not-a-chroma-db"- target.write_bytes(b"do not open")- (directory / "chroma.sqlite3").symlink_to(target)- with pytest.raises(VectorStoreError):- RawVectorStore.open_existing(directory, generation_id="generation-a", dimensions=3)- assert target.read_bytes() == b"do not open"---def test_writer_reopen_resumes_idempotently_but_query_handle_still_cannot_write(tmp_path):- directory = tmp_path / "retrieval-vectors"- original = RawVectorStore.create(directory, generation_id="generation-a", dimensions=3)- try:- original.upsert([record(vector=(1.0, 1.0, 0.0))])- finally:- original.close()- writer = RawVectorStore.open_writer_existing(- directory, generation_id="generation-a", dimensions=3- )- try:- writer.upsert([record(vector=(1.0, 1.0, 0.0))])- writer.verify([record(vector=(1.0, 1.0, 0.0))])- with pytest.raises(VectorStoreError, match="immutable_conflict"):- writer.upsert([record(build="other", vector=(1.0, 1.0, 0.0))])- with pytest.raises(VectorStoreError, match="immutable_conflict"):- writer.upsert([record(vector=(2.0, 2.0, 0.0))])- assert writer._collection.count() == 1- finally:- writer.close()- reader = RawVectorStore.open_existing(directory, generation_id="generation-a", dimensions=3)- try:- with pytest.raises(VectorStoreError, match="read_only"):- reader.upsert([record("new")])- result = reader.search((1.0, 1.0, 0.0), ["chunk-a"], 1, monotonic() + 10)- assert result.complete and result.hits[0].distance == pytest.approx(0, abs=1e-5)- finally:- reader.close()---def test_writer_existing_does_not_create_missing_store_or_collection(tmp_path, store):- missing = tmp_path / "writer-missing"- with pytest.raises(VectorStoreError, match="not_found"):- RawVectorStore.open_writer_existing(missing, generation_id="generation-a", dimensions=3)- assert not missing.exists()- with pytest.raises(VectorStoreError, match="not_found"):- RawVectorStore.open_writer_existing(store.directory, generation_id="other", dimensions=3)---def test_partial_keeps_only_completed_eligible_batches(store, monkeypatch):- ids = [f"present-{i}" for i in range(2048)] + ["last"]- store.upsert([*(record(chunk, vector=(1.0, float(i + 1), 0.0)) for i, chunk in enumerate(ids))])- now = [0.0]- store._clock = lambda: now[0]- get = store._collection.get- calls = []-- def timed(**kwargs):- calls.append(kwargs)- result = get(**kwargs)- if len(calls) == 2:- now[0] = 2.0- return result-- monkeypatch.setattr(store._collection, "get", timed)- result = store.search((1.0, 0.0, 0.0), ids, 2, 1.0)- assert not result.complete and result.reason == "deadline"- assert result.searched_count == 2048 and result.eligible_count == 2049- assert [hit.chunk_id for hit in result.hits] == ["present-0", "present-1"]--[email protected]("mutation", ["metadata", "dimension", "hash", "extra_id"])-def test_malformed_search_response_is_rejected_not_partial_success(store, monkeypatch, mutation):- store.upsert([record()])- get = store._collection.get-- def malformed(**kwargs):- result = get(**kwargs)- if mutation == "metadata":- result["metadatas"][0][0]["generation_id"] = "wrong"- elif mutation == "dimension":- result["embeddings"] = [[1.0, 0.0]]- elif mutation == "hash":- result["metadatas"][0]["vector_sha256"] = "0" * 64- else:- result["ids"] = result["ids"] * 2- result["metadatas"] = result["metadatas"] * 2- result["embeddings"] = [result["embeddings"][0], result["embeddings"][0]]- return result-- monkeypatch.setattr(store._collection, "get", malformed)- with pytest.raises(VectorStoreError):- store.search((1.0, 0.0, 0.0), ["chunk-a"], 1, monotonic() + 10)--- "ids,limit,deadline",- [- (["chunk-a", "chunk-a"], 1, 100.0),- ("chunk-a", 1, 100.0),- (["chunk-a"], 0, 100.0),- (["chunk-a"], True, 100.0),- (["chunk-a"], 1, math.inf),- (["chunk-a"], 1, math.nan),- ],-)-def test_invalid_query_parameters_do_not_reach_chroma(store, monkeypatch, ids, limit, deadline):- monkeypatch.setattr(store._collection, "query", lambda **kwargs: pytest.fail("invalid query"))- with pytest.raises(VectorStoreError):- store.search((1.0, 0.0, 0.0), ids, limit, deadline)---def test_query_backend_error_is_unavailable_not_complete_empty(store, monkeypatch):- def broken(**kwargs):- raise RuntimeError("backend request payload")-- monkeypatch.setattr(store._collection, "query", broken)- with pytest.raises(VectorStoreError, match="^vector_query_unavailable$"):- store.search((1.0, 0.0, 0.0), ["chunk-a"], 1, monotonic() + 10)---def test_last_owned_client_close_releases_chroma_system(tmp_path):- from chromadb.api.shared_system_client import SharedSystemClient-- directory = tmp_path / "retrieval-vectors"- writer = RawVectorStore.create(directory, generation_id="close-test", dimensions=3)- reader = RawVectorStore.open_existing(directory, generation_id="close-test", dimensions=3)- identifier = writer._client._identifier- try:- assert identifier in SharedSystemClient._identifier_to_system- writer.close()- assert identifier in SharedSystemClient._identifier_to_system- reader.close()- assert identifier not in SharedSystemClient._identifier_to_system- reader.close()- finally:- reader.close()- writer.close()---def test_open_existing_missing_collection_does_not_construct_a_client(store, monkeypatch):- import chromadb-- monkeypatch.setattr(- chromadb, "PersistentClient", lambda *args, **kwargs: pytest.fail("must preflight first")- )- with pytest.raises(VectorStoreError, match="collection_not_found"):- RawVectorStore.open_existing(store.directory, generation_id="absent", dimensions=3)---def test_existing_dimension_mismatch_does_not_modify_collection(store):- before = store._collection.metadata.copy()- with pytest.raises(VectorStoreError):- RawVectorStore.open_writer_existing(- store.directory, generation_id="generation-a", dimensions=4- )- assert store._collection.metadata == before- store.upsert([record()])- store.verify([record()])---def test_verify_candidates_reader_checks_every_batch_without_writing(store, monkeypatch):- rows = [record(f"candidate-{i}") for i in range(2050)]- store.upsert(rows)- reader = RawVectorStore.open_existing(- store.directory, generation_id="generation-a", dimensions=3- )- try:- calls = []- get = reader._collection.get-- def tracked(**kwargs):- calls.append(kwargs)- return get(**kwargs)-- monkeypatch.setattr(reader._collection, "get", tracked)- for name in ("add", "upsert", "update", "delete", "query"):- monkeypatch.setattr(- reader._collection, name, lambda **kwargs: pytest.fail("unexpected backend call")- )- assert (- reader.verify_candidates(- [(row.chunk_id, row.build_id) for row in rows], deadline=monotonic() + 30- )- is True- )- assert [len(call["ids"]) for call in calls] == [2048, 2]- assert all(call["include"] == ["embeddings", "metadatas"] for call in calls)- finally:- reader.close()---def test_verify_candidates_empty_and_expired_do_not_read(store, monkeypatch):- monkeypatch.setattr(store._collection, "get", lambda **kwargs: pytest.fail("unexpected read"))- store._clock = lambda: 10.0- assert store.verify_candidates([], deadline=0.0) is True- assert store.verify_candidates([("chunk-a", "build-a")], deadline=10.0) is False--[email protected]("expires", ["after_read", "between_batches"])-def test_verify_candidates_deadline_stops_before_another_batch(store, monkeypatch, expires):- rows = [record(f"candidate-{i}") for i in range(2049)]- store.upsert(rows)- calls = []- get = store._collection.get-- def tracked(**kwargs):- calls.append(kwargs)- return get(**kwargs)-- ticks = iter([0.0, 2.0] if expires == "after_read" else [0.0, 0.0, 2.0])- store._clock = lambda: next(ticks)- monkeypatch.setattr(store._collection, "get", tracked)- assert (- store.verify_candidates([(row.chunk_id, row.build_id) for row in rows], deadline=1.0)- is False- )- assert len(calls) == 1 and len(calls[0]["ids"]) == 2048--[email protected]("pair", [("missing", "build-a"), ("chunk-a", "other-build")])-def test_verify_candidates_missing_or_wrong_build_refuses(store, pair):- store.upsert([record()])- with pytest.raises(VectorStoreError, match="missing|identity_mismatch"):- store.verify_candidates([pair], deadline=monotonic() + 10)--[email protected]("mutation", ["generation", "dimension", "hash", "extra_id"])-def test_verify_candidates_corrupt_backend_record_refuses(store, monkeypatch, mutation):- store.upsert([record()])- get = store._collection.get-- def corrupt(**kwargs):- data = get(**kwargs)- if mutation == "generation":- data["metadatas"][0]["generation_id"] = "other-generation"- elif mutation == "dimension":- data["embeddings"] = [[1.0, 0.0]]- elif mutation == "hash":- data["metadatas"][0]["vector_sha256"] = "0" * 64- else:- data["ids"][0] = "not-eligible"- return data-- monkeypatch.setattr(store._collection, "get", corrupt)- with pytest.raises(VectorStoreError):- store.verify_candidates([("chunk-a", "build-a")], deadline=monotonic() + 10)--- "candidates",- [- [("chunk-a", "build-a"), ("chunk-a", "build-a")],- [("chunk-a", "build-a"), ("chunk-a", "other")],- [("../escape", "build-a")],- [("chunk-a", "")],- [("chunk-a", 123)],- [("chunk-a",)],- ["ab"],- "ab",- None,- ],-)-def test_verify_candidates_invalid_input_refuses_before_read(store, monkeypatch, candidates):- monkeypatch.setattr(store._collection, "get", lambda **kwargs: pytest.fail("unexpected read"))- with pytest.raises(VectorStoreError):- store.verify_candidates(candidates, deadline=monotonic() + 10)--[email protected]("deadline", [math.nan, math.inf, True, "soon"])-def test_verify_candidates_invalid_deadline_refuses_even_empty(store, deadline):- with pytest.raises(VectorStoreError, match="invalid_deadline"):- store.verify_candidates([], deadline=deadline)