Python
fix(memhub): metadata repair never runs beside an SDK client
alvinfeiaaebe642449d
diff --git a/memhub/apps/core/src/one_memory/api.py b/memhub/apps/core/src/one_memory/api.pyindex de06478..68ff6e6 100644--- a/memhub/apps/core/src/one_memory/api.py+++ b/memhub/apps/core/src/one_memory/api.py@@ -95,6 +95,8 @@ from one_memory.hy_sdk_runtime import (SdkClientHandle,builtin_retrieval_snapshot,close_all_shared_runtimes,+ hold_stores_offline,+ release_stores,sdk_relevance_floor,)from one_memory.hy_sdk_types import (@@ -223,6 +225,7 @@ from one_memory.sdk_memory_qa import (SdkMemoryAnswerer,)from one_memory.sdk_memory_runtime import SdkMemoryRuntime+from one_memory.sdk_migration_log import error_chain_text, error_framesfrom one_memory.sdk_write_processor import SdkWriteCoordinatorfrom one_memory.secrets import (KeyringSecretStore,@@ -710,7 +713,8 @@ def create_app(raise SdkMigrationRequired()if app.state.memory_authority is MemoryAuthority.SDK:# The metadata repair after a cutover reads every bundle: it is- # recovery work, so it runs after readiness (`recover_after_ready`).+ # recovery work, so it runs after readiness (`recover_after_ready`)+ # while the SDK's stores are held offline (see there).app.state.memory_governance_store = RetiredGovernanceStore(database_path)app.state.evidence_store = RetiredEvidenceStore(database_path)else:@@ -1187,10 +1191,14 @@ def create_app(except Exception as error:failed.append(name)app.state.recovery_failed_steps = tuple(failed)+ # Where it failed, as `file:line:function` frames of+ # the exception chain, never its message: a bare+ # `error_type=KeyError` named no location.logger.warning(- "core_recovery_step_failed step=%s error_type=%s",+ "core_recovery_step_failed step=%s error_type=%s chain=%s",name,type(error).__name__,+ error_chain_text(error),)diagnostic_event(level="warn",@@ -1198,20 +1206,42 @@ def create_app(event="core.recovery_step_failed",outcome="failed",error_type=type(error).__name__,+ safe_stack=error_frames(error) or None,duration_ms=elapsed_ms(step_started),details={"phase": name},)async def repair_sdk_metadata() -> None:+ # `sdk-migrate` repairs right after its cutover and writes+ # the receipt, so this normally finds it and returns. It+ # stays for homes whose repair failed before that (no+ # receipt). It opens its own Chroma client on the SDK's+ # vector store, and Chroma's shared System is not safe+ # beside another client: a first library request that+ # opened the SDK client meanwhile lost it (KeyError).+ # The stores were held offline before readiness, so any+ # SDK client waits for this step.from one_memory.hy_sdk_migration import (repair_sdk_metadata_from_bundle,)- repair = await asyncio.to_thread(- repair_sdk_metadata_from_bundle,- data_dir=runtime_settings.data_dir,- database_path=database_path,+ if not app.state.sdk_stores_held:+ logger.warning("sdk_metadata_repair_deferred reason=sdk_client_open")+ return+ # The hold is this step's from here; shutdown leaves it.+ app.state.sdk_stores_held = False+ work = asyncio.ensure_future(+ asyncio.to_thread(+ repair_sdk_metadata_from_bundle,+ data_dir=runtime_settings.data_dir,+ database_path=database_path,+ ))+ # Released when the thread ends, not when this coroutine+ # is cancelled: a cancelled await leaves the thread+ # writing Chroma.+ work.add_done_callback(lambda _work: release_stores(runtime_settings.data_dir))+ repair = await asyncio.shield(work)if repair.vector_rows or repair.coding_rows:logger.info("sdk_metadata_repair_completed mapped=%d vector=%d coding=%d",@@ -1335,7 +1365,12 @@ def create_app()app.state.recovery_task = None+ app.state.sdk_stores_held = Falsetry:+ if app.state.memory_authority is MemoryAuthority.SDK:+ # Before the first request can open an SDK client; the+ # metadata repair step releases it.+ app.state.sdk_stores_held = hold_stores_offline(runtime_settings.data_dir)diagnostic_event(level="info",component="core.lifecycle",@@ -1392,6 +1427,10 @@ def create_app(if app.state.live_sync_watch is not None:await app.state.live_sync_watch.shutdown()await app.state.live_sync_supervisor.shutdown()+ if app.state.sdk_stores_held:+ # Recovery never reached the metadata repair step.+ release_stores(runtime_settings.data_dir)+ app.state.sdk_stores_held = Falseawait app.state.import_supervisor.shutdown()app.state.memory_write_drain.stop()if app.state.memory_write_drain_task is not None:diff --git a/memhub/apps/core/src/one_memory/hy_sdk_migration.py b/memhub/apps/core/src/one_memory/hy_sdk_migration.pyindex b955d14..cf43a29 100644--- a/memhub/apps/core/src/one_memory/hy_sdk_migration.py+++ b/memhub/apps/core/src/one_memory/hy_sdk_migration.py@@ -82,7 +82,10 @@ def repair_sdk_metadata_from_bundle(database_path: Path,now: datetime | None = None,) -> MetadataRepairResult:- """Restore metadata lost by historical cutovers from their verified bundles."""+ """Restore metadata lost by historical cutovers from their verified bundles.++ Offline only: no SDK client may be open on this home (see `_repair_vector`).+ """migration_id = _sdk_migration_id(database_path)if migration_id is None:return MetadataRepairResult(None, 0, 0, 0)@@ -329,8 +332,11 @@ def _repair_vector(from one_memory.sdk_vector_reattach import open_offline_chromaclient = open_offline_chroma(directory)- # The SDK client is not open yet (startup has not created its handle), so- # this is the only writer; release the store before the SDK takes it.+ # This must be the only Chroma client on the directory: Chroma shares one+ # System per path, and this client's close() drops it from under any+ # other (KeyError on its next lookup). `sdk-migrate` runs it after the+ # cutover with Runtime stopped; Core's startup fallback runs it while+ # `hold_stores_offline` keeps every SDK client waiting.try:repaired = 0ids = tuple(mapping)diff --git a/memhub/apps/core/src/one_memory/hy_sdk_runtime.py b/memhub/apps/core/src/one_memory/hy_sdk_runtime.pyindex 522884a..854bb48 100644--- a/memhub/apps/core/src/one_memory/hy_sdk_runtime.py+++ b/memhub/apps/core/src/one_memory/hy_sdk_runtime.py@@ -56,6 +56,16 @@ class _RuntimeEntry:_RUNTIMES: dict[Path, _RuntimeEntry] = {}+# Homes whose SDK-owned local stores are held offline for maintenance, and+# the client constructions in flight per home. Chroma keeps one process-wide+# System per path with no lock around create, lookup and release, so a+# second client that opens or closes beside the SDK's pulls the System out+# from under it (KeyError on the next lookup). Maintenance therefore runs+# only while no SDK client exists for that home, and new clients wait.+_STORE_GATE = threading.Condition()+_OFFLINE_STORES: set[Path] = set()+_OPENING: dict[Path, int] = {}+class LocalOnnxEmbeddingService:"""Public ``EmbeddingService`` implementation backed by MemHub's ONNX model."""@@ -307,7 +317,60 @@ def acquire_client(data_dir: Path,http_transport_factory: Callable[[], Any] | None = None,) -> ClientLease:- """Construct a public SDK client and return its explicit runtime lease."""+ """Construct a public SDK client and return its explicit runtime lease.++ Waits while `hold_stores_offline` holds this home's stores.+ """+ key = data_dir.resolve()+ with _STORE_GATE:+ while key in _OFFLINE_STORES:+ _STORE_GATE.wait()+ _OPENING[key] = _OPENING.get(key, 0) + 1+ try:+ return _acquire_client(+ profile,+ api_key=api_key,+ data_dir=data_dir,+ http_transport_factory=http_transport_factory,+ )+ finally:+ with _STORE_GATE:+ _OPENING[key] -= 1+ if not _OPENING[key]:+ del _OPENING[key]+ _STORE_GATE.notify_all()+++def hold_stores_offline(data_dir: Path) -> bool:+ """Hold this home's SDK stores for offline maintenance; False if a client is open.++ While held, `acquire_client` for the home waits. Every SDK client in Core+ is built there, so nothing reads or writes Chroma beside the maintenance.+ `release_stores` ends the hold; it is idempotent.+ """+ key = data_dir.resolve()+ with _STORE_GATE:+ with _RUNTIME_LOCK:+ open_runtime = key in _RUNTIMES+ if key in _OFFLINE_STORES or _OPENING.get(key) or open_runtime:+ return False+ _OFFLINE_STORES.add(key)+ return True+++def release_stores(data_dir: Path) -> None:+ with _STORE_GATE:+ _OFFLINE_STORES.discard(data_dir.resolve())+ _STORE_GATE.notify_all()+++def _acquire_client(+ profile: ModelProfileSnapshot,+ *,+ api_key: str | None,+ data_dir: Path,+ http_transport_factory: Callable[[], Any] | None,+) -> ClientLease:config = build_sdk_config(profile, api_key=api_key, data_dir=data_dir)config.http_transport_factory = http_transport_factorymismatch = inspect_graph_width_mismatch(config)diff --git a/memhub/apps/core/src/one_memory/sdk_migration_log.py b/memhub/apps/core/src/one_memory/sdk_migration_log.pyindex 1642d49..8b0b67e 100644--- a/memhub/apps/core/src/one_memory/sdk_migration_log.py+++ b/memhub/apps/core/src/one_memory/sdk_migration_log.py@@ -125,6 +125,31 @@ def describe_error(error: BaseException) -> dict[str, object]:return {"chain": chain}+_PY_FRAME = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.\-]*\.py:\d{1,7}:[A-Za-z_][A-Za-z0-9_]*$")+++def error_frames(error: BaseException, *, limit: int = 16) -> tuple[str, ...]:+ """The raised error's innermost `file.py:line:function` frames, for a stack field.++ Library frames are kept (a KeyError raised inside Chroma is located by+ them); `<module>`-style names are dropped because stack fields refuse them.+ """+ chain = describe_error(error)["chain"]+ frames = chain[0].get("frames", ()) if chain else () # type: ignore[index,union-attr]+ return tuple(frame for frame in frames if _PY_FRAME.fullmatch(frame))[-limit:]+++def error_chain_text(error: BaseException) -> str:+ """`describe_error` as one log token: `Type@file:line:function>...;Cause@...`.++ For a plain text log (core.log). Types and frames only, never a message.+ """+ return ";".join(+ f"{entry['type']}@{'>'.join(entry.get('frames', ()))}" # type: ignore[arg-type]+ for entry in describe_error(error)["chain"] # type: ignore[union-attr]+ )++def _build() -> dict[str, object]:from one_memory import __version__diff --git a/memhub/apps/core/src/one_memory/sdk_migration_runner.py b/memhub/apps/core/src/one_memory/sdk_migration_runner.pyindex b8a305c..203eddc 100644--- a/memhub/apps/core/src/one_memory/sdk_migration_runner.py+++ b/memhub/apps/core/src/one_memory/sdk_migration_runner.py@@ -829,6 +829,7 @@ def run_startup_migration(ignore_errors=True,)discard_failed_state(data_dir)+ _repair_metadata_after_cutover(data_dir, database_path, log)progress("cleanup")with log.step("cleanup.drop_legacy_tables") as step:dropped = resume_legacy_cleanup(data_dir=data_dir, database_path=database_path)@@ -847,6 +848,25 @@ def run_startup_migration()+def _repair_metadata_after_cutover(data_dir: Path, database_path: Path, log: _StepLog) -> None:+ """Restore lost metadata here, where Runtime is stopped and no Chroma client is open.++ Writes the repair receipt, so Core's startup fallback finds it and does+ nothing. Authority is already committed: a failure is logged and left to+ that fallback instead of failing the migration.+ """+ from one_memory.hy_sdk_migration import repair_sdk_metadata_from_bundle++ try:+ with log.step("cutover.metadata_repair") as step:+ repair = repair_sdk_metadata_from_bundle(data_dir=data_dir, database_path=database_path)+ step["mapped"] = repair.mapped+ step["vector_rows"] = repair.vector_rows+ step["coding_rows"] = repair.coding_rows+ except Exception:+ return++def _restore_after_failure(*,data_dir: Path,diff --git a/memhub/apps/core/tests/test_core_ready_before_recovery.py b/memhub/apps/core/tests/test_core_ready_before_recovery.pyindex e1ea8b9..27e84a5 100644--- a/memhub/apps/core/tests/test_core_ready_before_recovery.py+++ b/memhub/apps/core/tests/test_core_ready_before_recovery.py@@ -142,3 +142,112 @@ def test_shutdown_during_recovery_stops_it_without_starting_more(tmp_path, monkerelease.set()asyncio.run(scenario())+++def test_metadata_repair_never_overlaps_an_sdk_client(tmp_path, monkeypatch):+ """The repair's own Chroma client dropped the SDK's shared System (KeyError).++ The first library request opened the SDK client while the repair ran+ after readiness. Now every SDK client of the home waits for the repair.+ """+ from one_memory import hy_sdk_runtime+ from one_memory.hy_sdk_runtime import ClientLease++ release = threading.Event()+ entered = threading.Event()+ events: list[str] = []++ def slow_repair(**_):+ entered.set()+ events.append("repair_started")+ release.wait(10)+ events.append("repair_finished")+ return MetadataRepairResult(None, 0, 0, 0)++ def open_client(profile, *, api_key, data_dir, http_transport_factory):+ events.append("client_opened")+ return ClientLease(object(), data_dir.resolve())++ monkeypatch.setattr(hy_sdk_migration, "repair_sdk_metadata_from_bundle", slow_repair)+ monkeypatch.setattr(hy_sdk_runtime, "_acquire_client", open_client)+ monkeypatch.setattr(ClientLease, "close", lambda self: None)++ async def scenario():+ app = application(tmp_path)+ try:+ async with app.router.lifespan_context(app), client_for(app) as client:+ assert await asyncio.to_thread(entered.wait, 5)+ handle = hy_sdk_runtime.SdkClientHandle(+ profile=hy_sdk_runtime.builtin_retrieval_snapshot(),+ api_key=None,+ data_dir=app.state.settings.data_dir,+ )+ opening = asyncio.create_task(handle.client())+ await asyncio.sleep(0.2)+ # Readiness and reads answer; the SDK client waits.+ assert (await health(client))["status"] == "ok"+ assert not opening.done()+ release.set()+ await asyncio.wait_for(opening, 5)+ await wait_for_recovery(client)+ finally:+ release.set()++ asyncio.run(scenario())+ assert events == ["repair_started", "repair_finished", "client_opened"]+++def test_stores_are_not_held_beside_an_open_sdk_client(tmp_path):+ from one_memory import hy_sdk_runtime++ key = tmp_path.resolve()+ with hy_sdk_runtime._STORE_GATE:+ hy_sdk_runtime._OPENING[key] = 1+ try:+ assert hy_sdk_runtime.hold_stores_offline(tmp_path) is False+ finally:+ with hy_sdk_runtime._STORE_GATE:+ del hy_sdk_runtime._OPENING[key]+ assert hy_sdk_runtime.hold_stores_offline(tmp_path) is True+ assert hy_sdk_runtime.hold_stores_offline(tmp_path) is False+ hy_sdk_runtime.release_stores(tmp_path)+ hy_sdk_runtime.release_stores(tmp_path)+ assert hy_sdk_runtime.hold_stores_offline(tmp_path) is True+ hy_sdk_runtime.release_stores(tmp_path)+++def test_a_failed_recovery_step_records_where_it_failed(tmp_path, monkeypatch, caplog):+ """`error_type=KeyError` alone named no location; frames, never the message."""+ from one_memory import api++ def lookup(systems):+ return systems["private-identifier"]++ def broken_repair(**_):+ return lookup({})++ events: list[dict] = []+ monkeypatch.setattr(hy_sdk_migration, "repair_sdk_metadata_from_bundle", broken_repair)+ monkeypatch.setattr(api, "diagnostic_event", lambda **values: events.append(values))++ async def scenario():+ app = application(tmp_path)+ async with app.router.lifespan_context(app), client_for(app) as client:+ await wait_for_recovery(client)++ with caplog.at_level("WARNING"):+ asyncio.run(scenario())++ [line] = [+ record.getMessage()+ for record in caplog.records+ if "core_recovery_step_failed" in record.getMessage()+ ]+ assert "step=sdk_metadata_repair error_type=KeyError chain=KeyError@" in line+ assert "test_core_ready_before_recovery.py:" in line and ":lookup" in line+ assert "private-identifier" not in line+ [failed] = [event for event in events if event.get("event") == "core.recovery_step_failed"]+ assert failed["error_type"] == "KeyError"+ assert failed["safe_stack"][-1].startswith("test_core_ready_before_recovery.py:")+ assert failed["safe_stack"][-1].endswith(":lookup")+ assert "private-identifier" not in repr(failed)diff --git a/memhub/apps/core/tests/test_sdk_data_plane_migration.py b/memhub/apps/core/tests/test_sdk_data_plane_migration.pyindex 797d1b5..31d41f7 100644--- a/memhub/apps/core/tests/test_sdk_data_plane_migration.py+++ b/memhub/apps/core/tests/test_sdk_data_plane_migration.py@@ -705,6 +705,29 @@ def test_cutover_waits_until_the_new_runtime_reads_every_chat_memory(store, actiassert store.authority() is MemoryAuthority.SDK+def test_sdk_migrate_repairs_metadata_after_cutover_and_writes_the_receipt(store, active_profile):+ """76/76 memories showed source=default after an upgrade whose repair failed.++ The repair now runs here, offline, right after the cutover commits; Core's+ startup finds the receipt and does nothing.+ """+ _confirmed(store, "m1")+ home = store.database_path.parent++ result = run_startup_migration(+ data_dir=home,+ runtime_build="21",+ sdk_version="2.0.5",+ namespace="governed-v2",+ open_client=_reading({"m1"}),+ )++ assert result.state == "cutover"+ receipt = home / f".sdk-metadata-repair-v2-{result.migration_id}.json"+ assert receipt.is_file()+ assert json.loads(receipt.read_text())["migration_id"] == result.migration_id++def test_retired_coding_experiences_do_not_block_but_an_empty_store_does(store, active_profile):_confirmed(store, "chat-1")_confirmed(store, "exp-kept", "exp-retired", kind="coding")