Skip to content
Python

fix(memhub): metadata repair never runs beside an SDK client

alvinfeiaaebe642449d
diff --git a/memhub/apps/core/src/one_memory/api.py b/memhub/apps/core/src/one_memory/api.py
index de06478..68ff6e6 100644
--- a/memhub/apps/core/src/one_memory/api.py
+++ b/memhub/apps/core/src/one_memory/api.py
@@ -95,6 +95,8 @@ from one_memory.hy_sdk_runtime import (
SdkClientHandle,
builtin_retrieval_snapshot,
close_all_shared_runtimes,
+ hold_stores_offline,
+ release_stores,
sdk_relevance_floor,
)
from one_memory.hy_sdk_types import (
@@ -223,6 +225,7 @@ from one_memory.sdk_memory_qa import (
SdkMemoryAnswerer,
)
from one_memory.sdk_memory_runtime import SdkMemoryRuntime
+from one_memory.sdk_migration_log import error_chain_text, error_frames
from one_memory.sdk_write_processor import SdkWriteCoordinator
from one_memory.secrets import (
KeyringSecretStore,
@@ -710,7 +713,8 @@ def create_app(
raise SdkMigrationRequired()
if app.state.memory_authority is MemoryAuthority.SDK:
# The metadata repair after a cutover reads every bundle: it is
- # recovery work, so it runs after readiness (`recover_after_ready`).
+ # recovery work, so it runs after readiness (`recover_after_ready`)
+ # while the SDK's stores are held offline (see there).
app.state.memory_governance_store = RetiredGovernanceStore(database_path)
app.state.evidence_store = RetiredEvidenceStore(database_path)
else:
@@ -1187,10 +1191,14 @@ def create_app(
except Exception as error:
failed.append(name)
app.state.recovery_failed_steps = tuple(failed)
+ # Where it failed, as `file:line:function` frames of
+ # the exception chain, never its message: a bare
+ # `error_type=KeyError` named no location.
logger.warning(
- "core_recovery_step_failed step=%s error_type=%s",
+ "core_recovery_step_failed step=%s error_type=%s chain=%s",
name,
type(error).__name__,
+ error_chain_text(error),
)
diagnostic_event(
level="warn",
@@ -1198,20 +1206,42 @@ def create_app(
event="core.recovery_step_failed",
outcome="failed",
error_type=type(error).__name__,
+ safe_stack=error_frames(error) or None,
duration_ms=elapsed_ms(step_started),
details={"phase": name},
)
async def repair_sdk_metadata() -> None:
+ # `sdk-migrate` repairs right after its cutover and writes
+ # the receipt, so this normally finds it and returns. It
+ # stays for homes whose repair failed before that (no
+ # receipt). It opens its own Chroma client on the SDK's
+ # vector store, and Chroma's shared System is not safe
+ # beside another client: a first library request that
+ # opened the SDK client meanwhile lost it (KeyError).
+ # The stores were held offline before readiness, so any
+ # SDK client waits for this step.
from one_memory.hy_sdk_migration import (
repair_sdk_metadata_from_bundle,
)
- repair = await asyncio.to_thread(
- repair_sdk_metadata_from_bundle,
- data_dir=runtime_settings.data_dir,
- database_path=database_path,
+ if not app.state.sdk_stores_held:
+ logger.warning("sdk_metadata_repair_deferred reason=sdk_client_open")
+ return
+ # The hold is this step's from here; shutdown leaves it.
+ app.state.sdk_stores_held = False
+ work = asyncio.ensure_future(
+ asyncio.to_thread(
+ repair_sdk_metadata_from_bundle,
+ data_dir=runtime_settings.data_dir,
+ database_path=database_path,
+ )
)
+ # Released when the thread ends, not when this coroutine
+ # is cancelled: a cancelled await leaves the thread
+ # writing Chroma.
+ work.add_done_callback(lambda _work: release_stores(runtime_settings.data_dir))
+ repair = await asyncio.shield(work)
if repair.vector_rows or repair.coding_rows:
logger.info(
"sdk_metadata_repair_completed mapped=%d vector=%d coding=%d",
@@ -1335,7 +1365,12 @@ def create_app(
)
app.state.recovery_task = None
+ app.state.sdk_stores_held = False
try:
+ if app.state.memory_authority is MemoryAuthority.SDK:
+ # Before the first request can open an SDK client; the
+ # metadata repair step releases it.
+ app.state.sdk_stores_held = hold_stores_offline(runtime_settings.data_dir)
diagnostic_event(
level="info",
component="core.lifecycle",
@@ -1392,6 +1427,10 @@ def create_app(
if app.state.live_sync_watch is not None:
await app.state.live_sync_watch.shutdown()
await app.state.live_sync_supervisor.shutdown()
+ if app.state.sdk_stores_held:
+ # Recovery never reached the metadata repair step.
+ release_stores(runtime_settings.data_dir)
+ app.state.sdk_stores_held = False
await app.state.import_supervisor.shutdown()
app.state.memory_write_drain.stop()
if app.state.memory_write_drain_task is not None:
diff --git a/memhub/apps/core/src/one_memory/hy_sdk_migration.py b/memhub/apps/core/src/one_memory/hy_sdk_migration.py
index b955d14..cf43a29 100644
--- a/memhub/apps/core/src/one_memory/hy_sdk_migration.py
+++ b/memhub/apps/core/src/one_memory/hy_sdk_migration.py
@@ -82,7 +82,10 @@ def repair_sdk_metadata_from_bundle(
database_path: Path,
now: datetime | None = None,
) -> MetadataRepairResult:
- """Restore metadata lost by historical cutovers from their verified bundles."""
+ """Restore metadata lost by historical cutovers from their verified bundles.
+
+ Offline only: no SDK client may be open on this home (see `_repair_vector`).
+ """
migration_id = _sdk_migration_id(database_path)
if migration_id is None:
return MetadataRepairResult(None, 0, 0, 0)
@@ -329,8 +332,11 @@ def _repair_vector(
from one_memory.sdk_vector_reattach import open_offline_chroma
client = open_offline_chroma(directory)
- # The SDK client is not open yet (startup has not created its handle), so
- # this is the only writer; release the store before the SDK takes it.
+ # This must be the only Chroma client on the directory: Chroma shares one
+ # System per path, and this client's close() drops it from under any
+ # other (KeyError on its next lookup). `sdk-migrate` runs it after the
+ # cutover with Runtime stopped; Core's startup fallback runs it while
+ # `hold_stores_offline` keeps every SDK client waiting.
try:
repaired = 0
ids = tuple(mapping)
diff --git a/memhub/apps/core/src/one_memory/hy_sdk_runtime.py b/memhub/apps/core/src/one_memory/hy_sdk_runtime.py
index 522884a..854bb48 100644
--- a/memhub/apps/core/src/one_memory/hy_sdk_runtime.py
+++ b/memhub/apps/core/src/one_memory/hy_sdk_runtime.py
@@ -56,6 +56,16 @@ class _RuntimeEntry:
_RUNTIMES: dict[Path, _RuntimeEntry] = {}
+# Homes whose SDK-owned local stores are held offline for maintenance, and
+# the client constructions in flight per home. Chroma keeps one process-wide
+# System per path with no lock around create, lookup and release, so a
+# second client that opens or closes beside the SDK's pulls the System out
+# from under it (KeyError on the next lookup). Maintenance therefore runs
+# only while no SDK client exists for that home, and new clients wait.
+_STORE_GATE = threading.Condition()
+_OFFLINE_STORES: set[Path] = set()
+_OPENING: dict[Path, int] = {}
+
class LocalOnnxEmbeddingService:
"""Public ``EmbeddingService`` implementation backed by MemHub's ONNX model."""
@@ -307,7 +317,60 @@ def acquire_client(
data_dir: Path,
http_transport_factory: Callable[[], Any] | None = None,
) -> ClientLease:
- """Construct a public SDK client and return its explicit runtime lease."""
+ """Construct a public SDK client and return its explicit runtime lease.
+
+ Waits while `hold_stores_offline` holds this home's stores.
+ """
+ key = data_dir.resolve()
+ with _STORE_GATE:
+ while key in _OFFLINE_STORES:
+ _STORE_GATE.wait()
+ _OPENING[key] = _OPENING.get(key, 0) + 1
+ try:
+ return _acquire_client(
+ profile,
+ api_key=api_key,
+ data_dir=data_dir,
+ http_transport_factory=http_transport_factory,
+ )
+ finally:
+ with _STORE_GATE:
+ _OPENING[key] -= 1
+ if not _OPENING[key]:
+ del _OPENING[key]
+ _STORE_GATE.notify_all()
+
+
+def hold_stores_offline(data_dir: Path) -> bool:
+ """Hold this home's SDK stores for offline maintenance; False if a client is open.
+
+ While held, `acquire_client` for the home waits. Every SDK client in Core
+ is built there, so nothing reads or writes Chroma beside the maintenance.
+ `release_stores` ends the hold; it is idempotent.
+ """
+ key = data_dir.resolve()
+ with _STORE_GATE:
+ with _RUNTIME_LOCK:
+ open_runtime = key in _RUNTIMES
+ if key in _OFFLINE_STORES or _OPENING.get(key) or open_runtime:
+ return False
+ _OFFLINE_STORES.add(key)
+ return True
+
+
+def release_stores(data_dir: Path) -> None:
+ with _STORE_GATE:
+ _OFFLINE_STORES.discard(data_dir.resolve())
+ _STORE_GATE.notify_all()
+
+
+def _acquire_client(
+ profile: ModelProfileSnapshot,
+ *,
+ api_key: str | None,
+ data_dir: Path,
+ http_transport_factory: Callable[[], Any] | None,
+) -> ClientLease:
config = build_sdk_config(profile, api_key=api_key, data_dir=data_dir)
config.http_transport_factory = http_transport_factory
mismatch = inspect_graph_width_mismatch(config)
diff --git a/memhub/apps/core/src/one_memory/sdk_migration_log.py b/memhub/apps/core/src/one_memory/sdk_migration_log.py
index 1642d49..8b0b67e 100644
--- a/memhub/apps/core/src/one_memory/sdk_migration_log.py
+++ b/memhub/apps/core/src/one_memory/sdk_migration_log.py
@@ -125,6 +125,31 @@ def describe_error(error: BaseException) -> dict[str, object]:
return {"chain": chain}
+_PY_FRAME = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.\-]*\.py:\d{1,7}:[A-Za-z_][A-Za-z0-9_]*$")
+
+
+def error_frames(error: BaseException, *, limit: int = 16) -> tuple[str, ...]:
+ """The raised error's innermost `file.py:line:function` frames, for a stack field.
+
+ Library frames are kept (a KeyError raised inside Chroma is located by
+ them); `<module>`-style names are dropped because stack fields refuse them.
+ """
+ chain = describe_error(error)["chain"]
+ frames = chain[0].get("frames", ()) if chain else () # type: ignore[index,union-attr]
+ return tuple(frame for frame in frames if _PY_FRAME.fullmatch(frame))[-limit:]
+
+
+def error_chain_text(error: BaseException) -> str:
+ """`describe_error` as one log token: `Type@file:line:function>...;Cause@...`.
+
+ For a plain text log (core.log). Types and frames only, never a message.
+ """
+ return ";".join(
+ f"{entry['type']}@{'>'.join(entry.get('frames', ()))}" # type: ignore[arg-type]
+ for entry in describe_error(error)["chain"] # type: ignore[union-attr]
+ )
+
+
def _build() -> dict[str, object]:
from one_memory import __version__
diff --git a/memhub/apps/core/src/one_memory/sdk_migration_runner.py b/memhub/apps/core/src/one_memory/sdk_migration_runner.py
index b8a305c..203eddc 100644
--- a/memhub/apps/core/src/one_memory/sdk_migration_runner.py
+++ b/memhub/apps/core/src/one_memory/sdk_migration_runner.py
@@ -829,6 +829,7 @@ def run_startup_migration(
ignore_errors=True,
)
discard_failed_state(data_dir)
+ _repair_metadata_after_cutover(data_dir, database_path, log)
progress("cleanup")
with log.step("cleanup.drop_legacy_tables") as step:
dropped = resume_legacy_cleanup(data_dir=data_dir, database_path=database_path)
@@ -847,6 +848,25 @@ def run_startup_migration(
)
+def _repair_metadata_after_cutover(data_dir: Path, database_path: Path, log: _StepLog) -> None:
+ """Restore lost metadata here, where Runtime is stopped and no Chroma client is open.
+
+ Writes the repair receipt, so Core's startup fallback finds it and does
+ nothing. Authority is already committed: a failure is logged and left to
+ that fallback instead of failing the migration.
+ """
+ from one_memory.hy_sdk_migration import repair_sdk_metadata_from_bundle
+
+ try:
+ with log.step("cutover.metadata_repair") as step:
+ repair = repair_sdk_metadata_from_bundle(data_dir=data_dir, database_path=database_path)
+ step["mapped"] = repair.mapped
+ step["vector_rows"] = repair.vector_rows
+ step["coding_rows"] = repair.coding_rows
+ except Exception:
+ return
+
+
def _restore_after_failure(
*,
data_dir: Path,
diff --git a/memhub/apps/core/tests/test_core_ready_before_recovery.py b/memhub/apps/core/tests/test_core_ready_before_recovery.py
index e1ea8b9..27e84a5 100644
--- a/memhub/apps/core/tests/test_core_ready_before_recovery.py
+++ b/memhub/apps/core/tests/test_core_ready_before_recovery.py
@@ -142,3 +142,112 @@ def test_shutdown_during_recovery_stops_it_without_starting_more(tmp_path, monke
release.set()
asyncio.run(scenario())
+
+
+def test_metadata_repair_never_overlaps_an_sdk_client(tmp_path, monkeypatch):
+ """The repair's own Chroma client dropped the SDK's shared System (KeyError).
+
+ The first library request opened the SDK client while the repair ran
+ after readiness. Now every SDK client of the home waits for the repair.
+ """
+ from one_memory import hy_sdk_runtime
+ from one_memory.hy_sdk_runtime import ClientLease
+
+ release = threading.Event()
+ entered = threading.Event()
+ events: list[str] = []
+
+ def slow_repair(**_):
+ entered.set()
+ events.append("repair_started")
+ release.wait(10)
+ events.append("repair_finished")
+ return MetadataRepairResult(None, 0, 0, 0)
+
+ def open_client(profile, *, api_key, data_dir, http_transport_factory):
+ events.append("client_opened")
+ return ClientLease(object(), data_dir.resolve())
+
+ monkeypatch.setattr(hy_sdk_migration, "repair_sdk_metadata_from_bundle", slow_repair)
+ monkeypatch.setattr(hy_sdk_runtime, "_acquire_client", open_client)
+ monkeypatch.setattr(ClientLease, "close", lambda self: None)
+
+ async def scenario():
+ app = application(tmp_path)
+ try:
+ async with app.router.lifespan_context(app), client_for(app) as client:
+ assert await asyncio.to_thread(entered.wait, 5)
+ handle = hy_sdk_runtime.SdkClientHandle(
+ profile=hy_sdk_runtime.builtin_retrieval_snapshot(),
+ api_key=None,
+ data_dir=app.state.settings.data_dir,
+ )
+ opening = asyncio.create_task(handle.client())
+ await asyncio.sleep(0.2)
+ # Readiness and reads answer; the SDK client waits.
+ assert (await health(client))["status"] == "ok"
+ assert not opening.done()
+ release.set()
+ await asyncio.wait_for(opening, 5)
+ await wait_for_recovery(client)
+ finally:
+ release.set()
+
+ asyncio.run(scenario())
+ assert events == ["repair_started", "repair_finished", "client_opened"]
+
+
+def test_stores_are_not_held_beside_an_open_sdk_client(tmp_path):
+ from one_memory import hy_sdk_runtime
+
+ key = tmp_path.resolve()
+ with hy_sdk_runtime._STORE_GATE:
+ hy_sdk_runtime._OPENING[key] = 1
+ try:
+ assert hy_sdk_runtime.hold_stores_offline(tmp_path) is False
+ finally:
+ with hy_sdk_runtime._STORE_GATE:
+ del hy_sdk_runtime._OPENING[key]
+ assert hy_sdk_runtime.hold_stores_offline(tmp_path) is True
+ assert hy_sdk_runtime.hold_stores_offline(tmp_path) is False
+ hy_sdk_runtime.release_stores(tmp_path)
+ hy_sdk_runtime.release_stores(tmp_path)
+ assert hy_sdk_runtime.hold_stores_offline(tmp_path) is True
+ hy_sdk_runtime.release_stores(tmp_path)
+
+
+def test_a_failed_recovery_step_records_where_it_failed(tmp_path, monkeypatch, caplog):
+ """`error_type=KeyError` alone named no location; frames, never the message."""
+ from one_memory import api
+
+ def lookup(systems):
+ return systems["private-identifier"]
+
+ def broken_repair(**_):
+ return lookup({})
+
+ events: list[dict] = []
+ monkeypatch.setattr(hy_sdk_migration, "repair_sdk_metadata_from_bundle", broken_repair)
+ monkeypatch.setattr(api, "diagnostic_event", lambda **values: events.append(values))
+
+ async def scenario():
+ app = application(tmp_path)
+ async with app.router.lifespan_context(app), client_for(app) as client:
+ await wait_for_recovery(client)
+
+ with caplog.at_level("WARNING"):
+ asyncio.run(scenario())
+
+ [line] = [
+ record.getMessage()
+ for record in caplog.records
+ if "core_recovery_step_failed" in record.getMessage()
+ ]
+ assert "step=sdk_metadata_repair error_type=KeyError chain=KeyError@" in line
+ assert "test_core_ready_before_recovery.py:" in line and ":lookup" in line
+ assert "private-identifier" not in line
+ [failed] = [event for event in events if event.get("event") == "core.recovery_step_failed"]
+ assert failed["error_type"] == "KeyError"
+ assert failed["safe_stack"][-1].startswith("test_core_ready_before_recovery.py:")
+ assert failed["safe_stack"][-1].endswith(":lookup")
+ assert "private-identifier" not in repr(failed)
diff --git a/memhub/apps/core/tests/test_sdk_data_plane_migration.py b/memhub/apps/core/tests/test_sdk_data_plane_migration.py
index 797d1b5..31d41f7 100644
--- a/memhub/apps/core/tests/test_sdk_data_plane_migration.py
+++ b/memhub/apps/core/tests/test_sdk_data_plane_migration.py
@@ -705,6 +705,29 @@ def test_cutover_waits_until_the_new_runtime_reads_every_chat_memory(store, acti
assert store.authority() is MemoryAuthority.SDK
+def test_sdk_migrate_repairs_metadata_after_cutover_and_writes_the_receipt(store, active_profile):
+ """76/76 memories showed source=default after an upgrade whose repair failed.
+
+ The repair now runs here, offline, right after the cutover commits; Core's
+ startup finds the receipt and does nothing.
+ """
+ _confirmed(store, "m1")
+ home = store.database_path.parent
+
+ result = run_startup_migration(
+ data_dir=home,
+ runtime_build="21",
+ sdk_version="2.0.5",
+ namespace="governed-v2",
+ open_client=_reading({"m1"}),
+ )
+
+ assert result.state == "cutover"
+ receipt = home / f".sdk-metadata-repair-v2-{result.migration_id}.json"
+ assert receipt.is_file()
+ assert json.loads(receipt.read_text())["migration_id"] == result.migration_id
+
+
def test_retired_coding_experiences_do_not_block_but_an_empty_store_does(store, active_profile):
_confirmed(store, "chat-1")
_confirmed(store, "exp-kept", "exp-retired", kind="coding")