Python
fix(memhub): serialize SDK client startup, log failure frames, settle pre-upgrade unknown writes
alvinfei352fffb3dadf
diff --git a/memhub/apps/core/src/one_memory/api.py b/memhub/apps/core/src/one_memory/api.pyindex 50513be..611ecc8 100644--- a/memhub/apps/core/src/one_memory/api.py+++ b/memhub/apps/core/src/one_memory/api.py@@ -3150,25 +3150,25 @@ def create_app(raise HTTPException(status_code=422, detail="memory_task_requires_experience")namespace = _namespace_kwargs(request.kind, request.variant)try:- client = await app.state.sdk_write_coordinator.client()- update_kwargs: dict[str, object] = dict(namespace)- if request.task is not None:- update_kwargs["task"] = request.task- result = await asyncio.to_thread(- client.update, memory_id, request.content, **update_kwargs- )- if not isinstance(result, dict):- raise HySdkError("hy_memory_update_rejected")- if result.get("success") is not True:- if "not found" in str(result.get("error_message") or ""):- raise HTTPException(status_code=404, detail="memory_not_found")- raise HySdkError("hy_memory_update_rejected")- # The SDK names the namespace it wrote; read that one back.- if result.get("kind") in {"experience", "fact"}:- namespace = {"kind": result["kind"]}- if result.get("variant") in {"working", "coding"}:- namespace["variant"] = result["variant"]- raw = await asyncio.to_thread(client.get, memory_id, **namespace)+ async with app.state.sdk_write_coordinator.use_client() as client:+ update_kwargs: dict[str, object] = dict(namespace)+ if request.task is not None:+ update_kwargs["task"] = request.task+ result = await asyncio.to_thread(+ client.update, memory_id, request.content, **update_kwargs+ )+ if not isinstance(result, dict):+ raise HySdkError("hy_memory_update_rejected")+ if result.get("success") is not True:+ if "not found" in str(result.get("error_message") or ""):+ raise HTTPException(status_code=404, detail="memory_not_found")+ raise HySdkError("hy_memory_update_rejected")+ # The SDK names the namespace it wrote; read that one back.+ if result.get("kind") in {"experience", "fact"}:+ namespace = {"kind": result["kind"]}+ if result.get("variant") in {"working", "coding"}:+ namespace["variant"] = result["variant"]+ raw = await asyncio.to_thread(client.get, memory_id, **namespace)memory = None if raw is None else HySdkMemoryRecord.from_sdk(raw)except HTTPException:raise@@ -3197,10 +3197,10 @@ def create_app() -> Response:_check_memory_namespace(kind, variant)try:- client = await app.state.sdk_write_coordinator.client()- result = await asyncio.to_thread(- client.delete, memory_id, **_namespace_kwargs(kind, variant)- )+ async with app.state.sdk_write_coordinator.use_client() as client:+ result = await asyncio.to_thread(+ client.delete, memory_id, **_namespace_kwargs(kind, variant)+ )if not isinstance(result, dict):raise HySdkError("hy_memory_delete_rejected")error_code = result.get("error_code")@@ -3442,9 +3442,9 @@ def create_app("""Retry unknown writes only behind the user's explicit confirmation."""del requestreconciled = await _sdk_memory_runtime(app).reconcile_unknown_outputs()- from one_memory.memory_server.durable_queue import DurableWriteQueue-- queue = DurableWriteQueue(app.state.memory_write_queue_path)+ # The startup queue knows the authority. A fresh legacy-mode queue+ # would recreate the job-evidence table the SDK cutover dropped.+ queue = app.state.memory_write_queueunknown = queue.unknown_jobs(limit=2048)if app.state.memory_authority is MemoryAuthority.LEGACY:for job in unknown:diff --git a/memhub/apps/core/src/one_memory/hy_sdk_runtime.py b/memhub/apps/core/src/one_memory/hy_sdk_runtime.pyindex 4b4d2a6..3b4f626 100644--- a/memhub/apps/core/src/one_memory/hy_sdk_runtime.py+++ b/memhub/apps/core/src/one_memory/hy_sdk_runtime.py@@ -67,6 +67,16 @@ _STORE_GATE = threading.Condition()_OFFLINE_STORES: set[Path] = set()_OPENING: dict[Path, int] = {}+# One client constructs at a time per home, its first-time store setup+# included. The constructor opens Chroma's `PersistentClient`, and two first+# opens on one directory race over that same unlocked System cache. On a+# clean install the write drain's client and a UI read client started in the+# same second; neither reached `VectorStore ready`, and the first write job+# was lost as unknown. Taken inside `acquire_client`, after the offline gate+# and never while `_STORE_GATE` is held, so it cannot deadlock with+# `hold_stores_offline`, which refuses while `_OPENING` counts a client.+_CONSTRUCTING: dict[Path, threading.Lock] = {}+class LocalOnnxEmbeddingService:"""Public ``EmbeddingService`` implementation backed by MemHub's ONNX model."""@@ -382,13 +392,16 @@ def _acquire_client(embedding_service: LocalOnnxEmbeddingService | None = Noneif (profile.embedding_provider_kind or profile.provider_kind) is ProviderKind.LOCAL_ONNX:embedding_service = LocalOnnxEmbeddingService()+ with _STORE_GATE:+ constructing = _CONSTRUCTING.setdefault(key, threading.Lock())try:- client = HyMemoryClient(- config=config,- mode="ultra" if graph_store_enabled() else "pro",- runtime=runtime,- embedding_service=embedding_service,- )+ with constructing:+ client = HyMemoryClient(+ config=config,+ mode="ultra" if graph_store_enabled() else "pro",+ runtime=runtime,+ embedding_service=embedding_service,+ )except BaseException:_release_runtime(key)raisediff --git a/memhub/apps/core/src/one_memory/memory_server/durable_queue.py b/memhub/apps/core/src/one_memory/memory_server/durable_queue.pyindex b755ba9..5d74014 100644--- a/memhub/apps/core/src/one_memory/memory_server/durable_queue.py+++ b/memhub/apps/core/src/one_memory/memory_server/durable_queue.py@@ -52,6 +52,7 @@ from one_memory.memory_tasks import (from one_memory.memory_tasks import (POLICY_STOPPED_CODES as _TASK_POLICY_STOPPED_CODES,)+from one_memory.sdk_migration_log import error_chain_textfrom one_memory.service_levels import ServiceLevel, service_level_contextlogger = logging.getLogger(__name__)@@ -241,14 +242,16 @@ PERMISSION_REQUIRED_CODES = frozenset()#: Model-side codes the SDK write coordinator raises before it can lease a-#: client (no active profile, missing credential, secure storage down). They prove-#: nothing was sent, so the write parks instead of being lost. Transport codes-#: that can arrive after a request was in flight stay `unknown`.+#: client (no active profile, missing credential, secure storage down, or the+#: client failed to construct). They prove nothing was sent, so the write+#: parks instead of being lost. Transport codes that can arrive after a+#: request was in flight stay `unknown`.MODEL_NOT_READY_CODES = frozenset({"model_profile_not_active","model_secret_unavailable","secure_storage_unavailable",+ "hy_memory_client_unavailable",})@@ -292,6 +295,23 @@ _VALID_JOB_STATES = frozenset(#: A row in one of these states can never be moved back to a live one._TERMINAL_WRITE_STATES = frozenset({COMPLETED, FAILED, DEAD, CANCELLED})+#: The legacy task rows a write job can be linked to. The SDK cutover drops+#: them (`target_schema.LEGACY_TABLES_IN_DROP_ORDER`, inputs before tasks), so+#: the job-only paths must not assume any of them.+_LEGACY_TASK_LINK_TABLES = ("memory_tasks", "memory_task_inputs", "memory_write_job_evidence")+++def _legacy_task_links_present(connection: sqlite3.Connection) -> bool:+ present = {+ str(row[0])+ for row in connection.execute(+ "SELECT name FROM sqlite_master WHERE type='table' AND name IN (?,?,?)",+ _LEGACY_TASK_LINK_TABLES,+ )+ }+ return present == set(_LEGACY_TASK_LINK_TABLES)++#: Columns a current schema declares but an older `memory_write_jobs` may lack.#: `CREATE TABLE IF NOT EXISTS` is a no-op for an existing table, so without#: this an upgraded install would keep an older queue that every claim/lease@@ -1982,21 +2002,24 @@ class DurableWriteQueue:or str(row["last_error"]) in NON_RETRYABLE_POST_WRITE_CODES):return 0- connection.execute(- """- UPDATE memory_tasks- SET status='pending', runner_token=NULL, attempts=0,- last_error_code='model_content_manual_retry_requested', updated_at=?- WHERE status='unknown' AND task_id IN (- SELECT inputs.task_id- FROM memory_task_inputs AS inputs, json_each(inputs.refs_json) AS ref- JOIN memory_write_job_evidence AS linked- ON linked.evidence_id=json_extract(ref.value, '$.evidence_id')- WHERE linked.job_id=?+ # After the SDK cutover the legacy task tables are gone and the+ # job is the only row to reset.+ if not self.sdk_authority and _legacy_task_links_present(connection):+ connection.execute(+ """+ UPDATE memory_tasks+ SET status='pending', runner_token=NULL, attempts=0,+ last_error_code='model_content_manual_retry_requested', updated_at=?+ WHERE status='unknown' AND task_id IN (+ SELECT inputs.task_id+ FROM memory_task_inputs AS inputs, json_each(inputs.refs_json) AS ref+ JOIN memory_write_job_evidence AS linked+ ON linked.evidence_id=json_extract(ref.value, '$.evidence_id')+ WHERE linked.job_id=?+ )+ """,+ (now, job_id),)- """,- (now, job_id),- )changed = connection.execute("""UPDATE memory_write_jobs@@ -2098,9 +2121,7 @@ class DurableWriteQueue:f"WHERE state=? AND job_id IN ({placeholders})",(PENDING, now, UNKNOWN, *job_ids),)- if connection.execute(- "SELECT 1 FROM sqlite_master WHERE type='table' AND name='memory_tasks'"- ).fetchone():+ if _legacy_task_links_present(connection):connection.execute(f"UPDATE memory_tasks SET status=?, runner_token=NULL, "f"last_error_code='model_content_reconciled_absent_retry', updated_at=? "@@ -2158,9 +2179,7 @@ class DurableWriteQueue:f"WHERE state=? AND attempts=1 AND job_id IN ({placeholders})",(PENDING, now, UNKNOWN, *job_ids),)- if connection.execute(- "SELECT 1 FROM sqlite_master WHERE type='table' AND name='memory_tasks'"- ).fetchone():+ if _legacy_task_links_present(connection):connection.execute(f"UPDATE memory_tasks SET status=?, runner_token=NULL, "f"last_error_code='model_content_unknown_auto_retry', updated_at=? "@@ -2177,6 +2196,14 @@ class DurableWriteQueue:def resolve_unknown_from_linked_output(self, job_id: str) -> bool:"""Close an unknown durable row when local governance output proves success."""with main_database(self.database_path).read(row_factory=sqlite3.Row) as connection:+ if not (+ _legacy_task_links_present(connection)+ and connection.execute(+ "SELECT 1 FROM sqlite_master WHERE type='table' AND name='memory_task_outputs'"+ ).fetchone()+ ):+ # No legacy governance output survives the SDK cutover.+ return Falserow = connection.execute("""SELECT 1@@ -2559,9 +2586,22 @@ async def execute_job(if code == "model_profile_not_active":logger.debug("durable_write_job_parked job_id=%s error_code=%s", job_id, code)elif state in PARKED_STATES:- logger.info("durable_write_job_parked job_id=%s error_code=%s", job_id, code)+ logger.info(+ "durable_write_job_parked job_id=%s error_code=%s chain=%s",+ job_id,+ code,+ error_chain_text(error),+ )else:- logger.error("durable_write_job_failed job_id=%s error_code=%s", job_id, code)+ # Where it failed, as `file:line:function` frames of the exception+ # chain, never its message: `error_code=AttributeError` alone+ # named no location.+ logger.error(+ "durable_write_job_failed job_id=%s error_code=%s chain=%s",+ job_id,+ code,+ error_chain_text(error),+ )returnawait in_the_background(queue.settle,diff --git a/memhub/apps/core/src/one_memory/sdk_memory_runtime.py b/memhub/apps/core/src/one_memory/sdk_memory_runtime.pyindex fd247e5..4e13689 100644--- a/memhub/apps/core/src/one_memory/sdk_memory_runtime.py+++ b/memhub/apps/core/src/one_memory/sdk_memory_runtime.py@@ -5,7 +5,8 @@ from __future__ import annotationsimport asyncioimport loggingimport re-from contextlib import suppress+from collections.abc import AsyncIterator+from contextlib import AsyncExitStack, asynccontextmanager, suppressfrom datetime import UTC, datetime, timedeltafrom pathlib import Pathfrom typing import Any@@ -164,44 +165,45 @@ class SdkMemoryRuntime:if str(error) == "model_profile_not_active":return HyGraphSnapshot(consulted=False)raise- try:- client = await self.coordinator.client()- except Exception as error:- raise HySdkError("hy_memory_graph_read_failed") from error- return await asyncio.to_thread(- read_graph_snapshot,- client,- "local",- "*",- limit,- )+ async with AsyncExitStack() as stack:+ try:+ client = await stack.enter_async_context(self.coordinator.use_client())+ except Exception as error:+ raise HySdkError("hy_memory_graph_read_failed") from error+ return await asyncio.to_thread(+ read_graph_snapshot,+ client,+ "local",+ "*",+ limit,+ )async def digest(self) -> dict[str, object]:- client = await self.coordinator.client()- try:- raw = await asyncio.to_thread(- client.list_memories,- user_id="local",- limit=1_000,- offset=0,- )- agent_ids = _agent_ids(raw)- digests = [- {- "agent_id": agent_id,- "schemas_recorded": 0,- **dict(- await asyncio.to_thread(- client.evolve,- user_id="local",- agent_id=agent_id,- )- ),- }- for agent_id in agent_ids- ]- except Exception as error:- raise HySdkError("hy_memory_digest_failed") from error+ async with self.coordinator.use_client() as client:+ try:+ raw = await asyncio.to_thread(+ client.list_memories,+ user_id="local",+ limit=1_000,+ offset=0,+ )+ agent_ids = _agent_ids(raw)+ digests = [+ {+ "agent_id": agent_id,+ "schemas_recorded": 0,+ **dict(+ await asyncio.to_thread(+ client.evolve,+ user_id="local",+ agent_id=agent_id,+ )+ ),+ }+ for agent_id in agent_ids+ ]+ except Exception as error:+ raise HySdkError("hy_memory_digest_failed") from errorreturn {"success": all(result.get("success") is not False for result in digests),"scope_count": len(digests),@@ -240,36 +242,39 @@ class SdkMemoryRuntime:decision=decision,)- async def _mutation_client(self) -> Any:+ @asynccontextmanager+ async def _mutation_client(self) -> AsyncIterator[Any]:"""The write client, with no-model and keychain refusals as SDK codes.Callers record these against the edit or deletion instead of failingthe request that already committed it."""- try:- return await self.coordinator.client()- except RuntimeError as error:- code = str(error)- if not _STABLE_CODE.fullmatch(code):- code = "hy_memory_client_unavailable"- raise HySdkError(code) from error+ async with AsyncExitStack() as stack:+ try:+ client = await stack.enter_async_context(self.coordinator.use_client())+ except RuntimeError as error:+ code = str(error)+ if not _STABLE_CODE.fullmatch(code):+ code = "hy_memory_client_unavailable"+ raise HySdkError(code) from error+ yield clientasync def update_hy_memory(self, memory_id: str, content: str) -> dict[str, object]:- client = await self._mutation_client()- try:- result = await asyncio.to_thread(client.update, memory_id, content)- except Exception as error:- raise HySdkError("hy_memory_update_failed") from error+ async with self._mutation_client() as client:+ try:+ result = await asyncio.to_thread(client.update, memory_id, content)+ except Exception as error:+ raise HySdkError("hy_memory_update_failed") from errorif not isinstance(result, dict) or result.get("success") is not True:raise HySdkError("hy_memory_update_rejected")return resultasync def finish_deletion(self, memory_id: str) -> None:- client = await self._mutation_client()- try:- result = await asyncio.to_thread(client.delete, memory_id)- except Exception as error:- raise HySdkError("hy_memory_delete_failed") from error+ async with self._mutation_client() as client:+ try:+ result = await asyncio.to_thread(client.delete, memory_id)+ except Exception as error:+ raise HySdkError("hy_memory_delete_failed") from errorif not isinstance(result, dict):raise HySdkError("hy_memory_delete_rejected")error_code = result.get("error_code")diff --git a/memhub/apps/core/src/one_memory/sdk_migration_runner.py b/memhub/apps/core/src/one_memory/sdk_migration_runner.pyindex 203eddc..a98ef46 100644--- a/memhub/apps/core/src/one_memory/sdk_migration_runner.py+++ b/memhub/apps/core/src/one_memory/sdk_migration_runner.py@@ -35,6 +35,8 @@ from one_memory.memory_server.durable_queue import (COMPLETED,FAILED,REAPED_PAYLOAD_MARKER,+ RUNNING,+ UNKNOWN,DurableWriteQueue,)from one_memory.memory_server.unified_entry import UnifiedWriteIntake@@ -317,11 +319,45 @@ def move_pending_dispatches(database_path: Path) -> int:return len(conversations)+#: Why a pre-cutover write job that may have reached HY was closed.+ABANDONED_AT_CUTOVER = "abandoned_at_cutover"+++def settle_legacy_unknown_write_jobs(database_path: Path) -> int:+ """Close write jobs the legacy pipeline left possibly-sent. Safe to repeat.++ They ran under the legacy pipeline, so the SDK's `get_write_outcome` knows+ none of them and the unknown reconcile can never settle them: a migrated+ 1.0.4 home showed 1000+ unknown forever. They are not retried, since they+ may already have written and the SDK has no idempotency key. `cancelled`,+ not `failed`: the upgrade ended them, nothing broke, and a `failed` count+ would put the home in the failed state for good. `hy_invoked` is kept, so+ no readmission path can pick one up again.+ """+ with main_database(database_path).read(row_factory=sqlite3.Row) as database:+ if "memory_write_jobs" not in _tables(database):+ return 0+ now = datetime.now(UTC).isoformat()+ with main_database(database_path).write(autocommit=True) as database:+ # A `running` row already marked invoked would be reclaimed to+ # `unknown` on the next open, so it is the same case.+ return int(+ database.execute(+ "UPDATE memory_write_jobs SET state=?,lease_owner=NULL,"+ "lease_expires_at=NULL,last_error=?,updated_at=? "+ "WHERE state=? OR (state=? AND hy_invoked=1)",+ (CANCELLED, ABANDONED_AT_CUTOVER, now, UNKNOWN, RUNNING),+ ).rowcount+ )++def abandoned_legacy_work(database_path: Path) -> dict[str, int]:"""Count legacy work that ends with the cutover instead of blocking it.A task that may have reached the SDK is not replayed: the SDK already holdswhatever it committed. Pending user deletions are not carried over.+ Settled write jobs are counted by their code, so a repeated preparation+ reports the same number."""abandoned: dict[str, int] = {}with main_database(database_path).read(row_factory=sqlite3.Row) as database:@@ -332,6 +368,11 @@ def abandoned_legacy_work(database_path: Path) -> dict[str, int]:"memory_tasks","status='unknown' OR (status='running' AND hy_invocation_count > 0)",),+ (+ "unknown_write_jobs",+ "memory_write_jobs",+ f"state='{CANCELLED}' AND last_error='{ABANDONED_AT_CUTOVER}'",+ ),("deletions", "memory_deletions", "status IN ('pending','running')"),)for name, table, where in checks:@@ -354,6 +395,10 @@ def prepare_legacy_work_for_cutover(running, so a failure names it."""trace = {} if trace is None else trace+ # First: a possibly-sent job never runs again, so its input is not+ # frozen, and input released since must not block the cutover.+ trace["sub_step"] = "unknown_write_jobs"+ settle_legacy_unknown_write_jobs(database_path)trace["sub_step"] = "write_queue"frozen, moved = prepare_legacy_write_queue_for_cutover(database_path)trace["sub_step"] = "pending_dispatches"diff --git a/memhub/apps/core/src/one_memory/sdk_write_processor.py b/memhub/apps/core/src/one_memory/sdk_write_processor.pyindex f6e4d98..dd5d94a 100644--- a/memhub/apps/core/src/one_memory/sdk_write_processor.py+++ b/memhub/apps/core/src/one_memory/sdk_write_processor.py@@ -3,7 +3,8 @@from __future__ import annotationsimport asyncio-from collections.abc import Awaitable, Callable, Mapping, Sequence+from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence+from contextlib import asynccontextmanagerfrom dataclasses import dataclassfrom datetime import datetimefrom typing import Any@@ -56,7 +57,12 @@ class SdkWriteProcessor:scope = _write_scope(records)metadata = _write_metadata(records, scope)- client = await self._client_handle.client()+ try:+ client = await self._client_handle.client()+ except Exception as error:+ # Before `before_hy_invocation`: nothing was sent, so the job+ # waits and retries instead of being left `unknown`.+ raise RuntimeError("hy_memory_client_unavailable") from errorif extracted:contents = [sanitize_model_input(record.verbatim) for record in records]@@ -125,7 +131,14 @@ class SdkWriteProcessor:class SdkWriteCoordinator:- """Resolve the active model profile, then execute one public SDK write."""+ """Resolve the active model profile, then execute one public SDK write.++ A model profile change retires the previous profile's handle, but only+ closes it once nothing uses it. Closing it at once tore the client out+ from under a write still running on it: the SDK cleared its stores and+ the SharedRuntime closed mid-`add`, the job failed with AttributeError+ after HY was invoked, and it was left `unknown`.+ """def __init__(self,@@ -142,6 +155,9 @@ class SdkWriteCoordinator:self.transport_factory = transport_factoryself.sedimentation_enabled = sedimentation_enabledself._handles: dict[tuple[str, int], SdkClientHandle] = {}+ #: Uses in flight per handle; a retired handle closes at zero.+ self._uses: dict[SdkClientHandle, int] = {}+ self._retired: set[SdkClientHandle] = set()self._lock = asyncio.Lock()async def process_sdk_job(@@ -153,28 +169,30 @@ class SdkWriteCoordinator:) -> SdkWriteReceipt:if not self.sedimentation_enabled():raise RuntimeError("model_sedimentation_disabled")- handle = await self._active_handle()- return await SdkWriteProcessor(handle).process_sdk_job(- records,- request_id=request_id,- before_hy_invocation=before_hy_invocation,- )+ async with self._active_handle() as handle:+ return await SdkWriteProcessor(handle).process_sdk_job(+ records,+ request_id=request_id,+ before_hy_invocation=before_hy_invocation,+ )async def inquire_write_outcome(self, request_id: str) -> SdkWriteReceipt | None:- return await SdkWriteProcessor(- await self._active_handle()- ).inquire_write_outcome(request_id)+ async with self._active_handle() as handle:+ return await SdkWriteProcessor(handle).inquire_write_outcome(request_id)async def close(self) -> None:async with self._lock:- handles, self._handles = tuple(self._handles.values()), {}+ handles = (*self._handles.values(), *self._retired)+ self._handles, self._retired, self._uses = {}, set(), {}await asyncio.gather(*(handle.close() for handle in handles))- async def client(self) -> Any:- """Return the public SDK client selected by the active write profile."""- return await (await self._active_handle()).client()+ @asynccontextmanager+ async def use_client(self) -> AsyncIterator[Any]:+ """The public SDK client of the active write profile, kept open while used."""+ async with self._active_handle() as handle:+ yield await handle.client()async def active_profile_credentials(self,@@ -183,7 +201,9 @@ class SdkWriteCoordinator:_identity, profile, secret = await asyncio.to_thread(self._read_active_credentials)return profile, secret- async def _active_handle(self) -> SdkClientHandle:+ @asynccontextmanager+ async def _active_handle(self) -> AsyncIterator[SdkClientHandle]:+ """Use the active profile's handle; retire the others, closing the unused."""identity, profile, secret = await asyncio.to_thread(self._read_active_credentials)@@ -197,14 +217,25 @@ class SdkWriteCoordinator:http_transport_factory=self.transport_factory(profile),)self._handles[identity] = handle- stale = [- value- for key, value in self._handles.items()- if key != identity- ]+ self._retired.update(value for key, value in self._handles.items() if key != identity)self._handles = {identity: handle}- await asyncio.gather(*(value.close() for value in stale))- return handle+ self._uses[handle] = self._uses.get(handle, 0) + 1+ unused = {value for value in self._retired if not self._uses.get(value)}+ self._retired -= unused+ await asyncio.gather(*(value.close() for value in unused))+ try:+ yield handle+ finally:+ # Bookkeeping without an await, so a cancelled caller still+ # releases its use.+ remaining = self._uses.get(handle, 0) - 1+ if remaining > 0:+ self._uses[handle] = remaining+ else:+ self._uses.pop(handle, None)+ if remaining <= 0 and handle in self._retired:+ self._retired.discard(handle)+ await handle.close()def _read_active_credentials(self,diff --git a/memhub/apps/core/tests/TEST_INDEX.md b/memhub/apps/core/tests/TEST_INDEX.mdindex 7f2d910..b38c5bb 100644--- a/memhub/apps/core/tests/TEST_INDEX.md+++ b/memhub/apps/core/tests/TEST_INDEX.md@@ -246,6 +246,7 @@| `test_schema_gate.py` | `contract` | 覆盖 fresh database records the build schema、future schema refuses to open、legacy database without a ledger is left untouched 场景。 || `test_schema_migrations.py` | `contract` | 覆盖 evidence store adds source ordinal to previous schema、import store migrates previous job and error schema、old active job index lets a sync sit beside paused history 等 4 个场景。 || `test_schema_ownership_gate.py` | `contract` | A1-03/A1-04: schema ownership, and the hard part of telling DDL from reads. |+| `test_sdk_client_construction.py` | `contract` | SDK clients of one home are constructed one at a time. || `test_sdk_data_plane_migration.py` | `contract` | SDK-authority migration ledger, aliases and atomic cutover marker. || `test_sdk_memory_qa.py` | `contract` | 覆盖 sdk ask propagates sdk search failure without legacy fallback、sdk ask revalidates memory before returning extract、sdk ask uses native sdk memory without governance fields 场景。 || `test_sdk_migration_bundle.py` | `contract` | SDK-authority cutover requires a verified management + SDK data-plane bundle. |@@ -308,4 +309,4 @@| `test_workspace_recall.py` | `contract` | Workspace memories were unreachable from every host; these pin the way in. || `test_write_queue_lease_renewal.py` | `contract` | Rule 8: a lease must outlive the work it covers, and be renewed while it runs. |-共 303 个测试文件。+共 304 个测试文件。diff --git a/memhub/apps/core/tests/core_test_classification.py b/memhub/apps/core/tests/core_test_classification.pyindex 805b8f4..7c0b208 100644--- a/memhub/apps/core/tests/core_test_classification.py+++ b/memhub/apps/core/tests/core_test_classification.py@@ -14,8 +14,8 @@ from pathlib import Pathimport pytestPRIMARY_MARKERS = ("unit", "contract", "integration", "performance", "live")-EXPECTED_TEST_FILE_COUNT = 303-EXPECTED_TEST_FILES_SHA256 = "7993203411697fc0bf42b92fa316f222a66e6826c5b4a0132dbb0a1ca6da4e32"+EXPECTED_TEST_FILE_COUNT = 304+EXPECTED_TEST_FILES_SHA256 = "b28f022b9688c96ca563c59de63e95d344d2f01ba3b6020b422bf4dfb27520d6"# Pure transformations, validation models, parity checks and state machines.# This list is intentionally small: uncertain files default to contract.diff --git a/memhub/apps/core/tests/test_memory_library_all_scenes.py b/memhub/apps/core/tests/test_memory_library_all_scenes.pyindex 1d07cb4..d53dd41 100644--- a/memhub/apps/core/tests/test_memory_library_all_scenes.py+++ b/memhub/apps/core/tests/test_memory_library_all_scenes.py@@ -6,6 +6,7 @@ and delete pass the memory's kind/variant through to the SDK."""import asyncio+from contextlib import asynccontextmanagerimport pytestfrom httpx import ASGITransport, AsyncClient@@ -310,12 +311,13 @@ def run_with_sdk(tmp_path, monkeypatch, scenario):async def close(self):return None+ @asynccontextmanagerasync def write_client():- return sdk+ yield sdkasync with app.router.lifespan_context(app):app.state.sdk_client_handle = Handle()- monkeypatch.setattr(app.state.sdk_write_coordinator, "client", write_client)+ monkeypatch.setattr(app.state.sdk_write_coordinator, "use_client", write_client)async with client_for(app) as client:await scenario(app, sdk, client)diff --git a/memhub/apps/core/tests/test_memory_server_durable_queue.py b/memhub/apps/core/tests/test_memory_server_durable_queue.pyindex df5d410..062accc 100644--- a/memhub/apps/core/tests/test_memory_server_durable_queue.py+++ b/memhub/apps/core/tests/test_memory_server_durable_queue.py@@ -1187,6 +1187,167 @@ def test_the_drain_tick_writes_a_quiet_live_conversation(tmp_path, monkeypatch)assert flushed+class _SwitchableProfiles:+ """Active profile identity that a test can change mid-write."""++ def __init__(self) -> None:+ self.identity = ("profile-a", 1)++ def active_identity(self):+ return self.identity++ def snapshot(self, profile_id, revision):+ return type(+ "Snapshot", (), {"profile_id": profile_id, "revision": revision, "has_api_key": False}+ )()++ def resolve_snapshot_secret(self, *_args):+ return None+++class _TrackedHandle:+ def __init__(self, profile, **_kwargs) -> None:+ self.profile_id = profile.profile_id+ self.closed = False++ async def client(self):+ return self++ async def close(self):+ self.closed = True+++def _coordinator(monkeypatch, opened: list[_TrackedHandle]):+ from one_memory import sdk_write_processor++ def handle(profile, **kwargs):+ value = _TrackedHandle(profile, **kwargs)+ opened.append(value)+ return value++ monkeypatch.setattr(sdk_write_processor, "SdkClientHandle", handle)+ profiles = _SwitchableProfiles()+ coordinator = sdk_write_processor.SdkWriteCoordinator(+ profiles=profiles, # type: ignore[arg-type]+ secrets=None, # type: ignore[arg-type]+ data_dir=None,+ transport_factory=lambda _profile: None,+ sedimentation_enabled=lambda: True,+ )+ return coordinator, profiles+++def test_a_profile_change_does_not_close_the_client_of_a_write_in_flight(monkeypatch) -> None:+ """Build 35 lost a clean install's first write as unknown (AttributeError).++ Saving a model profile changed the active identity while the write ran on+ the old profile's client. The next coordinator call closed that client at+ once: the SDK cleared its stores and the SharedRuntime closed mid-`add`.+ """+ from one_memory import sdk_write_processor++ opened: list[_TrackedHandle] = []+ coordinator, profiles = _coordinator(monkeypatch, opened)+ seen: list[tuple[str, bool]] = []+ writing = asyncio.Event()++ async def process(self, records, *, request_id, before_hy_invocation):+ client = await self._client_handle.client()+ await before_hy_invocation()+ seen.append((client.profile_id, client.closed))+ if client.profile_id == "profile-a":+ await writing.wait()+ # Still open while its own write runs.+ seen.append((client.profile_id, client.closed))+ return sdk_write_processor.SdkWriteReceipt()++ monkeypatch.setattr(sdk_write_processor.SdkWriteProcessor, "process_sdk_job", process)++ async def noop():+ return None++ async def scenario():+ first = asyncio.create_task(+ coordinator.process_sdk_job((), request_id="a", before_hy_invocation=noop)+ )+ while not seen:+ await asyncio.sleep(0)+ profiles.identity = ("profile-b", 2)+ # A second job and an inquiry under the new profile.+ await coordinator.process_sdk_job((), request_id="b", before_hy_invocation=noop)+ assert [handle.closed for handle in opened] == [False, False]+ writing.set()+ await first+ old, new = opened+ # Retired after its last use, and new work used the new profile.+ assert old.closed is True and new.closed is False+ await coordinator.close()+ assert new.closed is True++ asyncio.run(scenario())+ assert seen == [+ ("profile-a", False),+ ("profile-b", False),+ ("profile-b", False),+ ("profile-a", False),+ ]+++def test_an_unused_stale_handle_still_closes_at_once(monkeypatch) -> None:+ opened: list[_TrackedHandle] = []+ coordinator, profiles = _coordinator(monkeypatch, opened)++ async def scenario():+ async with coordinator.use_client() as client:+ assert client.profile_id == "profile-a"+ assert opened[0].closed is False+ profiles.identity = ("profile-b", 2)+ async with coordinator.use_client() as client:+ assert client.profile_id == "profile-b"+ assert opened[0].closed is True+ assert opened[1].closed is False+ await coordinator.close()+ assert opened[1].closed is True++ asyncio.run(scenario())+++def test_a_client_that_fails_to_construct_parks_the_job_instead_of_unknown(tmp_path) -> None:+ """Nothing was sent when the client could not be built, so it retries on its own."""+ from one_memory.sdk_write_processor import SdkWriteProcessor++ class _BrokenHandle:+ async def client(self):+ raise AttributeError("'RustBindingsAPI' object has no attribute 'bindings'")++ path = tmp_path / "one-memory.sqlite3"+ queue = DurableWriteQueue(path)+ record = queue.submit(_request())+ _claim_through_drain(queue, record.job_id)++ class _Supervisor:+ async def process_sdk_job(self, _records, **kwargs):+ return await SdkWriteProcessor(_BrokenHandle()).process_sdk_job( # type: ignore[arg-type]+ (_sdk_evidence("e1"),), **kwargs+ )++ asyncio.run(+ execute_job(+ record.job_id,+ queue=queue,+ supervisor=_Supervisor(),+ evidence=_EvidenceStore({"e1"}),+ )+ )++ [row] = _rows(path)+ assert (row["state"], row["hy_invoked"]) == (WAITING_FOR_MODEL, 0)+ assert row["last_error"] == "hy_memory_client_unavailable"+ later = datetime.now(UTC) + timedelta(minutes=5)+ assert queue.readmit_model_waiters(now=later) == 1+ assert _state(path, record.job_id) == PENDING++def test_sdk_unknown_outcome_inquiry_uses_only_public_receipt() -> None:from one_memory.sdk_write_processor import SdkWriteProcessor, SdkWriteReceipt@@ -1699,6 +1860,46 @@ def test_a_raw_provider_message_is_never_stored_as_the_error(tmp_path) -> None:assert stored == "RuntimeError"+def test_a_failed_job_logs_where_it_failed_but_never_the_message(tmp_path, caplog) -> None:+ """`error_code=AttributeError` alone named no location; frames, never the message."""+ path = tmp_path / "one-memory.sqlite3"+ queue = DurableWriteQueue(path)+ record = queue.submit(_request())+ _claim_through_drain(queue, record.job_id)++ def closed_store(client):+ return client.private_store_name.write++ class _Failing:+ async def process_sdk_job(self, _records, **kwargs):+ await kwargs["before_hy_invocation"]()+ try:+ closed_store(type("Client", (), {"private_store_name": None})())+ except AttributeError as error:+ raise RuntimeError("quoting 'my secret diary'") from error++ with caplog.at_level(logging.ERROR, logger="one_memory.memory_server.durable_queue"):+ asyncio.run(+ execute_job(+ record.job_id,+ queue=queue,+ supervisor=_Failing(),+ evidence=_EvidenceStore({"e1"}),+ )+ )++ [line] = [+ message.getMessage()+ for message in caplog.records+ if "durable_write_job_failed" in message.getMessage()+ ]+ assert f"job_id={record.job_id} error_code=RuntimeError chain=RuntimeError@" in line+ assert ";AttributeError@" in line+ assert "test_memory_server_durable_queue.py:" in line and ":closed_store" in line+ assert "secret diary" not in line+ assert "private_store_name" not in line++def test_a_machine_error_code_is_preserved(tmp_path) -> None:"""Recognised codes survive; only free-form text is dropped."""from one_memory.memory_server.durable_queue import safe_error_codediff --git a/memhub/apps/core/tests/test_sdk_client_construction.py b/memhub/apps/core/tests/test_sdk_client_construction.pynew file mode 100644index 0000000..93f828a--- /dev/null+++ b/memhub/apps/core/tests/test_sdk_client_construction.py@@ -0,0 +1,61 @@+"""SDK clients of one home are constructed one at a time.++Build 35, clean install: right after the model was activated, the write+drain's client and a UI read client started in the same second. Each opened+Chroma on the same empty vector directory, Chroma's shared System cache has+no lock around that first open, and neither client reached `VectorStore+ready`. The first write job was lost as unknown.+"""++import threading+import time++from one_memory import hy_sdk_runtime+from one_memory.hy_sdk_runtime import acquire_client, builtin_retrieval_snapshot+++def test_two_clients_on_a_fresh_home_are_constructed_one_at_a_time(tmp_path, monkeypatch):+ real = hy_sdk_runtime.HyMemoryClient+ state = {"inside": 0, "most": 0}+ count = threading.Lock()++ def recording_client(**options):+ with count:+ state["inside"] += 1+ state["most"] = max(state["most"], state["inside"])+ try:+ # Widen the window two unserialized constructors would share.+ time.sleep(0.2)+ return real(**options)+ finally:+ with count:+ state["inside"] -= 1++ monkeypatch.setattr(hy_sdk_runtime, "HyMemoryClient", recording_client)+ start = threading.Barrier(2)+ leases: list[hy_sdk_runtime.ClientLease] = []+ failures: list[BaseException] = []++ def open_client():+ start.wait()+ try:+ leases.append(+ acquire_client(builtin_retrieval_snapshot(), api_key=None, data_dir=tmp_path)+ )+ except BaseException as error: # noqa: BLE001 - reported below+ failures.append(error)++ threads = [threading.Thread(target=open_client) for _ in range(2)]+ for thread in threads:+ thread.start()+ for thread in threads:+ thread.join(120)+ try:+ assert failures == []+ assert len(leases) == 2+ assert state["most"] == 1+ for lease in leases:+ assert lease.client.memory_stats(user_id="local") is not None+ finally:+ for lease in leases:+ lease.close()diff --git a/memhub/apps/core/tests/test_sdk_data_plane_migration.py b/memhub/apps/core/tests/test_sdk_data_plane_migration.pyindex e628e2a..b7d6bcb 100644--- a/memhub/apps/core/tests/test_sdk_data_plane_migration.py+++ b/memhub/apps/core/tests/test_sdk_data_plane_migration.py@@ -37,6 +37,7 @@ from one_memory.sdk_migration_runner import (abandoned_legacy_work,commit_sdk_cutover,move_pending_dispatches,+ prepare_legacy_work_for_cutover,prepare_legacy_write_queue_for_cutover,recover_abandoned_staging,resume_legacy_cleanup,@@ -896,6 +897,116 @@ def test_legacy_work_that_may_have_reached_the_sdk_does_not_block_cutover(tmp_paassert store.authority() is MemoryAuthority.SDK+def test_pre_upgrade_unknown_writes_are_settled_at_cutover_not_retried(store):+ """A migrated 1.0.4 home kept 1000+ jobs unknown forever.++ The legacy pipeline invoked them, so the SDK's write-outcome inquiry knows+ none of them and the automatic reconcile could never settle one.+ """+ from one_memory.memory_server.durable_queue import (+ CANCELLED,+ PENDING,+ RUNNING,+ UNKNOWN,+ )++ queue = DurableWriteQueue(store.database_path)++ def job(key: str, state: str, *, invoked: bool) -> str:+ identifier = queue.submit(+ WriteRequest(+ internal_session_id="session-1",+ source_id="cursor",+ batch_identity=BatchIdentity(+ internal_session_id="session-1", first_round=1, last_round=1+ ),+ evidence_ids=["gone"],+ idempotency_key=key,+ )+ ).job_id+ with main_database(store.database_path).write() as database, database:+ database.execute(+ "UPDATE memory_write_jobs SET state=?,hy_invoked=?,"+ "last_error='model_content_outcome_unknown' WHERE job_id=?",+ (state, int(invoked), identifier),+ )+ return identifier++ # Their Evidence is gone too, which must not block the cutover.+ unknown = [job(f"unknown-{index}", UNKNOWN, invoked=True) for index in range(3)]+ sent_running = job("sent-running", RUNNING, invoked=True)+ finished = job("finished", "completed", invoked=True)++ first = prepare_legacy_work_for_cutover(store.database_path)+ assert first.abandoned == {"unknown_write_jobs": 4}+ # Resumed after an interruption: nothing changes, the count is the same.+ again = prepare_legacy_work_for_cutover(store.database_path)+ assert again.as_document() == first.as_document()++ with main_database(store.database_path).read() as database:+ rows = dict(+ (str(row[0]), (str(row[1]), str(row[2]), int(row[3])))+ for row in database.execute(+ "SELECT job_id,state,last_error,hy_invoked FROM memory_write_jobs"+ )+ )+ for identifier in (*unknown, sent_running):+ assert rows[identifier] == (CANCELLED, "abandoned_at_cutover", 1)+ assert rows[finished][0] == "completed"+ surviving = DurableWriteQueue(store.database_path, sdk_authority=True)+ assert surviving.unknown_jobs() == ()+ # Never requeued: they may already have written.+ assert surviving.retry_unknown_confirmed(unknown[0]) == 0+ assert surviving.readmit_parked() == 0+ assert PENDING not in {state for state, _, _ in rows.values()}+++def test_retrying_unknown_writes_works_after_the_legacy_tables_are_dropped(store):+ """The retry button raised `no such table: memory_tasks` after an upgrade."""+ from one_memory.memory_server.durable_queue import PENDING, UNKNOWN+ from one_memory.target_schema import drop_legacy_tables++ queue = DurableWriteQueue(store.database_path)+ identifier = queue.submit(+ WriteRequest(+ internal_session_id="session-1",+ source_id="cursor",+ batch_identity=BatchIdentity(+ internal_session_id="session-1", first_round=1, last_round=1+ ),+ evidence_ids=["e1"],+ idempotency_key="after-upgrade",+ )+ ).job_id+ drop_legacy_tables(store.database_path)+ assert legacy_tables_present(store.database_path) == ()+ old = datetime(2000, 1, 1, tzinfo=UTC).isoformat()++ def make_unknown(code: str, attempts: int) -> None:+ with main_database(store.database_path).write() as database, database:+ database.execute(+ "UPDATE memory_write_jobs SET state=?,hy_invoked=1,last_error=?,"+ "attempts=?,updated_at=? WHERE job_id=?",+ (UNKNOWN, code, attempts, old, identifier),+ )++ for sdk_authority in (True, False):+ surviving = DurableWriteQueue(store.database_path, sdk_authority=sdk_authority)+ make_unknown("model_content_outcome_unknown", 1)+ assert surviving.resolve_unknown_from_linked_output(identifier) is False+ assert surviving.retry_unknown_confirmed(identifier) == 1+ assert surviving.get("after-upgrade").state == PENDING++ make_unknown("hy_memory_timeout", 1)+ since = datetime.now(UTC)+ assert surviving.retry_timed_out_unknown_after_reconcile(older_than=since) == 1+ make_unknown("model_content_outcome_unknown", 1)+ assert surviving.retry_first_unknown_after_reconcile(older_than=since) == 1+ if sdk_authority:+ # The SDK-authority queue never brings a dropped table back.+ assert legacy_tables_present(store.database_path) == ()++def _protective(home: Path) -> Path:return home / "hy-memory" / ".governed-v2.pre-migration-0123456789abcdef"