Skip to content
Python

fix(memhub): serialize SDK client startup, log failure frames, settle pre-upgrade unknown writes

alvinfei352fffb3dadf
diff --git a/memhub/apps/core/src/one_memory/api.py b/memhub/apps/core/src/one_memory/api.py
index 50513be..611ecc8 100644
--- a/memhub/apps/core/src/one_memory/api.py
+++ b/memhub/apps/core/src/one_memory/api.py
@@ -3150,25 +3150,25 @@ def create_app(
raise HTTPException(status_code=422, detail="memory_task_requires_experience")
namespace = _namespace_kwargs(request.kind, request.variant)
try:
- client = await app.state.sdk_write_coordinator.client()
- update_kwargs: dict[str, object] = dict(namespace)
- if request.task is not None:
- update_kwargs["task"] = request.task
- result = await asyncio.to_thread(
- client.update, memory_id, request.content, **update_kwargs
- )
- if not isinstance(result, dict):
- raise HySdkError("hy_memory_update_rejected")
- if result.get("success") is not True:
- if "not found" in str(result.get("error_message") or ""):
- raise HTTPException(status_code=404, detail="memory_not_found")
- raise HySdkError("hy_memory_update_rejected")
- # The SDK names the namespace it wrote; read that one back.
- if result.get("kind") in {"experience", "fact"}:
- namespace = {"kind": result["kind"]}
- if result.get("variant") in {"working", "coding"}:
- namespace["variant"] = result["variant"]
- raw = await asyncio.to_thread(client.get, memory_id, **namespace)
+ async with app.state.sdk_write_coordinator.use_client() as client:
+ update_kwargs: dict[str, object] = dict(namespace)
+ if request.task is not None:
+ update_kwargs["task"] = request.task
+ result = await asyncio.to_thread(
+ client.update, memory_id, request.content, **update_kwargs
+ )
+ if not isinstance(result, dict):
+ raise HySdkError("hy_memory_update_rejected")
+ if result.get("success") is not True:
+ if "not found" in str(result.get("error_message") or ""):
+ raise HTTPException(status_code=404, detail="memory_not_found")
+ raise HySdkError("hy_memory_update_rejected")
+ # The SDK names the namespace it wrote; read that one back.
+ if result.get("kind") in {"experience", "fact"}:
+ namespace = {"kind": result["kind"]}
+ if result.get("variant") in {"working", "coding"}:
+ namespace["variant"] = result["variant"]
+ raw = await asyncio.to_thread(client.get, memory_id, **namespace)
memory = None if raw is None else HySdkMemoryRecord.from_sdk(raw)
except HTTPException:
raise
@@ -3197,10 +3197,10 @@ def create_app(
) -> Response:
_check_memory_namespace(kind, variant)
try:
- client = await app.state.sdk_write_coordinator.client()
- result = await asyncio.to_thread(
- client.delete, memory_id, **_namespace_kwargs(kind, variant)
- )
+ async with app.state.sdk_write_coordinator.use_client() as client:
+ result = await asyncio.to_thread(
+ client.delete, memory_id, **_namespace_kwargs(kind, variant)
+ )
if not isinstance(result, dict):
raise HySdkError("hy_memory_delete_rejected")
error_code = result.get("error_code")
@@ -3442,9 +3442,9 @@ def create_app(
"""Retry unknown writes only behind the user's explicit confirmation."""
del request
reconciled = await _sdk_memory_runtime(app).reconcile_unknown_outputs()
- from one_memory.memory_server.durable_queue import DurableWriteQueue
-
- queue = DurableWriteQueue(app.state.memory_write_queue_path)
+ # The startup queue knows the authority. A fresh legacy-mode queue
+ # would recreate the job-evidence table the SDK cutover dropped.
+ queue = app.state.memory_write_queue
unknown = queue.unknown_jobs(limit=2048)
if app.state.memory_authority is MemoryAuthority.LEGACY:
for job in unknown:
diff --git a/memhub/apps/core/src/one_memory/hy_sdk_runtime.py b/memhub/apps/core/src/one_memory/hy_sdk_runtime.py
index 4b4d2a6..3b4f626 100644
--- a/memhub/apps/core/src/one_memory/hy_sdk_runtime.py
+++ b/memhub/apps/core/src/one_memory/hy_sdk_runtime.py
@@ -67,6 +67,16 @@ _STORE_GATE = threading.Condition()
_OFFLINE_STORES: set[Path] = set()
_OPENING: dict[Path, int] = {}
+# One client constructs at a time per home, its first-time store setup
+# included. The constructor opens Chroma's `PersistentClient`, and two first
+# opens on one directory race over that same unlocked System cache. On a
+# clean install the write drain's client and a UI read client started in the
+# same second; neither reached `VectorStore ready`, and the first write job
+# was lost as unknown. Taken inside `acquire_client`, after the offline gate
+# and never while `_STORE_GATE` is held, so it cannot deadlock with
+# `hold_stores_offline`, which refuses while `_OPENING` counts a client.
+_CONSTRUCTING: dict[Path, threading.Lock] = {}
+
class LocalOnnxEmbeddingService:
"""Public ``EmbeddingService`` implementation backed by MemHub's ONNX model."""
@@ -382,13 +392,16 @@ def _acquire_client(
embedding_service: LocalOnnxEmbeddingService | None = None
if (profile.embedding_provider_kind or profile.provider_kind) is ProviderKind.LOCAL_ONNX:
embedding_service = LocalOnnxEmbeddingService()
+ with _STORE_GATE:
+ constructing = _CONSTRUCTING.setdefault(key, threading.Lock())
try:
- client = HyMemoryClient(
- config=config,
- mode="ultra" if graph_store_enabled() else "pro",
- runtime=runtime,
- embedding_service=embedding_service,
- )
+ with constructing:
+ client = HyMemoryClient(
+ config=config,
+ mode="ultra" if graph_store_enabled() else "pro",
+ runtime=runtime,
+ embedding_service=embedding_service,
+ )
except BaseException:
_release_runtime(key)
raise
diff --git a/memhub/apps/core/src/one_memory/memory_server/durable_queue.py b/memhub/apps/core/src/one_memory/memory_server/durable_queue.py
index b755ba9..5d74014 100644
--- a/memhub/apps/core/src/one_memory/memory_server/durable_queue.py
+++ b/memhub/apps/core/src/one_memory/memory_server/durable_queue.py
@@ -52,6 +52,7 @@ from one_memory.memory_tasks import (
from one_memory.memory_tasks import (
POLICY_STOPPED_CODES as _TASK_POLICY_STOPPED_CODES,
)
+from one_memory.sdk_migration_log import error_chain_text
from one_memory.service_levels import ServiceLevel, service_level_context
logger = logging.getLogger(__name__)
@@ -241,14 +242,16 @@ PERMISSION_REQUIRED_CODES = frozenset(
)
#: Model-side codes the SDK write coordinator raises before it can lease a
-#: client (no active profile, missing credential, secure storage down). They prove
-#: nothing was sent, so the write parks instead of being lost. Transport codes
-#: that can arrive after a request was in flight stay `unknown`.
+#: client (no active profile, missing credential, secure storage down, or the
+#: client failed to construct). They prove nothing was sent, so the write
+#: parks instead of being lost. Transport codes that can arrive after a
+#: request was in flight stay `unknown`.
MODEL_NOT_READY_CODES = frozenset(
{
"model_profile_not_active",
"model_secret_unavailable",
"secure_storage_unavailable",
+ "hy_memory_client_unavailable",
}
)
@@ -292,6 +295,23 @@ _VALID_JOB_STATES = frozenset(
#: A row in one of these states can never be moved back to a live one.
_TERMINAL_WRITE_STATES = frozenset({COMPLETED, FAILED, DEAD, CANCELLED})
+#: The legacy task rows a write job can be linked to. The SDK cutover drops
+#: them (`target_schema.LEGACY_TABLES_IN_DROP_ORDER`, inputs before tasks), so
+#: the job-only paths must not assume any of them.
+_LEGACY_TASK_LINK_TABLES = ("memory_tasks", "memory_task_inputs", "memory_write_job_evidence")
+
+
+def _legacy_task_links_present(connection: sqlite3.Connection) -> bool:
+ present = {
+ str(row[0])
+ for row in connection.execute(
+ "SELECT name FROM sqlite_master WHERE type='table' AND name IN (?,?,?)",
+ _LEGACY_TASK_LINK_TABLES,
+ )
+ }
+ return present == set(_LEGACY_TASK_LINK_TABLES)
+
+
#: Columns a current schema declares but an older `memory_write_jobs` may lack.
#: `CREATE TABLE IF NOT EXISTS` is a no-op for an existing table, so without
#: this an upgraded install would keep an older queue that every claim/lease
@@ -1982,21 +2002,24 @@ class DurableWriteQueue:
or str(row["last_error"]) in NON_RETRYABLE_POST_WRITE_CODES
):
return 0
- connection.execute(
- """
- UPDATE memory_tasks
- SET status='pending', runner_token=NULL, attempts=0,
- last_error_code='model_content_manual_retry_requested', updated_at=?
- WHERE status='unknown' AND task_id IN (
- SELECT inputs.task_id
- FROM memory_task_inputs AS inputs, json_each(inputs.refs_json) AS ref
- JOIN memory_write_job_evidence AS linked
- ON linked.evidence_id=json_extract(ref.value, '$.evidence_id')
- WHERE linked.job_id=?
+ # After the SDK cutover the legacy task tables are gone and the
+ # job is the only row to reset.
+ if not self.sdk_authority and _legacy_task_links_present(connection):
+ connection.execute(
+ """
+ UPDATE memory_tasks
+ SET status='pending', runner_token=NULL, attempts=0,
+ last_error_code='model_content_manual_retry_requested', updated_at=?
+ WHERE status='unknown' AND task_id IN (
+ SELECT inputs.task_id
+ FROM memory_task_inputs AS inputs, json_each(inputs.refs_json) AS ref
+ JOIN memory_write_job_evidence AS linked
+ ON linked.evidence_id=json_extract(ref.value, '$.evidence_id')
+ WHERE linked.job_id=?
+ )
+ """,
+ (now, job_id),
)
- """,
- (now, job_id),
- )
changed = connection.execute(
"""
UPDATE memory_write_jobs
@@ -2098,9 +2121,7 @@ class DurableWriteQueue:
f"WHERE state=? AND job_id IN ({placeholders})",
(PENDING, now, UNKNOWN, *job_ids),
)
- if connection.execute(
- "SELECT 1 FROM sqlite_master WHERE type='table' AND name='memory_tasks'"
- ).fetchone():
+ if _legacy_task_links_present(connection):
connection.execute(
f"UPDATE memory_tasks SET status=?, runner_token=NULL, "
f"last_error_code='model_content_reconciled_absent_retry', updated_at=? "
@@ -2158,9 +2179,7 @@ class DurableWriteQueue:
f"WHERE state=? AND attempts=1 AND job_id IN ({placeholders})",
(PENDING, now, UNKNOWN, *job_ids),
)
- if connection.execute(
- "SELECT 1 FROM sqlite_master WHERE type='table' AND name='memory_tasks'"
- ).fetchone():
+ if _legacy_task_links_present(connection):
connection.execute(
f"UPDATE memory_tasks SET status=?, runner_token=NULL, "
f"last_error_code='model_content_unknown_auto_retry', updated_at=? "
@@ -2177,6 +2196,14 @@ class DurableWriteQueue:
def resolve_unknown_from_linked_output(self, job_id: str) -> bool:
"""Close an unknown durable row when local governance output proves success."""
with main_database(self.database_path).read(row_factory=sqlite3.Row) as connection:
+ if not (
+ _legacy_task_links_present(connection)
+ and connection.execute(
+ "SELECT 1 FROM sqlite_master WHERE type='table' AND name='memory_task_outputs'"
+ ).fetchone()
+ ):
+ # No legacy governance output survives the SDK cutover.
+ return False
row = connection.execute(
"""
SELECT 1
@@ -2559,9 +2586,22 @@ async def execute_job(
if code == "model_profile_not_active":
logger.debug("durable_write_job_parked job_id=%s error_code=%s", job_id, code)
elif state in PARKED_STATES:
- logger.info("durable_write_job_parked job_id=%s error_code=%s", job_id, code)
+ logger.info(
+ "durable_write_job_parked job_id=%s error_code=%s chain=%s",
+ job_id,
+ code,
+ error_chain_text(error),
+ )
else:
- logger.error("durable_write_job_failed job_id=%s error_code=%s", job_id, code)
+ # Where it failed, as `file:line:function` frames of the exception
+ # chain, never its message: `error_code=AttributeError` alone
+ # named no location.
+ logger.error(
+ "durable_write_job_failed job_id=%s error_code=%s chain=%s",
+ job_id,
+ code,
+ error_chain_text(error),
+ )
return
await in_the_background(
queue.settle,
diff --git a/memhub/apps/core/src/one_memory/sdk_memory_runtime.py b/memhub/apps/core/src/one_memory/sdk_memory_runtime.py
index fd247e5..4e13689 100644
--- a/memhub/apps/core/src/one_memory/sdk_memory_runtime.py
+++ b/memhub/apps/core/src/one_memory/sdk_memory_runtime.py
@@ -5,7 +5,8 @@ from __future__ import annotations
import asyncio
import logging
import re
-from contextlib import suppress
+from collections.abc import AsyncIterator
+from contextlib import AsyncExitStack, asynccontextmanager, suppress
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
@@ -164,44 +165,45 @@ class SdkMemoryRuntime:
if str(error) == "model_profile_not_active":
return HyGraphSnapshot(consulted=False)
raise
- try:
- client = await self.coordinator.client()
- except Exception as error:
- raise HySdkError("hy_memory_graph_read_failed") from error
- return await asyncio.to_thread(
- read_graph_snapshot,
- client,
- "local",
- "*",
- limit,
- )
+ async with AsyncExitStack() as stack:
+ try:
+ client = await stack.enter_async_context(self.coordinator.use_client())
+ except Exception as error:
+ raise HySdkError("hy_memory_graph_read_failed") from error
+ return await asyncio.to_thread(
+ read_graph_snapshot,
+ client,
+ "local",
+ "*",
+ limit,
+ )
async def digest(self) -> dict[str, object]:
- client = await self.coordinator.client()
- try:
- raw = await asyncio.to_thread(
- client.list_memories,
- user_id="local",
- limit=1_000,
- offset=0,
- )
- agent_ids = _agent_ids(raw)
- digests = [
- {
- "agent_id": agent_id,
- "schemas_recorded": 0,
- **dict(
- await asyncio.to_thread(
- client.evolve,
- user_id="local",
- agent_id=agent_id,
- )
- ),
- }
- for agent_id in agent_ids
- ]
- except Exception as error:
- raise HySdkError("hy_memory_digest_failed") from error
+ async with self.coordinator.use_client() as client:
+ try:
+ raw = await asyncio.to_thread(
+ client.list_memories,
+ user_id="local",
+ limit=1_000,
+ offset=0,
+ )
+ agent_ids = _agent_ids(raw)
+ digests = [
+ {
+ "agent_id": agent_id,
+ "schemas_recorded": 0,
+ **dict(
+ await asyncio.to_thread(
+ client.evolve,
+ user_id="local",
+ agent_id=agent_id,
+ )
+ ),
+ }
+ for agent_id in agent_ids
+ ]
+ except Exception as error:
+ raise HySdkError("hy_memory_digest_failed") from error
return {
"success": all(result.get("success") is not False for result in digests),
"scope_count": len(digests),
@@ -240,36 +242,39 @@ class SdkMemoryRuntime:
decision=decision,
)
- async def _mutation_client(self) -> Any:
+ @asynccontextmanager
+ async def _mutation_client(self) -> AsyncIterator[Any]:
"""The write client, with no-model and keychain refusals as SDK codes.
Callers record these against the edit or deletion instead of failing
the request that already committed it.
"""
- try:
- return await self.coordinator.client()
- except RuntimeError as error:
- code = str(error)
- if not _STABLE_CODE.fullmatch(code):
- code = "hy_memory_client_unavailable"
- raise HySdkError(code) from error
+ async with AsyncExitStack() as stack:
+ try:
+ client = await stack.enter_async_context(self.coordinator.use_client())
+ except RuntimeError as error:
+ code = str(error)
+ if not _STABLE_CODE.fullmatch(code):
+ code = "hy_memory_client_unavailable"
+ raise HySdkError(code) from error
+ yield client
async def update_hy_memory(self, memory_id: str, content: str) -> dict[str, object]:
- client = await self._mutation_client()
- try:
- result = await asyncio.to_thread(client.update, memory_id, content)
- except Exception as error:
- raise HySdkError("hy_memory_update_failed") from error
+ async with self._mutation_client() as client:
+ try:
+ result = await asyncio.to_thread(client.update, memory_id, content)
+ except Exception as error:
+ raise HySdkError("hy_memory_update_failed") from error
if not isinstance(result, dict) or result.get("success") is not True:
raise HySdkError("hy_memory_update_rejected")
return result
async def finish_deletion(self, memory_id: str) -> None:
- client = await self._mutation_client()
- try:
- result = await asyncio.to_thread(client.delete, memory_id)
- except Exception as error:
- raise HySdkError("hy_memory_delete_failed") from error
+ async with self._mutation_client() as client:
+ try:
+ result = await asyncio.to_thread(client.delete, memory_id)
+ except Exception as error:
+ raise HySdkError("hy_memory_delete_failed") from error
if not isinstance(result, dict):
raise HySdkError("hy_memory_delete_rejected")
error_code = result.get("error_code")
diff --git a/memhub/apps/core/src/one_memory/sdk_migration_runner.py b/memhub/apps/core/src/one_memory/sdk_migration_runner.py
index 203eddc..a98ef46 100644
--- a/memhub/apps/core/src/one_memory/sdk_migration_runner.py
+++ b/memhub/apps/core/src/one_memory/sdk_migration_runner.py
@@ -35,6 +35,8 @@ from one_memory.memory_server.durable_queue import (
COMPLETED,
FAILED,
REAPED_PAYLOAD_MARKER,
+ RUNNING,
+ UNKNOWN,
DurableWriteQueue,
)
from one_memory.memory_server.unified_entry import UnifiedWriteIntake
@@ -317,11 +319,45 @@ def move_pending_dispatches(database_path: Path) -> int:
return len(conversations)
+#: Why a pre-cutover write job that may have reached HY was closed.
+ABANDONED_AT_CUTOVER = "abandoned_at_cutover"
+
+
+def settle_legacy_unknown_write_jobs(database_path: Path) -> int:
+ """Close write jobs the legacy pipeline left possibly-sent. Safe to repeat.
+
+ They ran under the legacy pipeline, so the SDK's `get_write_outcome` knows
+ none of them and the unknown reconcile can never settle them: a migrated
+ 1.0.4 home showed 1000+ unknown forever. They are not retried, since they
+ may already have written and the SDK has no idempotency key. `cancelled`,
+ not `failed`: the upgrade ended them, nothing broke, and a `failed` count
+ would put the home in the failed state for good. `hy_invoked` is kept, so
+ no readmission path can pick one up again.
+ """
+ with main_database(database_path).read(row_factory=sqlite3.Row) as database:
+ if "memory_write_jobs" not in _tables(database):
+ return 0
+ now = datetime.now(UTC).isoformat()
+ with main_database(database_path).write(autocommit=True) as database:
+ # A `running` row already marked invoked would be reclaimed to
+ # `unknown` on the next open, so it is the same case.
+ return int(
+ database.execute(
+ "UPDATE memory_write_jobs SET state=?,lease_owner=NULL,"
+ "lease_expires_at=NULL,last_error=?,updated_at=? "
+ "WHERE state=? OR (state=? AND hy_invoked=1)",
+ (CANCELLED, ABANDONED_AT_CUTOVER, now, UNKNOWN, RUNNING),
+ ).rowcount
+ )
+
+
def abandoned_legacy_work(database_path: Path) -> dict[str, int]:
"""Count legacy work that ends with the cutover instead of blocking it.
A task that may have reached the SDK is not replayed: the SDK already holds
whatever it committed. Pending user deletions are not carried over.
+ Settled write jobs are counted by their code, so a repeated preparation
+ reports the same number.
"""
abandoned: dict[str, int] = {}
with main_database(database_path).read(row_factory=sqlite3.Row) as database:
@@ -332,6 +368,11 @@ def abandoned_legacy_work(database_path: Path) -> dict[str, int]:
"memory_tasks",
"status='unknown' OR (status='running' AND hy_invocation_count > 0)",
),
+ (
+ "unknown_write_jobs",
+ "memory_write_jobs",
+ f"state='{CANCELLED}' AND last_error='{ABANDONED_AT_CUTOVER}'",
+ ),
("deletions", "memory_deletions", "status IN ('pending','running')"),
)
for name, table, where in checks:
@@ -354,6 +395,10 @@ def prepare_legacy_work_for_cutover(
running, so a failure names it.
"""
trace = {} if trace is None else trace
+ # First: a possibly-sent job never runs again, so its input is not
+ # frozen, and input released since must not block the cutover.
+ trace["sub_step"] = "unknown_write_jobs"
+ settle_legacy_unknown_write_jobs(database_path)
trace["sub_step"] = "write_queue"
frozen, moved = prepare_legacy_write_queue_for_cutover(database_path)
trace["sub_step"] = "pending_dispatches"
diff --git a/memhub/apps/core/src/one_memory/sdk_write_processor.py b/memhub/apps/core/src/one_memory/sdk_write_processor.py
index f6e4d98..dd5d94a 100644
--- a/memhub/apps/core/src/one_memory/sdk_write_processor.py
+++ b/memhub/apps/core/src/one_memory/sdk_write_processor.py
@@ -3,7 +3,8 @@
from __future__ import annotations
import asyncio
-from collections.abc import Awaitable, Callable, Mapping, Sequence
+from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence
+from contextlib import asynccontextmanager
from dataclasses import dataclass
from datetime import datetime
from typing import Any
@@ -56,7 +57,12 @@ class SdkWriteProcessor:
scope = _write_scope(records)
metadata = _write_metadata(records, scope)
- client = await self._client_handle.client()
+ try:
+ client = await self._client_handle.client()
+ except Exception as error:
+ # Before `before_hy_invocation`: nothing was sent, so the job
+ # waits and retries instead of being left `unknown`.
+ raise RuntimeError("hy_memory_client_unavailable") from error
if extracted:
contents = [sanitize_model_input(record.verbatim) for record in records]
@@ -125,7 +131,14 @@ class SdkWriteProcessor:
class SdkWriteCoordinator:
- """Resolve the active model profile, then execute one public SDK write."""
+ """Resolve the active model profile, then execute one public SDK write.
+
+ A model profile change retires the previous profile's handle, but only
+ closes it once nothing uses it. Closing it at once tore the client out
+ from under a write still running on it: the SDK cleared its stores and
+ the SharedRuntime closed mid-`add`, the job failed with AttributeError
+ after HY was invoked, and it was left `unknown`.
+ """
def __init__(
self,
@@ -142,6 +155,9 @@ class SdkWriteCoordinator:
self.transport_factory = transport_factory
self.sedimentation_enabled = sedimentation_enabled
self._handles: dict[tuple[str, int], SdkClientHandle] = {}
+ #: Uses in flight per handle; a retired handle closes at zero.
+ self._uses: dict[SdkClientHandle, int] = {}
+ self._retired: set[SdkClientHandle] = set()
self._lock = asyncio.Lock()
async def process_sdk_job(
@@ -153,28 +169,30 @@ class SdkWriteCoordinator:
) -> SdkWriteReceipt:
if not self.sedimentation_enabled():
raise RuntimeError("model_sedimentation_disabled")
- handle = await self._active_handle()
- return await SdkWriteProcessor(handle).process_sdk_job(
- records,
- request_id=request_id,
- before_hy_invocation=before_hy_invocation,
- )
+ async with self._active_handle() as handle:
+ return await SdkWriteProcessor(handle).process_sdk_job(
+ records,
+ request_id=request_id,
+ before_hy_invocation=before_hy_invocation,
+ )
async def inquire_write_outcome(
self, request_id: str
) -> SdkWriteReceipt | None:
- return await SdkWriteProcessor(
- await self._active_handle()
- ).inquire_write_outcome(request_id)
+ async with self._active_handle() as handle:
+ return await SdkWriteProcessor(handle).inquire_write_outcome(request_id)
async def close(self) -> None:
async with self._lock:
- handles, self._handles = tuple(self._handles.values()), {}
+ handles = (*self._handles.values(), *self._retired)
+ self._handles, self._retired, self._uses = {}, set(), {}
await asyncio.gather(*(handle.close() for handle in handles))
- async def client(self) -> Any:
- """Return the public SDK client selected by the active write profile."""
- return await (await self._active_handle()).client()
+ @asynccontextmanager
+ async def use_client(self) -> AsyncIterator[Any]:
+ """The public SDK client of the active write profile, kept open while used."""
+ async with self._active_handle() as handle:
+ yield await handle.client()
async def active_profile_credentials(
self,
@@ -183,7 +201,9 @@ class SdkWriteCoordinator:
_identity, profile, secret = await asyncio.to_thread(self._read_active_credentials)
return profile, secret
- async def _active_handle(self) -> SdkClientHandle:
+ @asynccontextmanager
+ async def _active_handle(self) -> AsyncIterator[SdkClientHandle]:
+ """Use the active profile's handle; retire the others, closing the unused."""
identity, profile, secret = await asyncio.to_thread(
self._read_active_credentials
)
@@ -197,14 +217,25 @@ class SdkWriteCoordinator:
http_transport_factory=self.transport_factory(profile),
)
self._handles[identity] = handle
- stale = [
- value
- for key, value in self._handles.items()
- if key != identity
- ]
+ self._retired.update(value for key, value in self._handles.items() if key != identity)
self._handles = {identity: handle}
- await asyncio.gather(*(value.close() for value in stale))
- return handle
+ self._uses[handle] = self._uses.get(handle, 0) + 1
+ unused = {value for value in self._retired if not self._uses.get(value)}
+ self._retired -= unused
+ await asyncio.gather(*(value.close() for value in unused))
+ try:
+ yield handle
+ finally:
+ # Bookkeeping without an await, so a cancelled caller still
+ # releases its use.
+ remaining = self._uses.get(handle, 0) - 1
+ if remaining > 0:
+ self._uses[handle] = remaining
+ else:
+ self._uses.pop(handle, None)
+ if remaining <= 0 and handle in self._retired:
+ self._retired.discard(handle)
+ await handle.close()
def _read_active_credentials(
self,
diff --git a/memhub/apps/core/tests/TEST_INDEX.md b/memhub/apps/core/tests/TEST_INDEX.md
index 7f2d910..b38c5bb 100644
--- a/memhub/apps/core/tests/TEST_INDEX.md
+++ b/memhub/apps/core/tests/TEST_INDEX.md
@@ -246,6 +246,7 @@
| `test_schema_gate.py` | `contract` | 覆盖 fresh database records the build schema、future schema refuses to open、legacy database without a ledger is left untouched 场景。 |
| `test_schema_migrations.py` | `contract` | 覆盖 evidence store adds source ordinal to previous schema、import store migrates previous job and error schema、old active job index lets a sync sit beside paused history 等 4 个场景。 |
| `test_schema_ownership_gate.py` | `contract` | A1-03/A1-04: schema ownership, and the hard part of telling DDL from reads. |
+| `test_sdk_client_construction.py` | `contract` | SDK clients of one home are constructed one at a time. |
| `test_sdk_data_plane_migration.py` | `contract` | SDK-authority migration ledger, aliases and atomic cutover marker. |
| `test_sdk_memory_qa.py` | `contract` | 覆盖 sdk ask propagates sdk search failure without legacy fallback、sdk ask revalidates memory before returning extract、sdk ask uses native sdk memory without governance fields 场景。 |
| `test_sdk_migration_bundle.py` | `contract` | SDK-authority cutover requires a verified management + SDK data-plane bundle. |
@@ -308,4 +309,4 @@
| `test_workspace_recall.py` | `contract` | Workspace memories were unreachable from every host; these pin the way in. |
| `test_write_queue_lease_renewal.py` | `contract` | Rule 8: a lease must outlive the work it covers, and be renewed while it runs. |
-共 303 个测试文件。
+共 304 个测试文件。
diff --git a/memhub/apps/core/tests/core_test_classification.py b/memhub/apps/core/tests/core_test_classification.py
index 805b8f4..7c0b208 100644
--- a/memhub/apps/core/tests/core_test_classification.py
+++ b/memhub/apps/core/tests/core_test_classification.py
@@ -14,8 +14,8 @@ from pathlib import Path
import pytest
PRIMARY_MARKERS = ("unit", "contract", "integration", "performance", "live")
-EXPECTED_TEST_FILE_COUNT = 303
-EXPECTED_TEST_FILES_SHA256 = "7993203411697fc0bf42b92fa316f222a66e6826c5b4a0132dbb0a1ca6da4e32"
+EXPECTED_TEST_FILE_COUNT = 304
+EXPECTED_TEST_FILES_SHA256 = "b28f022b9688c96ca563c59de63e95d344d2f01ba3b6020b422bf4dfb27520d6"
# Pure transformations, validation models, parity checks and state machines.
# This list is intentionally small: uncertain files default to contract.
diff --git a/memhub/apps/core/tests/test_memory_library_all_scenes.py b/memhub/apps/core/tests/test_memory_library_all_scenes.py
index 1d07cb4..d53dd41 100644
--- a/memhub/apps/core/tests/test_memory_library_all_scenes.py
+++ b/memhub/apps/core/tests/test_memory_library_all_scenes.py
@@ -6,6 +6,7 @@ and delete pass the memory's kind/variant through to the SDK.
"""
import asyncio
+from contextlib import asynccontextmanager
import pytest
from httpx import ASGITransport, AsyncClient
@@ -310,12 +311,13 @@ def run_with_sdk(tmp_path, monkeypatch, scenario):
async def close(self):
return None
+ @asynccontextmanager
async def write_client():
- return sdk
+ yield sdk
async with app.router.lifespan_context(app):
app.state.sdk_client_handle = Handle()
- monkeypatch.setattr(app.state.sdk_write_coordinator, "client", write_client)
+ monkeypatch.setattr(app.state.sdk_write_coordinator, "use_client", write_client)
async with client_for(app) as client:
await scenario(app, sdk, client)
diff --git a/memhub/apps/core/tests/test_memory_server_durable_queue.py b/memhub/apps/core/tests/test_memory_server_durable_queue.py
index df5d410..062accc 100644
--- a/memhub/apps/core/tests/test_memory_server_durable_queue.py
+++ b/memhub/apps/core/tests/test_memory_server_durable_queue.py
@@ -1187,6 +1187,167 @@ def test_the_drain_tick_writes_a_quiet_live_conversation(tmp_path, monkeypatch)
assert flushed
+class _SwitchableProfiles:
+ """Active profile identity that a test can change mid-write."""
+
+ def __init__(self) -> None:
+ self.identity = ("profile-a", 1)
+
+ def active_identity(self):
+ return self.identity
+
+ def snapshot(self, profile_id, revision):
+ return type(
+ "Snapshot", (), {"profile_id": profile_id, "revision": revision, "has_api_key": False}
+ )()
+
+ def resolve_snapshot_secret(self, *_args):
+ return None
+
+
+class _TrackedHandle:
+ def __init__(self, profile, **_kwargs) -> None:
+ self.profile_id = profile.profile_id
+ self.closed = False
+
+ async def client(self):
+ return self
+
+ async def close(self):
+ self.closed = True
+
+
+def _coordinator(monkeypatch, opened: list[_TrackedHandle]):
+ from one_memory import sdk_write_processor
+
+ def handle(profile, **kwargs):
+ value = _TrackedHandle(profile, **kwargs)
+ opened.append(value)
+ return value
+
+ monkeypatch.setattr(sdk_write_processor, "SdkClientHandle", handle)
+ profiles = _SwitchableProfiles()
+ coordinator = sdk_write_processor.SdkWriteCoordinator(
+ profiles=profiles, # type: ignore[arg-type]
+ secrets=None, # type: ignore[arg-type]
+ data_dir=None,
+ transport_factory=lambda _profile: None,
+ sedimentation_enabled=lambda: True,
+ )
+ return coordinator, profiles
+
+
+def test_a_profile_change_does_not_close_the_client_of_a_write_in_flight(monkeypatch) -> None:
+ """Build 35 lost a clean install's first write as unknown (AttributeError).
+
+ Saving a model profile changed the active identity while the write ran on
+ the old profile's client. The next coordinator call closed that client at
+ once: the SDK cleared its stores and the SharedRuntime closed mid-`add`.
+ """
+ from one_memory import sdk_write_processor
+
+ opened: list[_TrackedHandle] = []
+ coordinator, profiles = _coordinator(monkeypatch, opened)
+ seen: list[tuple[str, bool]] = []
+ writing = asyncio.Event()
+
+ async def process(self, records, *, request_id, before_hy_invocation):
+ client = await self._client_handle.client()
+ await before_hy_invocation()
+ seen.append((client.profile_id, client.closed))
+ if client.profile_id == "profile-a":
+ await writing.wait()
+ # Still open while its own write runs.
+ seen.append((client.profile_id, client.closed))
+ return sdk_write_processor.SdkWriteReceipt()
+
+ monkeypatch.setattr(sdk_write_processor.SdkWriteProcessor, "process_sdk_job", process)
+
+ async def noop():
+ return None
+
+ async def scenario():
+ first = asyncio.create_task(
+ coordinator.process_sdk_job((), request_id="a", before_hy_invocation=noop)
+ )
+ while not seen:
+ await asyncio.sleep(0)
+ profiles.identity = ("profile-b", 2)
+ # A second job and an inquiry under the new profile.
+ await coordinator.process_sdk_job((), request_id="b", before_hy_invocation=noop)
+ assert [handle.closed for handle in opened] == [False, False]
+ writing.set()
+ await first
+ old, new = opened
+ # Retired after its last use, and new work used the new profile.
+ assert old.closed is True and new.closed is False
+ await coordinator.close()
+ assert new.closed is True
+
+ asyncio.run(scenario())
+ assert seen == [
+ ("profile-a", False),
+ ("profile-b", False),
+ ("profile-b", False),
+ ("profile-a", False),
+ ]
+
+
+def test_an_unused_stale_handle_still_closes_at_once(monkeypatch) -> None:
+ opened: list[_TrackedHandle] = []
+ coordinator, profiles = _coordinator(monkeypatch, opened)
+
+ async def scenario():
+ async with coordinator.use_client() as client:
+ assert client.profile_id == "profile-a"
+ assert opened[0].closed is False
+ profiles.identity = ("profile-b", 2)
+ async with coordinator.use_client() as client:
+ assert client.profile_id == "profile-b"
+ assert opened[0].closed is True
+ assert opened[1].closed is False
+ await coordinator.close()
+ assert opened[1].closed is True
+
+ asyncio.run(scenario())
+
+
+def test_a_client_that_fails_to_construct_parks_the_job_instead_of_unknown(tmp_path) -> None:
+ """Nothing was sent when the client could not be built, so it retries on its own."""
+ from one_memory.sdk_write_processor import SdkWriteProcessor
+
+ class _BrokenHandle:
+ async def client(self):
+ raise AttributeError("'RustBindingsAPI' object has no attribute 'bindings'")
+
+ path = tmp_path / "one-memory.sqlite3"
+ queue = DurableWriteQueue(path)
+ record = queue.submit(_request())
+ _claim_through_drain(queue, record.job_id)
+
+ class _Supervisor:
+ async def process_sdk_job(self, _records, **kwargs):
+ return await SdkWriteProcessor(_BrokenHandle()).process_sdk_job( # type: ignore[arg-type]
+ (_sdk_evidence("e1"),), **kwargs
+ )
+
+ asyncio.run(
+ execute_job(
+ record.job_id,
+ queue=queue,
+ supervisor=_Supervisor(),
+ evidence=_EvidenceStore({"e1"}),
+ )
+ )
+
+ [row] = _rows(path)
+ assert (row["state"], row["hy_invoked"]) == (WAITING_FOR_MODEL, 0)
+ assert row["last_error"] == "hy_memory_client_unavailable"
+ later = datetime.now(UTC) + timedelta(minutes=5)
+ assert queue.readmit_model_waiters(now=later) == 1
+ assert _state(path, record.job_id) == PENDING
+
+
def test_sdk_unknown_outcome_inquiry_uses_only_public_receipt() -> None:
from one_memory.sdk_write_processor import SdkWriteProcessor, SdkWriteReceipt
@@ -1699,6 +1860,46 @@ def test_a_raw_provider_message_is_never_stored_as_the_error(tmp_path) -> None:
assert stored == "RuntimeError"
+def test_a_failed_job_logs_where_it_failed_but_never_the_message(tmp_path, caplog) -> None:
+ """`error_code=AttributeError` alone named no location; frames, never the message."""
+ path = tmp_path / "one-memory.sqlite3"
+ queue = DurableWriteQueue(path)
+ record = queue.submit(_request())
+ _claim_through_drain(queue, record.job_id)
+
+ def closed_store(client):
+ return client.private_store_name.write
+
+ class _Failing:
+ async def process_sdk_job(self, _records, **kwargs):
+ await kwargs["before_hy_invocation"]()
+ try:
+ closed_store(type("Client", (), {"private_store_name": None})())
+ except AttributeError as error:
+ raise RuntimeError("quoting 'my secret diary'") from error
+
+ with caplog.at_level(logging.ERROR, logger="one_memory.memory_server.durable_queue"):
+ asyncio.run(
+ execute_job(
+ record.job_id,
+ queue=queue,
+ supervisor=_Failing(),
+ evidence=_EvidenceStore({"e1"}),
+ )
+ )
+
+ [line] = [
+ message.getMessage()
+ for message in caplog.records
+ if "durable_write_job_failed" in message.getMessage()
+ ]
+ assert f"job_id={record.job_id} error_code=RuntimeError chain=RuntimeError@" in line
+ assert ";AttributeError@" in line
+ assert "test_memory_server_durable_queue.py:" in line and ":closed_store" in line
+ assert "secret diary" not in line
+ assert "private_store_name" not in line
+
+
def test_a_machine_error_code_is_preserved(tmp_path) -> None:
"""Recognised codes survive; only free-form text is dropped."""
from one_memory.memory_server.durable_queue import safe_error_code
diff --git a/memhub/apps/core/tests/test_sdk_client_construction.py b/memhub/apps/core/tests/test_sdk_client_construction.py
new file mode 100644
index 0000000..93f828a
--- /dev/null
+++ b/memhub/apps/core/tests/test_sdk_client_construction.py
@@ -0,0 +1,61 @@
+"""SDK clients of one home are constructed one at a time.
+
+Build 35, clean install: right after the model was activated, the write
+drain's client and a UI read client started in the same second. Each opened
+Chroma on the same empty vector directory, Chroma's shared System cache has
+no lock around that first open, and neither client reached `VectorStore
+ready`. The first write job was lost as unknown.
+"""
+
+import threading
+import time
+
+from one_memory import hy_sdk_runtime
+from one_memory.hy_sdk_runtime import acquire_client, builtin_retrieval_snapshot
+
+
+def test_two_clients_on_a_fresh_home_are_constructed_one_at_a_time(tmp_path, monkeypatch):
+ real = hy_sdk_runtime.HyMemoryClient
+ state = {"inside": 0, "most": 0}
+ count = threading.Lock()
+
+ def recording_client(**options):
+ with count:
+ state["inside"] += 1
+ state["most"] = max(state["most"], state["inside"])
+ try:
+ # Widen the window two unserialized constructors would share.
+ time.sleep(0.2)
+ return real(**options)
+ finally:
+ with count:
+ state["inside"] -= 1
+
+ monkeypatch.setattr(hy_sdk_runtime, "HyMemoryClient", recording_client)
+ start = threading.Barrier(2)
+ leases: list[hy_sdk_runtime.ClientLease] = []
+ failures: list[BaseException] = []
+
+ def open_client():
+ start.wait()
+ try:
+ leases.append(
+ acquire_client(builtin_retrieval_snapshot(), api_key=None, data_dir=tmp_path)
+ )
+ except BaseException as error: # noqa: BLE001 - reported below
+ failures.append(error)
+
+ threads = [threading.Thread(target=open_client) for _ in range(2)]
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join(120)
+ try:
+ assert failures == []
+ assert len(leases) == 2
+ assert state["most"] == 1
+ for lease in leases:
+ assert lease.client.memory_stats(user_id="local") is not None
+ finally:
+ for lease in leases:
+ lease.close()
diff --git a/memhub/apps/core/tests/test_sdk_data_plane_migration.py b/memhub/apps/core/tests/test_sdk_data_plane_migration.py
index e628e2a..b7d6bcb 100644
--- a/memhub/apps/core/tests/test_sdk_data_plane_migration.py
+++ b/memhub/apps/core/tests/test_sdk_data_plane_migration.py
@@ -37,6 +37,7 @@ from one_memory.sdk_migration_runner import (
abandoned_legacy_work,
commit_sdk_cutover,
move_pending_dispatches,
+ prepare_legacy_work_for_cutover,
prepare_legacy_write_queue_for_cutover,
recover_abandoned_staging,
resume_legacy_cleanup,
@@ -896,6 +897,116 @@ def test_legacy_work_that_may_have_reached_the_sdk_does_not_block_cutover(tmp_pa
assert store.authority() is MemoryAuthority.SDK
+def test_pre_upgrade_unknown_writes_are_settled_at_cutover_not_retried(store):
+ """A migrated 1.0.4 home kept 1000+ jobs unknown forever.
+
+ The legacy pipeline invoked them, so the SDK's write-outcome inquiry knows
+ none of them and the automatic reconcile could never settle one.
+ """
+ from one_memory.memory_server.durable_queue import (
+ CANCELLED,
+ PENDING,
+ RUNNING,
+ UNKNOWN,
+ )
+
+ queue = DurableWriteQueue(store.database_path)
+
+ def job(key: str, state: str, *, invoked: bool) -> str:
+ identifier = queue.submit(
+ WriteRequest(
+ internal_session_id="session-1",
+ source_id="cursor",
+ batch_identity=BatchIdentity(
+ internal_session_id="session-1", first_round=1, last_round=1
+ ),
+ evidence_ids=["gone"],
+ idempotency_key=key,
+ )
+ ).job_id
+ with main_database(store.database_path).write() as database, database:
+ database.execute(
+ "UPDATE memory_write_jobs SET state=?,hy_invoked=?,"
+ "last_error='model_content_outcome_unknown' WHERE job_id=?",
+ (state, int(invoked), identifier),
+ )
+ return identifier
+
+ # Their Evidence is gone too, which must not block the cutover.
+ unknown = [job(f"unknown-{index}", UNKNOWN, invoked=True) for index in range(3)]
+ sent_running = job("sent-running", RUNNING, invoked=True)
+ finished = job("finished", "completed", invoked=True)
+
+ first = prepare_legacy_work_for_cutover(store.database_path)
+ assert first.abandoned == {"unknown_write_jobs": 4}
+ # Resumed after an interruption: nothing changes, the count is the same.
+ again = prepare_legacy_work_for_cutover(store.database_path)
+ assert again.as_document() == first.as_document()
+
+ with main_database(store.database_path).read() as database:
+ rows = dict(
+ (str(row[0]), (str(row[1]), str(row[2]), int(row[3])))
+ for row in database.execute(
+ "SELECT job_id,state,last_error,hy_invoked FROM memory_write_jobs"
+ )
+ )
+ for identifier in (*unknown, sent_running):
+ assert rows[identifier] == (CANCELLED, "abandoned_at_cutover", 1)
+ assert rows[finished][0] == "completed"
+ surviving = DurableWriteQueue(store.database_path, sdk_authority=True)
+ assert surviving.unknown_jobs() == ()
+ # Never requeued: they may already have written.
+ assert surviving.retry_unknown_confirmed(unknown[0]) == 0
+ assert surviving.readmit_parked() == 0
+ assert PENDING not in {state for state, _, _ in rows.values()}
+
+
+def test_retrying_unknown_writes_works_after_the_legacy_tables_are_dropped(store):
+ """The retry button raised `no such table: memory_tasks` after an upgrade."""
+ from one_memory.memory_server.durable_queue import PENDING, UNKNOWN
+ from one_memory.target_schema import drop_legacy_tables
+
+ queue = DurableWriteQueue(store.database_path)
+ identifier = queue.submit(
+ WriteRequest(
+ internal_session_id="session-1",
+ source_id="cursor",
+ batch_identity=BatchIdentity(
+ internal_session_id="session-1", first_round=1, last_round=1
+ ),
+ evidence_ids=["e1"],
+ idempotency_key="after-upgrade",
+ )
+ ).job_id
+ drop_legacy_tables(store.database_path)
+ assert legacy_tables_present(store.database_path) == ()
+ old = datetime(2000, 1, 1, tzinfo=UTC).isoformat()
+
+ def make_unknown(code: str, attempts: int) -> None:
+ with main_database(store.database_path).write() as database, database:
+ database.execute(
+ "UPDATE memory_write_jobs SET state=?,hy_invoked=1,last_error=?,"
+ "attempts=?,updated_at=? WHERE job_id=?",
+ (UNKNOWN, code, attempts, old, identifier),
+ )
+
+ for sdk_authority in (True, False):
+ surviving = DurableWriteQueue(store.database_path, sdk_authority=sdk_authority)
+ make_unknown("model_content_outcome_unknown", 1)
+ assert surviving.resolve_unknown_from_linked_output(identifier) is False
+ assert surviving.retry_unknown_confirmed(identifier) == 1
+ assert surviving.get("after-upgrade").state == PENDING
+
+ make_unknown("hy_memory_timeout", 1)
+ since = datetime.now(UTC)
+ assert surviving.retry_timed_out_unknown_after_reconcile(older_than=since) == 1
+ make_unknown("model_content_outcome_unknown", 1)
+ assert surviving.retry_first_unknown_after_reconcile(older_than=since) == 1
+ if sdk_authority:
+ # The SDK-authority queue never brings a dropped table back.
+ assert legacy_tables_present(store.database_path) == ()
+
+
def _protective(home: Path) -> Path:
return home / "hy-memory" / ".governed-v2.pre-migration-0123456789abcdef"