Skip to content
Python

fix(memhub): migrate to SDK authority without copying memory

alvinfei7cbc2fd63258
diff --git a/memhub/apps/core/src/one_memory/api.py b/memhub/apps/core/src/one_memory/api.py
index f528213..b42e0a1 100644
--- a/memhub/apps/core/src/one_memory/api.py
+++ b/memhub/apps/core/src/one_memory/api.py
@@ -1430,6 +1430,14 @@ def create_app(
app.state.memory_backfill = None
app.state.memory_visibility_task = None
app.state.deletion_sync = None
+ if app.state.memory_authority is MemoryAuthority.LEGACY:
+ _start_backfill(app)
+ _start_visibility_recompute(app)
+ app.state.deletion_sync = watched_task(
+ asyncio.get_running_loop(),
+ lambda: _keep_legacy_deletions_in_sync(app),
+ name="one-memory-deletion-sync",
+ )
# Both times the loop was blocked here it was found from outside,
# by hand, days later. This says so at the time, and says what was
# holding it.
@@ -4226,7 +4234,7 @@ def create_app(
if action == "delete":
deletion = _memory_store(app).deletion(memory_id, user_id="local")
if deletion is None or deletion.status is not DeletionStatus.COMPLETED:
- await _sdk_memory_runtime(app).finish_deletion(memory_id)
+ await _finish_deletion(app, memory_id)
return memory
except KeyError:
raise HTTPException(status_code=404, detail="memory_not_found") from None
@@ -4924,6 +4932,54 @@ def _secret_store_detail(error: SecretStoreUnavailableError) -> str:
return str(getattr(error, "code", None) or SecretStoreErrorCode.STORE_UNAVAILABLE)
+async def _finish_deletion(app: FastAPI, memory_id: str) -> None:
+ """Delete from the SDK and, on legacy authority, settle the deletion row.
+
+ Without the settle the legacy row stays `pending` for good, and the person
+ is shown a deletion that never finishes.
+ """
+ if app.state.memory_authority is MemoryAuthority.SDK:
+ await _sdk_memory_runtime(app).finish_deletion(memory_id)
+ return
+ await _work_legacy_deletions(app, memory_id=memory_id)
+
+
+async def _work_legacy_deletions(app: FastAPI, *, memory_id: str | None = None) -> None:
+ store = _memory_store(app)
+ runtime = _sdk_memory_runtime(app)
+ claims = await asyncio.to_thread(store.claim_deletions, user_id="local", memory_id=memory_id)
+ for claim in claims:
+ try:
+ await runtime.finish_deletion(claim.memory_id)
+ except HySdkError as error:
+ code: str | None = error.code
+ except Exception as error: # noqa: BLE001 - recorded so it waits its turn
+ logger.warning("memory_deletion_unexpected_failure: %s", type(error).__name__)
+ code = "hy_memory_delete_unexpected"
+ else:
+ code = None
+ await asyncio.to_thread(store.settle_deletion, claim, user_id="local", error_code=code)
+
+
+_LEGACY_DELETION_SYNC_IDLE_SECONDS = 60.0
+
+
+async def _keep_legacy_deletions_in_sync(app: FastAPI) -> None:
+ """Retry legacy deletions the SDK did not take; nothing else asks again."""
+ store = _memory_store(app)
+ while True:
+ wait = _LEGACY_DELETION_SYNC_IDLE_SECONDS
+ try:
+ await _work_legacy_deletions(app)
+ due = await asyncio.to_thread(store.next_deletion_due, user_id="local")
+ except Exception as error: # noqa: BLE001 - reported, then retried
+ report_exception("memory.deletion_sync", "memory_deletion_sync.failed", error)
+ else:
+ if due is not None:
+ wait = min(wait, max(0.0, (due - store.clock()).total_seconds()))
+ await asyncio.sleep(wait)
+
+
def _memory_store(app: FastAPI) -> MemoryGovernanceStore:
return cast(MemoryGovernanceStore, app.state.memory_governance_store)
diff --git a/memhub/apps/core/src/one_memory/cli.py b/memhub/apps/core/src/one_memory/cli.py
index e4a764a..91fee4b 100644
--- a/memhub/apps/core/src/one_memory/cli.py
+++ b/memhub/apps/core/src/one_memory/cli.py
@@ -40,10 +40,9 @@ from typing import TextIO # noqa: E402
from one_memory.config import load_settings # noqa: E402
from one_memory.runtime_client import RuntimeStatus, inspect_runtime # noqa: E402
-# A single `sdk_migration_backfill_failed` told the person nothing: running out
-# of disk, a model that was never configured, and a Coding/Working-Fact gap are
-# three different next steps, and the startup dialog can only act on the name it
-# is given. These are the names that reach it.
+# The startup dialog can only act on the name it is given: running out of disk
+# and a damaged backup are different next steps. These are the names that reach
+# it.
_SAFE_MIGRATION_CODE = re.compile(r"^[a-z][a-z0-9_]{0,95}$")
_OUT_OF_SPACE = frozenset({errno.ENOSPC, getattr(errno, "EDQUOT", errno.ENOSPC)})
@@ -58,7 +57,7 @@ def _exception_chain(error: BaseException) -> Iterator[BaseException]:
def migration_failure_code(
- error: BaseException, *, fallback: str = "sdk_migration_backfill_failed"
+ error: BaseException, *, fallback: str = "sdk_migration_preflight_failed"
) -> str:
"""Name a migration failure; never guess a code from free text.
@@ -75,14 +74,6 @@ def migration_failure_code(
continue
if code.startswith(("sdk_data_plane_", "sdk_migration_")):
return code
- if code.startswith("hy_sdk_productivity_"):
- # The Coding inventory could not be read or classified; that is a
- # data-shape problem the person cannot fix by retrying.
- return "sdk_migration_coding_inventory_failed"
- if code == "hy_sdk_memory_import_failed" and any(
- "working_fact" in str(part) for part in chain
- ):
- return "sdk_migration_working_fact_conflict"
return fallback
@@ -416,22 +407,17 @@ def migration_bundle(options: Sequence[str], stream: TextIO) -> int:
def cutover_failure_code(error: BaseException) -> str:
- """The cutover's own codes, minus the counts it appends to one of them.
-
- `sdk_data_plane_cutover_work_pending` carries a JSON body of table names and
- counts. That belongs in the diagnostic event, not in the single string the
- startup dialog reads, so it collapses to one name here.
- """
code = getattr(error, "code", "")
- if isinstance(code, str) and code.startswith("sdk_data_plane_cutover_work_pending"):
- return "sdk_migration_work_pending"
if isinstance(code, str) and _SAFE_MIGRATION_CODE.match(code):
return code
return "sdk_migration_cutover_failed"
def sdk_migrate(options: Sequence[str], stream: TextIO) -> int:
- """Backfill legacy memory/raw into SDK while keeping legacy authority."""
+ """Make legacy queue work self-contained while keeping legacy authority.
+
+ The SDK already holds every memory and raw turn, so nothing is copied.
+ """
migration_id: str | None = None
bundle_id: str | None = None
confirmed = False
@@ -485,28 +471,24 @@ def sdk_migrate(options: Sequence[str], stream: TextIO) -> int:
stream,
as_json,
document,
- "预览通过;加 --yes 后执行可续跑 backfill,但不会切换 authority。",
+ "预览通过;加 --yes 后准备迁移,但不会切换 authority。",
)
return 4
from one_memory.database_access import close_main_databases
from one_memory.hy_runtime_config import HY_MEMORY_NAMESPACE, HY_MEMORY_VERSION
- from one_memory.hy_sdk_migration import HySdkMigration
- from one_memory.hy_sdk_runtime import acquire_client
from one_memory.memory_server.migration_runner import SqliteMigrationStore, migrate
from one_memory.memory_server.migrations import MIGRATIONS
- from one_memory.model_profiles import ModelProfileStore
+ from one_memory.sdk_data_plane_migration import SdkDataPlaneMigrationStore
from one_memory.sdk_migration_bundle import (
MigrationBundleRejected,
inspect_sdk_migration_bundle,
)
from one_memory.sdk_migration_runner import (
- BackfillResult,
- prepare_sdk_migration_staging,
- run_sdk_data_plane_backfill,
- sdk_migration_staging,
+ PREPARED_PHASE,
+ prepare_legacy_work_for_cutover,
+ recover_abandoned_staging,
)
- from one_memory.secrets import KeyringSecretStore, SecretStoreUnavailableError
database = settings.data_dir / "one-memory.sqlite3"
try:
@@ -516,118 +498,46 @@ def sdk_migrate(options: Sequence[str], stream: TextIO) -> int:
)
asyncio.run(migrate(SqliteMigrationStore(database), MIGRATIONS))
close_main_databases()
- profiles = ModelProfileStore(database)
- identity = profiles.active_identity()
- if identity is None:
- raise RuntimeError("model_profile_not_active")
- profile = profiles.snapshot(*identity)
- secrets = KeyringSecretStore()
- secret = profiles.resolve_snapshot_secret(*identity, secrets)
- if profile.has_api_key and secret is None:
- raise RuntimeError("model_secret_unavailable")
+ recover_abandoned_staging(
+ data_dir=settings.data_dir,
+ database_path=database,
+ namespace=HY_MEMORY_NAMESPACE,
+ )
+ store = SdkDataPlaneMigrationStore(database)
+ store.start(
+ migration_id=migration_id,
+ source_runtime_build=str(manifest.get("runtime_build") or ""),
+ target_sdk_version=HY_MEMORY_VERSION,
+ source_schema_version=MIGRATIONS[-1].version,
+ namespace_fingerprint=hashlib.sha256(HY_MEMORY_NAMESPACE.encode()).hexdigest(),
+ backend_fingerprint=hashlib.sha256(b"chroma:kuzu:sqlite").hexdigest(),
+ backup_bundle_id=bundle_id,
+ )
+ preparation = prepare_legacy_work_for_cutover(database)
+ store.set_phase(migration_id, phase=PREPARED_PHASE, state="ready")
except MigrationBundleRejected as error:
code = error.code
- except SecretStoreUnavailableError:
- code = "secure_storage_unavailable"
except Exception as error:
- message = str(error)
- code = (
- message
- if message
- in {
- "model_profile_not_active",
- "model_secret_unavailable",
- }
- else migration_failure_code(error, fallback="sdk_migration_preflight_failed")
- )
+ code = migration_failure_code(error, fallback="sdk_migration_preflight_failed")
else:
- namespace_fingerprint = hashlib.sha256(HY_MEMORY_NAMESPACE.encode()).hexdigest()
- backend_material = json.dumps(
- {
- "namespace": HY_MEMORY_NAMESPACE,
- "embedding_provider": str(profile.embedding_provider_kind or ""),
- "embedding_model": profile.embedding_model,
- "embedding_dimensions": profile.embedding_dimensions,
- "vector": "chroma",
- "cache": "sqlite",
- "history": "sqlite",
- "coding": "sqlite",
- },
- sort_keys=True,
- separators=(",", ":"),
- )
- backend_fingerprint = hashlib.sha256(backend_material.encode()).hexdigest()
- staging = sdk_migration_staging(
- data_dir=settings.data_dir,
- migration_id=migration_id,
- namespace=HY_MEMORY_NAMESPACE,
- )
- try:
- staging_data_dir = prepare_sdk_migration_staging(
- paths=staging,
- database_path=database,
- )
- except Exception as error:
- code = migration_failure_code(error, fallback="sdk_migration_staging_failed")
- staging_data_dir = None
-
- async def run_backfill() -> BackfillResult:
- assert staging_data_dir is not None
- lease = await asyncio.to_thread(
- acquire_client,
- profile,
- api_key=secret,
- data_dir=staging_data_dir,
- )
- try:
- return await run_sdk_data_plane_backfill(
- database_path=database,
- client=HySdkMigration(lease.client),
- migration_id=migration_id,
- bundle_id=bundle_id,
- source_runtime_build=str(manifest.get("runtime_build") or ""),
- target_sdk_version=HY_MEMORY_VERSION,
- source_schema_version=MIGRATIONS[-1].version,
- namespace_fingerprint=namespace_fingerprint,
- backend_fingerprint=backend_fingerprint,
- )
- finally:
- await asyncio.to_thread(lease.close)
-
- try:
- result = asyncio.run(run_backfill()) if staging_data_dir is not None else None
- except Exception as error:
- code = migration_failure_code(error)
- else:
- if result is None:
- document = {"schema_version": 1, "state": "failed", "code": code}
- _write_restore(stream, as_json, document, f"SDK backfill 失败:{code}")
- return 1
- document = {
- "schema_version": 1,
- "state": result.state,
- "phase": result.phase,
- "migration_id": result.migration_id,
- "counts": result.counts,
- "memory_items": result.memory_items,
- "raw_items": result.raw_items,
- "commits_cutover": False,
- }
- _write_restore(
- stream,
- as_json,
- document,
- "SDK backfill 已完成;authority 尚未切换。",
- )
- return 0 if result.state == "ready" else 4
+ document = {
+ "schema_version": 1,
+ "state": "ready",
+ "phase": PREPARED_PHASE,
+ "migration_id": migration_id,
+ **preparation.as_document(),
+ "commits_cutover": False,
+ }
+ _write_restore(stream, as_json, document, "迁移准备已完成;authority 尚未切换。")
+ return 0
document = {"schema_version": 1, "state": "failed", "code": code}
- _write_restore(stream, as_json, document, f"SDK backfill 失败:{code}")
+ _write_restore(stream, as_json, document, f"迁移准备失败:{code}")
return 1
def sdk_cutover(options: Sequence[str], stream: TextIO) -> int:
- """Commit SDK authority only after offline reconciliation and work drain."""
+ """Commit SDK authority once the backup is verified and the queue is self-contained."""
migration_id: str | None = None
bundle_id: str | None = None
confirmed = False
@@ -678,33 +588,19 @@ def sdk_cutover(options: Sequence[str], stream: TextIO) -> int:
_write_restore(stream, as_json, document, "预览通过;加 --yes 后提交 SDK authority。")
return 4
- from one_memory.hy_runtime_config import HY_MEMORY_NAMESPACE, HY_MEMORY_VERSION
+ from one_memory.hy_runtime_config import HY_MEMORY_VERSION
from one_memory.sdk_data_plane_migration import DataPlaneMigrationError
from one_memory.sdk_migration_bundle import MigrationBundleRejected
- from one_memory.sdk_migration_runner import (
- activate_sdk_migration_staging,
- commit_reconciled_cutover,
- resume_legacy_cleanup,
- sdk_migration_staging,
- )
+ from one_memory.sdk_migration_runner import commit_sdk_cutover, resume_legacy_cleanup
try:
database_path = settings.data_dir / "one-memory.sqlite3"
- staging = sdk_migration_staging(
+ marker = commit_sdk_cutover(
data_dir=settings.data_dir,
- migration_id=migration_id,
- namespace=HY_MEMORY_NAMESPACE,
- )
- marker = activate_sdk_migration_staging(
- paths=staging,
database_path=database_path,
- commit=lambda: commit_reconciled_cutover(
- data_dir=settings.data_dir,
- database_path=database_path,
- migration_id=migration_id,
- bundle_id=bundle_id,
- sdk_version=HY_MEMORY_VERSION,
- ),
+ migration_id=migration_id,
+ bundle_id=bundle_id,
+ sdk_version=HY_MEMORY_VERSION,
)
resume_legacy_cleanup(
data_dir=settings.data_dir,
@@ -757,7 +653,7 @@ def sdk_migration_status(options: Sequence[str], stream: TextIO) -> int:
MemoryAuthority,
SdkDataPlaneMigrationStore,
)
- from one_memory.sdk_migration_runner import sdk_migration_staging
+ from one_memory.sdk_migration_runner import abandoned_staging_present
from one_memory.target_schema import legacy_tables_present
migration_store = SdkDataPlaneMigrationStore(database_path)
@@ -767,14 +663,10 @@ def sdk_migration_status(options: Sequence[str], stream: TextIO) -> int:
generation = hashlib.sha256(f"{stat.st_size}:{stat.st_mtime_ns}".encode()).hexdigest()[:16]
if authority is MemoryAuthority.SDK:
legacy_tables = legacy_tables_present(database_path)
- staging_cleanup = False
- if marker is not None:
- staging = sdk_migration_staging(
- data_dir=settings.data_dir,
- migration_id=marker.migration_id,
- namespace=HY_MEMORY_NAMESPACE,
- )
- staging_cleanup = staging.protective_root.exists() or staging.staging_home.exists()
+ staging_cleanup = abandoned_staging_present(
+ data_dir=settings.data_dir,
+ namespace=HY_MEMORY_NAMESPACE,
+ )
document = {
"schema_version": 1,
"authority": "sdk",
diff --git a/memhub/apps/core/src/one_memory/hy_sdk_migration.py b/memhub/apps/core/src/one_memory/hy_sdk_migration.py
index 71c05e9..a518725 100644
--- a/memhub/apps/core/src/one_memory/hy_sdk_migration.py
+++ b/memhub/apps/core/src/one_memory/hy_sdk_migration.py
@@ -1,18 +1,13 @@
-"""Privileged offline access to the HY SDK data plane during App migration.
+"""Offline snapshot and restore of the HY SDK data plane for the App migration.
-Runtime product code must use public ``HyMemoryClient`` methods. This module
-is the deliberately isolated exception: App startup has stopped all writers,
-so the one-time legacy migration may access SDK stores directly without
-turning migration mechanics into public Client APIs.
+Runtime product code must use public ``HyMemoryClient`` methods. The startup
+migration bundle is the isolated exception: all writers are stopped, so it may
+copy and restore the SDK-owned local stores as files.
"""
from __future__ import annotations
-import asyncio
-from collections.abc import Callable, Mapping
-from datetime import datetime
from pathlib import Path
-from typing import Any, Protocol
from hy_memory import ( # type: ignore[import-untyped]
DataPlaneRestore,
@@ -20,537 +15,6 @@ from hy_memory import ( # type: ignore[import-untyped]
restore_local_data_plane,
snapshot_local_data_plane,
)
-from hy_memory.models.memory import ( # type: ignore[import-untyped]
- MemoryLayer,
- MemoryNode,
- MemoryStatus,
- SourceType,
-)
-from hy_memory.utils.source_filter import normalize_source # type: ignore[import-untyped]
-
-from one_memory.hy_sdk_types import (
- HySdkMemoryPage,
- HySdkMemoryRecord,
- HySdkMigrationError,
- HySdkRawPage,
- HySdkRawRecord,
-)
-
-
-class MigrationClient(Protocol):
- _loop_thread: Any
- _vector_store: Any
- _embed_service: Any
- _graph_store: Any
-
- def list_memories(
- self,
- *,
- user_id: str,
- agent_id: str | None = None,
- statuses: list[str] | None = None,
- limit: int = 100,
- offset: int = 0,
- order: str = "desc",
- ) -> Mapping[str, object]: ...
-
- def delete(self, memory_id: str) -> Mapping[str, object]: ...
-
- def get_write_outcome(self, request_id: str) -> Mapping[str, object] | None: ...
-
- def get(
- self,
- memory_id: str,
- *,
- kind: str | None = None,
- variant: str | None = None,
- ) -> Mapping[str, object] | None: ...
-
- async def _coding_store_for_exact_get(self, variant: str) -> Any: ...
-
-
-class HySdkMigration:
- """Typed offline adapter over one initialized SDK data plane."""
-
- def __init__(self, client: MigrationClient) -> None:
- self.client = client
- self._loop_thread = client._loop_thread
- self._vector_store = client._vector_store
- self._embed_service = client._embed_service
- self._graph_store = client._graph_store
-
- async def list_memories(
- self,
- *,
- user_id: str,
- agent_id: str | None = None,
- statuses: tuple[str, ...] = ("active",),
- limit: int = 100,
- offset: int = 0,
- order: str = "desc",
- ) -> HySdkMemoryPage:
- raw = await _sdk_call(
- "hy_sdk_memory_list_failed",
- self.client.list_memories,
- user_id=user_id,
- agent_id=agent_id,
- statuses=list(statuses),
- limit=limit,
- offset=offset,
- order=order,
- )
- return _memory_page(raw, limit=limit, offset=offset)
-
- async def list_productivity_memories(self) -> tuple[dict[str, object], ...]:
- raw = await _sdk_call(
- "hy_sdk_productivity_inventory_failed",
- self._loop_thread.run,
- self._list_productivity_memories(),
- )
- return tuple(raw)
-
- async def _list_productivity_memories(self) -> list[dict[str, object]]:
- records: list[dict[str, object]] = []
- for variant in ("working", "coding"):
- store = await self.client._coding_store_for_exact_get(variant)
- if store is None:
- continue
- with store._lock:
- experiences = store._conn.execute(
- "SELECT experience_id,user_id,agent_id,solution "
- "FROM coding_memory_meta ORDER BY experience_id"
- ).fetchall()
- facts = store._conn.execute(
- "SELECT fact_id,user_id,agent_id,statement,status "
- "FROM working_fact ORDER BY fact_id"
- ).fetchall()
- records.extend(
- {
- "memory_id": str(row["experience_id"]),
- "content": str(row["solution"]),
- "status": "active",
- "user_id": str(row["user_id"]),
- "agent_id": str(row["agent_id"]),
- "kind": "experience",
- "variant": variant,
- }
- for row in experiences
- )
- records.extend(
- {
- "memory_id": str(row["fact_id"]),
- "content": str(row["statement"]),
- "status": str(row["status"]),
- "user_id": str(row["user_id"]),
- "agent_id": str(row["agent_id"]),
- "kind": "fact",
- "variant": variant,
- }
- for row in facts
- )
- return records
-
- async def read_productivity_memory(
- self,
- memory_id: str,
- *,
- kind: str,
- variant: str | None,
- ) -> tuple[HySdkMemoryRecord, str] | None:
- raw = await _sdk_call(
- "hy_sdk_productivity_get_failed",
- self.client.get,
- memory_id,
- kind=kind,
- variant=variant,
- )
- if raw is None:
- return None
- if not isinstance(raw, Mapping) or raw.get("variant") not in {
- "working",
- "coding",
- }:
- raise HySdkMigrationError("hy_sdk_productivity_variant_invalid")
- return HySdkMemoryRecord.from_sdk(raw), str(raw["variant"])
-
- async def delete_productivity_memory(
- self,
- memory_id: str,
- *,
- kind: str,
- variant: str | None,
- user_id: str,
- ) -> dict[str, object]:
- raw = await _sdk_call(
- "hy_sdk_productivity_delete_failed",
- self._loop_thread.run,
- self._delete_productivity_memory(
- memory_id,
- kind=kind,
- variant=variant,
- user_id=user_id,
- ),
- )
- return _successful_result(raw, "hy_sdk_productivity_delete_failed")
-
- async def _delete_productivity_memory(
- self,
- memory_id: str,
- *,
- kind: str,
- variant: str | None,
- user_id: str,
- ) -> dict[str, object]:
- if kind not in {"experience", "fact"}:
- raise ValueError("productivity kind must be experience or fact")
- matches: list[tuple[Any, str]] = []
- for candidate in ((variant,) if variant is not None else ("working", "coding")):
- store = await self.client._coding_store_for_exact_get(candidate)
- if store is None:
- continue
- existing = (
- await store.get_by_id(memory_id, user_id=user_id)
- if kind == "experience"
- else await store.get_fact(memory_id, user_id=user_id)
- )
- if existing is not None:
- matches.append((store, candidate))
- if len(matches) > 1:
- raise ValueError("productivity memory id is ambiguous across variants")
- if not matches:
- return {"success": True, "deleted_count": 0}
- store, _actual_variant = matches[0]
- if kind == "experience":
- deleted = await store.delete(memory_id, user_id=user_id)
- elif kind == "fact":
- await store._fact_vec.delete(memory_id)
- with store._lock:
- store._conn.execute(
- "DELETE FROM working_fact WHERE fact_id=? AND user_id=?",
- (memory_id, user_id),
- )
- store._conn.commit()
- deleted = True
- return {"success": True, "deleted_count": int(deleted)}
-
- async def write_legacy_memory(
- self,
- *,
- content: str,
- memory_id: str,
- user_id: str,
- agent_id: str,
- session_id: str,
- source: str,
- layer: str,
- status: str,
- owner: str | None,
- tags: tuple[str, ...],
- supersedes: tuple[str, ...],
- superseded_by: tuple[str, ...],
- is_latest: bool,
- memory_at: datetime | None,
- metadata: Mapping[str, object],
- request_id: str,
- ) -> dict[str, object]:
- raw = await _sdk_call(
- "hy_sdk_memory_import_failed",
- self._loop_thread.run,
- self._write_legacy_memory(
- content=content,
- memory_id=memory_id,
- user_id=user_id,
- agent_id=agent_id,
- session_id=session_id,
- source=source,
- layer=layer,
- status=status,
- owner=owner,
- tags=tags,
- supersedes=supersedes,
- superseded_by=superseded_by,
- is_latest=is_latest,
- memory_at=memory_at,
- metadata=metadata,
- request_id=request_id,
- ),
- )
- return _successful_result(raw, "hy_sdk_memory_import_failed")
-
- async def _write_legacy_memory(
- self,
- *,
- content: str,
- memory_id: str,
- user_id: str,
- agent_id: str,
- session_id: str,
- source: str,
- layer: str,
- status: str,
- owner: str | None,
- tags: tuple[str, ...],
- supersedes: tuple[str, ...],
- superseded_by: tuple[str, ...],
- is_latest: bool,
- memory_at: datetime | None,
- metadata: Mapping[str, object],
- request_id: str,
- ) -> dict[str, object]:
- if not content or not memory_id or not user_id:
- raise ValueError("content, memory_id and user_id are required")
- normalized_layer = MemoryLayer.from_string(layer)
- if normalized_layer is MemoryLayer.L1_RAW:
- raise ValueError("raw content must use the raw migration lane")
- normalized_status = MemoryStatus(status.lower())
- normalized_source = normalize_source(source)
- existing = await self._vector_store.get_by_id(memory_id)
- if existing is not None:
- identity = (
- existing.content == content,
- existing.user_id == user_id,
- existing.agent_id == agent_id,
- existing.session_id == session_id,
- existing.source == normalized_source,
- existing.layer == normalized_layer,
- existing.status == normalized_status,
- existing.owner == owner,
- list(existing.tags or []) == list(tags),
- list(existing.supersedes or []) == list(supersedes),
- list(existing.superseded_by or []) == list(superseded_by),
- existing.is_latest == is_latest,
- )
- if not all(identity):
- raise ValueError("memory_id is already bound to different memory content")
- await self._project_graph(existing)
- return {
- "success": True,
- "memory_id": memory_id,
- "memory_ids": [memory_id],
- "request_id": request_id,
- "replayed": True,
- }
- custom = dict(metadata)
- custom["migration_request_id"] = request_id
- embedding = await self._embed_service.embed(content)
- node = MemoryNode(
- node_id=memory_id,
- user_id=user_id,
- agent_id=agent_id,
- session_id=session_id,
- source=normalized_source,
- layer=normalized_layer,
- content=content,
- source_type=SourceType.EXPLICIT,
- status=normalized_status,
- owner=owner,
- tags=list(tags),
- supersedes=list(supersedes),
- superseded_by=list(superseded_by),
- is_latest=is_latest,
- embedding=embedding,
- memory_at=memory_at,
- custom=custom,
- )
- stored_id = await self._vector_store.upsert(node)
- if stored_id != memory_id:
- raise RuntimeError("exact migration changed the requested memory identity")
- await self._project_graph(node)
- return {
- "success": True,
- "memory_id": memory_id,
- "memory_ids": [memory_id],
- "request_id": request_id,
- }
-
- async def _project_graph(self, node: MemoryNode) -> None:
- if self._graph_store is not None:
- await self._graph_store.upsert_memory_node(node)
-
- async def read_raw_inventory(
- self,
- *,
- user_id: str,
- agent_id: str | None = None,
- session_id: str | None = None,
- source: str | None = None,
- limit: int = 100,
- offset: int = 0,
- order: str = "desc",
- ) -> HySdkRawPage:
- raw = await _sdk_call(
- "hy_sdk_raw_list_failed",
- self._loop_thread.run,
- self._read_raw_inventory(
- user_id=user_id,
- agent_id=agent_id,
- session_id=session_id,
- source=source,
- limit=limit,
- offset=offset,
- order=order,
- ),
- )
- return _raw_page(raw, limit=limit, offset=offset)
-
- async def _read_raw_inventory(
- self,
- *,
- user_id: str,
- agent_id: str | None,
- session_id: str | None,
- source: str | None,
- limit: int,
- offset: int,
- order: str,
- ) -> dict[str, object]:
- if not user_id:
- raise ValueError("user_id is required")
- nodes = await self._vector_store.list_by_user(
- user_id=user_id,
- agent_id=agent_id,
- status_filter=list(MemoryStatus),
- layers=[MemoryLayer.L1_RAW],
- include_sources=[source] if source else None,
- limit=100_000,
- only_latest=False,
- with_vectors=False,
- allow_raw_layer=True,
- )
- if session_id:
- nodes = [node for node in nodes if node.session_id == session_id]
- nodes.sort(
- key=_node_timestamp,
- reverse=order.lower() != "asc",
- )
- return {
- "raw": [_raw_node(node) for node in nodes[offset : offset + limit]],
- "total": len(nodes),
- "limit": limit,
- "offset": offset,
- }
-
- async def write_legacy_raw(
- self,
- *,
- content: str,
- raw_id: str,
- user_id: str,
- agent_id: str,
- session_id: str,
- source: str,
- memory_at: datetime | None,
- metadata: Mapping[str, object],
- request_id: str,
- ) -> dict[str, object]:
- raw = await _sdk_call(
- "hy_sdk_raw_import_failed",
- self._loop_thread.run,
- self._write_legacy_raw(
- content=content,
- raw_id=raw_id,
- user_id=user_id,
- agent_id=agent_id,
- session_id=session_id,
- source=source,
- memory_at=memory_at,
- metadata=metadata,
- request_id=request_id,
- ),
- )
- return _successful_result(raw, "hy_sdk_raw_import_failed")
-
- async def delete_raw(self, raw_id: str) -> dict[str, object]:
- raw = await _sdk_call(
- "hy_sdk_raw_delete_failed",
- self._loop_thread.run,
- self._delete_raw(raw_id),
- )
- return _successful_result(raw, "hy_sdk_raw_delete_failed")
-
- async def _delete_raw(self, raw_id: str) -> dict[str, object]:
- existing = await self._vector_store.get_by_id(raw_id)
- if existing is None or existing.layer is not MemoryLayer.L1_RAW:
- return {"success": True, "deleted_count": 0}
- deleted = await self._vector_store.delete(raw_id)
- return {"success": True, "deleted_count": int(bool(deleted))}
-
- async def _write_legacy_raw(
- self,
- *,
- content: str,
- raw_id: str,
- user_id: str,
- agent_id: str,
- session_id: str,
- source: str,
- memory_at: datetime | None,
- metadata: Mapping[str, object],
- request_id: str,
- ) -> dict[str, object]:
- if not content or not raw_id or not user_id:
- raise ValueError("content, raw_id and user_id are required")
- normalized_source = normalize_source(source)
- existing = await self._vector_store.get_by_id(raw_id)
- if existing is not None:
- identity = (
- existing.layer is MemoryLayer.L1_RAW,
- existing.content == content,
- existing.user_id == user_id,
- existing.agent_id == agent_id,
- existing.session_id == session_id,
- existing.source == normalized_source,
- )
- if not all(identity):
- raise ValueError("raw_id is already bound to different source content")
- return {
- "success": True,
- "raw_id": raw_id,
- "request_id": request_id,
- "replayed": True,
- }
- custom = dict(metadata)
- custom["migration_request_id"] = request_id
- embedding = await self._embed_service.embed(content)
- node = MemoryNode(
- node_id=raw_id,
- user_id=user_id,
- agent_id=agent_id,
- session_id=session_id,
- source=normalized_source,
- layer=MemoryLayer.L1_RAW,
- content=content,
- source_type=SourceType.EXPLICIT,
- status=MemoryStatus.ACTIVE,
- is_latest=True,
- embedding=embedding,
- memory_at=memory_at,
- custom=custom,
- )
- stored_id = await self._vector_store.upsert(node)
- if stored_id != raw_id:
- raise RuntimeError("exact migration changed the requested raw identity")
- return {"success": True, "raw_id": raw_id, "request_id": request_id}
-
- async def delete(self, memory_id: str) -> dict[str, object]:
- raw = await _sdk_call(
- "hy_sdk_memory_delete_failed",
- self.client.delete,
- memory_id,
- )
- return _successful_result(raw, "hy_sdk_memory_delete_failed")
-
- async def get_write_outcome(self, request_id: str) -> dict[str, object] | None:
- raw = await _sdk_call(
- "hy_sdk_write_outcome_failed",
- self.client.get_write_outcome,
- request_id,
- )
- if raw is None:
- return None
- if not isinstance(raw, Mapping):
- raise HySdkMigrationError("hy_sdk_write_outcome_shape_invalid")
- return dict(raw)
def snapshot_data_plane(
@@ -581,176 +45,3 @@ def restore_data_plane(
writers_stopped=writers_stopped,
protective_root=protective_root,
)
-
-
-async def _sdk_call[T](
- code: str,
- call: Callable[..., T],
- *args: object,
- **kwargs: object,
-) -> T:
- pending = asyncio.create_task(asyncio.to_thread(call, *args, **kwargs))
- try:
- return await asyncio.shield(pending)
- except asyncio.CancelledError:
- await _finish_cancelled_call(pending)
- raise
- except Exception as error:
- raise HySdkMigrationError(code) from error
-
-
-async def _finish_cancelled_call[T](pending: asyncio.Task[T]) -> None:
- while not pending.done():
- try:
- await asyncio.shield(pending)
- except asyncio.CancelledError:
- continue
- except Exception:
- return
- if not pending.cancelled():
- pending.exception()
-
-
-def _memory_page(raw: object, *, limit: int, offset: int) -> HySdkMemoryPage:
- if not isinstance(raw, Mapping):
- raise HySdkMigrationError("hy_sdk_memory_page_shape_invalid")
- rows: list[object] = []
- total = 0
- for bucket_name, row_name in (("vdb", "memories"), ("graph", "nodes")):
- bucket = raw.get(bucket_name)
- if bucket is None and bucket_name == "graph":
- continue
- if not isinstance(bucket, Mapping):
- raise HySdkMigrationError("hy_sdk_memory_page_shape_invalid")
- bucket_rows = bucket.get(row_name)
- bucket_total = bucket.get("total")
- if not isinstance(bucket_rows, list) or type(bucket_total) is not int or bucket_total < 0:
- raise HySdkMigrationError("hy_sdk_memory_page_shape_invalid")
- rows.extend(bucket_rows)
- total += bucket_total
-
- records: dict[str, HySdkMemoryRecord] = {}
- for row in rows:
- record = _memory_record(row)
- records.setdefault(record.memory_id, record)
- return HySdkMemoryPage(
- memories=tuple(records.values()),
- total=total,
- limit=limit,
- offset=offset,
- )
-
-
-def _memory_record(raw: object) -> HySdkMemoryRecord:
- if not isinstance(raw, Mapping):
- raise HySdkMigrationError("hy_sdk_memory_record_shape_invalid")
- required = ("memory_id", "content", "layer", "status", "user_id", "agent_id")
- if any(not isinstance(raw.get(name), str) or not raw[name] for name in required):
- raise HySdkMigrationError("hy_sdk_memory_record_shape_invalid")
- custom = raw.get("custom")
- source_raw_ids: list[object] = []
- if isinstance(custom, Mapping) and isinstance(custom.get("source_raw_memory_ids"), list):
- source_raw_ids.extend(custom["source_raw_memory_ids"])
- if isinstance(raw.get("source_raw_memory_id"), str):
- source_raw_ids.append(raw["source_raw_memory_id"])
- try:
- return HySdkMemoryRecord(
- memory_id=raw["memory_id"],
- content=raw["content"],
- layer=raw["layer"],
- status=raw["status"],
- is_latest=bool(raw.get("is_latest", True)),
- supersedes=_string_tuple(raw.get("supersedes")),
- superseded_by=_string_tuple(raw.get("superseded_by")),
- tags=_string_tuple(raw.get("tags")),
- owner=raw.get("owner") if raw.get("owner") in {"user", "agent"} else None,
- source_raw_ids=_string_tuple(source_raw_ids),
- user_id=raw["user_id"],
- agent_id=raw["agent_id"],
- session_id=_optional_string(raw.get("session_id")),
- source=_optional_string(raw.get("source")),
- memory_at=_optional_int(raw.get("memory_at")),
- gmt_created=_optional_int(raw.get("gmt_created")),
- )
- except (TypeError, ValueError) as error:
- raise HySdkMigrationError("hy_sdk_memory_record_shape_invalid") from error
-
-
-def _raw_page(raw: object, *, limit: int, offset: int) -> HySdkRawPage:
- if not isinstance(raw, Mapping):
- raise HySdkMigrationError("hy_sdk_raw_page_shape_invalid")
- rows = raw.get("raw")
- total = raw.get("total")
- if not isinstance(rows, list) or type(total) is not int or total < 0:
- raise HySdkMigrationError("hy_sdk_raw_page_shape_invalid")
- return HySdkRawPage(
- raw=tuple(_raw_record(row) for row in rows),
- total=total,
- limit=limit,
- offset=offset,
- )
-
-
-def _node_timestamp(node: Any) -> float:
- value = getattr(node, "memory_at", None) or getattr(node, "gmt_created", None)
- return value.timestamp() if isinstance(value, datetime) else 0.0
-
-
-def _raw_node(node: Any) -> dict[str, object]:
- def timestamp(value: object) -> int | None:
- return int(value.timestamp()) if isinstance(value, datetime) else None
-
- return {
- "raw_id": node.node_id,
- "content": node.content,
- "user_id": node.user_id,
- "agent_id": node.agent_id,
- "session_id": node.session_id,
- "source": node.source,
- "memory_at": timestamp(node.memory_at),
- "gmt_created": timestamp(node.gmt_created),
- "custom": node.custom or {},
- }
-
-
-def _raw_record(raw: object) -> HySdkRawRecord:
- if not isinstance(raw, Mapping):
- raise HySdkMigrationError("hy_sdk_raw_record_shape_invalid")
- required = ("raw_id", "content", "user_id", "agent_id", "session_id", "source")
- if any(not isinstance(raw.get(name), str) or not raw[name] for name in required):
- raise HySdkMigrationError("hy_sdk_raw_record_shape_invalid")
- custom = raw.get("custom")
- try:
- return HySdkRawRecord(
- raw_id=raw["raw_id"],
- content=raw["content"],
- user_id=raw["user_id"],
- agent_id=raw["agent_id"],
- session_id=raw["session_id"],
- source=raw["source"],
- memory_at=_optional_int(raw.get("memory_at")),
- gmt_created=_optional_int(raw.get("gmt_created")),
- custom=dict(custom) if isinstance(custom, Mapping) else {},
- )
- except (TypeError, ValueError) as error:
- raise HySdkMigrationError("hy_sdk_raw_record_shape_invalid") from error
-
-
-def _successful_result(raw: object, code: str) -> dict[str, object]:
- if not isinstance(raw, Mapping) or raw.get("success") is not True:
- raise HySdkMigrationError(code)
- return dict(raw)
-
-
-def _string_tuple(value: object) -> tuple[str, ...]:
- if not isinstance(value, list | tuple):
- return ()
- return tuple(item for item in value if isinstance(item, str) and item)
-
-
-def _optional_string(value: object) -> str | None:
- return value if isinstance(value, str) else None
-
-
-def _optional_int(value: object) -> int | None:
- return value if type(value) is int else None
diff --git a/memhub/apps/core/src/one_memory/memory_server/sqlite_semantics.py b/memhub/apps/core/src/one_memory/memory_server/sqlite_semantics.py
index 4eb9046..c58edba 100644
--- a/memhub/apps/core/src/one_memory/memory_server/sqlite_semantics.py
+++ b/memhub/apps/core/src/one_memory/memory_server/sqlite_semantics.py
@@ -64,7 +64,6 @@ EXEMPT_PATHS = frozenset(
"memory_server/sqlite_semantics.py",
"memory_tasks.py",
"sdk_data_plane_migration.py",
- "sdk_migration_inventory.py",
"target_schema.py",
# D93 A: the visibility cache is SQLite-only by construction. It is a
# derived table with triggers on the six tables the read check reads —
diff --git a/memhub/apps/core/src/one_memory/sdk_data_plane_migration.py b/memhub/apps/core/src/one_memory/sdk_data_plane_migration.py
index 11cd74a..ced4278 100644
--- a/memhub/apps/core/src/one_memory/sdk_data_plane_migration.py
+++ b/memhub/apps/core/src/one_memory/sdk_data_plane_migration.py
@@ -2,7 +2,6 @@
from __future__ import annotations
-import json
import sqlite3
from dataclasses import dataclass
from datetime import UTC, datetime
@@ -40,10 +39,6 @@ class MigrationProgress:
migration_id: str
state: str
phase: str
- counts: dict[str, int]
-
-
-TERMINAL_ITEM_STATES = frozenset({"verified", "adopted", "deleted", "skipped"})
class SdkDataPlaneMigrationStore:
@@ -136,37 +131,14 @@ class SdkDataPlaneMigrationStore:
""",
values,
)
- else:
- identity = (
- source_runtime_build,
- target_sdk_version,
- source_schema_version,
- namespace_fingerprint,
- backend_fingerprint,
- )
- existing_identity = tuple(
- existing[key]
- for key in (
- "source_runtime_build",
- "target_sdk_version",
- "source_schema_version",
- "namespace_fingerprint",
- "backend_fingerprint",
- )
+ elif backup_bundle_id and existing["backup_bundle_id"] is None:
+ connection.execute(
+ "UPDATE sdk_data_plane_migrations "
+ "SET backup_bundle_id=?,updated_at=? WHERE migration_id=?",
+ (backup_bundle_id, now, migration_id),
)
- if existing_identity != identity:
- raise DataPlaneMigrationError("sdk_data_plane_migration_identity_conflict")
- if backup_bundle_id and existing["backup_bundle_id"] not in {
- None,
- backup_bundle_id,
- }:
- raise DataPlaneMigrationError("sdk_data_plane_migration_bundle_conflict")
- if backup_bundle_id and existing["backup_bundle_id"] is None:
- connection.execute(
- "UPDATE sdk_data_plane_migrations "
- "SET backup_bundle_id=?,updated_at=? WHERE migration_id=?",
- (backup_bundle_id, now, migration_id),
- )
+ elif backup_bundle_id and existing["backup_bundle_id"] != backup_bundle_id:
+ raise DataPlaneMigrationError("sdk_data_plane_migration_bundle_conflict")
connection.execute("COMMIT")
return self.progress(migration_id)
@@ -182,211 +154,18 @@ class SdkDataPlaneMigrationStore:
raise DataPlaneMigrationError("sdk_data_plane_migration_not_mutable")
connection.execute("COMMIT")
- def upsert_item(
- self,
- *,
- migration_id: str,
- item_key: str,
- source_kind: str,
- source_id: str,
- source_hash: str,
- operation_key: str,
- source_revision: str | None = None,
- state: str = "pending",
- ) -> None:
- now = self._now()
- with main_database(self.database_path).write(
- autocommit=True, row_factory=sqlite3.Row
- ) as connection:
- connection.execute("BEGIN IMMEDIATE")
- row = connection.execute(
- "SELECT * FROM sdk_data_plane_migration_items WHERE migration_id=? AND item_key=?",
- (migration_id, item_key),
- ).fetchone()
- if row is None:
- try:
- connection.execute(
- """
- INSERT INTO sdk_data_plane_migration_items (
- migration_id,item_key,source_kind,source_id,
- source_revision,source_hash,operation_key,state,updated_at
- ) VALUES (?,?,?,?,?,?,?,?,?)
- """,
- (
- migration_id,
- item_key,
- source_kind,
- source_id,
- source_revision,
- source_hash,
- operation_key,
- state,
- now,
- ),
- )
- except sqlite3.IntegrityError as error:
- raise DataPlaneMigrationError(
- "sdk_data_plane_migration_operation_conflict"
- ) from error
- else:
- if str(row["state"]) in TERMINAL_ITEM_STATES and (
- source_id,
- source_revision,
- source_hash,
- ) == (
- row["source_id"],
- row["source_revision"],
- row["source_hash"],
- ):
- # A crash after an SDK side effect can make the next
- # inventory classify the same source as verified/adopted
- # instead of import/delete. Keep the settled operation as
- # the durable audit identity.
- connection.execute("COMMIT")
- return
- identity = (
- source_kind,
- source_id,
- source_revision,
- source_hash,
- operation_key,
- )
- if identity != tuple(
- row[key]
- for key in (
- "source_kind",
- "source_id",
- "source_revision",
- "source_hash",
- "operation_key",
- )
- ):
- raise DataPlaneMigrationError("sdk_data_plane_migration_item_conflict")
- connection.execute("COMMIT")
-
- def settle_item(
- self,
- *,
- migration_id: str,
- item_key: str,
- state: str,
- target_memory_id: str | None = None,
- target_hash: str | None = None,
- error_code: str | None = None,
- ) -> None:
- with main_database(self.database_path).write(autocommit=True) as connection:
- connection.execute("BEGIN IMMEDIATE")
- cursor = connection.execute(
- """
- UPDATE sdk_data_plane_migration_items
- SET state=?,target_memory_id=?,target_hash=?,error_code=?,
- attempts=attempts+1,updated_at=?
- WHERE migration_id=? AND item_key=?
- """,
- (
- state,
- target_memory_id,
- target_hash,
- error_code,
- self._now(),
- migration_id,
- item_key,
- ),
- )
- if cursor.rowcount != 1:
- raise DataPlaneMigrationError("sdk_data_plane_migration_item_not_found")
- connection.execute("COMMIT")
-
- def record_alias(
- self,
- *,
- migration_id: str,
- legacy_memory_id: str,
- canonical_memory_id: str,
- ) -> None:
- with main_database(self.database_path).write(
- autocommit=True, row_factory=sqlite3.Row
- ) as connection:
- connection.execute("BEGIN IMMEDIATE")
- existing = connection.execute(
- "SELECT canonical_memory_id,migration_id FROM memory_id_aliases "
- "WHERE legacy_memory_id=?",
- (legacy_memory_id,),
- ).fetchone()
- if existing is None:
- connection.execute(
- """
- INSERT INTO memory_id_aliases (
- legacy_memory_id,canonical_memory_id,migration_id,created_at
- ) VALUES (?,?,?,?)
- """,
- (
- legacy_memory_id,
- canonical_memory_id,
- migration_id,
- self._now(),
- ),
- )
- elif existing["canonical_memory_id"] != canonical_memory_id:
- raise DataPlaneMigrationError("sdk_data_plane_alias_conflict")
- connection.execute("COMMIT")
-
- def record_raw_alias(
- self,
- *,
- migration_id: str,
- legacy_evidence_id: str,
- canonical_raw_id: str,
- item_index: int = 0,
- ) -> None:
- with main_database(self.database_path).write(
- autocommit=True,
- row_factory=sqlite3.Row,
- ) as connection:
- connection.execute("BEGIN IMMEDIATE")
- existing = connection.execute(
- "SELECT canonical_raw_id FROM raw_id_aliases "
- "WHERE legacy_evidence_id=? AND item_index=?",
- (legacy_evidence_id, item_index),
- ).fetchone()
- if existing is None:
- connection.execute(
- "INSERT INTO raw_id_aliases "
- "(legacy_evidence_id,item_index,canonical_raw_id,migration_id,created_at) "
- "VALUES (?,?,?,?,?)",
- (
- legacy_evidence_id,
- item_index,
- canonical_raw_id,
- migration_id,
- self._now(),
- ),
- )
- elif existing["canonical_raw_id"] != canonical_raw_id:
- raise DataPlaneMigrationError("sdk_data_plane_raw_alias_conflict")
- connection.execute("COMMIT")
-
def progress(self, migration_id: str) -> MigrationProgress:
with main_database(self.database_path).read(row_factory=sqlite3.Row) as connection:
migration = connection.execute(
"SELECT state,phase FROM sdk_data_plane_migrations WHERE migration_id=?",
(migration_id,),
).fetchone()
- if migration is None:
- raise DataPlaneMigrationError("sdk_data_plane_migration_not_found")
- counts = {
- str(row["state"]): int(row["count"])
- for row in connection.execute(
- "SELECT state,COUNT(*) AS count "
- "FROM sdk_data_plane_migration_items WHERE migration_id=? GROUP BY state",
- (migration_id,),
- )
- }
+ if migration is None:
+ raise DataPlaneMigrationError("sdk_data_plane_migration_not_found")
return MigrationProgress(
migration_id=migration_id,
state=str(migration["state"]),
phase=str(migration["phase"]),
- counts=counts,
)
def commit_cutover(
@@ -409,23 +188,8 @@ class SdkDataPlaneMigrationStore:
).fetchone()
if migration is None:
raise DataPlaneMigrationError("sdk_data_plane_migration_not_found")
- if migration["phase"] != "reconciled" or migration["state"] != "ready":
+ if migration["phase"] != "prepared" or migration["state"] != "ready":
raise DataPlaneMigrationError("sdk_data_plane_migration_not_ready")
- unresolved = connection.execute(
- "SELECT state,COUNT(*) AS count FROM sdk_data_plane_migration_items "
- "WHERE migration_id=? GROUP BY state",
- (migration_id,),
- ).fetchall()
- blocked = {
- str(row["state"]): int(row["count"])
- for row in unresolved
- if str(row["state"]) not in TERMINAL_ITEM_STATES
- }
- if blocked:
- raise DataPlaneMigrationError(
- "sdk_data_plane_migration_items_unresolved:"
- + json.dumps(blocked, sort_keys=True, separators=(",", ":"))
- )
if migration["backup_bundle_id"] != backup_bundle_id:
raise DataPlaneMigrationError("sdk_data_plane_migration_bundle_conflict")
existing = connection.execute(
diff --git a/memhub/apps/core/src/one_memory/sdk_memory_runtime.py b/memhub/apps/core/src/one_memory/sdk_memory_runtime.py
index 98338eb..fd247e5 100644
--- a/memhub/apps/core/src/one_memory/sdk_memory_runtime.py
+++ b/memhub/apps/core/src/one_memory/sdk_memory_runtime.py
@@ -4,6 +4,7 @@ from __future__ import annotations
import asyncio
import logging
+import re
from contextlib import suppress
from datetime import UTC, datetime, timedelta
from pathlib import Path
@@ -27,6 +28,9 @@ from one_memory.memory_server.durable_queue import COMPLETED, DurableWriteQueue
from one_memory.sdk_write_processor import SdkWriteCoordinator
from one_memory.semantic_deadline import SemanticMemoryHit
+# Only a code-shaped message is a code; anything else may carry provider text.
+_STABLE_CODE = re.compile(r"[a-z][a-z0-9_]{0,95}")
+
logger = logging.getLogger(__name__)
RAW_RETENTION_DAYS = 3
RAW_RETENTION_CHECK_SECONDS = 6 * 60 * 60
@@ -236,8 +240,22 @@ class SdkMemoryRuntime:
decision=decision,
)
+ async def _mutation_client(self) -> Any:
+ """The write client, with no-model and keychain refusals as SDK codes.
+
+ Callers record these against the edit or deletion instead of failing
+ the request that already committed it.
+ """
+ try:
+ return await self.coordinator.client()
+ except RuntimeError as error:
+ code = str(error)
+ if not _STABLE_CODE.fullmatch(code):
+ code = "hy_memory_client_unavailable"
+ raise HySdkError(code) from error
+
async def update_hy_memory(self, memory_id: str, content: str) -> dict[str, object]:
- client = await self.coordinator.client()
+ client = await self._mutation_client()
try:
result = await asyncio.to_thread(client.update, memory_id, content)
except Exception as error:
@@ -247,7 +265,7 @@ class SdkMemoryRuntime:
return result
async def finish_deletion(self, memory_id: str) -> None:
- client = await self.coordinator.client()
+ client = await self._mutation_client()
try:
result = await asyncio.to_thread(client.delete, memory_id)
except Exception as error:
diff --git a/memhub/apps/core/src/one_memory/sdk_migration_inventory.py b/memhub/apps/core/src/one_memory/sdk_migration_inventory.py
deleted file mode 100644
index a929141..0000000
--- a/memhub/apps/core/src/one_memory/sdk_migration_inventory.py
+++ /dev/null
@@ -1,911 +0,0 @@
-"""Legacy Governance to SDK inventory classification for cutover planning."""
-
-from __future__ import annotations
-
-import hashlib
-import json
-import sqlite3
-from collections.abc import Awaitable, Mapping
-from dataclasses import dataclass
-from datetime import UTC, datetime
-from pathlib import Path
-from typing import Protocol
-
-from one_memory.database_access import main_database
-from one_memory.hy_sdk_types import HySdkMemoryPage, HySdkMemoryRecord, HySdkRawPage
-from one_memory.input_sanitization import sanitize_model_input
-from one_memory.sdk_data_plane_migration import SdkDataPlaneMigrationStore
-
-
-def content_hash(content: str) -> str:
- return hashlib.sha256(content.encode("utf-8")).hexdigest()
-
-
-@dataclass(frozen=True)
-class LegacyMemoryInventory:
- legacy_memory_id: str
- canonical_memory_id: str
- user_id: str
- agent_id: str
- session_id: str
- content: str
- content_hash: str
- status: str
- layer: str | None
- owner: str | None
- tags: tuple[str, ...]
- supersedes: tuple[str, ...]
- superseded_by: tuple[str, ...]
- productivity_kind: str | None = None
- productivity_variant: str | None = None
-
-
-@dataclass(frozen=True)
-class SdkMemoryInventory:
- memory_id: str
- user_id: str
- agent_id: str
- content_hash: str
- status: str
- productivity_kind: str | None = None
- productivity_variant: str | None = None
-
-
-@dataclass(frozen=True)
-class MemoryMigrationDecision:
- item_key: str
- action: str
- legacy_memory_id: str | None
- canonical_memory_id: str
- reason: str
-
-
-@dataclass(frozen=True)
-class LegacyRawInventory:
- evidence_id: str
- user_id: str
- agent_id: str
- session_id: str
- source: str
- content: str
- content_hash: str
- memory_at: datetime | None
- metadata: dict[str, object]
-
-
-@dataclass(frozen=True)
-class SdkRawInventory:
- raw_id: str
- user_id: str
- agent_id: str
- session_id: str
- source: str
- content_hash: str
-
-
-@dataclass(frozen=True)
-class RawMigrationDecision:
- item_key: str
- action: str
- evidence_id: str | None
- raw_id: str
- reason: str
-
-
-_KIND_LAYER = {
- "chat": "l2_fact",
- "schema": "l6_schema",
- "intention": "l7_intention",
-}
-_PRODUCTIVITY_LAYER = {
- "coding_experience": ("experience", "coding"),
- "working_experience": ("experience", "working"),
- "coding_fact": ("fact", "coding"),
- # Historical receipts called every productivity fact "working_fact",
- # including facts stored beside a coding-variant experience. Resolve the
- # actual variant from the SDK exact lookup instead of trusting this label.
- "working_fact": ("fact", None),
-}
-_RAW_TURN_MAX_CHARACTERS = 60_000
-_RAW_USER_MAX_CHARACTERS = 30_000
-
-SDK_LIFECYCLE_STATUSES = (
- "active",
- "superseded",
- "shadow",
- "negated",
- "conflicted",
- "archived",
-)
-
-
-class SdkMigrationClient(Protocol):
- def list_memories(
- self,
- *,
- user_id: str,
- agent_id: str | None = None,
- statuses: tuple[str, ...] = ("active",),
- limit: int = 100,
- offset: int = 0,
- order: str = "desc",
- ) -> Awaitable[HySdkMemoryPage]: ...
-
- def write_legacy_memory(
- self,
- *,
- content: str,
- memory_id: str,
- user_id: str,
- agent_id: str,
- session_id: str,
- layer: str,
- status: str,
- owner: str | None,
- tags: tuple[str, ...],
- supersedes: tuple[str, ...],
- superseded_by: tuple[str, ...],
- is_latest: bool,
- request_id: str,
- ) -> Awaitable[dict[str, object]]: ...
-
- def read_raw_inventory(
- self,
- *,
- user_id: str,
- agent_id: str | None = None,
- session_id: str | None = None,
- source: str | None = None,
- limit: int = 100,
- offset: int = 0,
- order: str = "desc",
- ) -> Awaitable[HySdkRawPage]: ...
-
- def write_legacy_raw(
- self,
- *,
- content: str,
- raw_id: str,
- user_id: str,
- agent_id: str,
- session_id: str,
- source: str,
- memory_at: datetime | None,
- metadata: Mapping[str, object],
- request_id: str,
- ) -> Awaitable[dict[str, object]]: ...
-
- def delete(self, memory_id: str) -> Awaitable[dict[str, object]]: ...
-
- def read_productivity_memory(
- self,
- memory_id: str,
- *,
- kind: str,
- variant: str | None,
- ) -> Awaitable[tuple[HySdkMemoryRecord, str] | None]: ...
-
- def list_productivity_memories(
- self,
- ) -> Awaitable[tuple[dict[str, object], ...]]: ...
-
- def delete_productivity_memory(
- self,
- memory_id: str,
- *,
- kind: str,
- variant: str | None,
- user_id: str,
- ) -> Awaitable[dict[str, object]]: ...
-
- def delete_raw(self, raw_id: str) -> Awaitable[dict[str, object]]: ...
-
-
-def read_legacy_memory_inventory(database_path: Path) -> tuple[LegacyMemoryInventory, ...]:
- """Read legacy memory intent without constructing a Governance writer."""
- with main_database(database_path).read(row_factory=sqlite3.Row) as connection:
- mappings: dict[str, tuple[str, str | None, str | None]] = {}
- for row in connection.execute("SELECT body FROM memory_task_outputs"):
- try:
- body = json.loads(row["body"])
- except (TypeError, ValueError):
- continue
- legacy_id = body.get("memory_id")
- sdk_id = body.get("hy_memory_id")
- if isinstance(legacy_id, str) and isinstance(sdk_id, str):
- mappings[legacy_id] = (
- sdk_id,
- body.get("layer") if isinstance(body.get("layer"), str) else None,
- body.get("owner") if body.get("owner") in {"user", "agent"} else None,
- )
-
- supersedes: dict[str, list[str]] = {}
- superseded_by: dict[str, list[str]] = {}
- for row in connection.execute(
- "SELECT memory_id,superseded_by FROM governed_memory_supersessions "
- "ORDER BY memory_id,superseded_by"
- ):
- older, newer = str(row["memory_id"]), str(row["superseded_by"])
- superseded_by.setdefault(older, []).append(newer)
- supersedes.setdefault(newer, []).append(older)
-
- records: list[LegacyMemoryInventory] = []
- for row in connection.execute(
- "SELECT memory_id,user_id,kind,status,content,tags_json "
- "FROM governed_memories ORDER BY memory_id"
- ):
- legacy_id = str(row["memory_id"])
- mapped_id, mapped_layer, mapped_owner = mappings.get(legacy_id, (legacy_id, None, None))
- try:
- tags_value = json.loads(row["tags_json"] or "[]")
- except (TypeError, ValueError):
- tags_value = []
- tags = tuple(value for value in tags_value if isinstance(value, str))
- kind = str(row["kind"])
- status = str(row["status"])
- productivity = _PRODUCTIVITY_LAYER.get(mapped_layer or "")
- records.append(
- LegacyMemoryInventory(
- legacy_memory_id=legacy_id,
- canonical_memory_id=mapped_id,
- user_id=str(row["user_id"]),
- agent_id="one-memory",
- session_id="default_session",
- content=str(row["content"]),
- content_hash=content_hash(str(row["content"])),
- status=(
- "active"
- if status == "confirmed"
- else "superseded"
- if status == "superseded"
- else "deleted"
- ),
- layer=mapped_layer or _KIND_LAYER.get(kind),
- owner=mapped_owner,
- tags=tags,
- supersedes=tuple(supersedes.get(legacy_id, ())),
- superseded_by=tuple(superseded_by.get(legacy_id, ())),
- productivity_kind=productivity[0] if productivity else None,
- productivity_variant=productivity[1] if productivity else None,
- )
- )
- return tuple(records)
-
-
-def _parse_datetime(value: object) -> datetime | None:
- if not isinstance(value, str) or not value:
- return None
- try:
- return datetime.fromisoformat(value)
- except ValueError:
- return None
-
-
-def read_legacy_raw_inventory(
- database_path: Path,
- *,
- cutoff: datetime,
-) -> tuple[LegacyRawInventory, ...]:
- """Rebuild recent legacy messages into one user/assistant raw per turn."""
- if cutoff.tzinfo is None:
- cutoff = cutoff.replace(tzinfo=UTC)
- with main_database(database_path).read(row_factory=sqlite3.Row) as connection:
- rows = connection.execute(
- """
- SELECT evidence_id,user_id,source_id,workspace,project,
- conversation_id,message_id,source_ordinal,content_kind,
- role,written_by,created_at,imported_at,verbatim,content_hash,revision
- FROM evidence
- WHERE content_kind='raw_conversation'
- AND (
- (role='user' AND written_by='person')
- OR role='assistant'
- )
- ORDER BY source_id,conversation_id,workspace,
- COALESCE(created_at,imported_at),
- COALESCE(ingest_seq,rowid),evidence_id
- """
- ).fetchall()
- turns: list[LegacyRawInventory] = []
- current_key: tuple[str, str, str, str] | None = None
- current: list[sqlite3.Row] = []
-
- def finish() -> None:
- if not current or str(current[0]["role"]) != "user":
- return
- assistants = [row for row in current[1:] if str(row["role"]) == "assistant"]
- if not assistants:
- return
- rows_for_turn = [current[0], *assistants]
- completed_at = _parse_datetime(
- assistants[-1]["created_at"] or assistants[-1]["imported_at"]
- )
- if completed_at is not None and completed_at.tzinfo is None:
- completed_at = completed_at.replace(tzinfo=UTC)
- if completed_at is None or completed_at.astimezone(UTC) < cutoff.astimezone(UTC):
- return
- evidence_ids = [str(row["evidence_id"]) for row in rows_for_turn]
- source = str(current[0]["source_id"])
- session_id = str(current[0]["conversation_id"])
- digest = hashlib.sha256(
- "\0".join((source, session_id, *evidence_ids)).encode()
- ).hexdigest()
- raw_user_text = str(rows_for_turn[0]["verbatim"] or "")
- user_text = sanitize_model_input(raw_user_text)[:_RAW_USER_MAX_CHARACTERS]
- content_parts = [f"[user]: {user_text}"]
- truncated = len(raw_user_text) > len(user_text)
- for row in assistants:
- sanitized = sanitize_model_input(str(row["verbatim"] or ""))
- text = sanitized
- prefix = "[assistant]: "
- remaining = _RAW_TURN_MAX_CHARACTERS - len("\n".join(content_parts)) - 1
- if remaining <= len(prefix):
- truncated = True
- break
- available = remaining - len(prefix)
- if len(sanitized) > available:
- text = text[:available]
- truncated = True
- content_parts.append(f"{prefix}{text}")
- if len(text) < len(sanitized):
- break
- content = "\n".join(content_parts)
- if not content:
- return
- turns.append(
- LegacyRawInventory(
- evidence_id=f"turn:{digest}",
- user_id=str(current[0]["user_id"]),
- agent_id="all-apps",
- session_id=session_id,
- source=source,
- content=content,
- content_hash=content_hash(content),
- memory_at=_parse_datetime(
- current[0]["created_at"] or current[0]["imported_at"]
- ),
- metadata={
- "evidence_ids": evidence_ids,
- "workspace": current[0]["workspace"],
- "project": current[0]["project"],
- "input_kind": "legacy_turn",
- "truncated": truncated,
- },
- )
- )
-
- for row in rows:
- key = (
- str(row["source_id"]),
- str(row["conversation_id"]),
- str(row["user_id"]),
- str(row["workspace"] or ""),
- )
- if key != current_key:
- finish()
- current_key, current = key, []
- if str(row["role"]) == "user":
- finish()
- current = [row]
- elif current:
- current.append(row)
- finish()
- return tuple(turns)
-
-
-def classify_memory_inventory(
- legacy: tuple[LegacyMemoryInventory, ...],
- sdk: tuple[SdkMemoryInventory, ...],
-) -> tuple[MemoryMigrationDecision, ...]:
- """Classify every legacy and SDK memory; no item may silently disappear."""
- sdk_by_id: dict[str, list[SdkMemoryInventory]] = {}
- for sdk_item in sdk:
- sdk_by_id.setdefault(sdk_item.memory_id, []).append(sdk_item)
- claimed_sdk_items: set[tuple[str, str | None, str | None]] = set()
- decisions: list[MemoryMigrationDecision] = []
- for item in legacy:
- candidates = [
- candidate
- for candidate in sdk_by_id.get(item.canonical_memory_id, ())
- if (
- candidate.productivity_kind == item.productivity_kind
- and (
- item.productivity_kind is None
- or item.productivity_variant is None
- or candidate.productivity_variant == item.productivity_variant
- )
- )
- ]
- if len(candidates) > 1:
- claimed_sdk_items.update(_sdk_inventory_key(candidate) for candidate in candidates)
- decisions.append(
- MemoryMigrationDecision(
- item_key=f"memory:{item.legacy_memory_id}",
- action="quarantine",
- legacy_memory_id=item.legacy_memory_id,
- canonical_memory_id=item.canonical_memory_id,
- reason="sdk_memory_id_ambiguous",
- )
- )
- continue
- current = candidates[0] if candidates else None
- if current is not None:
- claimed_sdk_items.add(_sdk_inventory_key(current))
- if item.status == "deleted":
- decisions.append(
- MemoryMigrationDecision(
- item_key=f"memory:{item.legacy_memory_id}",
- action="delete" if current is not None else "verified_deleted",
- legacy_memory_id=item.legacy_memory_id,
- canonical_memory_id=item.canonical_memory_id,
- reason="legacy_user_delete",
- )
- )
- continue
- if item.layer is None:
- decisions.append(
- MemoryMigrationDecision(
- item_key=f"memory:{item.legacy_memory_id}",
- action="quarantine",
- legacy_memory_id=item.legacy_memory_id,
- canonical_memory_id=item.canonical_memory_id,
- reason="legacy_layer_unknown",
- )
- )
- continue
- if current is None:
- if item.productivity_kind is not None:
- decisions.append(
- MemoryMigrationDecision(
- item_key=f"memory:{item.legacy_memory_id}",
- action="quarantine",
- legacy_memory_id=item.legacy_memory_id,
- canonical_memory_id=item.canonical_memory_id,
- reason="mapped_productivity_memory_missing",
- )
- )
- continue
- decisions.append(
- MemoryMigrationDecision(
- item_key=f"memory:{item.legacy_memory_id}",
- action="import_exact",
- legacy_memory_id=item.legacy_memory_id,
- canonical_memory_id=item.canonical_memory_id,
- reason="governance_only",
- )
- )
- continue
- if item.productivity_kind is not None and (
- current.user_id == item.user_id
- and current.productivity_kind == item.productivity_kind
- and (
- item.productivity_variant is None
- or current.productivity_variant == item.productivity_variant
- )
- ):
- action = (
- "matched" if item.legacy_memory_id == item.canonical_memory_id else "record_alias"
- )
- reason = "existing_productivity_identity"
- elif (
- current.content_hash == item.content_hash
- and current.status == item.status
- and current.user_id == item.user_id
- and current.agent_id == item.agent_id
- ):
- action = (
- "matched" if item.legacy_memory_id == item.canonical_memory_id else "record_alias"
- )
- reason = "same_identity" if action == "matched" else "mapped_identity"
- else:
- action, reason = "quarantine", "content_owner_or_lifecycle_diverged"
- decisions.append(
- MemoryMigrationDecision(
- item_key=f"memory:{item.legacy_memory_id}",
- action=action,
- legacy_memory_id=item.legacy_memory_id,
- canonical_memory_id=item.canonical_memory_id,
- reason=reason,
- )
- )
-
- for sdk_item in sdk:
- if _sdk_inventory_key(sdk_item) not in claimed_sdk_items:
- decisions.append(
- MemoryMigrationDecision(
- item_key=_sdk_item_key(sdk_item),
- action="adopt",
- legacy_memory_id=None,
- canonical_memory_id=sdk_item.memory_id,
- reason="sdk_only",
- )
- )
- return tuple(decisions)
-
-
-def _sdk_inventory_key(
- item: SdkMemoryInventory,
-) -> tuple[str, str | None, str | None]:
- return (
- item.memory_id,
- item.productivity_kind,
- item.productivity_variant,
- )
-
-
-def _sdk_item_key(item: SdkMemoryInventory) -> str:
- namespace = (
- f"{item.productivity_variant}:{item.productivity_kind}"
- if item.productivity_kind is not None
- else "chat"
- )
- return f"sdk:{namespace}:{item.memory_id}"
-
-
-async def read_sdk_memory_inventory(
- client: SdkMigrationClient,
- *,
- legacy: tuple[LegacyMemoryInventory, ...] = (),
- user_id: str = "local",
- page_size: int = 500,
-) -> tuple[SdkMemoryInventory, ...]:
- records: list[SdkMemoryInventory] = []
- offset = 0
- while True:
- page = await client.list_memories(
- user_id=user_id,
- statuses=SDK_LIFECYCLE_STATUSES,
- limit=page_size,
- offset=offset,
- order="asc",
- )
- for memory in page.memories:
- records.append(
- SdkMemoryInventory(
- memory_id=memory.memory_id,
- user_id=memory.user_id,
- agent_id=memory.agent_id,
- content_hash=content_hash(memory.content),
- status=memory.status,
- )
- )
- consumed = len(page.memories)
- offset += consumed
- if consumed == 0 or offset >= page.total:
- break
- del legacy
- for memory in await client.list_productivity_memories():
- try:
- records.append(
- SdkMemoryInventory(
- memory_id=str(memory["memory_id"]),
- user_id=str(memory["user_id"]),
- agent_id=str(memory["agent_id"]),
- content_hash=content_hash(str(memory["content"])),
- status=str(memory["status"]),
- productivity_kind=str(memory["kind"]),
- productivity_variant=str(memory["variant"]),
- )
- )
- except KeyError as error:
- raise RuntimeError("invalid productivity inventory row") from error
- return tuple(
- sorted(
- records,
- key=lambda item: (
- item.memory_id,
- item.productivity_kind or "",
- item.productivity_variant or "",
- ),
- )
- )
-
-
-def record_memory_migration_plan(
- store: SdkDataPlaneMigrationStore,
- *,
- migration_id: str,
- legacy: tuple[LegacyMemoryInventory, ...],
- sdk: tuple[SdkMemoryInventory, ...],
-) -> tuple[MemoryMigrationDecision, ...]:
- decisions = classify_memory_inventory(legacy, sdk)
- legacy_by_id = {item.legacy_memory_id: item for item in legacy}
- sdk_by_item_key = {_sdk_item_key(item): item for item in sdk}
- for decision in decisions:
- source_hash = (
- legacy_by_id[decision.legacy_memory_id].content_hash
- if decision.legacy_memory_id is not None
- else sdk_by_item_key[decision.item_key].content_hash
- )
- operation_key = hashlib.sha256(
- f"{migration_id}:{decision.item_key}:{decision.action}:{source_hash}".encode()
- ).hexdigest()
- store.upsert_item(
- migration_id=migration_id,
- item_key=decision.item_key,
- source_kind=f"memory:{decision.action}",
- source_id=(
- decision.legacy_memory_id
- if decision.legacy_memory_id is not None
- else decision.canonical_memory_id
- ),
- source_hash=source_hash,
- operation_key=operation_key,
- )
- return decisions
-
-
-async def execute_memory_migration_plan(
- client: SdkMigrationClient,
- store: SdkDataPlaneMigrationStore,
- *,
- migration_id: str,
- legacy: tuple[LegacyMemoryInventory, ...],
- decisions: tuple[MemoryMigrationDecision, ...],
-) -> None:
- legacy_by_id = {item.legacy_memory_id: item for item in legacy}
- for decision in decisions:
- item = (
- legacy_by_id.get(decision.legacy_memory_id)
- if decision.legacy_memory_id is not None
- else None
- )
- if decision.action == "quarantine":
- store.settle_item(
- migration_id=migration_id,
- item_key=decision.item_key,
- state="quarantine",
- error_code=decision.reason,
- )
- continue
- if decision.action == "adopt":
- store.settle_item(
- migration_id=migration_id,
- item_key=decision.item_key,
- state="adopted",
- target_memory_id=decision.canonical_memory_id,
- )
- continue
- if decision.action in {"delete", "verified_deleted"}:
- if decision.action == "delete":
- if item is not None and item.productivity_kind is not None:
- await client.delete_productivity_memory(
- decision.canonical_memory_id,
- kind=item.productivity_kind,
- variant=item.productivity_variant,
- user_id=item.user_id,
- )
- else:
- await client.delete(decision.canonical_memory_id)
- store.settle_item(
- migration_id=migration_id,
- item_key=decision.item_key,
- state="deleted",
- target_memory_id=decision.canonical_memory_id,
- )
- continue
- if item is None:
- raise RuntimeError(f"missing legacy inventory for {decision.item_key}")
- if decision.action == "import_exact":
- assert item.layer is not None
- request_id = hashlib.sha256(
- f"{migration_id}:{decision.item_key}:import_exact".encode()
- ).hexdigest()
- await client.write_legacy_memory(
- content=item.content,
- memory_id=item.canonical_memory_id,
- user_id=item.user_id,
- agent_id=item.agent_id,
- session_id=item.session_id,
- source="legacy-governance",
- layer=item.layer,
- status=item.status,
- owner=item.owner,
- tags=item.tags,
- supersedes=item.supersedes,
- superseded_by=item.superseded_by,
- is_latest=item.status == "active",
- memory_at=None,
- metadata={},
- request_id=request_id,
- )
- if decision.action == "record_alias":
- store.record_alias(
- migration_id=migration_id,
- legacy_memory_id=item.legacy_memory_id,
- canonical_memory_id=item.canonical_memory_id,
- )
- store.settle_item(
- migration_id=migration_id,
- item_key=decision.item_key,
- state="verified",
- target_memory_id=decision.canonical_memory_id,
- target_hash=item.content_hash,
- )
-
-
-async def read_sdk_raw_inventory(
- client: SdkMigrationClient,
- *,
- user_id: str = "local",
- page_size: int = 500,
-) -> tuple[SdkRawInventory, ...]:
- records: dict[str, SdkRawInventory] = {}
- offset = 0
- while True:
- page = await client.read_raw_inventory(
- user_id=user_id,
- limit=page_size,
- offset=offset,
- order="asc",
- )
- for raw in page.raw:
- raw_id = str(raw.raw_id)
- records[raw_id] = SdkRawInventory(
- raw_id=raw_id,
- user_id=str(raw.user_id),
- agent_id=str(raw.agent_id),
- session_id=str(raw.session_id),
- source=str(raw.source),
- content_hash=content_hash(str(raw.content)),
- )
- consumed = len(page.raw)
- offset += consumed
- if consumed == 0 or offset >= page.total:
- break
- return tuple(records[key] for key in sorted(records))
-
-
-def classify_raw_inventory(
- legacy: tuple[LegacyRawInventory, ...],
- sdk: tuple[SdkRawInventory, ...],
-) -> tuple[RawMigrationDecision, ...]:
- sdk_by_identity: dict[tuple[str, str, str, str, str], list[SdkRawInventory]] = {}
- for raw in sdk:
- identity = (
- raw.user_id,
- raw.agent_id,
- raw.session_id,
- raw.source,
- raw.content_hash,
- )
- sdk_by_identity.setdefault(identity, []).append(raw)
- claimed: set[str] = set()
- decisions: list[RawMigrationDecision] = []
- for item in legacy:
- identity = (
- item.user_id,
- item.agent_id,
- item.session_id,
- item.source,
- item.content_hash,
- )
- candidates = sorted(
- sdk_by_identity.get(identity, ()),
- key=lambda candidate: candidate.raw_id,
- )
- existing = next(
- (candidate for candidate in candidates if candidate.raw_id not in claimed),
- None,
- )
- if existing is not None:
- claimed.add(existing.raw_id)
- decisions.append(
- RawMigrationDecision(
- item_key=f"raw:{item.evidence_id}",
- action="matched",
- evidence_id=item.evidence_id,
- raw_id=existing.raw_id,
- reason="same_content_and_scope",
- )
- )
- else:
- decisions.append(
- RawMigrationDecision(
- item_key=f"raw:{item.evidence_id}",
- action="import_exact",
- evidence_id=item.evidence_id,
- raw_id=f"evidence:{item.evidence_id}",
- reason="evidence_only",
- )
- )
- for raw in sdk:
- if raw.raw_id not in claimed:
- decisions.append(
- RawMigrationDecision(
- item_key=f"sdk-raw:{raw.raw_id}",
- action="delete",
- evidence_id=None,
- raw_id=raw.raw_id,
- reason="outside_canonical_three_day_turns",
- )
- )
- return tuple(decisions)
-
-
-def record_raw_migration_plan(
- store: SdkDataPlaneMigrationStore,
- *,
- migration_id: str,
- legacy: tuple[LegacyRawInventory, ...],
- sdk: tuple[SdkRawInventory, ...],
-) -> tuple[RawMigrationDecision, ...]:
- decisions = classify_raw_inventory(legacy, sdk)
- legacy_by_id = {item.evidence_id: item for item in legacy}
- sdk_by_id = {item.raw_id: item for item in sdk}
- for decision in decisions:
- source_hash = (
- legacy_by_id[decision.evidence_id].content_hash
- if decision.evidence_id is not None
- else sdk_by_id[decision.raw_id].content_hash
- )
- operation_key = hashlib.sha256(
- f"{migration_id}:{decision.item_key}:{decision.action}:{source_hash}".encode()
- ).hexdigest()
- store.upsert_item(
- migration_id=migration_id,
- item_key=decision.item_key,
- source_kind=f"raw:{decision.action}",
- source_id=decision.evidence_id or decision.raw_id,
- source_hash=source_hash,
- operation_key=operation_key,
- )
- return decisions
-
-
-async def execute_raw_migration_plan(
- client: SdkMigrationClient,
- store: SdkDataPlaneMigrationStore,
- *,
- migration_id: str,
- legacy: tuple[LegacyRawInventory, ...],
- decisions: tuple[RawMigrationDecision, ...],
-) -> None:
- legacy_by_id = {item.evidence_id: item for item in legacy}
- for decision in decisions:
- if decision.action == "delete":
- await client.delete_raw(decision.raw_id)
- store.settle_item(
- migration_id=migration_id,
- item_key=decision.item_key,
- state="deleted",
- target_memory_id=decision.raw_id,
- )
- continue
- item = legacy_by_id.get(decision.evidence_id or "")
- if item is None:
- raise RuntimeError(f"missing legacy raw inventory for {decision.item_key}")
- if decision.action == "import_exact":
- request_id = hashlib.sha256(
- f"{migration_id}:{decision.item_key}:write_legacy_raw".encode()
- ).hexdigest()
- await client.write_legacy_raw(
- content=item.content,
- raw_id=decision.raw_id,
- user_id=item.user_id,
- agent_id=item.agent_id,
- session_id=item.session_id,
- source=item.source,
- memory_at=item.memory_at,
- metadata=item.metadata,
- request_id=request_id,
- )
- store.settle_item(
- migration_id=migration_id,
- item_key=decision.item_key,
- state="verified",
- target_memory_id=decision.raw_id,
- target_hash=item.content_hash,
- )
- evidence_ids = item.metadata.get("evidence_ids")
- for evidence_id in (
- evidence_ids
- if isinstance(evidence_ids, list)
- else [item.evidence_id]
- ):
- if isinstance(evidence_id, str) and evidence_id:
- store.record_raw_alias(
- migration_id=migration_id,
- legacy_evidence_id=evidence_id,
- canonical_raw_id=decision.raw_id,
- )
diff --git a/memhub/apps/core/src/one_memory/sdk_migration_runner.py b/memhub/apps/core/src/one_memory/sdk_migration_runner.py
index 85e3b2b..f0ecf9e 100644
--- a/memhub/apps/core/src/one_memory/sdk_migration_runner.py
+++ b/memhub/apps/core/src/one_memory/sdk_migration_runner.py
@@ -1,4 +1,10 @@
-"""Offline, resumable backfill runner for SDK memory authority."""
+"""Offline startup cutover from the legacy Governance mirror to SDK authority.
+
+Every released build wrote memory and raw turns through the SDK first; the
+Evidence/Governance tables only mirrored them. The cutover therefore copies no
+memory. It makes the surviving write queue independent of the legacy tables,
+commits SDK authority and drops those tables.
+"""
from __future__ import annotations
@@ -6,13 +12,11 @@ import hashlib
import json
import shutil
import sqlite3
-from collections.abc import Callable
-from contextlib import suppress
from dataclasses import dataclass
-from datetime import UTC, datetime, timedelta
from pathlib import Path
from one_memory.database_access import main_database
+from one_memory.imports.models import EvidenceRecord
from one_memory.imports.storage import EvidenceStore
from one_memory.memory_server.contracts import (
BatchIdentity,
@@ -20,177 +24,80 @@ from one_memory.memory_server.contracts import (
internal_session_id_for,
)
from one_memory.memory_server.durable_queue import DurableWriteQueue
+from one_memory.memory_server.unified_entry import UnifiedWriteIntake
from one_memory.sdk_data_plane_migration import (
CutoverMarker,
DataPlaneMigrationError,
- MigrationProgress,
SdkDataPlaneMigrationStore,
)
from one_memory.sdk_migration_bundle import inspect_sdk_migration_bundle
-from one_memory.sdk_migration_inventory import (
- SdkMigrationClient,
- execute_memory_migration_plan,
- execute_raw_migration_plan,
- read_legacy_memory_inventory,
- read_legacy_raw_inventory,
- read_sdk_memory_inventory,
- read_sdk_raw_inventory,
- record_memory_migration_plan,
- record_raw_migration_plan,
-)
from one_memory.target_schema import drop_legacy_tables
-
-@dataclass(frozen=True)
-class BackfillResult:
- migration_id: str
- state: str
- phase: str
- counts: dict[str, int]
- memory_items: int
- raw_items: int
+PREPARED_PHASE = "prepared"
@dataclass(frozen=True)
-class SdkMigrationStaging:
- migration_id: str
- staging_home: Path
- live_root: Path
- staging_root: Path
- protective_root: Path
- state_file: Path
+class CutoverPreparation:
+ """What the cutover did to legacy work, for the receipt and the dialog."""
+
+ frozen_jobs: int
+ moved_tasks: int
+ moved_dispatches: int
+ abandoned: dict[str, int]
+
+ def as_document(self) -> dict[str, object]:
+ return {
+ "frozen_jobs": self.frozen_jobs,
+ "moved_tasks": self.moved_tasks,
+ "moved_dispatches": self.moved_dispatches,
+ "abandoned": dict(self.abandoned),
+ }
-def sdk_migration_staging(
- *,
- data_dir: Path,
- migration_id: str,
- namespace: str,
-) -> SdkMigrationStaging:
- """Return same-filesystem paths without trusting the migration ID as a path."""
- token = hashlib.sha256(migration_id.encode()).hexdigest()[:16]
- home = data_dir.resolve()
- staging_home = home / ".sdk-migration" / token
- live_root = home / "hy-memory" / namespace
- return SdkMigrationStaging(
- migration_id=migration_id,
- staging_home=staging_home,
- live_root=live_root,
- staging_root=staging_home / "hy-memory" / namespace,
- protective_root=live_root.parent / f".{namespace}.pre-migration-{token}",
- state_file=staging_home / "swap-state.json",
- )
+def _tables(database: sqlite3.Connection) -> set[str]:
+ return {
+ str(row[0])
+ for row in database.execute("SELECT name FROM sqlite_master WHERE type='table'")
+ }
-def _write_staging_state(paths: SdkMigrationStaging, phase: str) -> None:
- paths.staging_home.mkdir(parents=True, exist_ok=True)
- temporary = paths.state_file.with_suffix(".tmp")
- temporary.write_text(
- json.dumps(
- {"schema_version": 1, "migration_id": paths.migration_id, "phase": phase},
- sort_keys=True,
- ),
- encoding="utf-8",
- )
- temporary.replace(paths.state_file)
+def recover_abandoned_staging(*, data_dir: Path, database_path: Path, namespace: str) -> None:
+ """Undo or discard the SDK staging copy that builds through 19 created.
-
-def recover_sdk_migration_swap(
- *,
- paths: SdkMigrationStaging,
- database_path: Path,
-) -> None:
- """Recover an interrupted directory swap before opening any SDK store."""
- if not paths.protective_root.exists():
- return
+ Those builds staged a full SDK copy and swapped directories at cutover. The
+ live root was untouched until the swap, so before SDK authority the
+ protective root is the original store; after it, the protective root is
+ stale.
+ """
+ home = data_dir.resolve()
+ live_root = home / "hy-memory" / namespace
authority = SdkDataPlaneMigrationStore(database_path).authority()
- if authority.value == "sdk":
- if not paths.live_root.is_dir():
- raise DataPlaneMigrationError("sdk_migration_committed_store_missing")
- shutil.rmtree(paths.protective_root)
- shutil.rmtree(paths.staging_home, ignore_errors=True)
- return
- if paths.live_root.exists():
- if paths.staging_root.exists():
- raise DataPlaneMigrationError("sdk_migration_swap_state_ambiguous")
- paths.staging_root.parent.mkdir(parents=True, exist_ok=True)
- paths.live_root.replace(paths.staging_root)
- paths.protective_root.replace(paths.live_root)
- _write_staging_state(paths, "recovered")
-
-
-def prepare_sdk_migration_staging(
- *,
- paths: SdkMigrationStaging,
- database_path: Path,
-) -> Path:
- """Build a private copy; incomplete retries restart from the unchanged live root."""
- recover_sdk_migration_swap(paths=paths, database_path=database_path)
- migration_store = SdkDataPlaneMigrationStore(database_path)
- try:
- progress = migration_store.progress(paths.migration_id)
- reuse_ready = progress.state == "ready" and progress.phase == "reconciled"
- except Exception:
- reuse_ready = False
- if paths.staging_root.exists():
- if paths.staging_root.is_symlink() or not paths.staging_root.is_dir():
- raise DataPlaneMigrationError("sdk_migration_staging_invalid")
- if reuse_ready:
- return paths.staging_home
- shutil.rmtree(paths.staging_root)
- paths.live_root.mkdir(parents=True, exist_ok=True)
- from one_memory.hy_sdk_migration import snapshot_data_plane
-
- snapshot_data_plane(
- source_root=paths.live_root,
- target_root=paths.staging_root,
- writers_stopped=True,
- )
- _write_staging_state(paths, "prepared")
- return paths.staging_home
+ protective_roots = sorted(live_root.parent.glob(f".{namespace}.pre-migration-*"))
+ if authority.value != "sdk" and len(protective_roots) > 1:
+ raise DataPlaneMigrationError("sdk_migration_swap_state_ambiguous")
+ for protective_root in protective_roots:
+ if authority.value == "sdk":
+ if not live_root.is_dir():
+ raise DataPlaneMigrationError("sdk_migration_committed_store_missing")
+ shutil.rmtree(protective_root)
+ continue
+ if live_root.exists():
+ shutil.rmtree(live_root)
+ protective_root.replace(live_root)
+ shutil.rmtree(home / ".sdk-migration", ignore_errors=True)
-def activate_sdk_migration_staging(
- *,
- paths: SdkMigrationStaging,
- database_path: Path,
- commit: Callable[[], CutoverMarker],
-) -> CutoverMarker:
- """Atomically expose staged SDK files and roll them back if cutover fails."""
- recover_sdk_migration_swap(paths=paths, database_path=database_path)
- if not paths.staging_root.is_dir():
- raise DataPlaneMigrationError("sdk_migration_staging_missing")
- paths.live_root.mkdir(parents=True, exist_ok=True)
- _write_staging_state(paths, "activating")
- paths.live_root.replace(paths.protective_root)
- try:
- paths.staging_root.replace(paths.live_root)
- _write_staging_state(paths, "staging_live")
- marker = commit()
- except BaseException:
- if paths.live_root.exists():
- paths.staging_root.parent.mkdir(parents=True, exist_ok=True)
- paths.live_root.replace(paths.staging_root)
- if paths.protective_root.exists():
- paths.protective_root.replace(paths.live_root)
- _write_staging_state(paths, "rolled_back")
- raise
- with suppress(OSError):
- _write_staging_state(paths, "committed")
- # Authority is already committed; leftover paths are detected by
- # sdk-migration-status and removed by sdk-cleanup on the next startup.
- shutil.rmtree(paths.protective_root, ignore_errors=True)
- shutil.rmtree(paths.staging_home, ignore_errors=True)
- return marker
+def abandoned_staging_present(*, data_dir: Path, namespace: str) -> bool:
+ home = data_dir.resolve()
+ return (home / ".sdk-migration").exists() or any(
+ (home / "hy-memory").glob(f".{namespace}.pre-migration-*")
+ )
def prepare_legacy_write_queue_for_cutover(database_path: Path) -> tuple[int, int]:
"""Freeze old jobs and move unstarted task rows into the surviving queue."""
with main_database(database_path).read(row_factory=sqlite3.Row) as database:
- tables = {
- str(row["name"])
- for row in database.execute("SELECT name FROM sqlite_master WHERE type='table'")
- }
+ tables = _tables(database)
if "evidence" not in tables:
return (0, 0)
jobs = (
@@ -278,30 +185,55 @@ def prepare_legacy_write_queue_for_cutover(database_path: Path) -> tuple[int, in
return (frozen, moved)
-def sdk_cutover_blockers(database_path: Path) -> dict[str, int]:
- """Count legacy work that could write or duplicate data after cutover."""
- blockers: dict[str, int] = {}
+def move_pending_dispatches(database_path: Path) -> int:
+ """Queue Evidence that was waiting for its source file to finish.
+
+ Its source cursor has already moved past these rows, so History catch-up
+ after the cutover would not read them again.
+ """
with main_database(database_path).read(row_factory=sqlite3.Row) as database:
- tables = {
- str(row["name"])
- for row in database.execute("SELECT name FROM sqlite_master WHERE type='table'")
- }
+ if not {"evidence", "evidence_pending_memory_dispatch"} <= _tables(database):
+ return 0
+ identifiers = tuple(
+ str(row["evidence_id"])
+ for row in database.execute(
+ "SELECT DISTINCT pending.evidence_id "
+ "FROM evidence_pending_memory_dispatch pending "
+ "JOIN evidence USING(evidence_id) "
+ "ORDER BY COALESCE(evidence.created_at,evidence.imported_at),"
+ "COALESCE(evidence.ingest_seq,evidence.rowid),pending.evidence_id"
+ )
+ )
+ evidence = EvidenceStore(database_path)
+ conversations: dict[tuple[str, str, str, str | None], list[EvidenceRecord]] = {}
+ for identifier in identifiers:
+ record = evidence.get(identifier)
+ if record is None:
+ continue
+ key = (record.user_id, record.source_id, record.conversation_id, record.workspace)
+ conversations.setdefault(key, []).append(record)
+ intake = UnifiedWriteIntake(DurableWriteQueue(database_path))
+ for records in conversations.values():
+ intake.submit_records(tuple(records))
+ return len(conversations)
+
+
+def abandoned_legacy_work(database_path: Path) -> dict[str, int]:
+ """Count legacy work that ends with the cutover instead of blocking it.
+
+ A task that may have reached the SDK is not replayed: the SDK already holds
+ whatever it committed. Pending user deletions are not carried over.
+ """
+ abandoned: dict[str, int] = {}
+ with main_database(database_path).read(row_factory=sqlite3.Row) as database:
+ tables = _tables(database)
checks = (
(
- "memory_tasks",
+ "invoked_tasks",
"memory_tasks",
"status='unknown' OR (status='running' AND hy_invocation_count > 0)",
),
- (
- "deletions",
- "memory_deletions",
- "status IN ('pending','running')",
- ),
- (
- "pending_dispatch",
- "evidence_pending_memory_dispatch",
- "1=1",
- ),
+ ("deletions", "memory_deletions", "status IN ('pending','running')"),
)
for name, table, where in checks:
if table not in tables:
@@ -310,11 +242,23 @@ def sdk_cutover_blockers(database_path: Path) -> dict[str, int]:
database.execute(f"SELECT COUNT(*) FROM {table} WHERE {where}").fetchone()[0]
)
if count:
- blockers[name] = count
- return blockers
+ abandoned[name] = count
+ return abandoned
+
+
+def prepare_legacy_work_for_cutover(database_path: Path) -> CutoverPreparation:
+ """Make the surviving queue self-contained. Safe to repeat."""
+ frozen, moved = prepare_legacy_write_queue_for_cutover(database_path)
+ dispatches = move_pending_dispatches(database_path)
+ return CutoverPreparation(
+ frozen_jobs=frozen,
+ moved_tasks=moved,
+ moved_dispatches=dispatches,
+ abandoned=abandoned_legacy_work(database_path),
+ )
-def commit_reconciled_cutover(
+def commit_sdk_cutover(
*,
data_dir: Path,
database_path: Path,
@@ -323,38 +267,25 @@ def commit_reconciled_cutover(
sdk_version: str,
store_version: str = "1",
) -> CutoverMarker:
- """Verify backup/work quiescence and atomically commit SDK authority."""
+ """Verify the backup, settle legacy work and commit SDK authority."""
inspect_sdk_migration_bundle(data_dir=data_dir, bundle_id=bundle_id)
- prepare_legacy_write_queue_for_cutover(database_path)
- blockers = sdk_cutover_blockers(database_path)
- if blockers:
- raise DataPlaneMigrationError(
- "sdk_data_plane_cutover_work_pending:"
- + json.dumps(blockers, sort_keys=True, separators=(",", ":"))
- )
- with main_database(database_path).read(row_factory=sqlite3.Row) as database:
- rows = database.execute(
- "SELECT item_key,state,target_memory_id,target_hash "
- "FROM sdk_data_plane_migration_items WHERE migration_id=? "
- "ORDER BY item_key",
- (migration_id,),
- ).fetchall()
+ # Repeated here because a CLI-managed Runtime may have run on legacy
+ # authority since `sdk-migrate`.
+ preparation = prepare_legacy_work_for_cutover(database_path)
digest = hashlib.sha256(
json.dumps(
- [dict(row) for row in rows],
+ preparation.as_document(),
sort_keys=True,
separators=(",", ":"),
).encode()
).hexdigest()
- marker = SdkDataPlaneMigrationStore(database_path).commit_cutover(
+ return SdkDataPlaneMigrationStore(database_path).commit_cutover(
migration_id=migration_id,
sdk_version=sdk_version,
store_version=store_version,
backup_bundle_id=bundle_id,
reconciliation_digest=digest,
)
- inspect_sdk_migration_bundle(data_dir=data_dir, bundle_id=marker.backup_bundle_id)
- return marker
def resume_legacy_cleanup(*, data_dir: Path, database_path: Path) -> tuple[str, ...]:
@@ -368,100 +299,11 @@ def resume_legacy_cleanup(*, data_dir: Path, database_path: Path) -> tuple[str,
data_dir=data_dir,
bundle_id=marker.backup_bundle_id,
)
- from one_memory.hy_runtime_config import HY_MEMORY_NAMESPACE
+ from one_memory.hy_runtime_config import HY_MEMORY_NAMESPACE
- recover_sdk_migration_swap(
- paths=sdk_migration_staging(
- data_dir=data_dir,
- migration_id=marker.migration_id,
- namespace=HY_MEMORY_NAMESPACE,
- ),
- database_path=database_path,
- )
- return drop_legacy_tables(database_path)
-
-
-async def run_sdk_data_plane_backfill(
- *,
- database_path: Path,
- client: SdkMigrationClient,
- migration_id: str,
- bundle_id: str,
- source_runtime_build: str,
- target_sdk_version: str,
- source_schema_version: int,
- namespace_fingerprint: str,
- backend_fingerprint: str,
-) -> BackfillResult:
- """Backfill both lanes but never commit the authority cutover marker."""
- store = SdkDataPlaneMigrationStore(database_path)
- store.start(
- migration_id=migration_id,
- source_runtime_build=source_runtime_build,
- target_sdk_version=target_sdk_version,
- source_schema_version=source_schema_version,
- namespace_fingerprint=namespace_fingerprint,
- backend_fingerprint=backend_fingerprint,
- backup_bundle_id=bundle_id,
- )
- store.set_phase(migration_id, phase="inventory", state="running")
-
- legacy_memories = read_legacy_memory_inventory(database_path)
- sdk_memories = await read_sdk_memory_inventory(client, legacy=legacy_memories)
- memory_decisions = record_memory_migration_plan(
- store,
- migration_id=migration_id,
- legacy=legacy_memories,
- sdk=sdk_memories,
- )
- with main_database(database_path).read(row_factory=sqlite3.Row) as database:
- started = database.execute(
- "SELECT created_at FROM sdk_data_plane_migrations WHERE migration_id=?",
- (migration_id,),
- ).fetchone()
- if started is None:
- raise DataPlaneMigrationError("sdk_data_plane_migration_start_missing")
- started_at = datetime.fromisoformat(str(started["created_at"]))
- if started_at.tzinfo is None:
- started_at = started_at.replace(tzinfo=UTC)
- legacy_raw = read_legacy_raw_inventory(
- database_path,
- cutoff=started_at.astimezone(UTC) - timedelta(days=3),
- )
- sdk_raw = await read_sdk_raw_inventory(client)
- raw_decisions = record_raw_migration_plan(
- store,
- migration_id=migration_id,
- legacy=legacy_raw,
- sdk=sdk_raw,
- )
-
- store.set_phase(migration_id, phase="backfill", state="running")
- await execute_memory_migration_plan(
- client,
- store,
- migration_id=migration_id,
- legacy=legacy_memories,
- decisions=memory_decisions,
- )
- await execute_raw_migration_plan(
- client,
- store,
- migration_id=migration_id,
- legacy=legacy_raw,
- decisions=raw_decisions,
- )
- progress: MigrationProgress = store.progress(migration_id)
- if progress.counts.get("quarantine", 0):
- store.set_phase(migration_id, phase="review_required", state="blocked")
- else:
- store.set_phase(migration_id, phase="reconciled", state="ready")
- progress = store.progress(migration_id)
- return BackfillResult(
- migration_id=migration_id,
- state=progress.state,
- phase=progress.phase,
- counts=progress.counts,
- memory_items=len(memory_decisions),
- raw_items=len(raw_decisions),
+ recover_abandoned_staging(
+ data_dir=data_dir,
+ database_path=database_path,
+ namespace=HY_MEMORY_NAMESPACE,
)
+ return drop_legacy_tables(database_path)
diff --git a/memhub/apps/core/src/one_memory/target_schema.py b/memhub/apps/core/src/one_memory/target_schema.py
index ea3481b..27ac76b 100644
--- a/memhub/apps/core/src/one_memory/target_schema.py
+++ b/memhub/apps/core/src/one_memory/target_schema.py
@@ -54,6 +54,8 @@ LEGACY_TABLES_IN_DROP_ORDER = (
"evidence_origins",
"evidence_versions",
"evidence_repairs",
+ # FTS5 index over Evidence text; dropping it removes its shadow tables.
+ "evidence_search",
"evidence",
"memory_id_aliases",
"raw_id_aliases",
diff --git a/memhub/apps/core/tests/test_cli_migration_error_codes.py b/memhub/apps/core/tests/test_cli_migration_error_codes.py
index 8252f75..260a94b 100644
--- a/memhub/apps/core/tests/test_cli_migration_error_codes.py
+++ b/memhub/apps/core/tests/test_cli_migration_error_codes.py
@@ -1,8 +1,7 @@
"""The startup dialog can only act on the code it is handed.
-`sdk_migration_backfill_failed` covered a full disk, a missing model profile and
-a Coding classification gap alike, so every one of them reached the person as
-the same unactionable sentence. These tests pin the names that replace it.
+A full disk and a damaged backup are different next steps, so they must reach
+the person under different names. These tests pin those names.
"""
from __future__ import annotations
@@ -50,37 +49,23 @@ def test_quota_exhaustion_counts_as_no_space(tmp_path: Path) -> None:
)
-def test_a_coding_inventory_gap_is_not_reported_as_a_generic_failure() -> None:
- assert (
- migration_failure_code(_Coded("hy_sdk_productivity_variant_invalid"))
- == "sdk_migration_coding_inventory_failed"
- )
-
-
-def test_the_legacy_working_fact_failure_keeps_its_own_name() -> None:
- error = _Coded("hy_sdk_memory_import_failed")
- error.__cause__ = ValueError("Invalid memory layer: working_fact")
-
- assert migration_failure_code(error) == "sdk_migration_working_fact_conflict"
-
-
def test_migration_state_codes_pass_through_because_they_are_already_actionable() -> None:
assert (
- migration_failure_code(_Coded("sdk_data_plane_migration_item_conflict"))
- == "sdk_data_plane_migration_item_conflict"
+ migration_failure_code(_Coded("sdk_data_plane_migration_bundle_conflict"))
+ == "sdk_data_plane_migration_bundle_conflict"
)
assert (
- migration_failure_code(_Coded("sdk_migration_staging_invalid"))
- == "sdk_migration_staging_invalid"
+ migration_failure_code(_Coded("sdk_migration_swap_state_ambiguous"))
+ == "sdk_migration_swap_state_ambiguous"
)
def test_unknown_failures_keep_their_message_out_of_the_code() -> None:
error = RuntimeError("/Users/someone/private/path blew up")
- assert migration_failure_code(error) == "sdk_migration_backfill_failed"
+ assert migration_failure_code(error) == "sdk_migration_preflight_failed"
assert (
- migration_failure_code(error, fallback="sdk_migration_preflight_failed")
- == "sdk_migration_preflight_failed"
+ migration_failure_code(error, fallback="sdk_migration_bundle_copy_failed")
+ == "sdk_migration_bundle_copy_failed"
)
@@ -88,7 +73,7 @@ def test_a_non_code_attribute_never_becomes_a_code() -> None:
class Odd(RuntimeError):
code = 17
- assert migration_failure_code(Odd("x")) == "sdk_migration_backfill_failed"
+ assert migration_failure_code(Odd("x")) == "sdk_migration_preflight_failed"
def test_the_chain_survives_a_cycle() -> None:
@@ -97,16 +82,16 @@ def test_the_chain_survives_a_cycle() -> None:
first.__cause__ = second
second.__cause__ = first
- assert migration_failure_code(first) == "sdk_migration_backfill_failed"
+ assert migration_failure_code(first) == "sdk_migration_preflight_failed"
@pytest.mark.parametrize(
"code",
[
"sdk_migration_disk_space_insufficient",
- "sdk_migration_working_fact_conflict",
- "sdk_migration_coding_inventory_failed",
- "sdk_migration_work_pending",
+ "sdk_migration_preflight_failed",
+ "sdk_migration_bundle_copy_failed",
+ "sdk_migration_cutover_failed",
],
)
def test_the_codes_the_desktop_translates_stay_in_sync(code: str) -> None:
@@ -116,11 +101,7 @@ def test_the_codes_the_desktop_translates_stay_in_sync(code: str) -> None:
assert f'"{code}"' in desktop.read_text(encoding="utf-8")
-def test_cutover_work_counts_do_not_become_the_dialog_text() -> None:
- """The blocker code carries a JSON body; the person gets one name."""
- blocked = _Coded('sdk_data_plane_cutover_work_pending:{"memory_tasks":591}')
-
- assert cutover_failure_code(blocked) == "sdk_migration_work_pending"
+def test_cutover_codes_never_carry_free_text() -> None:
assert (
cutover_failure_code(_Coded("sdk_data_plane_cutover_missing"))
== "sdk_data_plane_cutover_missing"
diff --git a/memhub/apps/core/tests/test_cli_restore.py b/memhub/apps/core/tests/test_cli_restore.py
index 7231294..d3a98e4 100644
--- a/memhub/apps/core/tests/test_cli_restore.py
+++ b/memhub/apps/core/tests/test_cli_restore.py
@@ -266,7 +266,8 @@ def test_sdk_migrate_preview_never_commits_cutover(tmp_path, monkeypatch) -> Non
assert '"commits_cutover":false' in document
-def test_sdk_migrate_runs_offline_backfill_without_cutover(tmp_path, monkeypatch) -> None:
+def test_sdk_migrate_prepares_without_a_model_or_cutover(tmp_path, monkeypatch) -> None:
+ """No model profile, no SDK client: the SDK already holds every memory."""
monkeypatch.setattr(cli, "load_settings", lambda: Settings(_env_file=None, data_dir=tmp_path))
monkeypatch.setattr(
cli,
@@ -277,66 +278,6 @@ def test_sdk_migrate_runs_offline_backfill_without_cutover(tmp_path, monkeypatch
"one_memory.sdk_migration_bundle.inspect_sdk_migration_bundle",
lambda **_kwargs: {"runtime_build": "v1.0.4"},
)
-
- async def fake_migrate(*_args, **_kwargs):
- return None
-
- monkeypatch.setattr("one_memory.memory_server.migration_runner.migrate", fake_migrate)
-
- profile = SimpleNamespace(
- has_api_key=False,
- embedding_provider_kind="openai",
- embedding_model="embedding",
- embedding_dimensions=1024,
- )
-
- class Profiles:
- def __init__(self, _database):
- pass
-
- def active_identity(self):
- return ("profile", 1)
-
- def snapshot(self, *_identity):
- return profile
-
- def resolve_snapshot_secret(self, *_args):
- return None
-
- class Secrets:
- pass
-
- class Lease:
- client = SimpleNamespace(
- _loop_thread=None,
- _vector_store=None,
- _embed_service=None,
- _graph_store=None,
- )
-
- def close(self):
- return None
-
- async def backfill(**_kwargs):
- return SimpleNamespace(
- state="ready",
- phase="reconciled",
- migration_id="migration-1",
- counts={"verified": 2},
- memory_items=1,
- raw_items=1,
- )
-
- monkeypatch.setattr("one_memory.model_profiles.ModelProfileStore", Profiles)
- monkeypatch.setattr("one_memory.secrets.KeyringSecretStore", Secrets)
- monkeypatch.setattr(
- "one_memory.hy_sdk_runtime.acquire_client",
- lambda *_args, **_kwargs: Lease(),
- )
- monkeypatch.setattr(
- "one_memory.sdk_migration_runner.run_sdk_data_plane_backfill",
- backfill,
- )
output = io.StringIO()
code = cli.run(
[
@@ -350,10 +291,11 @@ def test_sdk_migrate_runs_offline_backfill_without_cutover(tmp_path, monkeypatch
],
output=output,
)
- assert code == 0
- document = output.getvalue().replace(" ", "")
- assert '"state":"ready"' in document
- assert '"commits_cutover":false' in document
+ assert code == 0, output.getvalue()
+ document = json.loads(output.getvalue())
+ assert document["state"] == "ready"
+ assert document["commits_cutover"] is False
+ assert document["abandoned"] == {}
def test_sdk_cutover_is_offline_confirmed_and_returns_marker(tmp_path, monkeypatch) -> None:
@@ -364,7 +306,7 @@ def test_sdk_cutover_is_offline_confirmed_and_returns_marker(tmp_path, monkeypat
lambda _data_dir: RuntimeStatus("stopped", "runtime_receipt_missing"),
)
monkeypatch.setattr(
- "one_memory.sdk_migration_runner.commit_reconciled_cutover",
+ "one_memory.sdk_migration_runner.commit_sdk_cutover",
lambda **_kwargs: SimpleNamespace(
migration_id="migration-1",
backup_bundle_id="bundle-1",
@@ -372,10 +314,6 @@ def test_sdk_cutover_is_offline_confirmed_and_returns_marker(tmp_path, monkeypat
completed_at="2026-10-21T00:00:00Z",
),
)
- monkeypatch.setattr(
- "one_memory.sdk_migration_runner.activate_sdk_migration_staging",
- lambda *, commit, **_kwargs: commit(),
- )
monkeypatch.setattr(
"one_memory.sdk_migration_runner.resume_legacy_cleanup",
lambda **_kwargs: (),
diff --git a/memhub/apps/core/tests/test_fault_matrix.py b/memhub/apps/core/tests/test_fault_matrix.py
index 1b3b4a4..13f6afb 100644
--- a/memhub/apps/core/tests/test_fault_matrix.py
+++ b/memhub/apps/core/tests/test_fault_matrix.py
@@ -92,11 +92,13 @@ def test_each_fault_outcome_is_realised_by_the_durable_queue(tmp_path) -> None:
self.error = error
self.seen: list[str] = []
- async def process_records(self, records, **_kwargs):
+ async def process_sdk_job(self, records, **_kwargs):
+ # Refusals are classified before `before_hy_invocation`, as the
+ # SDK write processor does, so these never cross into HY.
self.seen.append("called")
if self.error is not None:
raise self.error
- return ()
+ return type("Result", (), {"raw_ids": (), "memory_ids": ()})()
# hy_succeeded_then_crash: marked invoked, then the lease lapses -> unknown.
path = tmp_path / "fault.sqlite3"
diff --git a/memhub/apps/core/tests/test_memory_server_unified_entry.py b/memhub/apps/core/tests/test_memory_server_unified_entry.py
index 4f2aabf..f9be9f4 100644
--- a/memhub/apps/core/tests/test_memory_server_unified_entry.py
+++ b/memhub/apps/core/tests/test_memory_server_unified_entry.py
@@ -6,6 +6,7 @@ import asyncio
import json
import sqlite3
from datetime import UTC, datetime
+from types import SimpleNamespace
import pytest
@@ -249,16 +250,17 @@ def test_old_branch_job_executes_its_frozen_revision_after_harness_rewind(tmp_pa
captured = []
class Supervisor:
- async def process_records(
+ async def process_sdk_job(
self,
records,
*,
+ request_id,
before_hy_invocation,
- source_admitted,
):
- assert source_admitted is True
+ assert request_id == old_job.request_id
captured.extend(records)
await before_hy_invocation()
+ return SimpleNamespace(raw_ids=(), memory_ids=())
asyncio.run(
execute_job(
diff --git a/memhub/apps/core/tests/test_sdk_data_plane_migration.py b/memhub/apps/core/tests/test_sdk_data_plane_migration.py
index 5586472..a864637 100644
--- a/memhub/apps/core/tests/test_sdk_data_plane_migration.py
+++ b/memhub/apps/core/tests/test_sdk_data_plane_migration.py
@@ -1,27 +1,23 @@
from __future__ import annotations
import asyncio
+import hashlib
+import io
import json
-import sqlite3
-import threading
from datetime import UTC, datetime, timedelta
from pathlib import Path
import pytest
-from hy_memory.models.memory import MemoryLayer, MemoryStatus
+from one_memory import cli
+from one_memory.config import Settings
from one_memory.database_access import close_main_databases, main_database
-from one_memory.hy_sdk_migration import HySdkMigration
-from one_memory.hy_sdk_types import (
- HySdkMemoryPage,
- HySdkMemoryRecord,
- HySdkRawPage,
- HySdkRawRecord,
-)
+from one_memory.hy_sdk_types import HySdkRawPage
from one_memory.memory_server.contracts import BatchIdentity, WriteRequest
from one_memory.memory_server.durable_queue import DurableWriteQueue
from one_memory.memory_server.migration_runner import SqliteMigrationStore, migrate
from one_memory.memory_server.migrations import MIGRATIONS
+from one_memory.runtime_client import RuntimeStatus
from one_memory.sdk_data_plane_migration import (
DataPlaneMigrationError,
MemoryAuthority,
@@ -29,37 +25,22 @@ from one_memory.sdk_data_plane_migration import (
)
from one_memory.sdk_memory_runtime import SdkMemoryRuntime
from one_memory.sdk_migration_bundle import create_sdk_migration_bundle
-from one_memory.sdk_migration_inventory import (
- LegacyMemoryInventory,
- LegacyRawInventory,
- SdkMemoryInventory,
- SdkRawInventory,
- classify_memory_inventory,
- classify_raw_inventory,
- content_hash,
- execute_memory_migration_plan,
- execute_raw_migration_plan,
- read_legacy_memory_inventory,
- read_legacy_raw_inventory,
- read_sdk_memory_inventory,
- read_sdk_raw_inventory,
- record_memory_migration_plan,
- record_raw_migration_plan,
-)
from one_memory.sdk_migration_runner import (
- activate_sdk_migration_staging,
- commit_reconciled_cutover,
+ PREPARED_PHASE,
+ abandoned_legacy_work,
+ commit_sdk_cutover,
+ move_pending_dispatches,
prepare_legacy_write_queue_for_cutover,
- prepare_sdk_migration_staging,
- recover_sdk_migration_swap,
+ recover_abandoned_staging,
resume_legacy_cleanup,
- run_sdk_data_plane_backfill,
- sdk_cutover_blockers,
- sdk_migration_staging,
)
from one_memory.target_schema import legacy_tables_present
+def content_hash(content: str) -> str:
+ return hashlib.sha256(content.encode("utf-8")).hexdigest()
+
+
@pytest.fixture
def store(tmp_path):
database = tmp_path / "one-memory.sqlite3"
@@ -80,6 +61,41 @@ def _start(store, migration_id="m1"):
)
+def _insert_evidence(store, evidence_id, *, conversation="session-1", role="user", seq=1):
+ verbatim = f"text of {evidence_id}"
+ with main_database(store.database_path).write() as database, database:
+ database.execute(
+ "INSERT INTO evidence (evidence_id,idempotency_key,user_id,source_id,"
+ "conversation_id,content_kind,role,written_by,verbatim,content_hash,"
+ "revision,imported_at,updated_at,ingest_seq) "
+ "VALUES (?,?,'local','cursor',?,'raw_conversation',?,?,?,?,1,"
+ "'2026-10-22T00:00:00+00:00','2026-10-22T00:00:00+00:00',?)",
+ (
+ evidence_id,
+ evidence_id,
+ conversation,
+ role,
+ "person" if role == "user" else "agent",
+ verbatim,
+ content_hash(verbatim),
+ seq,
+ ),
+ )
+
+
+def _insert_task(store, task_id, *, status, invocations):
+ with main_database(store.database_path).write() as database, database:
+ database.execute(
+ "INSERT INTO memory_tasks "
+ "(task_id,request_id,user_id,kind,input_ref,status,profile_id,"
+ "profile_revision,profile_snapshot_json,runner_token,attempts,"
+ "last_error_code,created_at,updated_at,duration_seconds,hy_invocation_count) "
+ "VALUES (?,?,'local','extract','e-none',?,'profile',1,'{}',NULL,0,NULL,"
+ "'now','now',NULL,?)",
+ (task_id, f"request-{task_id}", status, invocations),
+ )
+
+
def test_authority_stays_legacy_without_explicit_cutover(store):
assert store.authority() is MemoryAuthority.LEGACY
progress = _start(store)
@@ -88,107 +104,54 @@ def test_authority_stays_legacy_without_explicit_cutover(store):
assert store.authority() is MemoryAuthority.LEGACY
-def test_start_is_idempotent_but_refuses_identity_change(store):
+def test_start_resumes_an_earlier_run_but_keeps_its_bundle(store):
first = _start(store)
- second = _start(store)
- assert second == first
+ resumed = store.start(
+ migration_id="m1",
+ source_runtime_build="build-19",
+ target_sdk_version="2.0.6",
+ source_schema_version=60,
+ namespace_fingerprint="other",
+ backend_fingerprint="other",
+ backup_bundle_id="bundle-1",
+ )
+ assert resumed == first
with pytest.raises(DataPlaneMigrationError) as caught:
store.start(
migration_id="m1",
- source_runtime_build="different",
+ source_runtime_build="build-13",
target_sdk_version="2.0.5",
source_schema_version=55,
namespace_fingerprint="namespace",
backend_fingerprint="backend",
- backup_bundle_id="bundle-1",
- )
- assert caught.value.code == "sdk_data_plane_migration_identity_conflict"
-
-
-def test_item_operation_key_is_unique_and_identity_is_immutable(store):
- _start(store)
- store.upsert_item(
- migration_id="m1",
- item_key="governance:g1",
- source_kind="governance",
- source_id="g1",
- source_hash="hash-1",
- operation_key="operation-1",
- )
- store.upsert_item(
- migration_id="m1",
- item_key="governance:g1",
- source_kind="governance",
- source_id="g1",
- source_hash="hash-1",
- operation_key="operation-1",
- )
-
- with pytest.raises(DataPlaneMigrationError) as caught:
- store.upsert_item(
- migration_id="m1",
- item_key="governance:g2",
- source_kind="governance",
- source_id="g2",
- source_hash="hash-2",
- operation_key="operation-1",
+ backup_bundle_id="bundle-2",
)
- assert caught.value.code == "sdk_data_plane_migration_operation_conflict"
+ assert caught.value.code == "sdk_data_plane_migration_bundle_conflict"
-def test_cutover_requires_ready_phase_and_terminal_items(store):
+def test_cutover_requires_the_prepared_phase(store):
_start(store)
- store.upsert_item(
- migration_id="m1",
- item_key="raw:e1",
- source_kind="raw",
- source_id="e1",
- source_hash="hash",
- operation_key="operation",
- )
+ commit = {
+ "migration_id": "m1",
+ "sdk_version": "2.0.5",
+ "store_version": "1",
+ "backup_bundle_id": "bundle-1",
+ "reconciliation_digest": "digest",
+ }
with pytest.raises(DataPlaneMigrationError) as caught:
- store.commit_cutover(
- migration_id="m1",
- sdk_version="2.0.5",
- store_version="1",
- backup_bundle_id="bundle-1",
- reconciliation_digest="digest",
- )
+ store.commit_cutover(**commit)
assert caught.value.code == "sdk_data_plane_migration_not_ready"
- store.set_phase("m1", phase="reconciled", state="ready")
- with pytest.raises(DataPlaneMigrationError) as caught:
- store.commit_cutover(
- migration_id="m1",
- sdk_version="2.0.5",
- store_version="1",
- backup_bundle_id="bundle-1",
- reconciliation_digest="digest",
- )
- assert caught.value.code.startswith("sdk_data_plane_migration_items_unresolved:")
-
- store.settle_item(
- migration_id="m1",
- item_key="raw:e1",
- state="verified",
- target_memory_id="raw-1",
- target_hash="hash",
- )
- marker = store.commit_cutover(
- migration_id="m1",
- sdk_version="2.0.5",
- store_version="1",
- backup_bundle_id="bundle-1",
- reconciliation_digest="digest",
- )
+ store.set_phase("m1", phase=PREPARED_PHASE, state="ready")
+ marker = store.commit_cutover(**commit)
assert marker.migration_id == "m1"
assert store.authority() is MemoryAuthority.SDK
def test_startup_cutover_freezes_pending_job_input_and_preserves_the_queue(store):
_start(store)
- store.set_phase("m1", phase="reconciled", state="ready")
+ store.set_phase("m1", phase=PREPARED_PHASE, state="ready")
data_dir = store.database_path.parent
with main_database(store.database_path).write() as database, database:
database.execute(
@@ -219,7 +182,7 @@ def test_startup_cutover_freezes_pending_job_input_and_preserves_the_queue(store
runtime_build="build-13",
runtime_stopped=True,
)
- marker = commit_reconciled_cutover(
+ marker = commit_sdk_cutover(
data_dir=data_dir,
database_path=store.database_path,
migration_id="m1",
@@ -283,990 +246,191 @@ def test_unstarted_legacy_task_moves_to_surviving_write_queue(store):
).fetchone() == ("cancelled", "migrated_to_memory_write_job")
-def test_alias_is_idempotent_but_cannot_be_retargeted(store):
- _start(store)
- store.record_alias(
- migration_id="m1",
- legacy_memory_id="legacy",
- canonical_memory_id="sdk",
- )
- store.record_alias(
- migration_id="m2",
- legacy_memory_id="legacy",
- canonical_memory_id="sdk",
- )
- store.record_alias(
- migration_id="m1",
- legacy_memory_id="legacy",
- canonical_memory_id="sdk",
- )
- with pytest.raises(DataPlaneMigrationError) as caught:
- store.record_alias(
- migration_id="m1",
- legacy_memory_id="legacy",
- canonical_memory_id="other",
- )
- assert caught.value.code == "sdk_data_plane_alias_conflict"
-
-
-def test_cutover_ignores_unpublished_matter_placeholders(store):
+def test_pending_dispatch_becomes_one_write_job_per_conversation(store):
+ _insert_evidence(store, "a1", conversation="a", seq=1)
+ _insert_evidence(store, "a2", conversation="a", role="assistant", seq=2)
+ _insert_evidence(store, "b1", conversation="b", seq=3)
with main_database(store.database_path).write() as database, database:
- database.execute("INSERT INTO matters(matter_id,version,needs_update) VALUES ('m1',1,0)")
-
- assert "matter_references" not in sdk_cutover_blockers(store.database_path)
-
-
-def test_legacy_inventory_preserves_task_id_mapping_and_supersession(store):
- database = store.database_path
- with main_database(database).write() as connection, connection:
- for memory_id, status, content in (
- ("legacy-old", "superseded", "old"),
- ("legacy-new", "confirmed", "new"),
- ):
- connection.execute(
- """
- INSERT INTO governed_memories (
- memory_id,user_id,kind,status,content,scopes_json,origin,
- tags_json,revision,created_at,updated_at
- ) VALUES (?,?,?,?,?,'[\"all-apps\"]','extracted','[\"tag\"]',1,'now','now')
- """,
- (memory_id, "local", "chat", status, content),
+ for evidence_id in ("a1", "a2", "b1"):
+ database.execute(
+ "INSERT INTO evidence_pending_memory_dispatch "
+ "(user_id,source_id,file_key,evidence_id) VALUES ('local','cursor','f',?)",
+ (evidence_id,),
)
- connection.execute(
- "INSERT INTO governed_memory_supersessions VALUES ('legacy-old','legacy-new','now')"
- )
- connection.execute(
- "INSERT INTO memory_task_outputs VALUES (?,?,?)",
- (
- "task",
- "legacy-new",
- json.dumps(
- {
- "task_id": "task",
- "memory_id": "legacy-new",
- "memory_revision": 1,
- "hy_memory_id": "sdk-new",
- "owner": "user",
- "layer": "l2_fact",
- "engine": "hy-memory",
- "engine_version": "2.0.4",
- "support_refs": [],
- }
- ),
- ),
- )
-
- records = {item.legacy_memory_id: item for item in read_legacy_memory_inventory(database)}
- assert records["legacy-new"].canonical_memory_id == "sdk-new"
- assert records["legacy-new"].owner == "user"
- assert records["legacy-new"].supersedes == ("legacy-old",)
- assert records["legacy-old"].superseded_by == ("legacy-new",)
+ assert move_pending_dispatches(store.database_path) == 2
+ assert move_pending_dispatches(store.database_path) == 2
-def _legacy(
- legacy_id,
- *,
- canonical_id=None,
- content="body",
- status="active",
- layer="l2_fact",
-):
- productivity = {
- "coding_experience": ("experience", "coding"),
- "working_experience": ("experience", "working"),
- "coding_fact": ("fact", "coding"),
- "working_fact": ("fact", None),
- }.get(layer)
- return LegacyMemoryInventory(
- legacy_memory_id=legacy_id,
- canonical_memory_id=canonical_id or legacy_id,
- user_id="local",
- agent_id="one-memory",
- session_id="default_session",
- content=content,
- content_hash=content_hash(content),
- status=status,
- layer=layer,
- owner="user",
- tags=(),
- supersedes=(),
- superseded_by=(),
- productivity_kind=productivity[0] if productivity else None,
- productivity_variant=productivity[1] if productivity else None,
- )
-
-
-def _sdk(
- memory_id,
- *,
- content="body",
- status="active",
- productivity_kind=None,
- productivity_variant=None,
-):
- return SdkMemoryInventory(
- memory_id=memory_id,
- user_id="local",
- agent_id="one-memory",
- content_hash=content_hash(content),
- status=status,
- productivity_kind=productivity_kind,
- productivity_variant=productivity_variant,
- )
-
-
-def test_inventory_classification_covers_every_legacy_and_sdk_state():
- decisions = classify_memory_inventory(
- (
- _legacy("matched"),
- _legacy("mapped", canonical_id="sdk-mapped"),
- _legacy("missing"),
- _legacy("deleted", status="deleted"),
- _legacy("delete-existing", status="deleted"),
- _legacy("divergent"),
- _legacy("unknown-layer", layer=None),
- ),
- (
- _sdk("matched"),
- _sdk("sdk-mapped"),
- _sdk("delete-existing"),
- _sdk("divergent", content="different"),
- _sdk("sdk-only"),
- ),
- )
- by_key = {item.item_key: item for item in decisions}
- assert by_key["memory:matched"].action == "matched"
- assert by_key["memory:mapped"].action == "record_alias"
- assert by_key["memory:missing"].action == "import_exact"
- assert by_key["memory:deleted"].action == "verified_deleted"
- assert by_key["memory:delete-existing"].action == "delete"
- assert by_key["memory:divergent"].action == "quarantine"
- assert by_key["memory:unknown-layer"].reason == "legacy_layer_unknown"
- assert by_key["sdk:chat:sdk-only"].action == "adopt"
- assert len(decisions) == 8
-
-
-class _MigrationClient:
- def __init__(self, memories=()):
- self.memories = tuple(memories)
- self.imported = []
- self.imported_raw = []
- self.deleted = []
- self.deleted_productivity = []
- self.deleted_raw = []
- self.raw = ()
- self.productivity = {}
-
- async def list_memories(self, *, limit, offset, **_kwargs):
- page = self.memories[offset : offset + limit]
- return HySdkMemoryPage(
- memories=page,
- total=len(self.memories),
- limit=limit,
- offset=offset,
- )
-
- async def write_legacy_memory(self, **kwargs):
- self.imported.append(kwargs)
- return {"success": True, "memory_id": kwargs["memory_id"]}
-
- async def delete(self, memory_id):
- self.deleted.append(memory_id)
- return {"success": True, "deleted_count": 1}
-
- async def read_productivity_memory(self, memory_id, *, kind, variant):
- if variant is not None:
- memory = self.productivity.get((memory_id, kind, variant))
- return None if memory is None else (memory, variant)
- for candidate in ("working", "coding"):
- memory = self.productivity.get((memory_id, kind, candidate))
- if memory is not None:
- return memory, candidate
- return None
-
- async def list_productivity_memories(self):
- return tuple(
- {
- "memory_id": memory.memory_id,
- "content": memory.content,
- "status": memory.status or "active",
- "user_id": memory.user_id,
- "agent_id": memory.agent_id,
- "kind": kind,
- "variant": variant,
- }
- for (_memory_id, kind, variant), memory in self.productivity.items()
- )
-
- async def delete_productivity_memory(self, memory_id, *, kind, variant, user_id):
- self.deleted_productivity.append((memory_id, kind, variant, user_id))
- return {"success": True, "deleted_count": 1}
-
- async def read_raw_inventory(self, *, limit, offset, **_kwargs):
- rows = self.raw[offset : offset + limit]
- return HySdkRawPage(
- raw=rows,
- total=len(self.raw),
- limit=limit,
- offset=offset,
- )
-
- async def write_legacy_raw(self, **kwargs):
- self.imported_raw.append(kwargs)
- return {"success": True, "raw_id": kwargs["raw_id"]}
-
- async def delete_raw(self, raw_id):
- self.deleted_raw.append(raw_id)
- return {"success": True, "deleted_count": 1}
-
- async def get_sdk_memory(self, memory_id):
- return next(
- (memory for memory in self.memories if memory.memory_id == memory_id),
- None,
- )
-
- async def get_sdk_raw(self, raw_id):
- return next((raw for raw in self.raw if raw.raw_id == raw_id), None)
-
-
-def _record(memory_id, *, content="body", status="active"):
- return HySdkMemoryRecord(
- memory_id=memory_id,
- content=content,
- layer="l2_fact",
- status=status,
- is_latest=status == "active",
- user_id="local",
- agent_id="one-memory",
- )
+ with main_database(store.database_path).read() as database:
+ payloads = [
+ json.loads(row[0])
+ for row in database.execute("SELECT payload_json FROM memory_write_jobs")
+ ]
+ assert sorted(
+ [record["evidence_id"] for record in payload["input_records"]] for payload in payloads
+ ) == [["a1", "a2"], ["b1"]]
-def test_sdk_inventory_is_paged_without_dropping_lifecycle_records():
- client = _MigrationClient(
- (
- _record("active"),
- _record("old", status="superseded"),
- _record("shadow", status="shadow"),
+def test_legacy_work_that_may_have_reached_the_sdk_does_not_block_cutover(tmp_path, store):
+ """Build 19 stopped here forever: a blocker no startup could ever clear."""
+ _insert_task(store, "sent", status="unknown", invocations=1)
+ with main_database(store.database_path).write() as database, database:
+ database.execute(
+ "INSERT INTO governed_memories (memory_id,user_id,kind,status,content,"
+ "scopes_json,revision,created_at,updated_at) "
+ "VALUES ('g1','local','chat','confirmed','x','[]',1,'now','now')"
)
- )
- inventory = asyncio.run(read_sdk_memory_inventory(client, page_size=2))
- assert [(item.memory_id, item.status) for item in inventory] == [
- ("active", "active"),
- ("old", "superseded"),
- ("shadow", "shadow"),
- ]
-
-
-def test_sdk_inventory_adopts_mapped_productivity_ids_and_deletes_user_removed_fact(store):
- active = _legacy("exp", layer="coding_experience")
- deleted = _legacy("fact", layer="working_fact", status="deleted")
- client = _MigrationClient()
- client.productivity = {
- ("exp", "experience", "coding"): HySdkMemoryRecord(
- memory_id="exp",
- content="sdk experience",
- layer="experience",
- status="active",
- is_latest=True,
- user_id="local",
- agent_id="scope",
- ),
- ("fact", "fact", "coding"): HySdkMemoryRecord(
- memory_id="fact",
- content="sdk fact",
- layer="fact",
- status="active",
- is_latest=True,
- user_id="local",
- agent_id="scope",
- ),
- ("sdk-only", "experience", "working"): HySdkMemoryRecord(
- memory_id="sdk-only",
- content="sdk only",
- layer="experience",
- status="active",
- is_latest=True,
- user_id="local",
- agent_id="another-scope",
- ),
- }
- sdk = asyncio.run(read_sdk_memory_inventory(client, legacy=(active, deleted)))
- _start(store)
- decisions = record_memory_migration_plan(
- store,
- migration_id="m1",
- legacy=(active, deleted),
- sdk=sdk,
- )
- by_key = {decision.item_key: decision for decision in decisions}
- assert by_key["memory:exp"].action == "matched"
- assert by_key["memory:fact"].action == "delete"
- assert by_key["sdk:working:experience:sdk-only"].action == "adopt"
-
- asyncio.run(
- execute_memory_migration_plan(
- client,
- store,
- migration_id="m1",
- legacy=(active, deleted),
- decisions=decisions,
+ database.execute(
+ "INSERT INTO memory_deletions (memory_id,status,updated_at) "
+ "VALUES ('g1','pending','now')"
)
- )
- assert client.imported == []
- assert client.deleted_productivity == [("fact", "fact", None, "local")]
-
-
-def test_productivity_id_in_both_variants_is_quarantined():
- legacy = (_legacy("fact", layer="working_fact"),)
- sdk = (
- _sdk(
- "fact",
- productivity_kind="fact",
- productivity_variant="working",
- ),
- _sdk(
- "fact",
- productivity_kind="fact",
- productivity_variant="coding",
- ),
- )
- (decision, *_sdk_only) = classify_memory_inventory(legacy, sdk)
- assert decision.action == "quarantine"
- assert decision.reason == "sdk_memory_id_ambiguous"
-
+ with main_database(store.database_path).write() as database, database:
+ database.execute("INSERT INTO matters(matter_id,version,needs_update) VALUES ('m1',1,0)")
+ assert abandoned_legacy_work(store.database_path) == {"invoked_tasks": 1, "deletions": 1}
-def test_recorded_plan_executes_import_delete_alias_and_adoption(store):
_start(store)
- legacy = (
- _legacy("matched"),
- _legacy("missing"),
- _legacy("mapped", canonical_id="sdk-mapped"),
- _legacy("deleted", status="deleted"),
- _legacy("delete-existing", status="deleted"),
- )
- sdk = (
- _sdk("matched"),
- _sdk("sdk-mapped"),
- _sdk("delete-existing"),
- _sdk("sdk-only"),
+ store.set_phase("m1", phase=PREPARED_PHASE, state="ready")
+ create_sdk_migration_bundle(
+ data_dir=store.database_path.parent,
+ bundle_id="bundle-1",
+ runtime_build="build-13",
+ runtime_stopped=True,
)
- decisions = record_memory_migration_plan(
- store,
+ commit_sdk_cutover(
+ data_dir=store.database_path.parent,
+ database_path=store.database_path,
migration_id="m1",
- legacy=legacy,
- sdk=sdk,
- )
- client = _MigrationClient()
- asyncio.run(
- execute_memory_migration_plan(
- client,
- store,
- migration_id="m1",
- legacy=legacy,
- decisions=decisions,
- )
- )
-
- assert [item["memory_id"] for item in client.imported] == ["missing"]
- assert client.deleted == ["delete-existing"]
- progress = store.progress("m1")
- assert progress.counts == {"adopted": 1, "deleted": 2, "verified": 3}
-
-
-def test_legacy_raw_inventory_builds_recent_user_assistant_turns_without_tools(store):
- database = store.database_path
- with main_database(database).write() as connection, connection:
- connection.executemany(
- """
- INSERT INTO evidence (
- evidence_id,idempotency_key,user_id,source_id,workspace,project,
- conversation_id,message_id,source_ordinal,content_kind,role,
- written_by,created_at,verbatim,content_hash,revision,
- imported_at,updated_at,ingest_seq
- ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
- """,
- (
- (
- "u1",
- "k1",
- "local",
- "cursor",
- "/repo",
- "project",
- "session",
- "m1",
- "1",
- "raw_conversation",
- "user",
- "person",
- "2026-10-22T00:00:00+00:00",
- "question",
- content_hash("question"),
- 1,
- "2026-10-22T00:00:00+00:00",
- "2026-10-22T00:00:00+00:00",
- 1,
- ),
- (
- "t1",
- "k2",
- "local",
- "cursor",
- "/repo",
- "project",
- "session",
- "m2",
- "2",
- "raw_conversation",
- "tool",
- "tool",
- "2026-10-22T00:00:01+00:00",
- "secret tool output",
- content_hash("secret tool output"),
- 1,
- "2026-10-22T00:00:01+00:00",
- "2026-10-22T00:00:01+00:00",
- 2,
- ),
- (
- "a1",
- "k3",
- "local",
- "cursor",
- "/repo",
- "project",
- "session",
- "m3",
- "3",
- "raw_conversation",
- "assistant",
- "agent",
- "2026-10-22T00:00:02+00:00",
- "answer",
- content_hash("answer"),
- 1,
- "2026-10-22T00:00:02+00:00",
- "2026-10-22T00:00:02+00:00",
- 3,
- ),
- (
- "old-u",
- "k4",
- "local",
- "cursor",
- "/repo",
- "project",
- "old-session",
- "m4",
- "4",
- "raw_conversation",
- "user",
- "person",
- "2026-10-01T00:00:00+00:00",
- "old question",
- content_hash("old question"),
- 1,
- "2026-10-01T00:00:00+00:00",
- "2026-10-01T00:00:00+00:00",
- 4,
- ),
- (
- "old-a",
- "k5",
- "local",
- "cursor",
- "/repo",
- "project",
- "old-session",
- "m5",
- "5",
- "raw_conversation",
- "assistant",
- "agent",
- "2026-10-01T00:00:01+00:00",
- "old answer",
- content_hash("old answer"),
- 1,
- "2026-10-01T00:00:01+00:00",
- "2026-10-01T00:00:01+00:00",
- 5,
- ),
- ),
- )
- (raw,) = read_legacy_raw_inventory(
- database,
- cutoff=datetime.fromisoformat("2026-10-20T00:00:00+00:00"),
+ bundle_id="bundle-1",
+ sdk_version="2.0.5",
)
- assert raw.evidence_id.startswith("turn:")
- assert raw.session_id == "session"
- assert raw.source == "cursor"
- assert raw.agent_id == "all-apps"
- assert raw.content == "[user]: question\n[assistant]: answer"
- assert raw.metadata["evidence_ids"] == ["u1", "a1"]
- assert "tool" not in raw.content
- assert raw.memory_at.isoformat() == "2026-10-22T00:00:00+00:00"
+ assert store.authority() is MemoryAuthority.SDK
-def test_raw_turn_cutoff_uses_completion_time_and_keeps_workspaces_separate(store):
- database = store.database_path
- rows = (
- ("u0", "k0", "shared", "w0", "user", "person", "before", "2026-10-19T23:59:00+00:00", 1),
- ("a0", "k1", "shared", "w0", "assistant", "agent", "after", "2026-10-20T00:01:00+00:00", 2),
- ("u1", "k2", "same-session", "w1", "user", "person", "one", "2026-10-21T00:00:00+00:00", 3),
- (
- "a1",
- "k3",
- "same-session",
- "w1",
- "assistant",
- "agent",
- "first",
- "2026-10-21T00:01:00+00:00",
- 4,
- ),
- ("u2", "k4", "same-session", "w2", "user", "person", "two", "2026-10-21T00:02:00+00:00", 5),
- (
- "a2",
- "k5",
- "same-session",
- "w2",
- "assistant",
- "agent",
- "second",
- "2026-10-21T00:03:00+00:00",
- 6,
- ),
- )
- with main_database(database).write() as connection, connection:
- connection.executemany(
- """
- INSERT INTO evidence (
- evidence_id,idempotency_key,user_id,source_id,conversation_id,
- workspace,content_kind,role,written_by,verbatim,content_hash,
- revision,created_at,imported_at,updated_at,ingest_seq
- ) VALUES (?,?,'local','cursor',?,?,'raw_conversation',?,?,?,? ,1,?,?,?,?)
- """,
- tuple(
- (
- evidence_id,
- key,
- session,
- workspace,
- role,
- written_by,
- text,
- content_hash(text),
- timestamp,
- timestamp,
- timestamp,
- sequence,
- )
- for (
- evidence_id,
- key,
- session,
- workspace,
- role,
- written_by,
- text,
- timestamp,
- sequence,
- ) in rows
- ),
- )
+def _protective(home: Path) -> Path:
+ return home / "hy-memory" / ".governed-v2.pre-migration-0123456789abcdef"
- turns = read_legacy_raw_inventory(
- database,
- cutoff=datetime.fromisoformat("2026-10-20T00:00:00+00:00"),
- )
- assert {turn.content for turn in turns} == {
- "[user]: before\n[assistant]: after",
- "[user]: one\n[assistant]: first",
- "[user]: two\n[assistant]: second",
- }
- assert {turn.metadata["workspace"] for turn in turns} == {"w0", "w1", "w2"}
-
-
-def _legacy_raw(evidence_id, *, content="verbatim"):
- return LegacyRawInventory(
- evidence_id=evidence_id,
- user_id="local",
- agent_id="one-memory",
- session_id="session",
- source="cursor",
- content=content,
- content_hash=content_hash(content),
- memory_at=None,
- metadata={"role": "user"},
- )
+def test_interrupted_swap_before_authority_restores_the_original_store(tmp_path, store):
+ home = store.database_path.parent
+ live = home / "hy-memory" / "governed-v2"
+ live.mkdir(parents=True)
+ (live / "store.txt").write_text("staged", encoding="utf-8")
+ _protective(home).mkdir()
+ (_protective(home) / "store.txt").write_text("original", encoding="utf-8")
+ (home / ".sdk-migration" / "token").mkdir(parents=True)
-def _sdk_raw(raw_id, *, content="verbatim"):
- return SdkRawInventory(
- raw_id=raw_id,
- user_id="local",
- agent_id="one-memory",
- session_id="session",
- source="cursor",
- content_hash=content_hash(content),
+ recover_abandoned_staging(
+ data_dir=home,
+ database_path=store.database_path,
+ namespace="governed-v2",
)
+ assert (live / "store.txt").read_text(encoding="utf-8") == "original"
+ assert not _protective(home).exists()
+ assert not (home / ".sdk-migration").exists()
-def test_raw_inventory_matches_turns_imports_missing_and_deletes_noncanonical_sdk_raw(store):
- legacy = (_legacy_raw("e1"), _legacy_raw("e2", content="missing"))
- sdk = (_sdk_raw("raw-existing"), _sdk_raw("raw-sdk-only", content="sdk-only"))
- decisions = classify_raw_inventory(legacy, sdk)
- by_key = {item.item_key: item for item in decisions}
- assert by_key["raw:e1"].action == "matched"
- assert by_key["raw:e1"].raw_id == "raw-existing"
- assert by_key["raw:e2"].action == "import_exact"
- assert by_key["raw:e2"].raw_id == "evidence:e2"
- assert by_key["sdk-raw:raw-sdk-only"].action == "delete"
+def test_stale_protective_root_after_authority_is_discarded(tmp_path, store):
+ home = store.database_path.parent
+ live = home / "hy-memory" / "governed-v2"
+ live.mkdir(parents=True)
+ (live / "store.txt").write_text("live", encoding="utf-8")
+ _protective(home).mkdir()
_start(store)
- recorded = record_raw_migration_plan(
- store,
+ store.set_phase("m1", phase=PREPARED_PHASE, state="ready")
+ store.commit_cutover(
migration_id="m1",
- legacy=legacy,
- sdk=sdk,
- )
- client = _MigrationClient()
- asyncio.run(
- execute_raw_migration_plan(
- client,
- store,
- migration_id="m1",
- legacy=legacy,
- decisions=recorded,
- )
+ sdk_version="2.0.5",
+ store_version="1",
+ backup_bundle_id="bundle-1",
+ reconciliation_digest="digest",
)
- assert [item["raw_id"] for item in client.imported_raw] == ["evidence:e2"]
- assert client.deleted_raw == ["raw-sdk-only"]
- assert store.progress("m1").counts == {"deleted": 1, "verified": 2}
-
-def test_sdk_raw_inventory_pages_all_records():
- client = _MigrationClient()
- client.raw = (
- HySdkRawRecord(
- raw_id="r1",
- content="one",
- user_id="local",
- agent_id="one-memory",
- session_id="s1",
- source="cursor",
- ),
- HySdkRawRecord(
- raw_id="r2",
- content="two",
- user_id="local",
- agent_id="one-memory",
- session_id="s2",
- source="claude",
- ),
+ recover_abandoned_staging(
+ data_dir=home,
+ database_path=store.database_path,
+ namespace="governed-v2",
)
- inventory = asyncio.run(read_sdk_raw_inventory(client, page_size=1))
- assert [item.raw_id for item in inventory] == ["r1", "r2"]
+ assert (live / "store.txt").read_text(encoding="utf-8") == "live"
+ assert not _protective(home).exists()
-def test_offline_backfill_reaches_ready_without_committing_cutover(store):
- database = store.database_path
- with main_database(database).write() as connection, connection:
- connection.execute(
- """
- INSERT INTO governed_memories (
- memory_id,user_id,kind,status,content,scopes_json,origin,
- tags_json,revision,created_at,updated_at
- ) VALUES (
- 'memory-1','local','chat','confirmed','preference',
- '[\"all-apps\"]','extracted','[]',1,'now','now'
- )
- """
- )
- connection.execute(
- """
- INSERT INTO evidence (
- evidence_id,idempotency_key,user_id,source_id,conversation_id,
- content_kind,role,written_by,verbatim,content_hash,revision,
- imported_at,updated_at,ingest_seq
- ) VALUES (
- 'e1','key','local','cursor','session','raw_conversation',
- 'user','person','verbatim',?,1,
- '2026-10-21T00:00:00+00:00','2026-10-21T00:00:00+00:00',1
- )
- """,
- (content_hash("verbatim"),),
- )
- connection.execute(
- """
- INSERT INTO evidence (
- evidence_id,idempotency_key,user_id,source_id,conversation_id,
- content_kind,role,written_by,verbatim,content_hash,revision,
- imported_at,updated_at,ingest_seq
- ) VALUES (
- 'a1','assistant-key','local','cursor','session','raw_conversation',
- 'assistant','agent','reply',?,1,
- '2026-10-21T00:00:01+00:00','2026-10-21T00:00:01+00:00',2
- )
- """,
- (content_hash("reply"),),
- )
- client = _MigrationClient()
- result = asyncio.run(
- run_sdk_data_plane_backfill(
- database_path=database,
- client=client,
- migration_id="migration",
- bundle_id="bundle",
- source_runtime_build="v1.0.4",
- target_sdk_version="2.0.5",
- source_schema_version=55,
- namespace_fingerprint="namespace",
- backend_fingerprint="backend",
- )
- )
- assert result.state == "ready"
- assert result.phase == "reconciled"
- assert result.counts == {"verified": 2}
- assert [item["memory_id"] for item in client.imported] == ["memory-1"]
- assert len(client.imported_raw) == 1
- assert client.imported_raw[0]["content"] == "[user]: verbatim\n[assistant]: reply"
- migrated_raw_id = client.imported_raw[0]["raw_id"]
- assert store.authority() is MemoryAuthority.LEGACY
- # Simulate a process dying after both SDK side effects but before startup
- # could finish. The next inventory observes them as matched rather than
- # import_exact; that reclassification must resume the settled ledger
- # instead of conflicting with its original operation key.
- client.memories = (_record("memory-1", content="preference"),)
- client.raw = (
- HySdkRawRecord(
- raw_id=migrated_raw_id,
- content="[user]: verbatim\n[assistant]: reply",
- user_id="local",
- agent_id="all-apps",
- session_id="session",
- source="cursor",
- custom={"input_kind": "legacy_turn"},
- ),
+def test_a_home_blocked_by_build_19_finishes_on_the_next_startup(tmp_path, monkeypatch, store):
+ """The quarantined ledger, its staging copy and pending queue work all resolve."""
+ home = store.database_path.parent
+ live = home / "hy-memory" / "governed-v2"
+ live.mkdir(parents=True)
+ (live / "coding.sqlite3").write_text("sdk data", encoding="utf-8")
+ staging = home / ".sdk-migration" / "token" / "hy-memory" / "governed-v2"
+ staging.mkdir(parents=True)
+ store.start(
+ migration_id="sdk-authority-v2-73b3561be69371e0",
+ source_runtime_build="19",
+ target_sdk_version="2.0.6",
+ source_schema_version=60,
+ namespace_fingerprint="old",
+ backend_fingerprint="old-with-embedding",
+ backup_bundle_id="sdk-authority-v2-73b3561be69371e0-pre-cutover",
+ )
+ store.set_phase(
+ "sdk-authority-v2-73b3561be69371e0",
+ phase="review_required",
+ state="blocked",
)
- resumed = asyncio.run(
- run_sdk_data_plane_backfill(
- database_path=database,
- client=client,
- migration_id="migration",
- bundle_id="bundle",
- source_runtime_build="v1.0.4",
- target_sdk_version="2.0.5",
- source_schema_version=55,
- namespace_fingerprint="namespace",
- backend_fingerprint="backend",
+ with main_database(store.database_path).write() as database, database:
+ database.execute(
+ "INSERT INTO sdk_data_plane_migration_items (migration_id,item_key,"
+ "source_kind,source_id,source_hash,operation_key,state,error_code,updated_at) "
+ "VALUES ('sdk-authority-v2-73b3561be69371e0','memory:exp-1',"
+ "'memory:quarantine','exp-1','hash','op','quarantine',"
+ "'mapped_productivity_memory_missing','now')"
+ )
+ _insert_evidence(store, "e1")
+ DurableWriteQueue(store.database_path).submit(
+ WriteRequest(
+ internal_session_id="session-1",
+ source_id="cursor",
+ batch_identity=BatchIdentity(
+ internal_session_id="session-1", first_round=1, last_round=1
+ ),
+ evidence_ids=["e1"],
+ idempotency_key="pending-job",
)
)
- assert resumed.state == "ready"
- assert resumed.counts == {"verified": 2}
- assert len(client.imported) == 1
- assert len(client.imported_raw) == 1
-
-
-def test_offline_backfill_blocks_cutover_when_an_item_is_quarantined(store):
- database = store.database_path
- with main_database(database).write() as connection, connection:
- connection.execute(
- """
- INSERT INTO governed_memories (
- memory_id,user_id,kind,status,content,scopes_json,origin,
- tags_json,revision,created_at,updated_at
- ) VALUES (
- 'memory-1','local','coding','confirmed','unknown layer',
- '[\"all-apps\"]','extracted','[]',1,'now','now'
- )
- """
- )
- result = asyncio.run(
- run_sdk_data_plane_backfill(
- database_path=database,
- client=_MigrationClient(),
- migration_id="migration",
- bundle_id="bundle",
- source_runtime_build="v1.0.4",
- target_sdk_version="2.0.5",
- source_schema_version=55,
- namespace_fingerprint="namespace",
- backend_fingerprint="backend",
- )
+ monkeypatch.setattr(cli, "load_settings", lambda: Settings(_env_file=None, data_dir=home))
+ monkeypatch.setattr(
+ cli,
+ "inspect_runtime",
+ lambda _data_dir: RuntimeStatus("stopped", "runtime_receipt_missing"),
)
+ migration = "sdk-authority-v2-73b3561be69371e0"
+ bundle = f"{migration}-pre-cutover"
- assert result.state == "blocked"
- assert result.phase == "review_required"
- assert result.counts == {"quarantine": 1}
- assert store.authority() is MemoryAuthority.LEGACY
-
-
-class _MigrationLoop:
- def run(self, coroutine):
- return asyncio.run(coroutine)
+ def run(*arguments):
+ output = io.StringIO()
+ code = cli.run([*arguments, "--yes", "--json"], output=output)
+ return code, json.loads(output.getvalue())
+ assert run("migration-bundle", "--bundle-id", bundle, "--runtime-build", "19")[0] == 0
+ code, prepared = run("sdk-migrate", "--migration-id", migration, "--bundle-id", bundle)
+ assert (code, prepared["state"]) == (0, "ready"), prepared
+ assert prepared["frozen_jobs"] == 1
+ code, cutover = run("sdk-cutover", "--migration-id", migration, "--bundle-id", bundle)
+ assert (code, cutover["state"]) == (0, "cutover"), cutover
-class _MigrationVectorStore:
- def __init__(self):
- self.nodes = {}
-
- async def get_by_id(self, memory_id):
- return self.nodes.get(memory_id)
-
- async def upsert(self, node):
- self.nodes[node.node_id] = node
- return node.node_id
-
- async def list_by_user(self, *, user_id, layers, **_kwargs):
- return [
- node for node in self.nodes.values() if node.user_id == user_id and node.layer in layers
- ]
-
-
-class _MigrationEmbedder:
- async def embed(self, _content):
- return [0.1, 0.2]
-
-
-class _MigrationGraphStore:
- def __init__(self):
- self.nodes = {}
-
- async def upsert_memory_node(self, node):
- self.nodes[node.node_id] = node
- return node.node_id
-
-
-class _MigrationFactVector:
- def __init__(self):
- self.deleted = []
-
- async def delete(self, memory_id):
- self.deleted.append(memory_id)
- return True
-
-
-class _MigrationCodingStore:
- def __init__(self):
- self._conn = sqlite3.connect(":memory:", check_same_thread=False)
- self._conn.row_factory = sqlite3.Row
- self._conn.execute(
- "CREATE TABLE coding_memory_meta ("
- "experience_id TEXT PRIMARY KEY,user_id TEXT,agent_id TEXT,solution TEXT)"
- )
- self._conn.execute(
- "CREATE TABLE working_fact ("
- "fact_id TEXT PRIMARY KEY,user_id TEXT,agent_id TEXT,"
- "statement TEXT,status TEXT)"
- )
- self._conn.execute(
- "INSERT INTO working_fact VALUES ('fact-1','local','scope','fact body','active')"
- )
- self._conn.commit()
- self._lock = threading.Lock()
- self._fact_vec = _MigrationFactVector()
-
- async def get_fact(self, memory_id, *, user_id):
- row = self._conn.execute(
- "SELECT 1 FROM working_fact WHERE fact_id=? AND user_id=?",
- (memory_id, user_id),
- ).fetchone()
- return object() if row is not None else None
-
-
-class _PrivilegedMigrationClient:
- def __init__(self):
- self._loop_thread = _MigrationLoop()
- self._vector_store = _MigrationVectorStore()
- self._embed_service = _MigrationEmbedder()
- self._graph_store = _MigrationGraphStore()
- self._coding = _MigrationCodingStore()
-
- async def _coding_store_for_exact_get(self, variant):
- return self._coding if variant == "coding" else None
-
- def list_memories(self, **_kwargs):
- return {"vdb": {"memories": [], "total": 0}, "request_id": "list"}
-
- def delete(self, _memory_id):
- return {"success": True, "deleted_count": 1}
-
- def get_write_outcome(self, _request_id):
- return None
-
-
-def test_offline_migration_writes_sdk_stores_without_public_client_import_methods():
- client = _PrivilegedMigrationClient()
- migration = HySdkMigration(client)
-
- raw_result = asyncio.run(
- migration.write_legacy_raw(
- content="verbatim",
- raw_id="raw-1",
- user_id="local",
- agent_id="one-memory",
- session_id="session",
- source="cursor",
- memory_at=None,
- metadata={"role": "user"},
- request_id="migrate-raw-1",
- )
- )
- memory_result = asyncio.run(
- migration.write_legacy_memory(
- content="prefers tea",
- memory_id="memory-1",
- user_id="local",
- agent_id="one-memory",
- session_id="session",
- source="legacy-governance",
- layer="l2_fact",
- status="active",
- owner="user",
- tags=("preference",),
- supersedes=(),
- superseded_by=(),
- is_latest=True,
- memory_at=None,
- metadata={},
- request_id="migrate-memory-1",
- )
- )
- raw_page = asyncio.run(migration.read_raw_inventory(user_id="local"))
- productivity = asyncio.run(migration.list_productivity_memories())
- deleted = asyncio.run(
- migration.delete_productivity_memory(
- "fact-1",
- kind="fact",
- variant=None,
- user_id="local",
- )
- )
-
- assert raw_result["raw_id"] == "raw-1"
- assert memory_result["memory_id"] == "memory-1"
- assert [item.raw_id for item in raw_page.raw] == ["raw-1"]
- assert productivity == (
- {
- "memory_id": "fact-1",
- "content": "fact body",
- "status": "active",
- "user_id": "local",
- "agent_id": "scope",
- "kind": "fact",
- "variant": "coding",
- },
- )
- assert client._vector_store.nodes["raw-1"].layer is MemoryLayer.L1_RAW
- assert client._vector_store.nodes["memory-1"].status is MemoryStatus.ACTIVE
- assert client._graph_store.nodes["memory-1"].content == "prefers tea"
- assert deleted["deleted_count"] == 1
- assert client._coding._fact_vec.deleted == ["fact-1"]
- assert client._coding._conn.execute("SELECT COUNT(*) FROM working_fact").fetchone()[0] == 0
- assert not hasattr(client, "import_raw")
- assert not hasattr(client, "import_memory_exact")
+ assert store.authority() is MemoryAuthority.SDK
+ assert legacy_tables_present(store.database_path) == ()
+ with main_database(store.database_path).read() as database:
+ assert not database.execute(
+ "SELECT name FROM sqlite_master WHERE name LIKE 'evidence%'"
+ ).fetchall()
+ assert (live / "coding.sqlite3").read_text(encoding="utf-8") == "sdk data"
+ assert not (home / ".sdk-migration").exists()
+ queue = DurableWriteQueue(store.database_path, sdk_authority=True)
+ job = queue.get("pending-job")
+ assert job is not None and job.state == "pending"
+ assert queue.payload_for(job.job_id).input_records[0].verbatim == "text of e1"
def test_raw_http_dto_uses_the_unified_memory_list_shape():
@@ -1347,84 +511,3 @@ def test_sdk_raw_retention_keeps_only_the_latest_three_days():
)
assert asyncio.run(runtime.prune_expired_raw(now=now)) == 1
assert [row["memory_id"] for row in client.rows] == ["recent"]
-
-
-def _staging_fixture(tmp_path, store):
- data_dir = tmp_path / "home"
- live = data_dir / "hy-memory" / "governed-v2"
- live.mkdir(parents=True)
- (live / "sdk-only.txt").write_text("old", encoding="utf-8")
- paths = sdk_migration_staging(
- data_dir=data_dir,
- migration_id="migration-1",
- namespace="governed-v2",
- )
- prepare_sdk_migration_staging(
- paths=paths,
- database_path=store.database_path,
- )
- return paths
-
-
-def test_staging_preserves_sdk_only_data_and_swaps_after_commit(tmp_path, store):
- paths = _staging_fixture(tmp_path, store)
- staged = paths.staging_root / "sdk-only.txt"
- assert staged.read_text(encoding="utf-8") == "old"
- staged.write_text("migrated", encoding="utf-8")
- marker = object()
-
- result = activate_sdk_migration_staging(
- paths=paths,
- database_path=store.database_path,
- commit=lambda: marker,
- )
-
- assert result is marker
- assert (paths.live_root / "sdk-only.txt").read_text(encoding="utf-8") == "migrated"
- assert not paths.protective_root.exists()
- assert not paths.staging_home.exists()
-
-
-def test_incomplete_staging_retry_restarts_from_unchanged_live_root(tmp_path, store):
- paths = _staging_fixture(tmp_path, store)
- (paths.staging_root / "sdk-only.txt").write_text("partial", encoding="utf-8")
-
- prepare_sdk_migration_staging(
- paths=paths,
- database_path=store.database_path,
- )
-
- assert (paths.staging_root / "sdk-only.txt").read_text(encoding="utf-8") == "old"
-
-
-def test_failed_cutover_restores_live_store_and_keeps_staging(tmp_path, store):
- paths = _staging_fixture(tmp_path, store)
- (paths.staging_root / "sdk-only.txt").write_text("migrated", encoding="utf-8")
-
- def fail():
- raise DataPlaneMigrationError("cutover_failed")
-
- with pytest.raises(DataPlaneMigrationError, match="cutover_failed"):
- activate_sdk_migration_staging(
- paths=paths,
- database_path=store.database_path,
- commit=fail,
- )
-
- assert (paths.live_root / "sdk-only.txt").read_text(encoding="utf-8") == "old"
- assert (paths.staging_root / "sdk-only.txt").read_text(encoding="utf-8") == "migrated"
-
-
-def test_restart_recovers_swap_when_authority_was_not_committed(tmp_path, store):
- paths = _staging_fixture(tmp_path, store)
- (paths.staging_root / "sdk-only.txt").write_text("migrated", encoding="utf-8")
- paths.live_root.replace(paths.protective_root)
- paths.staging_root.replace(paths.live_root)
-
- recover_sdk_migration_swap(
- paths=paths,
- database_path=store.database_path,
- )
-
- assert (paths.live_root / "sdk-only.txt").read_text(encoding="utf-8") == "old"
- assert (paths.staging_root / "sdk-only.txt").read_text(encoding="utf-8") == "migrated"
diff --git a/memhub/apps/desktop/src/stable-runtime.test.ts b/memhub/apps/desktop/src/stable-runtime.test.ts
index 25839e6..79f8dd0 100644
--- a/memhub/apps/desktop/src/stable-runtime.test.ts
+++ b/memhub/apps/desktop/src/stable-runtime.test.ts
@@ -285,18 +285,6 @@ describe("startup install errors", () => {
expect(noSpace).toContain("重新启动");
expect(noSpace).toContain("不会丢失");
expect(noSpace).not.toContain("sdk_migration_disk_space_insufficient");
-
- const noModel = stableRuntimeUserMessage(
- new Error("model_profile_not_active"),
- );
- expect(noModel).toContain("模型");
- expect(noModel).not.toContain("model_profile_not_active");
-
- const coding = stableRuntimeUserMessage(
- new Error("sdk_migration_working_fact_conflict"),
- );
- expect(coding).toContain("Coding");
- expect(coding).toContain("不会丢失");
});
});
diff --git a/memhub/apps/desktop/src/stable-runtime.ts b/memhub/apps/desktop/src/stable-runtime.ts
index 8d44689..af919f7 100644
--- a/memhub/apps/desktop/src/stable-runtime.ts
+++ b/memhub/apps/desktop/src/stable-runtime.ts
@@ -129,15 +129,8 @@ const SAFE_STABLE_RUNTIME_ERROR_CODES = new Set([
// Migration runs before the app window exists, so these are the only names
// a first launch can be given. They stay codes here and become sentences in
// STABLE_RUNTIME_USER_MESSAGES: a person reading the dialog never sees them.
- "model_profile_not_active",
- "model_secret_unavailable",
"sdk_migration_preflight_failed",
"sdk_migration_disk_space_insufficient",
- "sdk_migration_staging_failed",
- "sdk_migration_backfill_failed",
- "sdk_migration_working_fact_conflict",
- "sdk_migration_coding_inventory_failed",
- "sdk_migration_work_pending",
"sdk_migration_cutover_failed",
"sdk_migration_bundle_copy_failed",
"sdk_migration_response_invalid",
@@ -208,24 +201,10 @@ export const STABLE_RUNTIME_USER_MESSAGES: Record<string, string> = {
"已安装的运行环境比这个安装包更新,无法用旧安装包启动。请安装最新版本的 MemHub;你的记忆数据不会受到影响。",
install_current_unmanaged:
"运行环境的安装目录被改动过,无法安全接管。请重新安装 MemHub,你的记忆数据不会受到影响。",
- model_profile_not_active:
- "还没有配置可用的模型,无法升级记忆数据格式。请在旧版本中完成模型配置后重启;你的记忆数据不会受到影响。",
- model_secret_unavailable:
- "模型密钥读不到(可能被系统钥匙串拒绝)。请允许 MemHub 访问钥匙串后重新启动;你的记忆数据不会受到影响。",
sdk_migration_disk_space_insufficient:
"磁盘空间不足,升级记忆数据格式时中断了。请至少腾出与记忆数据同等大小的可用空间(可在「关于本机 → 储存空间」查看),然后重新启动 MemHub;升级会自动继续,原有记忆数据不会丢失。",
- sdk_migration_staging_failed:
- "升级准备工作没有完成,通常是磁盘空间不足或文件被占用。请清理磁盘空间后重新启动;原有记忆数据不会丢失。",
sdk_migration_bundle_copy_failed:
"升级前备份没有写完,通常是磁盘空间不足。请清理磁盘空间后重新启动;原有记忆数据不会丢失。",
- sdk_migration_working_fact_conflict:
- "旧版数据里有一条 Coding 记忆的记录类型在升级中无法安全对应。为避免写错你的记忆,升级已停止。请把这个提示反馈给我们;原有记忆数据不会丢失。",
- sdk_migration_coding_inventory_failed:
- "读取旧版 Coding 记忆时发现无法识别的结构,升级已停止。请把这个提示反馈给我们;原有记忆数据不会丢失。",
- sdk_migration_backfill_failed:
- "升级记忆数据格式时失败,已停止且没有切换数据源。请重新启动重试;如果仍然失败,请把这个提示反馈给我们,原有记忆数据不会丢失。",
- sdk_migration_work_pending:
- "记忆数据已经升级完成,但还有之前没处理完的记忆任务,所以暂时不能切换数据源。请保持 MemHub 打开几分钟,等这些任务处理完再重新启动;原有记忆数据不会丢失。",
sdk_migration_cutover_failed:
"切换数据源时失败,原有记忆数据仍然完整可用。请重新启动重试;如果仍然失败,请把这个提示反馈给我们。",
sdk_migration_preflight_failed:
diff --git a/memhub/docs/ARCHITECTURE.md b/memhub/docs/ARCHITECTURE.md
index 62cd86f..3cc5da8 100644
--- a/memhub/docs/ARCHITECTURE.md
+++ b/memhub/docs/ARCHITECTURE.md
@@ -1,6 +1,6 @@
# MemHub Architecture
-Updated: 2026-10-21.
+Updated: 2026-10-24.
Hy Memory 是 SDK / 引擎;MemHub 是产品与发布名。`one-memory` 仍可能出现在
兼容路径、模块名和服务标识中。
@@ -129,16 +129,13 @@ directly for low-latency Cypher projection and store-width administration.
This exception does not extend to VDB, Coding, prompt, reconcile, or lifecycle.
- Storage backends:仅 SDK config/env(Chroma/FAISS/Qdrant、Kuzu/Neo4j、cache/history sqlite|mysql、coding sqlite 等)。
-- Queue:pending `memory_write_jobs` 只表示尚未调用 SDK 的待处理请求,因此没有 SDK raw/memory ID。worker 单次调用 SDK `add()`;SDK 内部完成 raw 写入与 extraction/reconciliation,成功后 MemHub 仅将返回的 raw ID / memory IDs 记作 job outcome。旧用户启动迁移在 Runtime writers 停止期间直接操作 SDK 存储,不向公开 `HyMemoryClient` 增加迁移专用方法,也不拆分正常写路径。
-- Raw retention:SDK L1 raw 只保留最近三天;每个完整 turn 只保存真人 user 与 assistant,tool 不进入 raw。高层 Chat/Coding/Working memory 不受该窗口影响。
+- Queue:pending `memory_write_jobs` 只表示尚未调用 SDK 的待处理请求,因此没有 SDK raw/memory ID。worker 单次调用 SDK `add()`;SDK 内部完成 raw 写入与 extraction/reconciliation,成功后 MemHub 仅将返回的 raw ID / memory IDs 记作 job outcome。
+- Raw retention:SDK L1 raw 只保留最近三天,由 Runtime 的 `prune_expired_raw` 执行;每个完整 turn 只保存真人 user 与 assistant,tool 不进入 raw。高层 Chat/Coding/Working memory 不受该窗口影响。
- Recall:Host Bridge / Core 先做宿主安装凭据与 tool grant,再调 **SDK search/get**。当前产品没有逐 memory 的 A-harness→B-harness 可见性开关,默认允许跨 harness 搜索;未来若增加来源限制,直接透传 SDK `source` / `include_sources` / `exclude_sources`,不在 MemHub 建 memory ACL。
- Context wire:cutover marker 提交前只使用 legacy Context v1;提交后 Host Bridge 只使用 SDK-native Context v2。两种 schema 原样校验、原样返回,不把 SDK item 补造成 v1 的 `revision`、`evidence_ids`、`paths` 或 `adoption`,已选择的 v2 路径失败时也不回退 legacy 权威。
- Ask wire:marker 前保留 Governance Ask v1;marker 后 Web / Host Bridge 只调用 SDK Ask v2。v2 的引用只含 SDK 原生 `memory_id/content/layer/status/source/gmt_created`,生成过程中重复 `get` 校验同一 SDK 快照,不补造 revision、Evidence 或 scopes,SDK 失败不回退 v1。
- Matter:未形成已发布产品合同,不迁移占位数据;所有 `matter*` 表在升级清理时物理删除,fresh install 不创建,Runtime 不注册 Matter API。
-- 旧用户迁移:由 App 在可见的启动 loading 阶段统一执行;此时 Core 尚未对外提供产品读写。启动流程创建/校验完整 bundle、续跑 ledger、完成 SDK backfill 与 reconciliation,满足全部门禁后才提交 marker 并进入主界面。普通用户不需要另跑 CLI,也不按旧 App 内 SDK 版本分叉迁移。
-- SDK backfill 只写同文件系统的私有 staging 副本;验证后先原子切换 SDK
- root,再提交 authority marker。切换或 marker 提交失败必须恢复原 live root;
- App 不得在 migration `deferred` 状态继续启动 Runtime。
+- 旧用户迁移:由 App 在可见的启动 loading 阶段统一执行;此时 Core 尚未对外提供产品读写。所有已发布版本都先经 SDK 写入,Governance/Evidence 只是镜像,所以迁移**不复制、不比对任何 memory 或 raw**:只创建并校验完整 bundle、让写入队列脱离旧表、提交 marker、删除旧表。普通用户不需要另跑 CLI;App 不得在 migration `deferred` 状态继续启动 Runtime。
- 迁移期间宿主 capture hook 由 Desktop 活跃 PID marker 静默挡住,不再追加
Evidence;cutover 后先从宿主 History checkpoint 补齐遗漏 turn,再恢复 live sync。
- Matter 若未来重新开发,需以新的 SDK-native 合同重新设计,不继承当前占位 schema。
@@ -306,9 +303,9 @@ Test entry points: [`../CONTRIBUTING.md`](../CONTRIBUTING.md).
## Upgrade Compatibility
-Released installations require the offline, resumable compatibility protocol in
+Released installations require the offline compatibility protocol in
[`contracts/sdk-data-plane-migration-v1.md`](contracts/sdk-data-plane-migration-v1.md).
The App runs it during the startup loading phase before Core exposes product
-reads or writes. A cutover requires a complete bundle, resumable backfill,
-reconciliation, and verified cleanup; a SQLite-only backup or replaying legacy
-rows through extraction is not sufficient.
+reads or writes. A cutover requires a verified full bundle and a write queue
+that no longer depends on legacy tables; it copies no memory, because the SDK
+stores already hold every memory and raw turn.
diff --git a/memhub/docs/contracts/sdk-data-plane-migration-v1.md b/memhub/docs/contracts/sdk-data-plane-migration-v1.md
index a14f2ea..39e3923 100644
--- a/memhub/docs/contracts/sdk-data-plane-migration-v1.md
+++ b/memhub/docs/contracts/sdk-data-plane-migration-v1.md
@@ -1,300 +1,134 @@
# SDK Data Plane Migration v1
-Updated: 2026-10-21.
+Updated: 2026-10-24.
-This contract migrates released MemHub installations from the historical
+This contract moves released MemHub installations from the historical
Evidence/Governance authority to the Hy Memory SDK data plane defined by
[`../ARCHITECTURE.md`](../ARCHITECTURE.md).
-It is an offline, resumable migration executed inside the App's visible startup
-loading phase, before Core accepts product traffic. It is not a permanent
-dual-read or dual-write design and is not a separate manual upgrade flow for
-ordinary users.
+It is an offline migration executed inside the App's visible startup loading
+phase, before Core accepts product traffic. It is not a dual-read or dual-write
+design and not a separate manual upgrade flow for ordinary users.
-## 1. Authority before and after cutover
+## 1. Why no memory is copied
-Before the cutover marker is committed:
+Every released build (`v1.0.0` through build 19) wrote memory through SDK
+`add()` / `add_extracted()` first. The SDK stored the raw turn (L1 raw) and the
+extracted Chat / Coding Experience / Working Fact memories in its own stores.
+The Governance and Evidence tables were a mirror that MemHub read back from the
+SDK afterwards.
-- `evidence*` and `governed_*` remain the product read authority;
-- SDK stores are inspected and backfilled, but are not merged into live search;
-- new imports, writes, queue drains, corrections, deletes and session delivery
- are paused.
-- Host capture commands detect the live Desktop migration fence and return
- silently without writing Evidence. After cutover, History catch-up runs from
- the existing source cursor before ordinary live sync.
+The SDK stores are therefore already complete. The migration copies no memory
+and no raw turn, and does not compare the mirror with the SDK:
+
+- the mirror's layer labels did not record which Coding store held a row, and
+ the SDK physically deletes experiences its reconciler retires, so the mirror
+ cannot be trusted to decide what the SDK should contain;
+- the product has no user editing of memories in practice, so the mirror holds
+ no user intent the SDK lacks;
+- SDK L1 raw is retained for three days by the Runtime's own
+ `prune_expired_raw`, not by the migration.
+
+## 2. Authority before and after cutover
+
+Before the cutover marker is committed, legacy tables remain the product read
+authority and Core may still run on them (CLI-managed Runtime). The App does
+not start Runtime while migration is incomplete.
After the cutover marker is committed:
-- SDK memory/raw/vector/graph/coding stores are the only memory data authority;
+- SDK stores are the only memory data authority;
- MemHub stores management state and `memory_id` / raw references only;
-- legacy Evidence/Governance/retrieval/session-shadow tables are physically
+- legacy Evidence/Governance/retrieval/session-shadow and Matter tables are
dropped before the normal App window is revealed;
-- startup must not fall back to legacy authority because an SDK read failed.
-
-The surviving `memory_authority_state` singleton, not legacy table existence,
-selects the authority.
-
-### Fresh install
-
-A fresh home does not execute the historical application schema chain. It
-bootstraps the latest management-plane schema directly with SDK authority and
-never creates Evidence, Governance, legacy retrieval, any Matter, or
-session-shadow tables. Historical schema migrations are an input format reader
-for old homes only.
-
-## 2. Supported source state
-
-The migration preflight records and validates:
-
-- source MemHub Runtime tag/build and management schema version;
-- migration Runtime SDK capabilities and the actual on-disk store shape;
-- SDK namespace, vector/graph/cache/history/coding providers and locations;
-- embedding model identity and dimensions;
-- source grants, model profiles, config, queue leases, and pending operations.
-
-The migration fails closed when it cannot inspect every SDK memory namespace
-(Chat, Coding Experience, Working Fact), write a missing ordinary memory, or
-create and restore a consistent data-plane snapshot.
-
-MemHub tags `v1.0.0` through `v1.0.4` all use the `governed-v2` namespace and
-the same SDK SQL table families (`memory_operations_v2`, `pipeline_logs_v2`,
-`system_metrics`, `episode_state`, `memory_history`, `coding_memory_meta`,
-`working_fact`, and `coding_trace`). They therefore use one migration selected
-by inspected schema/store capabilities, not one branch per historical SDK pin.
-The old SDK version is informational only and is not a migration discriminator.
-
-The migration Runtime itself must expose the required APIs. Its exact SDK
-version is defined by the release lockfiles; old data is not rejected merely
-because an earlier App wrote it.
-
-## 3. Legacy-state classification
-
-While Core and Runtime writers are stopped, build four inventories:
-
-- `E`: completed user/assistant turns from the rolling three-day legacy window;
-- `G`: current Governance rows, statuses, supersession links, and latest content;
-- `M`: task-output mappings from legacy `memory_id` to SDK `hy_memory_id`;
-- `S`: complete SDK inventory, including Coding/Working and non-active records.
-
-Every item must end in exactly one class:
-
-| Class | Meaning | Required action |
-| --- | --- | --- |
-| matched | Governance/mapping and SDK identity, content, owner and lifecycle agree | verify only |
-| mapped mismatch | legacy ID maps to a different existing SDK ID | keep SDK ID; record alias |
-| Governance-only | current legacy memory has no SDK entity | exact import without extraction |
-| SDK-only | SDK entity has no legacy mirror | retain and adopt |
-| divergent | both exist but content, owner or lifecycle differs | reconcile explicit user intent or quarantine |
-| raw turn missing | canonical recent turn has no matching SDK raw | write one bounded L1 raw turn |
-| unknown outcome | a task may have reached SDK without a confirmed result | SDK outcome inquiry; never blind retry |
-
-An unclassified item blocks cutover.
-
-## 4. User edits, deletion and supersession
-
-The current product has no MemHub revision history or Undo.
-
-- The latest non-deleted user correction is imported as the current SDK body.
-- Historical `governed_memory_versions` remain in the compatibility backup, but
- are not recreated as product-visible SDK versions.
-- A legacy user deletion is applied as an SDK physical delete. Deleted
- Governance tombstones are not imported and no deleted-list UI survives.
-- SDK supersession remains an SDK lifecycle relationship. Existing SDK chains
- are retained. Governance links are used only to repair a missing SDK chain
- during exact import.
-- Mapped Coding Experience and Working Fact IDs are resolved in their native
- SDK stores and adopted without conversion to Chat layers. A mapped ID missing
- from its native store blocks cutover rather than fabricating another type.
-- Pending correction/delete records are settled before ordinary backfill.
-
-`update(memory_id, content)` modifies the same SDK ID. There is no product CAS,
-revision number, or restore operation.
-
-## 5. Legacy Evidence to bounded SDK raw
-
-Evidence rows are message-level legacy storage, not SDK raw chunks. Migration
-uses them only to rebuild completed turns from the three days preceding the
-stable migration start time:
-
-- one human `user` opener plus its following `assistant` records becomes one
- deterministic L1 raw node;
-- tool rows and agent-authored user envelopes are excluded;
-- a turn is bounded to 60,000 characters and records whether it was truncated;
-- existing SDK raw outside this canonical set is removed in staging;
-- high-level Chat/Coding/Working memories are retained independently of raw.
-
-No extraction or reconciliation is rerun while rebuilding raw. After cutover,
-the normal Cursor History catch-up scans forward from its source checkpoint and
+- startup never falls back to legacy authority.
+
+The surviving `memory_authority_state` singleton, not table existence, selects
+the authority.
+
+A fresh home bootstraps the latest management schema with SDK authority
+directly and never creates legacy tables.
+
+## 3. Startup steps
+
+The packaged App runs these Core commands while the startup placeholder is
+visible, under the Runtime maintenance fence and the capture fence (host
+capture hooks return silently instead of writing Evidence):
+
+1. `migration-bundle`: copy `one-memory.sqlite3`, the SDK data plane and the
+ non-secret configuration into a versioned bundle, then verify it by an
+ isolated restore. An existing verified bundle for the same migration is
+ reused.
+2. `sdk-migrate`: apply the management schema migrations, discard SDK staging
+ left by builds through 19 (see §5), and make legacy queue work
+ self-contained (see §4). It needs no model profile and no SDK client. The run
+ is recorded as `phase=prepared, state=ready`.
+3. `sdk-cutover`: re-verify the bundle, repeat §4 (a CLI-managed Runtime may
+ have run since step 2), commit SDK authority, and drop the legacy tables.
+
+Each step is safe to repeat. A retry resumes the same `sdk-authority-v2-*`
+migration ID and bundle, including a run that build 19 left in
+`review_required/blocked`; earlier per-item ledger rows are ignored and dropped
+with the legacy tables.
+
+After cutover, History catch-up scans forward from each source checkpoint and
recovers turns skipped while capture was fenced.
-## 6. Full migration bundle
-
-A successful preflight creates one versioned bundle containing:
-
-- an online backup of `one-memory.sqlite3`;
-- an SDK-consistent export/snapshot of raw, vector, graph, cache, history and
- coding stores;
-- `config.json` and non-secret configuration needed to locate those stores;
-- Runtime, SDK, schema, namespace, backend and embedding fingerprints;
-- object/file hashes, row counts, object counts and creation time;
-- migration ledger version and bundle ID.
-
-Secrets are not copied into a plaintext manifest. The bundle records only
-secret references and the secure-store identity required for restore.
-
-The bundle is published only after an isolated restore verifies hashes, counts
-and SDK reads. A remote backend without a supported snapshot/export mechanism
-blocks migration; SQLite-only backup is not reported as success.
-
-## 7. Authority state and migration ledger
-
-The MemHub management database owns:
-
-- `memory_authority_state`: the surviving singleton authority and migration ID;
-- `sdk_data_plane_migrations`: one run, source/target fingerprints, bundle,
- phase, cutover state and reconciliation digest;
-- `sdk_data_plane_migration_items`: source identity/revision/hash, target SDK
- identity/hash, deterministic operation key, state, attempts and error code;
-- `memory_id_aliases`: temporary historical ID to canonical SDK ID mapping for
- call-events, write-job receipts, and links;
-- `raw_id_aliases`: legacy Evidence ID/item index to canonical SDK raw ID.
-- `sdk_data_plane_cutover`: a temporary detailed cutover receipt containing
- SDK/backend/bundle/reconciliation fingerprints.
-
-Progress is committed only after SDK verification. Retrying an item uses the
-same deterministic SDK operation key and must not create another memory.
-Before cleanup, every surviving reference is rewritten to its canonical SDK
-ID. Alias, item-ledger, and detailed cutover tables are then dropped; the
-authority singleton and verified bundle remain.
-
-Backfill never mutates the live SDK root. Startup first copies the complete
-local SDK data plane, including SDK-only records, to a same-filesystem staging
-home. Exact legacy writes and reconciliation run there. Cutover renames the
-live root to a protective path, renames staging into place, and only then
-commits SDK authority. A failed commit restores the previous root; restart
-recovery resolves any interrupted rename before opening SDK stores.
-
-## 8. Queue and unknown outcomes
-
-Queue rows and leases remain MemHub management state.
-
-- A pending job has not called the SDK and therefore has no SDK raw or memory
- ID yet.
-- Its worker calls SDK `add()` once. The SDK owns raw persistence plus
- extraction/reconciliation; a successful job stores only the returned raw ID,
- canonical memory IDs and terminal outcome.
-- Legacy raw and semantic backfill is an App-owned privileged offline
- operation while Runtime writers are stopped. It writes SDK stores directly;
- no migration-only import/get/list method is added to public
- `HyMemoryClient`, and this is not the first half of a two-stage normal write
- path.
-
-- `hy_invoked=0` may be safely retried after normal lease recovery.
-- `memory_write_jobs.hy_invoked=1`, `state=unknown`, or any non-terminal SDK
- outcome must use `get_write_outcome(request_id)`.
-- `running`/`unknown` is not absence and must not be replayed.
-- confirmed SDK IDs become write-job outcomes/receipts; Governance publication is not part of
- completion after cutover.
-
-## 9. Call events and session delivery
-
-- Matter was an unpublished placeholder. Its rows are not migrated and every
- `matter*` table is deleted during cleanup.
-- Before cleanup, historical IDs resolve through `memory_id_aliases`; surviving
- records are rewritten to canonical SDK IDs before that table is dropped.
-- `call_events` remains metadata-only; reference titles/content resolve through
- SDK APIs.
-- Evidence-based session inbox/outbox is disabled at cutover unless replaced by
- a management-event or SDK-event contract. It must not send shadow memory
- bodies.
-
-## 10. Cutover
-
-The packaged App owns orchestration. While the startup placeholder is visible,
-it starts no product writer, creates or verifies the full bundle, resumes the
-durable migration ledger, reconciles SDK state, and only then commits cutover.
-The normal App window is revealed after this phase. A retry resumes the same
-migration and deterministic operation keys; it does not create a second run.
-
-If migration cannot safely complete, the marker remains absent and startup must
-report the durable blocked/deferred reason. It must not silently continue into
-an SDK/legacy dual-write mode. A missing model profile may defer migration while
-legacy authority is still intact so the user can configure the App; it never
-authorizes a partial cutover.
-
-Cutover requires:
-
-- no unclassified or quarantined item;
-- complete raw coverage;
-- no unsafe unknown task;
-- pending corrections and deletions settled;
-- call-event references resolvable;
-- exact counts and hashes reconciled;
-- deterministic full-ID get verification;
-- a verified full migration bundle.
-
-The final transaction writes the detailed cutover receipt and changes
-`memory_authority_state` to SDK authority. Cleanup may remove the detailed
-receipt after bundle verification; the authority singleton remains.
-
-After that transaction, legacy memory/raw writes are rejected.
-
-## 11. Rollback and version skew
-
-Before cutover, a failed run resumes from the ledger or restores the full
-bundle.
-
-After cutover but before any SDK-only write, rollback restores the full bundle
-and matching old Runtime together.
-
-After an SDK-only write:
-
-- automatic binary-only rollback is forbidden;
-- operators either forward-fix, or explicitly restore the pre-cutover bundle
- while acknowledging loss of post-cutover writes;
-- an old Runtime must fail closed on the post-cutover marker/schema.
-
-Changing only the Runtime `current` symlink is not data rollback.
-
-## 12. Legacy cleanup
-
-Legacy tables are not a compatibility runtime. After every memory/raw
-reference resolves, the verified bundle has passed restore validation, and the
-SDK cutover marker is durable, the offline startup transaction drops:
-
-- every `evidence*` and `governed_*` table;
-- legacy retrieval/index tables;
-- every `matter*` placeholder table;
-- legacy deletion/index-sync/task-output and session-shadow tables;
-- migration-only aliases and ledgers after their canonical references have
- been materialized into the surviving management tables.
-
-Before that DROP, every surviving `memory_write_jobs` row is made
-self-contained by freezing its source records into `payload_json`. Unstarted
-legacy `memory_tasks` rows are converted to deterministic
-`memory_write_jobs(idempotency_key="legacy-task:<task_id>")`; possibly-sent
-task rows remain a cutover blocker until SDK request-ID reconciliation proves
-their outcome.
-
-The verified bundle is the audit and rollback artifact; keeping duplicate
-plaintext tables in the live database is not. A cleanup failure keeps startup
-blocked and never exposes a half-cleaned schema.
-
-## 13. Acceptance matrix
+## 4. Queue work
+
+Queue rows remain MemHub management state. Before the legacy tables disappear:
+
+- each `memory_write_jobs` row that only names Evidence IDs gets those records
+ frozen into `payload_json`;
+- unstarted legacy `memory_tasks` become
+ `memory_write_jobs(idempotency_key="legacy-task:<task_id>")`;
+- Evidence waiting in `evidence_pending_memory_dispatch` (its source cursor has
+ already passed it) becomes one write job per conversation.
+
+Legacy work that cannot continue does not block cutover; its counts are
+reported in the `sdk-migrate` / cutover receipt as `abandoned`:
+
+- a legacy task that may already have reached the SDK (`unknown`, or `running`
+ with an SDK invocation) is not replayed: the SDK already holds whatever it
+ committed;
+- pending legacy user deletions are not carried over.
+
+Surviving `memory_write_jobs` with `hy_invoked=1` keep their normal
+post-cutover handling through SDK `get_write_outcome(request_id)`.
+
+## 5. Staging left by earlier builds
+
+Builds through 19 copied the SDK root to `<home>/.sdk-migration/` and swapped
+directories at cutover. Before SDK authority, a leftover
+`hy-memory/.<namespace>.pre-migration-*` directory is the original store and is
+restored; after SDK authority it is stale and removed. More than one such
+directory before authority is ambiguous and fails closed
+(`sdk_migration_swap_state_ambiguous`).
+
+## 6. Failure and rollback
+
+Any failure leaves the marker absent and reports a stable code to the startup
+dialog (for example `sdk_migration_disk_space_insufficient`,
+`sdk_migration_bundle_copy_failed`, `sdk_migration_preflight_failed`,
+`sdk_migration_cutover_failed`). The next startup retries.
+
+After cutover but before any SDK-only write, rollback restores the bundle
+together with the matching old Runtime. After an SDK-only write, automatic
+binary-only rollback is forbidden; restoring the pre-cutover bundle loses the
+post-cutover writes and must be explicit. An old Runtime must fail closed on a
+post-cutover home. Changing only the Runtime `current` symlink is not data
+rollback.
+
+## 7. Acceptance
The release gate covers:
-- a clean install with no legacy rows;
-- a clean install whose schema contains no legacy table names;
-- Governance-only, SDK-only, mapped-ID and divergent fixtures;
-- Evidence/raw-only content;
-- pending correction and physical deletion;
-- unknown SDK outcome and crash after SDK commit;
-- interruption before/after each ledger transition;
-- full bundle restore and hash/count mismatch;
-- call-event alias preservation;
-- old Runtime against a post-cutover home;
-- successful migration followed by physical absence of every legacy table;
-- local and supported remote backend snapshot behavior.
-
-No narrow unit test may be used as evidence that the full migration is safe.
+- a fresh install with no legacy table names;
+- a legacy home reaching cutover with its SDK stores byte-for-byte unchanged;
+- a home left blocked by build 19 (quarantined ledger and staging copy)
+ completing on the next startup;
+- pending job freezing, legacy task and pending dispatch conversion;
+- invoked/unknown legacy tasks and pending deletions not blocking cutover;
+- interrupted directory swap recovery before and after authority;
+- bundle creation, reuse and restore;
+- physical absence of every legacy table after cutover.