Python
fix(memhub): write conversations by one turn rule on every path
alvinfeif3b417ced4bb
diff --git a/memhub/apps/core/src/one_memory/api.py b/memhub/apps/core/src/one_memory/api.pyindex 68ff6e6..50513be 100644--- a/memhub/apps/core/src/one_memory/api.py+++ b/memhub/apps/core/src/one_memory/api.py@@ -47,8 +47,8 @@ from one_memory.background_work import shutdown_bulk_pool, warm_bulk_poolfrom one_memory.config import Settings, load_settingsfrom one_memory.context_routes import sdk_context_routerfrom one_memory.conversation_processing import (- _conversation_chunks,memory_task_timeout_seconds,+ turn_batches,)from one_memory.database_access import (MAIN_DATABASE_REPORT_SECONDS,@@ -1035,21 +1035,20 @@ def create_app(app.state.start_memory_backfill = lambda: _start_backfill(app)async def enqueue_stable_evidence(record: EvidenceRecord) -> None:- priority = "live" if live_sync_arrival.get() else "normal"+ live = live_sync_arrival.get()conversation = await asyncio.to_thread(app.state.evidence_store.list_conversation, record.evidence_id)- for chunk in _conversation_chunks(conversation, max_turns=12, completed_only=True):- if (- record not in chunk- or not any(item.role == "user" for item in chunk)- or not any(item.role == "assistant" for item in chunk)- ):- continue+ # Legacy authority. The turn rule groups the whole conversation+ # from its start, so a batch's records and key never change:+ # history (end of file) writes it all, live writes complete+ # batches and leaves the rest to catch-up (`_run_backfill`).+ batches = turn_batches(conversation, flush=not live).batches+ for batch in _batches_settled_by(batches, record):await asyncio.to_thread(app.state.memory_write_intake.submit_records,- chunk,- priority=priority,+ batch,+ priority="live" if live else "normal",)app.state.memory_addition_store = MemoryAdditionStore(@@ -4381,14 +4380,11 @@ async def _run_backfill(app: FastAPI) -> None:_evidence_store(app).list_conversation,evidence_id,)- for chunk in _conversation_chunks(- conversation,- max_turns=12,- completed_only=True,- ):- if not any(item.evidence_id in evidence_ids for item in chunk):+ # Catch-up is history: the last turn and a short batch are written.+ for batch in turn_batches(conversation, flush=True).batches:+ if not any(item.evidence_id in evidence_ids for item in batch):continue- await asyncio.to_thread(intake.submit_records, chunk, priority="normal")+ await asyncio.to_thread(intake.submit_records, batch, priority="normal")cursor = evidence_ids[-1]except asyncio.CancelledError:raise@@ -4407,6 +4403,22 @@ async def _run_backfill(app: FastAPI) -> None:logger.exception("memory catch-up failed, no further memories will be made")+def _batches_settled_by(+ batches: tuple[tuple[EvidenceRecord, ...], ...],+ record: EvidenceRecord,+) -> list[tuple[EvidenceRecord, ...]]:+ """The batches one arriving record can have made writable.++ Its own batch, and the last two: a user message completes the turn+ before it, which can close that turn's batch and, at the 8000-character+ cut, the batch before it. Submitting is idempotent, so an older batch+ named again is only a hit.+ """+ chosen = [batch for batch in batches if record in batch]+ chosen.extend(batch for batch in batches[-2:] if batch not in chosen)+ return chosen++def _sdk_memory_runtime(app: FastAPI) -> SdkMemoryRuntime:return cast(SdkMemoryRuntime,diff --git a/memhub/apps/core/src/one_memory/conversation_processing.py b/memhub/apps/core/src/one_memory/conversation_processing.pyindex ee4b76d..ce7323f 100644--- a/memhub/apps/core/src/one_memory/conversation_processing.py+++ b/memhub/apps/core/src/one_memory/conversation_processing.py@@ -4,9 +4,10 @@ from __future__ import annotationsimport osimport re+from collections.abc import Sequence+from dataclasses import dataclass-from one_memory.imports.models import EvidenceContentKind, EvidenceOrigin, EvidenceRecord-from one_memory.memory_tasks import segment_evidence_record+from one_memory.imports.models import EvidenceContentKind, EvidenceRecordMEMORY_TASK_TIMEOUT_MARGIN_SECONDS = 60.0MEMORY_TASK_TIMEOUT_MIN_SECONDS = 180.0@@ -29,10 +30,13 @@ _HOST_ENVELOPE = re.compile(r")",re.IGNORECASE | re.DOTALL,)-_WHOLE_TURN_CHARACTERS = 60_000-_WINDOW_BYTE_CEILING = 2_000_000-_HISTORY_MERGE_TURNS = 8-_HISTORY_MERGE_CHARACTERS = 8_000+#: The write rule (`turn_batches`): every 5 complete turns, or fewer when+#: the next turn would take the batch past 8000 HY-read characters.+TURN_BATCH_TURNS = 5+TURN_BATCH_CHARACTERS = 8_000+#: A live conversation's last turn counts as complete, and a short batch is+#: written, after this long without a new message in that conversation.+TURN_IDLE_FLUSH_SECONDS = 600def memory_task_timeout_seconds(model_timeout_seconds: float) -> float:@@ -57,180 +61,83 @@ def _coding_tool_result_max_bytes() -> int:_HY_TOOL_CHARACTERS = _coding_tool_result_max_bytes()-def _conversation_chunks(- records: list[EvidenceRecord],- *,- max_characters: int | None = None,- max_turns: int | None = None,- completed_only: bool = False,-) -> list[tuple[EvidenceRecord, ...]]:- """Build bounded, complete user/assistant/tool windows."""+@dataclass(frozen=True)+class TurnBatches:+ """What a conversation's records become: batches to write, records to keep."""++ #: Each batch is whole turns; every batch holds at least one user message.+ batches: tuple[tuple[EvidenceRecord, ...], ...]+ #: Not written yet: the open last turn, complete turns short of a batch,+ #: or records with no user message before them. Fed back in, in order,+ #: with the conversation's next records.+ carry: tuple[EvidenceRecord, ...]+++def conversation_turns(records: Sequence[EvidenceRecord]) -> list[tuple[EvidenceRecord, ...]]:+ """One user message and every record after it up to the next user message.++ Records before the first user message are not a turn of their own: they+ join the first turn (an assistant proposal and the user turn adopting it+ reach HY together). With no user message at all the result is one group+ that does not open with a user, which `turn_batches` never writes.+ """turns: list[list[EvidenceRecord]] = []for record in records:if not _is_hy_turn_record(record):continue- pending_proposal = bool(turns) and not _opens_turn(turns[-1][0])- if not turns or (_opens_turn(record) and not pending_proposal):+ # Until the first user message, the group is leading records only.+ leading = bool(turns) and not any(_opens_turn(item) for item in turns[-1])+ if not turns or (_opens_turn(record) and not leading):turns.append([record])else:turns[-1].append(record)-- if completed_only and turns and _trailing_user_turn_is_open(turns[-1]):- turns.pop()-- whole = _whole_turns() and max_characters is None- if whole:- size = _hy_read_length- max_characters = _WHOLE_TURN_CHARACTERS- max_turns = max_turns or 1_000_000- else:- size = lambda record: len(record.verbatim) # noqa: E731- max_characters = max_characters or 12_000- max_turns = max_turns or 24-- chunks: list[tuple[EvidenceRecord, ...]] = []- for turn in turns:- original_question = turn[0] if _opens_turn(turn[0]) else None- expanded: list[EvidenceRecord] = []- for record in turn:- reserve = (- size(original_question)- if original_question is not None- and record is not original_question- and size(original_question) < max_characters- else 0- )- segment_limit = max(1, max_characters - reserve)- if size(record) <= segment_limit:- expanded.append(record)- continue- spans = _semantic_segment_spans(record.verbatim, segment_limit)- expanded.extend(- segment_evidence_record(record, index=index, start=start, end=end)- if len(spans) > 1- else record- for index, (start, end) in enumerate(spans)- )- question = expanded[0] if expanded and _opens_turn(expanded[0]) else None- current: list[EvidenceRecord] = []- for record in expanded:- if current and (- len(current) >= max_turns- or sum(size(item) for item in current) + size(record) > max_characters- or sum(_window_bytes(item) for item in current) + _window_bytes(record)- > _WINDOW_BYTE_CEILING- ):- chunks.append(tuple(current))- current = (- [question]- if question is not None- and max_turns > 1- and size(question) + size(record) <= max_characters- else []- )- current.append(record)- chunks.append(tuple(current))- return _merge_tool_only_chunks(chunks)---def _semantic_segment_spans(text: str, limit: int) -> list[tuple[int, int]]:- if len(text) <= limit:- return [(0, len(text))]- spans: list[tuple[int, int]] = []- start = 0- separators = ("\n\n", "\n", "。", "!", "?", ". ", "! ", "? ")- while start < len(text):- hard_end = min(len(text), start + limit)- if hard_end == len(text):- end = hard_end- else:- floor = start + max(1, limit // 2)- candidates = [text.rfind(separator, floor, hard_end) for separator in separators]- boundary = max(candidates)- if boundary < floor:- end = hard_end- else:- matched = next(- separator for separator in separators if text.startswith(separator, boundary)- )- end = boundary + len(matched)- spans.append((start, end))- start = end- return spans---def _merge_tool_only_chunks(- chunks: list[tuple[EvidenceRecord, ...]],-) -> list[tuple[EvidenceRecord, ...]]:- if not any(chunk and all(record.role == "tool" for record in chunk) for chunk in chunks):- return chunks- merged: list[tuple[EvidenceRecord, ...]] = []- leading_tools: list[EvidenceRecord] = []- for chunk in chunks:- if not chunk:- continue- if all(record.role == "tool" for record in chunk):- if merged:- merged[-1] = (*merged[-1], *chunk)- else:- leading_tools.extend(chunk)- continue- if leading_tools:- chunk = (*leading_tools, *chunk)- leading_tools = []- merged.append(chunk)- return [] if leading_tools else merged---def _merge_historical_chunks(- chunks: list[tuple[EvidenceRecord, ...]],- *,- max_turns: int = _HISTORY_MERGE_TURNS,- max_characters: int = _HISTORY_MERGE_CHARACTERS,-) -> list[tuple[EvidenceRecord, ...]]:- if not chunks:- return []- merged: list[tuple[EvidenceRecord, ...]] = []- current: list[EvidenceRecord] = []- current_turns = 0+ return [tuple(turn) for turn in turns]+++def turn_batches(records: Sequence[EvidenceRecord], *, flush: bool) -> TurnBatches:+ """Group a conversation's records into write batches by the turn rule.++ - A turn is complete once the next user message follows it; `flush`+ completes the last one too (history at end of file, or 10 idle minutes).+ - Every 5 complete turns are one batch. Before 5, a turn that would take+ the batch past 8000 characters starts the next batch; one turn over+ 8000 is a batch by itself and is never split.+ - Characters are what HY reads: user/assistant text in full, a tool+ record up to `_HY_TOOL_CHARACTERS`.+ - Fewer than 5 complete turns within 8000 are written only on `flush`.+ - No batch is written without a user message; records with none before+ them are carried, and dropped on `flush`.+ """+ turns = conversation_turns(records)+ if turns and not any(_opens_turn(record) for record in turns[0]):+ # Records with no user message yet: nothing to write until one comes.+ return TurnBatches(batches=(), carry=() if flush else turns[0])+ complete = turns if flush else turns[:-1]+ open_tail = () if flush or not turns else turns[-1]+ batches: list[tuple[EvidenceRecord, ...]] = []+ current: list[tuple[EvidenceRecord, ...]] = []current_characters = 0- for chunk in chunks:- chunk_characters = sum(len(record.verbatim) for record in chunk)- if current and (- current_turns >= max_turns or current_characters + chunk_characters > max_characters- ):- merged.append(tuple(current))- current = []- current_turns = 0- current_characters = 0- current.extend(chunk)- current_turns += 1- current_characters += chunk_characters- if current:- merged.append(tuple(current))- return merged---def _chunk_input_ref(chunk: tuple[EvidenceRecord, ...]) -> str:- user_records = tuple(item for item in chunk if item.role == "user")- return user_records[-1].evidence_id if user_records else chunk[-1].evidence_id+ for turn in complete:+ characters = sum(_hy_read_length(record) for record in turn)+ if current and current_characters + characters > TURN_BATCH_CHARACTERS:+ batches.append(tuple(record for item in current for record in item))+ current, current_characters = [], 0+ current.append(turn)+ current_characters += characters+ if len(current) >= TURN_BATCH_TURNS or current_characters > TURN_BATCH_CHARACTERS:+ batches.append(tuple(record for item in current for record in item))+ current, current_characters = [], 0+ if current and flush:+ batches.append(tuple(record for item in current for record in item))+ current = []+ carry = tuple(record for item in current for record in item) + tuple(open_tail)+ return TurnBatches(batches=tuple(batches), carry=carry)def _opens_turn(record: EvidenceRecord) -> bool:return record.role == "user"-def _trailing_user_turn_is_open(turn: list[EvidenceRecord]) -> bool:- if len(turn) != 1:- return False- opener = turn[0]- return (- _opens_turn(opener)- and opener.content_kind is EvidenceContentKind.RAW_CONVERSATION- and not _is_hand_added_memory(opener)- )--def _is_hy_turn_record(record: EvidenceRecord) -> bool:if record.role == "tool":return record.content_kind is EvidenceContentKind.RAW_CONVERSATION and bool(@@ -254,23 +161,6 @@ def _is_hy_input_turn(record: EvidenceRecord) -> bool:return _HOST_ENVELOPE.search(source[:2_000]) is None-def _is_hand_added_memory(record: EvidenceRecord) -> bool:- meta = record.origin_meta.get(EvidenceOrigin.MANUAL_IMPORT)- return (- record.source_id == "one-memory-manual"- and meta is not None- and meta.raw_meta.get("input_mode") == "direct_memory_note"- )---def _window_bytes(record: EvidenceRecord) -> int:- return len(record.verbatim.encode("utf-8"))---def _whole_turns() -> bool:- return os.environ.get("ONE_MEMORY_SPLIT_TURNS", "").strip() != "1"--def _hy_read_length(record: EvidenceRecord) -> int:if record.role == "tool":return min(len(record.verbatim), _HY_TOOL_CHARACTERS)diff --git a/memhub/apps/core/src/one_memory/imports/runner.py b/memhub/apps/core/src/one_memory/imports/runner.pyindex e5d60b4..f45b1e6 100644--- a/memhub/apps/core/src/one_memory/imports/runner.py+++ b/memhub/apps/core/src/one_memory/imports/runner.py@@ -662,11 +662,15 @@ class ImportRunner:and self.sdk_queue is not Noneand event.cursor.read_to_end):+ # History ends at this end of file, so its last turn+ # and a short batch are written. Live polling keeps+ # them for the next increment or the idle flush.self.sdk_queue.finalize_sdk_file(user_id=current.user_id,source_id=current.source_id,file_key=event.cursor.file_key,connection=connection,+ flush=not self._live_import,)continueif self.on_stable_evidence is not None and event.cursor.read_to_end:diff --git a/memhub/apps/core/src/one_memory/live_capture.py b/memhub/apps/core/src/one_memory/live_capture.pyindex de57e21..0853390 100644--- a/memhub/apps/core/src/one_memory/live_capture.py+++ b/memhub/apps/core/src/one_memory/live_capture.py@@ -256,24 +256,19 @@ def record_capture_rejection(def _submit_live_memory(store: EvidenceStore, record: EvidenceRecord, *, changed: bool) -> None:+ """Add the record to its conversation's live assembling row.++ The turn rule decides what is written (`turn_batches`): a turn is+ complete when the next user message arrives, batches are 5 turns or+ 8000 characters, and Core writes the rest after 10 quiet minutes.+ """if not changed or not _sync_open(store.database_path, record.source_id):return- from one_memory.conversation_processing import _conversation_chunksfrom one_memory.memory_server.durable_queue import DurableWriteQueue- from one_memory.memory_server.unified_entry import UnifiedWriteIntake-- intake = UnifiedWriteIntake(DurableWriteQueue(store.database_path, evidence=store))- for chunk in _conversation_chunks(- store.list_conversation(record.evidence_id),- max_turns=12,- completed_only=True,- ):- if (- record in chunk- and any(item.role == "user" for item in chunk)- and any(item.role == "assistant" for item in chunk)- ):- intake.submit_records(chunk, priority="live")++ DurableWriteQueue(store.database_path, evidence=store).assemble_live_records(+ (record,), file_key=f"hook:{record.conversation_id}"+ )def _conversation(payload: dict[str, object]) -> str | None:diff --git a/memhub/apps/core/src/one_memory/memory_server/durable_queue.py b/memhub/apps/core/src/one_memory/memory_server/durable_queue.pyindex 65ff266..b755ba9 100644--- a/memhub/apps/core/src/one_memory/memory_server/durable_queue.py+++ b/memhub/apps/core/src/one_memory/memory_server/durable_queue.py@@ -24,10 +24,11 @@ from collections.abc import Awaitable, Callablefrom dataclasses import dataclassfrom datetime import UTC, datetime, timedeltafrom pathlib import Path-from typing import Any+from typing import Any, Literalfrom uuid import uuid4from one_memory.background_work import in_the_background+from one_memory.conversation_processing import TURN_IDLE_FLUSH_SECONDS, turn_batchesfrom one_memory.database_access import main_databasefrom one_memory.diagnostic_log import diagnostic_eventfrom one_memory.imports.models import EvidenceOrigin, EvidenceRecord@@ -894,16 +895,30 @@ class DurableWriteQueue:priority: str,connection: sqlite3.Connection,) -> None:- """Append fresh-install input to an assembling queue row.-- The row is not claimable until the scanner commits EOF and finalizes- it. No Evidence table is involved.+ """Append input to its conversation's assembling queue row.++ The row is not claimable. `finalize_sdk_file` turns what is complete+ into write jobs (`turn_batches`) and keeps the rest here for the next+ increment; `flush_idle_assemblies` writes a live row's rest once the+ conversation has been quiet. `updated_at` moves only when a record is+ new or revised, so a re-read does not keep a quiet conversation open.+ No Evidence table is involved."""assembly_key = (f"assembly:{record.user_id}:{record.source_id}:"f"{hashlib.sha256(file_key.encode()).hexdigest()}:"f"{hashlib.sha256(record.conversation_id.encode()).hexdigest()}")+ session_key = internal_session_id_for(record.conversation_id, record.source_id)+ written = connection.execute(+ "SELECT 1 FROM memory_write_jobs AS job, json_each(job.evidence_ids_json) AS ref "+ "WHERE job.session_key=? AND job.state!='assembling' AND ref.value=? LIMIT 1",+ (session_key, record.evidence_id),+ ).fetchone()+ if written is not None:+ # A re-read of a record a job already holds: staging it again+ # would regroup it with later turns into a second job.+ returnrow = connection.execute("SELECT job_id,payload_json,priority FROM memory_write_jobs ""WHERE idempotency_key=? AND state='assembling'",@@ -914,8 +929,20 @@ class DurableWriteQueue:payload = json.loads(str(row["payload_json"]))records = list(payload.get("input_records") or [])dumped = record.model_dump(mode="json")- if not any(item.get("evidence_id") == record.evidence_id for item in records):+ index = next(+ (+ position+ for position, item in enumerate(records)+ if item.get("evidence_id") == record.evidence_id+ ),+ None,+ )+ changed = index is None or records[index] != dumped+ if index is None:records.append(dumped)+ elif changed:+ # A revision replaces the carried copy in place.+ records[index] = dumpednow = datetime.now(UTC).isoformat()payload_json = json.dumps({@@ -926,15 +953,13 @@ class DurableWriteQueue:sort_keys=True,)if row is not None:+ merged_priority = "live" if "live" in (row["priority"], priority) else "normal"+ if not changed and merged_priority == row["priority"]:+ returnconnection.execute(- "UPDATE memory_write_jobs SET payload_json=?,priority=?,updated_at=? "- "WHERE job_id=?",- (- payload_json,- "live" if row["priority"] == "live" or priority == "live" else "normal",- now,- str(row["job_id"]),- ),+ "UPDATE memory_write_jobs SET payload_json=?,priority=?,"+ "updated_at=CASE WHEN ? THEN ? ELSE updated_at END WHERE job_id=?",+ (payload_json, merged_priority, int(changed), now, str(row["job_id"])),)returnqueue_seq = int(@@ -952,7 +977,7 @@ class DurableWriteQueue:(str(uuid4()),queue_seq,- internal_session_id_for(record.conversation_id, record.source_id),+ session_key,record.source_id,assembly_key,assembly_key,@@ -982,92 +1007,215 @@ class DurableWriteQueue:source_id: str,file_key: str,connection: sqlite3.Connection,+ flush: bool = True,) -> tuple[str, ...]:+ """Write what one file's assembling rows hold that the turn rule completes.++ `flush` (history at end of file) completes the last turn and writes a+ short batch; without it (live polling) those stay in the row.+ """rows = connection.execute("SELECT * FROM memory_write_jobs WHERE state='assembling' ""AND source_key=? ORDER BY queue_seq",(source_id,),).fetchall()- finalized = []+ finalized: list[str] = []for row in rows:payload = json.loads(str(row["payload_json"]))if payload.get("file_key") != file_key:continue- records = tuple(- EvidenceRecord.model_validate(item)- for item in payload.get("input_records") or []- if item.get("user_id") == user_id- )- if not records:- connection.execute(- "DELETE FROM memory_write_jobs WHERE job_id=?",- (str(row["job_id"]),),- )- continue- first = records[0]- session_id = internal_session_id_for(- first.conversation_id,- first.source_id,+ finalized.extend(+ self._write_assembled_turns(connection, row, user_id=user_id, flush=flush))- material = ":".join(f"{item.evidence_id}:{item.content_hash}" for item in records)- request_id = f"turn:v2:{hashlib.sha256(material.encode()).hexdigest()}"- existing = connection.execute(- "SELECT job_id FROM memory_write_jobs WHERE idempotency_key=? AND job_id!=?",- (request_id, str(row["job_id"])),- ).fetchone()- if existing is not None:- connection.execute(- "DELETE FROM memory_write_jobs WHERE job_id=?",- (str(row["job_id"]),),+ return tuple(finalized)++ def assemble_live_records(+ self,+ records: tuple[EvidenceRecord, ...],+ *,+ file_key: str,+ ) -> tuple[str, ...]:+ """Stage live records and write what is complete, in one transaction.++ For live capture outside the import runner (host hooks): the same+ assembling rows, turn rule and idle flush as live file polling.+ """+ if not records:+ return ()+ first = records[0]+ with (+ main_database(self.database_path).write(row_factory=sqlite3.Row) as connection,+ connection,+ ):+ for record in records:+ self.stage_sdk_input(+ record, file_key=file_key, priority="live", connection=connection)- finalized.append(str(existing["job_id"]))- continue- request = WriteRequest(- internal_session_id=session_id,+ return self.finalize_sdk_file(+ user_id=first.user_id,source_id=first.source_id,- batch_identity=BatchIdentity(- internal_session_id=session_id,- first_round=1,- last_round=1,- ),- evidence_ids=[item.evidence_id for item in records],- input_records=records,- idempotency_key=request_id,- priority="live" if str(row["priority"]) == "live" else "normal",- event_at=max(- (item.created_at for item in records if item.created_at),- default=first.imported_at,- ),- time_quality="exact" if any(item.created_at for item in records) else "approximate",+ file_key=file_key,+ connection=connection,+ flush=False,)++ def flush_idle_assemblies(self, *, now: datetime | None = None) -> tuple[str, ...]:+ """Write the rest of every live conversation quiet for 10 minutes.++ Its last turn counts as complete and a batch short of 5 turns is+ written. History rows are left alone: their end of file flushes them.+ """+ current = now or datetime.now(UTC)+ cutoff = (current - timedelta(seconds=TURN_IDLE_FLUSH_SECONDS)).isoformat()+ query = (+ "SELECT * FROM memory_write_jobs WHERE state='assembling' AND priority='live' "+ "AND updated_at<=? ORDER BY queue_seq"+ )+ # A read first: most ticks find nothing and must not take the writer.+ with main_database(self.database_path).read(row_factory=sqlite3.Row) as connection:+ if connection.execute(query, (cutoff,)).fetchone() is None:+ return ()+ finalized: list[str] = []+ with (+ main_database(self.database_path).write(row_factory=sqlite3.Row) as connection,+ connection,+ ):+ for row in connection.execute(query, (cutoff,)).fetchall():+ finalized.extend(self._write_assembled_turns(connection, row, flush=True))+ return tuple(finalized)++ def _write_assembled_turns(+ self,+ connection: sqlite3.Connection,+ row: sqlite3.Row,+ *,+ flush: bool,+ user_id: str | None = None,+ ) -> list[str]:+ """Turn one assembling row's complete batches into jobs; keep the carry."""+ payload = json.loads(str(row["payload_json"]))+ records = tuple(+ EvidenceRecord.model_validate(item)+ for item in payload.get("input_records") or []+ if user_id is None or item.get("user_id") == user_id+ )+ grouped = turn_batches(records, flush=flush)+ priority: Literal["live", "normal"] = "live" if str(row["priority"]) == "live" else "normal"+ job_ids = [+ self._insert_turn_batch(connection, batch, priority=priority)+ for batch in grouped.batches+ ]+ if not grouped.carry:connection.execute(- "UPDATE memory_write_jobs SET session_key=?,batch_id=?,"- "idempotency_key=?,payload_json=?,evidence_ids_json=?,state='pending',"- "event_at=?,time_quality=?,updated_at=? WHERE job_id=?",+ "DELETE FROM memory_write_jobs WHERE job_id=?", (str(row["job_id"]),)+ )+ elif grouped.batches or len(grouped.carry) != len(records):+ # `updated_at` stays: the idle clock counts new messages only.+ connection.execute(+ "UPDATE memory_write_jobs SET payload_json=? WHERE job_id=?",(- session_id,- request.batch_identity.batch_id,- request_id,json.dumps(- request.model_dump(- mode="json",- exclude={"event_at", "time_quality"},- ),+ {+ **payload,+ "input_records": [+ record.model_dump(mode="json") for record in grouped.carry+ ],+ },sort_keys=True,),- json.dumps(list(request.evidence_ids)),- (- request.event_at.isoformat()- if request.event_at- else datetime.now(UTC).isoformat()- ),- request.time_quality,- datetime.now(UTC).isoformat(),str(row["job_id"]),),)- finalized.append(str(row["job_id"]))- return tuple(finalized)+ return job_ids++ def _insert_turn_batch(+ self,+ connection: sqlite3.Connection,+ records: tuple[EvidenceRecord, ...],+ *,+ priority: Literal["live", "normal"],+ ) -> str:+ """One pending job for one batch of whole turns, keyed by its content.++ Same records, same key: a re-read returns the job already written.+ """+ first = records[0]+ session_id = internal_session_id_for(first.conversation_id, first.source_id)+ material = ":".join(f"{item.evidence_id}:{item.content_hash}" for item in records)+ request_id = f"turn:v2:{hashlib.sha256(material.encode()).hexdigest()}"+ existing = connection.execute(+ "SELECT job_id FROM memory_write_jobs WHERE idempotency_key=?",+ (request_id,),+ ).fetchone()+ if existing is not None:+ return str(existing["job_id"])+ request = WriteRequest(+ internal_session_id=session_id,+ source_id=first.source_id,+ batch_identity=BatchIdentity(+ internal_session_id=session_id,+ first_round=1,+ last_round=1,+ ),+ evidence_ids=[item.evidence_id for item in records],+ input_records=records,+ idempotency_key=request_id,+ priority=priority,+ )+ # The batch's time is its first user message's, as for every other job.+ event_at, time_quality = self._event_time_for_request(request)+ now = datetime.now(UTC)+ job_id = str(uuid4())+ queue_seq = int(+ connection.execute(+ "SELECT COALESCE(MAX(queue_seq),0)+1 FROM memory_write_jobs"+ ).fetchone()[0]+ )+ event = event_at or now+ if event.tzinfo is None:+ event = event.replace(tzinfo=UTC)+ connection.execute(+ "INSERT INTO memory_write_jobs ("+ "job_id,queue_seq,session_key,source_key,batch_id,idempotency_key,"+ "payload_json,evidence_ids_json,state,priority,attempts,max_attempts,"+ "lease_owner,lease_expires_at,hy_invoked,last_error,event_at,time_quality,"+ "created_at,updated_at,sdk_raw_ids_json,sdk_memory_ids_json"+ ") VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",+ (+ job_id,+ queue_seq,+ session_id,+ first.source_id,+ request.batch_identity.batch_id,+ request_id,+ json.dumps(+ request.model_dump(mode="json", exclude={"event_at", "time_quality"}),+ sort_keys=True,+ ),+ json.dumps(list(request.evidence_ids)),+ PENDING,+ priority,+ 0,+ 3,+ None,+ None,+ 0,+ None,+ event.astimezone(UTC).isoformat(),+ time_quality,+ now.isoformat(),+ now.isoformat(),+ "[]",+ "[]",+ ),+ )+ if not self.sdk_authority:+ # Legacy catch-up reads coverage from these references.+ connection.executemany(+ "INSERT INTO memory_write_job_evidence (job_id, evidence_id) VALUES (?,?)",+ ((job_id, evidence_id) for evidence_id in request.evidence_ids),+ )+ return job_iddef _freeze_request_inputs(self, request: WriteRequest) -> WriteRequest:if request.input_records or self.evidence is None:@@ -2500,6 +2648,9 @@ class DurableWriteDrain:#: a model outage clears on its own, and a queue that never goes idle#: must still re-admit the write once the wait has elapsed.self._last_model_readmit: datetime | None = None+ #: A live conversation quiet for 10 minutes has its last turn and short+ #: batch written (`flush_idle_assemblies`); checked on the same timer.+ self._last_assembly_flush: datetime | None = None#: Idle polls do not need to hit the single writer every 50ms. A#: 1m Evidence table makes a miss expensive enough that the tail#: call-event write can be behind it; back off only while idle and@@ -3032,6 +3183,15 @@ class DurableWriteDrain:self.queue.readmit_model_waiters,operation_name="readmit_model_waiters",)+ if (+ self._last_assembly_flush is None+ or tick_started - self._last_assembly_flush >= self._reap_interval+ ):+ self._last_assembly_flush = tick_started+ await self._run_idle_maintenance(+ self.queue.flush_idle_assemblies,+ operation_name="flush_idle_assemblies",+ )if self.reconcile_unknown is not None and (self._last_unknown_reconcile is Noneor tick_started - self._last_unknown_reconcilediff --git a/memhub/apps/core/src/one_memory/sdk_write_processor.py b/memhub/apps/core/src/one_memory/sdk_write_processor.pyindex cbd8674..f6e4d98 100644--- a/memhub/apps/core/src/one_memory/sdk_write_processor.py+++ b/memhub/apps/core/src/one_memory/sdk_write_processor.py@@ -16,6 +16,7 @@ from one_memory.secrets import SecretStore, SecretStoreUnavailableErrorPRODUCT_AGENT_ID = "all-apps"_MESSAGE_ROLES = frozenset({"user", "assistant", "tool"})+_CONVERSATION_ROLES = frozenset({"user", "assistant"})@dataclass(frozen=True)@@ -41,12 +42,23 @@ class SdkWriteProcessor:) -> SdkWriteReceipt:if not records:raise RuntimeError("hy_memory_task_without_inputs")+ extracted = all(_is_extracted_content(record) for record in records)+ if not extracted and not any(+ record.content_kind is EvidenceContentKind.RAW_CONVERSATION+ and record.role in _CONVERSATION_ROLES+ for record in records+ ):+ # Nothing to remember: tool records alone. The SDK strips them on+ # the chat path and answers 400 "content or messages is+ # required", which used to settle the job as unknown. Completed+ # with no memories, and the SDK is never called.+ return SdkWriteReceipt()scope = _write_scope(records)metadata = _write_metadata(records, scope)client = await self._client_handle.client()- if all(_is_extracted_content(record) for record in records):+ if extracted:contents = [sanitize_model_input(record.verbatim) for record in records]await before_hy_invocation()result = await asyncio.to_thread(diff --git a/memhub/apps/core/tests/test_conversation_turn_grouping.py b/memhub/apps/core/tests/test_conversation_turn_grouping.pyindex 7b492d5..5fb7fe1 100644--- a/memhub/apps/core/tests/test_conversation_turn_grouping.py+++ b/memhub/apps/core/tests/test_conversation_turn_grouping.py@@ -1,90 +1,199 @@-"""A proposal and the turn that adopts it must reach HY as one turn."""+"""The turn rule: what a conversation's records become before they reach HY.++A turn is one user message and everything up to the next one. Every 5+complete turns are written together, sooner when the next turn would pass+8000 characters; a short batch waits for the end of a history file or 10+quiet minutes. No batch is written without a user message.+"""from __future__ import annotationsfrom datetime import UTC, datetime-from one_memory.conversation_processing import _conversation_chunks+from one_memory.conversation_processing import (+ TURN_BATCH_CHARACTERS,+ TURN_BATCH_TURNS,+ conversation_turns,+ turn_batches,+)from one_memory.imports.models import EvidenceInput, EvidenceOrigin, OriginMetadata, WrittenByfrom one_memory.imports.storage import EvidenceStore+_WRITER = {"user": "person", "assistant": "agent", "tool": "tool"}+-def _records(tmp_path, *speakers: tuple[str, str, str]) -> list:+def _records(tmp_path, *speakers: tuple[str, str]) -> list:store = EvidenceStore(tmp_path / "one-memory.sqlite3")- records = []- for index, (role, written_by, text) in enumerate(speakers):- records.append(- store.put(- EvidenceInput(- source_id="cursor",- conversation_id="conversation",- message_id=f"m{index}",- role=role,- written_by=WrittenBy(written_by),- verbatim=text,- created_at=datetime.now(UTC),- origin_meta={EvidenceOrigin.FILE: OriginMetadata()},- )- ).record- )- return records---def test_an_unanswered_assistant_prefix_joins_the_turn_that_adopts_it(tmp_path) -> None:- """`assistant` proposal -> `user` explicit adoption is one turn, not two.-- Splitting it sends HY an answer without what it answers, and loses the- authority pair the author/adoption quality gate depends on.- """+ return [+ store.put(+ EvidenceInput(+ source_id="cursor",+ conversation_id="conversation",+ message_id=f"m{index}",+ role=role,+ written_by=WrittenBy(_WRITER[role]),+ verbatim=text,+ created_at=datetime.now(UTC),+ origin_meta={EvidenceOrigin.FILE: OriginMetadata()},+ )+ ).record+ for index, (role, text) in enumerate(speakers)+ ]+++def _turns(count: int, *, answer: str = "好的。") -> list[tuple[str, str]]:+ return [+ part+ for index in range(count)+ for part in (("user", f"第 {index} 个问题"), ("assistant", answer))+ ]+++def _roles(batches) -> list[list[str]]:+ return [[record.role for record in batch] for batch in batches]+++def test_rule_constants_are_the_product_rule() -> None:+ assert (TURN_BATCH_TURNS, TURN_BATCH_CHARACTERS) == (5, 8_000)+++def test_a_turn_runs_from_one_user_message_to_the_next(tmp_path) -> None:records = _records(tmp_path,- ("assistant", "agent", "建议把超时改成 30 秒。"),- ("user", "person", "就这么做,超时改成 30 秒。"),+ ("user", "跑一下测试"),+ ("assistant", "好"),+ ("tool", "3 passed"),+ ("assistant", "都通过了"),+ ("user", "提交吧"),+ ("assistant", "已提交"),)- chunks = _conversation_chunks(records)+ assert [len(turn) for turn in conversation_turns(records)] == [4, 2]- assert [[record.role for record in chunk] for chunk in chunks] == [["assistant", "user"]]+def test_records_before_the_first_user_message_join_the_first_turn(tmp_path) -> None:+ """An assistant proposal and the user turn adopting it reach HY together."""+ records = _records(+ tmp_path,+ ("assistant", "建议把超时改成 30 秒。"),+ ("user", "就这么做,超时改成 30 秒。"),+ ("assistant", "改好了。"),+ ("user", "再把重试改成 3 次。"),+ ("assistant", "也改好了。"),+ )-def test_a_lone_user_message_does_not_become_a_model_input(tmp_path) -> None:- """A user message with no reply yet is Evidence, not a completed turn.+ assert _roles(conversation_turns(records)) == [+ ["assistant", "user", "assistant"],+ ["user", "assistant"],+ ]- Concurrent-import fixtures write `user` then `assistant` pairs, but a lone- user message must not create an HY task: that is the complete-turn rule,- not a queue stall.- """- records = _records(tmp_path, ("user", "person", "把超时改成 30 秒。"))- assert _conversation_chunks(records, completed_only=True) == []+def test_five_complete_turns_are_one_batch_and_the_open_turn_waits(tmp_path) -> None:+ records = _records(tmp_path, *_turns(6))++ grouped = turn_batches(records, flush=False)++ assert [len(batch) for batch in grouped.batches] == [10]+ assert grouped.batches[0] == tuple(records[:10])+ assert grouped.carry == tuple(records[10:])+++def test_complete_turns_short_of_five_wait_for_flush(tmp_path) -> None:+ records = _records(tmp_path, *_turns(3))++ waiting = turn_batches(records, flush=False)+ assert waiting.batches == ()+ assert waiting.carry == tuple(records)+ flushed = turn_batches(records, flush=True)+ assert flushed.batches == (tuple(records),)+ assert flushed.carry == ()-def test_only_the_unanswered_trailing_turn_is_held_back(tmp_path) -> None:- """Earlier complete turns stay eligible while the open tail waits."""++def test_a_lone_user_message_is_not_written_until_flush(tmp_path) -> None:+ records = _records(tmp_path, ("user", "把超时改成 30 秒。"))++ assert turn_batches(records, flush=False).batches == ()+ assert turn_batches(records, flush=True).batches == (tuple(records),)+++def test_the_turn_that_would_pass_8000_characters_starts_the_next_batch(tmp_path) -> None:+ answer = "句" * 3_000+ records = _records(tmp_path, *_turns(4, answer=answer))++ grouped = turn_batches(records, flush=False)++ # Turns 1+2 fit; turn 3 would pass 8000, so 1+2 are written now. Turn 3+ # is complete but alone, and turn 4 is still open.+ assert grouped.batches == (tuple(records[:4]),)+ assert grouped.carry == tuple(records[4:])+ assert sum(len(record.verbatim) for record in grouped.batches[0]) <= TURN_BATCH_CHARACTERS+++def test_one_turn_over_8000_characters_is_a_batch_of_its_own_and_not_split(tmp_path) -> None:records = _records(tmp_path,- ("user", "person", "先把超时改成 30 秒。"),- ("assistant", "agent", "已经改好了。"),- ("user", "person", "接着把发布包传上去。"),+ ("user", "先问个小问题"),+ ("assistant", "好"),+ ("user", "写一份方案"),+ ("assistant", "段落。" * 7_000),+ ("user", "谢谢"),)- assert _conversation_chunks(records, completed_only=True) == [(records[0], records[1])]+ grouped = turn_batches(records, flush=False)++ assert grouped.batches == (tuple(records[:2]), tuple(records[2:4]))+ assert grouped.carry == (records[4],)+ assert all(":segment:" not in record.evidence_id for record in grouped.batches[1])+++def test_tool_output_counts_only_what_hy_reads(tmp_path) -> None:+ """A tool record counts up to `_HY_TOOL_CHARACTERS`, not its full text."""+ from one_memory.conversation_processing import _HY_TOOL_CHARACTERS+ parts = [+ part+ for index in range(5)+ for part in (("user", f"第 {index} 步"), ("tool", "x" * 50_000), ("assistant", "完成"))+ ]+ records = _records(tmp_path, *parts, ("user", "下一步"))++ grouped = turn_batches(records, flush=False)++ assert _HY_TOOL_CHARACTERS == 512+ assert [len(batch) for batch in grouped.batches] == [15]-def test_the_complete_turn_contract_used_for_concurrent_import_reds(tmp_path) -> None:- """Node-id level answer for `[BLOCK C2 -> C1]` concurrent-import reds.- Those fixtures write `user` turns and never emit the closing- `assistant`/tool event, so under the complete-turn rule no HY task is- created inside the 5s window. That is the contract, not a queue stall:- Evidence is still committed and searchable, and the task appears once the- closing event arrives.+def test_records_with_no_user_message_are_never_a_batch(tmp_path) -> None:+ """The SDK strips tool records on the chat path and refuses what is left.++ A tool-only batch came back 400 "content or messages is required" and+ was recorded unknown, 16 times. Carried until a user message arrives;+ dropped when the conversation is flushed without one."""- closed = _records(+ records = _records(tmp_path, ("tool", "ls output"), ("assistant", "看完了"))++ waiting = turn_batches(records, flush=False)+ assert waiting.batches == ()+ assert waiting.carry == tuple(records)+ assert turn_batches(records, flush=True) == turn_batches((), flush=True)++ later = _records(tmp_path, ("user", "那就开始吧"), ("assistant", "开始"))+ joined = turn_batches((*waiting.carry, *later), flush=True)+ assert _roles(joined.batches) == [["tool", "assistant", "user", "assistant"]]+++def test_every_batch_holds_a_user_message(tmp_path) -> None:+ records = _records(tmp_path,- ("user", "person", "把超时改成 30 秒。"),- ("assistant", "agent", "已改为 30 秒。"),+ ("tool", "前置输出"),+ *_turns(7, answer="答" * 2_500),+ ("tool", "尾部输出"),)- assert [[record.role for record in chunk] for chunk in _conversation_chunks(closed)] == [- ["user", "assistant"]- ]++ for flush in (False, True):+ grouped = turn_batches(records, flush=flush)+ assert grouped.batches+ assert all(any(record.role == "user" for record in batch) for batch in grouped.batches)+ written = [record for batch in grouped.batches for record in batch]+ assert written + list(grouped.carry) == recordsdiff --git a/memhub/apps/core/tests/test_import_runner.py b/memhub/apps/core/tests/test_import_runner.pyindex 88f2dbc..ddca793 100644--- a/memhub/apps/core/tests/test_import_runner.py+++ b/memhub/apps/core/tests/test_import_runner.py@@ -773,3 +773,131 @@ def test_cancel_during_callback_keeps_a_replayable_pending_marker(tmp_path) -> Nassert evidence_store.count() == 1assert len(_pending_markers(evidence_store.database_path)) == 1+++def said(message_id: str, role: str, text: str) -> EvidenceInput:+ writer = {"user": WrittenBy.PERSON, "assistant": WrittenBy.AGENT, "tool": WrittenBy.TOOL}+ return evidence(message_id).model_copy(+ update={"role": role, "written_by": writer[role], "verbatim": text}+ )+++def _sdk_runner(tmp_path):+ database = tmp_path / "one-memory.sqlite3"+ bootstrap_fresh_management_schema(database)+ job_store = ImportJobStore(database)+ queue = DurableWriteQueue(database, sdk_authority=True)+ runner = ImportRunner(+ evidence_store=RetiredEvidenceStore(database), # type: ignore[arg-type]+ job_store=job_store,+ sdk_queue=queue,+ sdk_authority=lambda: True,+ )+ return database, job_store, queue, runner+++def _read(runner, job_store, parts, *, operation: str, first_line: int = 1):+ events: list[ScanEvent] = [+ ScannedEvidence(evidence=part, cursor=cursor(f"line:{first_line + index}"))+ for index, part in enumerate(parts)+ ]+ last = f"line:{first_line + len(parts) - 1}"+ events.append(ScanCursorTouch(cursor=cursor(last).model_copy(update={"read_to_end": True})))+ job = job_store.create(user_id="local", source_id="fixture", operation=operation)+ asyncio.run(runner.run(adapter=FixtureAdapter(events), job_id=job.job_id))+++def _jobs(database) -> list[tuple[str, list[str]]]:+ import json++ with sqlite3.connect(database) as connection:+ rows = connection.execute(+ "SELECT state,payload_json FROM memory_write_jobs ORDER BY queue_seq"+ ).fetchall()+ return [+ (state, [item["role"] for item in json.loads(payload)["input_records"]])+ for state, payload in rows+ ]+++def _turns(count: int, *, start: int = 0) -> list[EvidenceInput]:+ return [+ said(f"{role[0]}{index}", role, f"{role} {index}")+ for index in range(start, start + count)+ for role in ("user", "assistant")+ ]+++def test_history_import_writes_five_turn_batches_and_flushes_the_rest_at_eof(tmp_path):+ """A first import used to be one giant job per conversation."""+ database, job_store, _queue, runner = _sdk_runner(tmp_path)++ _read(runner, job_store, _turns(7), operation="history")++ assert _jobs(database) == [+ ("pending", ["user", "assistant"] * 5),+ ("pending", ["user", "assistant"] * 2),+ ]+++def test_live_polling_carries_an_open_turn_and_a_tool_only_increment(tmp_path):+ """A transcript read mid-turn used to give tool-only jobs (16 unknown)."""+ database, job_store, queue, runner = _sdk_runner(tmp_path)++ _read(runner, job_store, _turns(2), operation="sync")+ assert _jobs(database) == [("assembling", ["user", "assistant"] * 2)]++ tools = [said("t1", "tool", "ls"), said("t2", "tool", "3 passed")]+ _read(runner, job_store, tools, operation="sync", first_line=5)+ assert _jobs(database) == [+ ("assembling", ["user", "assistant", "user", "assistant", "tool", "tool"])+ ]++ # Ten quiet minutes: the open turn is complete and the short batch goes.+ assert queue.flush_idle_assemblies(now=datetime.now(UTC) + timedelta(minutes=9)) == ()+ flushed = queue.flush_idle_assemblies(now=datetime.now(UTC) + timedelta(minutes=11))+ assert len(flushed) == 1+ assert _jobs(database) == [+ ("pending", ["user", "assistant", "user", "assistant", "tool", "tool"])+ ]+++def test_live_polling_writes_five_complete_turns_and_keeps_the_open_one(tmp_path):+ database, job_store, queue, runner = _sdk_runner(tmp_path)++ _read(runner, job_store, _turns(5), operation="sync")+ assert [state for state, _ in _jobs(database)] == ["assembling"]++ _read(runner, job_store, [said("u5", "user", "user 5")], operation="sync", first_line=11)+ assert _jobs(database) == [+ ("assembling", ["user"]),+ ("pending", ["user", "assistant"] * 5),+ ]+++def test_a_reread_increment_creates_no_second_job(tmp_path):+ database, job_store, queue, runner = _sdk_runner(tmp_path)++ _read(runner, job_store, _turns(6), operation="sync")+ _read(runner, job_store, _turns(6), operation="sync")+ queue.flush_idle_assemblies(now=datetime.now(UTC) + timedelta(minutes=11))+ _read(runner, job_store, _turns(6), operation="sync")+ queue.flush_idle_assemblies(now=datetime.now(UTC) + timedelta(minutes=11))++ assert _jobs(database) == [+ ("pending", ["user", "assistant"] * 5),+ ("pending", ["user", "assistant"]),+ ]+++def test_history_records_with_no_user_message_are_dropped_at_eof(tmp_path):+ database, job_store, _queue, runner = _sdk_runner(tmp_path)++ _read(+ runner,+ job_store,+ [said("t1", "tool", "ls"), said("a1", "assistant", "ok")],+ operation="history",+ )++ assert _jobs(database) == []diff --git a/memhub/apps/core/tests/test_live_capture.py b/memhub/apps/core/tests/test_live_capture.pyindex 47be928..75cf198 100644--- a/memhub/apps/core/tests/test_live_capture.py+++ b/memhub/apps/core/tests/test_live_capture.py@@ -5,6 +5,7 @@ import jsonimport osimport sqlite3from contextlib import closing+from datetime import UTC, datetime, timedeltafrom io import StringIOfrom pathlib import Path@@ -107,7 +108,15 @@ def test_same_message_id_from_the_file_does_not_duplicate(tmp_path) -> None:assert store.count() == 1+def _quiet_for_ten_minutes(database: Path, store: EvidenceStore) -> tuple[str, ...]:+ """What Core's drain does once a live conversation has been quiet."""+ return DurableWriteQueue(database, evidence=store).flush_idle_assemblies(+ now=datetime.now(UTC) + timedelta(minutes=11)+ )++def test_hook_submits_complete_turn_to_the_live_durable_lane(tmp_path) -> None:+ """The turn rule holds the turn until the next user message or 10 quiet minutes."""store = _grant(tmp_path, "workbuddy", sync=True)database = tmp_path / "one-memory.sqlite3"@@ -132,10 +141,13 @@ def test_hook_submits_complete_turn_to_the_live_durable_lane(tmp_path) -> None:database,)+ with closing(sqlite3.connect(database)) as db:+ assert db.execute("SELECT state FROM memory_write_jobs").fetchall() == [("assembling",)]+ assert len(_quiet_for_ten_minutes(database, store)) == 1with closing(sqlite3.connect(database)) as db:row = db.execute("SELECT priority,evidence_ids_json FROM memory_write_jobs "- "WHERE idempotency_key LIKE 'turn:v1:%'"+ "WHERE idempotency_key LIKE 'turn:v2:%'").fetchone()assert row[0] == "live"assert len(json.loads(row[1])) == 2@@ -161,6 +173,41 @@ def test_hook_submits_complete_turn_to_the_live_durable_lane(tmp_path) -> None:assert db.execute("SELECT COUNT(*) FROM evidence").fetchone() == (2,)+def test_hook_writes_five_turns_once_the_next_user_message_arrives(tmp_path) -> None:+ store = _grant(tmp_path, "workbuddy", sync=True)+ database = tmp_path / "one-memory.sqlite3"++ def say(event: str, field: str, message_id: str, text: str) -> None:+ assert capture_hook(+ "workbuddy",+ {+ "hook_event_name": event,+ "session_id": "sess-five",+ "message_id": message_id,+ field: text,+ },+ database,+ )++ for index in range(5):+ say("UserPromptSubmit", "prompt", f"u{index}", f"第 {index} 个偏好")+ say("Stop", "last_assistant_message", f"a{index}", f"记住第 {index} 个")+ with closing(sqlite3.connect(database)) as db:+ assert db.execute("SELECT state FROM memory_write_jobs").fetchall() == [("assembling",)]++ say("UserPromptSubmit", "prompt", "u5", "第 5 个偏好")++ with closing(sqlite3.connect(database)) as db:+ rows = db.execute(+ "SELECT state,evidence_ids_json FROM memory_write_jobs ORDER BY queue_seq"+ ).fetchall()+ assert [(state, len(json.loads(ids))) for state, ids in rows] == [+ ("assembling", 0),+ ("pending", 10),+ ]+ assert store.count() == 11++def test_file_adopts_the_hook_row_when_the_text_is_unique(tmp_path) -> None:store = _grant(tmp_path, "workbuddy", sync=True)database = tmp_path / "one-memory.sqlite3"@@ -211,6 +258,7 @@ def test_hook_job_completes_after_file_identity_adoption(tmp_path) -> None:},database,)+ assert len(_quiet_for_ten_minutes(database, store)) == 1for message_id, role, author, text in (("file-user", "user", WrittenBy.PERSON, "我喜欢蓝莓"),("file-assistant", "assistant", WrittenBy.AGENT, "我会记住你喜欢蓝莓"),diff --git a/memhub/apps/core/tests/test_memory_server_durable_queue.py b/memhub/apps/core/tests/test_memory_server_durable_queue.pyindex 2216276..df5d410 100644--- a/memhub/apps/core/tests/test_memory_server_durable_queue.py+++ b/memhub/apps/core/tests/test_memory_server_durable_queue.py@@ -1118,6 +1118,75 @@ def test_sdk_write_processor_uses_add_extracted_for_memory_content_and_additionsassert len(client.calls) == 1+def test_a_batch_with_no_user_or_assistant_message_is_nothing_to_remember(tmp_path) -> None:+ """Tool-only input came back 400 "content or messages is required" (16 unknown).++ The SDK strips tool records on the chat path. Such a batch completes with+ no memories; the SDK is never called and nothing is recorded unknown.+ """+ from one_memory.imports.models import WrittenBy+ from one_memory.sdk_write_processor import SdkWriteProcessor++ client = _SdkPublicClient()+ processor = SdkWriteProcessor(_SdkPublicHandle(client)) # type: ignore[arg-type]+ tools = tuple(+ _sdk_evidence(f"tool-{index}", text="3 passed").model_copy(+ update={"role": "tool", "written_by": WrittenBy.TOOL}+ )+ for index in range(2)+ )+ path = tmp_path / "one-memory.sqlite3"+ queue = DurableWriteQueue(path)+ record = queue.submit(+ WriteRequest(+ internal_session_id="sess-tools",+ source_id="cursor",+ batch_identity=BatchIdentity(+ internal_session_id="sess-tools", first_round=1, last_round=1+ ),+ evidence_ids=[item.evidence_id for item in tools],+ input_records=tools,+ idempotency_key="tools-only",+ )+ )+ _claim_through_drain(queue, record.job_id)++ asyncio.run(+ execute_job(+ record.job_id,+ queue=queue,+ supervisor=processor,+ evidence=_EvidenceStore(set()),+ )+ )++ assert client.calls == []+ assert client.events == []+ [row] = _rows(path)+ assert (row["state"], row["hy_invoked"], row["sdk_memory_ids_json"]) == (COMPLETED, 0, "[]")+++def test_the_drain_tick_writes_a_quiet_live_conversation(tmp_path, monkeypatch) -> None:+ """The 10-minute idle flush runs on the drain's maintenance timer."""+ path = tmp_path / "one-memory.sqlite3"+ queue = DurableWriteQueue(path)+ flushed: list[str] = []+ monkeypatch.setattr(queue, "flush_idle_assemblies", lambda: flushed.append("flush") or ())+ drain = DurableWriteDrain(queue, execute=_never_execute, reap_interval=timedelta(0))++ async def tick() -> None:+ task = asyncio.create_task(drain.run_forever())+ for _ in range(100):+ await asyncio.sleep(0.01)+ if flushed:+ break+ drain.stop()+ await task++ asyncio.run(tick())+ assert flushed++def test_sdk_unknown_outcome_inquiry_uses_only_public_receipt() -> None:from one_memory.sdk_write_processor import SdkWriteProcessor, SdkWriteReceiptdiff --git a/memhub/apps/core/tests/test_unified_write_entry_wiring.py b/memhub/apps/core/tests/test_unified_write_entry_wiring.pyindex fab49a7..cf08406 100644--- a/memhub/apps/core/tests/test_unified_write_entry_wiring.py+++ b/memhub/apps/core/tests/test_unified_write_entry_wiring.py@@ -16,6 +16,7 @@ from __future__ import annotationsimport asyncioimport contextlib+import jsonimport sqlite3from uuid import uuid4@@ -197,17 +198,20 @@ def test_live_import_callback_submits_realtime_priority(tmp_path) -> None:)app.state.config_store.set_sync(source_id, True)user = _committed_record(app, source_id=source_id, message_id="live-user")- record = app.state.evidence_store.put(- EvidenceInput(- source_id=source_id,- conversation_id=user.conversation_id,- message_id="live-assistant",- role="assistant",- written_by=WrittenBy.AGENT,- verbatim="Live reply",- origin_meta={EvidenceOrigin.FILE: OriginMetadata()},- )- ).record+ # Live writes by the turn rule: 5 complete turns, the fifth+ # completed by the next user message.+ for index in range(10):+ record = app.state.evidence_store.put(+ EvidenceInput(+ source_id=source_id,+ conversation_id=user.conversation_id,+ message_id=f"live-{index}",+ role="assistant" if index % 2 == 0 else "user",+ written_by=WrittenBy.AGENT if index % 2 == 0 else WrittenBy.PERSON,+ verbatim=f"Live line {index}",+ origin_meta={EvidenceOrigin.FILE: OriginMetadata()},+ )+ ).recordcallback = app.state.import_supervisor.on_stable_evidenceassert callback is not Nonetoken = live_sync_arrival.set(True)@@ -216,11 +220,12 @@ def test_live_import_callback_submits_realtime_priority(tmp_path) -> None:finally:live_sync_arrival.reset(token)with sqlite3.connect(app.state.evidence_store.database_path) as db:- priority = db.execute(- "SELECT priority FROM memory_write_jobs WHERE idempotency_key LIKE ?",+ rows = db.execute(+ "SELECT priority,evidence_ids_json FROM memory_write_jobs "+ "WHERE idempotency_key LIKE ?",("turn:v1:%",),- ).fetchone()[0]- assert priority == "live"+ ).fetchall()+ assert [(priority, len(json.loads(ids))) for priority, ids in rows] == [("live", 10)]asyncio.run(scenario())diff --git a/memhub/apps/core/tests/test_whole_turn_hy_input.py b/memhub/apps/core/tests/test_whole_turn_hy_input.pyindex ffdfcef..0f8e9bf 100644--- a/memhub/apps/core/tests/test_whole_turn_hy_input.py+++ b/memhub/apps/core/tests/test_whole_turn_hy_input.py@@ -1,8 +1,8 @@-"""D74: a turn reaches HY whole; only what HY will read counts toward the ceiling."""+"""D74: a turn reaches HY whole; only what HY will read counts toward the batch size."""from datetime import UTC, datetime-from one_memory.conversation_processing import _conversation_chunks+from one_memory.conversation_processing import turn_batchesfrom one_memory.imports.models import EvidenceInput, EvidenceOrigin, OriginMetadata, WrittenByfrom one_memory.imports.storage import EvidenceStore@@ -36,66 +36,33 @@ def test_tool_heavy_turn_is_one_window(tmp_path):*[("assistant", "定位中") for _ in range(30)],("assistant", "原因是 fixture 没清理" * 100),)- chunks = _conversation_chunks(records)- assert len(chunks) == 1- assert chunks[0] == tuple(records)+ batches = turn_batches(records, flush=True).batches+ assert batches == (tuple(records),)def test_long_answer_is_not_cut(tmp_path):records = _turn(tmp_path, ("user", "写一份方案"), ("assistant", "段落。" * 7_000))- chunks = _conversation_chunks(records)- assert chunks == [tuple(records)]- assert all(":segment:" not in record.evidence_id for record in chunks[0])+ batches = turn_batches(records, flush=True).batches+ assert batches == (tuple(records),)+ assert all(":segment:" not in record.evidence_id for record in batches[0])-def test_text_over_the_ceiling_still_splits(tmp_path):- records = _turn(tmp_path, ("user", "整理"), ("assistant", "句子。" * 25_000))- chunks = _conversation_chunks(records)- assert len(chunks) > 1- assert all(chunk[0].evidence_id == records[0].evidence_id for chunk in chunks)+def test_a_turn_far_over_the_batch_size_is_still_one_window(tmp_path):+ """The rule never splits a turn: it is written alone, whole.--def test_switch_restores_the_old_split(tmp_path, monkeypatch):- monkeypatch.setenv("ONE_MEMORY_SPLIT_TURNS", "1")- records = _turn(tmp_path, ("user", "写一份方案"), ("assistant", "段落。" * 7_000))- assert len(_conversation_chunks(records)) > 1---def test_explicit_limits_keep_old_semantics(tmp_path):- records = _turn(tmp_path, ("user", "q"), *[("assistant", "a" * 100) for _ in range(30)])- assert len(_conversation_chunks(records, max_turns=6, max_characters=1200)) > 1---def test_one_window_is_capped_in_raw_bytes(tmp_path):- """HY counts 2 KiB per tool record, but we still hold the raw text.-- `client.add` is in-process, so a window's raw bytes are what the process- actually holds even though HY will cut each tool record down before it- reaches the model. Past the byte ceiling the window is cut, so no single- call can grow without bound.+ The old windows cut text past 60k characters and raw bytes past 2 MB into+ segments that repeated the user question. HY reads at most+ `_HY_TOOL_CHARACTERS` of each tool record, so the window HY reads stays+ bounded by the turn's own text."""- from one_memory.conversation_processing import _WINDOW_BYTE_CEILING-records = _turn(tmp_path,("user", "跑一下全部测试"),*[("tool", "y" * 400_000) for _ in range(20)],+ ("assistant", "句子。" * 25_000),)- chunks = _conversation_chunks(records)- assert len(chunks) > 1- assert all(- sum(len(item.verbatim.encode("utf-8")) for item in chunk) <= _WINDOW_BYTE_CEILING- for chunk in chunks- )---def test_the_byte_ceiling_counts_bytes_not_characters(tmp_path):- """One CJK character is three bytes; a character count would let three- times as much through."""- from one_memory.conversation_processing import _WINDOW_BYTE_CEILING, _window_bytes-- assert _window_bytes(_turn(tmp_path, ("user", "中文"))[0]) > len("中文")- assert _WINDOW_BYTE_CEILING == 2_000_000+ batches = turn_batches(records, flush=True).batches+ assert batches == (tuple(records),)def test_the_sdk_strips_tool_messages_on_the_chat_path():