Python
fix(import): truncate oversized verbatim instead of wedging the job (#95)
u7-u775163a0d1ef3
diff --git a/memhub/apps/core/src/one_memory/imports/sources/common.py b/memhub/apps/core/src/one_memory/imports/sources/common.pyindex 6a6f9c1..1429d86 100644--- a/memhub/apps/core/src/one_memory/imports/sources/common.py+++ b/memhub/apps/core/src/one_memory/imports/sources/common.py@@ -44,6 +44,7 @@ from one_memory.imports.discovery import (discovered_off_loop,)from one_memory.imports.models import (+ MAX_VERBATIM_CHARACTERS,EvidenceContentKind,EvidenceInput,EvidenceOrigin,@@ -65,6 +66,20 @@ from one_memory.imports.sources.parsing import parse_timestamp_log = logging.getLogger(__name__)++def _clamp_verbatim(text: str) -> str:+ """Keep one pathological record from violating EvidenceInput's limit.++ A single pasted dump larger than MAX_VERBATIM_CHARACTERS used to fail+ validation, retry, and then wedge the whole history import on the same+ record. Keep the head and make the truncation explicit and actionable.+ """+ if len(text) <= MAX_VERBATIM_CHARACTERS:+ return text+ marker = f"\n…[truncated: original {len(text)} chars]"+ keep = max(0, MAX_VERBATIM_CHARACTERS - len(marker))+ return text[:keep] + marker+#: At most one read heartbeat per second. Faster would be more writes than the#: counts are worth; slower leaves the job's visible state stale._READ_PROGRESS_INTERVAL_SECONDS = 1.0@@ -1076,8 +1091,11 @@ def to_evidence(role=parsed.role,written_by=parsed.written_by,created_at=parsed.created_at,- verbatim=parsed.verbatim,- raw_verbatim=parsed.raw_verbatim,+ verbatim=_clamp_verbatim(parsed.verbatim),+ raw_verbatim=(+ None if parsed.raw_verbatim is None+ else _clamp_verbatim(parsed.raw_verbatim)+ ),origin_meta={EvidenceOrigin.FILE: OriginMetadata(source_version=origin.source_version or source_version,diff --git a/memhub/apps/core/tests/TEST_INDEX.md b/memhub/apps/core/tests/TEST_INDEX.mdindex b7949e0..6ca4079 100644--- a/memhub/apps/core/tests/TEST_INDEX.md+++ b/memhub/apps/core/tests/TEST_INDEX.md@@ -148,6 +148,7 @@| `test_import_api.py` | `integration` | 覆盖 resume during cleanup returns conflict without internal error、authorize detect import and revoke api flow、protected api rejects missing token and foreign origin 等 5 个场景。 || `test_import_control_races_review.py` | `contract` | Read-only review repro: real runner/store, synthetic delayed adapter cleanup. || `test_import_evidence_atomicity.py` | `contract` | 覆盖 counter failure rolls back evidence versions and cursor、same identity cursor rewind is blocked and logged、replaced identity may reset checkpoint 等 20 个场景。 |+| `test_import_oversized_verbatim.py` | `contract` | 覆盖 normal verbatim is unchanged、oversized verbatim is truncated instead of rejected 场景。 || `test_import_runner.py` | `contract` | 覆盖 job transitions are explicit、runner keeps per record errors and continues、sync slices reuse the same source listing 等 22 个场景。 || `test_import_skip_batching.py` | `contract` | 覆盖 batch limit and tail use fewer commits、batched skips keep their order against the next write、flushes before every nonmatching event 等 14 个场景。 || `test_import_snapshots.py` | `contract` | Atomic snapshot publication plus frozen manifest, path, mode, and per-file checksum guards. |@@ -388,4 +389,4 @@| `test_workspace_recall.py` | `contract` | Workspace memories were unreachable from every host; these pin the way in. || `test_write_queue_lease_renewal.py` | `contract` | Rule 8: a lease must outlive the work it covers, and be renewed while it runs. |-共 383 个测试文件。+共 384 个测试文件。diff --git a/memhub/apps/core/tests/core_test_classification.py b/memhub/apps/core/tests/core_test_classification.pyindex 8e07749..dd44d20 100644--- a/memhub/apps/core/tests/core_test_classification.py+++ b/memhub/apps/core/tests/core_test_classification.py@@ -14,8 +14,8 @@ from pathlib import Pathimport pytestPRIMARY_MARKERS = ("unit", "contract", "integration", "performance", "live")-EXPECTED_TEST_FILE_COUNT = 383-EXPECTED_TEST_FILES_SHA256 = "3c1a3d3febaf478e1ba005fbcd22ad23d76fc5925b32b8d8740e4e65c554b922"+EXPECTED_TEST_FILE_COUNT = 384+EXPECTED_TEST_FILES_SHA256 = "33cef58fc5a4eb505da9d519149e0444bf581a336232d47a3d70f3aade0ef8ed"# Pure transformations, validation models, parity checks and state machines.# This list is intentionally small: uncertain files default to contract.diff --git a/memhub/apps/core/tests/test_import_oversized_verbatim.py b/memhub/apps/core/tests/test_import_oversized_verbatim.pynew file mode 100644index 0000000..b74d44e--- /dev/null+++ b/memhub/apps/core/tests/test_import_oversized_verbatim.py@@ -0,0 +1,52 @@+from pathlib import Path++from one_memory.imports.access import AuthorizedSourceFile+from one_memory.imports.models import MAX_VERBATIM_CHARACTERS, WrittenBy+from one_memory.imports.sources.common import ParsedMessage, to_evidence+++def _source_file(tmp_path: Path) -> AuthorizedSourceFile:+ return AuthorizedSourceFile(+ source_id="claude-code",+ root=tmp_path,+ path=tmp_path / "session.jsonl",+ relative_path=Path("session.jsonl"),+ max_bytes=400_000_000,+ file_identity="identity",+ size=1,+ modified_ns=1,+ changed_ns=1,+ )+++def _evidence(tmp_path: Path, verbatim: str):+ return to_evidence(+ source_id="claude-code",+ source_file=_source_file(tmp_path),+ parsed=ParsedMessage(+ conversation_id="conversation",+ message_id="message",+ role="user",+ written_by=WrittenBy.PERSON,+ verbatim=verbatim,+ raw_verbatim=verbatim,+ ),+ ordinal="1",+ source_version=None,+ )+++def test_oversized_verbatim_is_truncated_instead_of_rejected(tmp_path):+ oversized = "x" * (MAX_VERBATIM_CHARACTERS + 5)+ evidence = _evidence(tmp_path, oversized)++ assert len(evidence.verbatim) <= MAX_VERBATIM_CHARACTERS+ assert "truncated: original 1000005 chars" in evidence.verbatim+ assert len(evidence.raw_verbatim) <= MAX_VERBATIM_CHARACTERS+ assert "truncated: original 1000005 chars" in evidence.raw_verbatim+++def test_normal_verbatim_is_unchanged(tmp_path):+ evidence = _evidence(tmp_path, "a normal message")+ assert evidence.verbatim == "a normal message"+ assert evidence.raw_verbatim == "a normal message"