Skip to content
Python

fix(import): truncate oversized verbatim instead of wedging the job (#95)

u7-u775163a0d1ef3
diff --git a/memhub/apps/core/src/one_memory/imports/sources/common.py b/memhub/apps/core/src/one_memory/imports/sources/common.py
index 6a6f9c1..1429d86 100644
--- a/memhub/apps/core/src/one_memory/imports/sources/common.py
+++ b/memhub/apps/core/src/one_memory/imports/sources/common.py
@@ -44,6 +44,7 @@ from one_memory.imports.discovery import (
discovered_off_loop,
)
from one_memory.imports.models import (
+ MAX_VERBATIM_CHARACTERS,
EvidenceContentKind,
EvidenceInput,
EvidenceOrigin,
@@ -65,6 +66,20 @@ from one_memory.imports.sources.parsing import parse_timestamp
_log = logging.getLogger(__name__)
+
+def _clamp_verbatim(text: str) -> str:
+ """Keep one pathological record from violating EvidenceInput's limit.
+
+ A single pasted dump larger than MAX_VERBATIM_CHARACTERS used to fail
+ validation, retry, and then wedge the whole history import on the same
+ record. Keep the head and make the truncation explicit and actionable.
+ """
+ if len(text) <= MAX_VERBATIM_CHARACTERS:
+ return text
+ marker = f"\n…[truncated: original {len(text)} chars]"
+ keep = max(0, MAX_VERBATIM_CHARACTERS - len(marker))
+ return text[:keep] + marker
+
#: At most one read heartbeat per second. Faster would be more writes than the
#: counts are worth; slower leaves the job's visible state stale.
_READ_PROGRESS_INTERVAL_SECONDS = 1.0
@@ -1076,8 +1091,11 @@ def to_evidence(
role=parsed.role,
written_by=parsed.written_by,
created_at=parsed.created_at,
- verbatim=parsed.verbatim,
- raw_verbatim=parsed.raw_verbatim,
+ verbatim=_clamp_verbatim(parsed.verbatim),
+ raw_verbatim=(
+ None if parsed.raw_verbatim is None
+ else _clamp_verbatim(parsed.raw_verbatim)
+ ),
origin_meta={
EvidenceOrigin.FILE: OriginMetadata(
source_version=origin.source_version or source_version,
diff --git a/memhub/apps/core/tests/TEST_INDEX.md b/memhub/apps/core/tests/TEST_INDEX.md
index b7949e0..6ca4079 100644
--- a/memhub/apps/core/tests/TEST_INDEX.md
+++ b/memhub/apps/core/tests/TEST_INDEX.md
@@ -148,6 +148,7 @@
| `test_import_api.py` | `integration` | 覆盖 resume during cleanup returns conflict without internal error、authorize detect import and revoke api flow、protected api rejects missing token and foreign origin 等 5 个场景。 |
| `test_import_control_races_review.py` | `contract` | Read-only review repro: real runner/store, synthetic delayed adapter cleanup. |
| `test_import_evidence_atomicity.py` | `contract` | 覆盖 counter failure rolls back evidence versions and cursor、same identity cursor rewind is blocked and logged、replaced identity may reset checkpoint 等 20 个场景。 |
+| `test_import_oversized_verbatim.py` | `contract` | 覆盖 normal verbatim is unchanged、oversized verbatim is truncated instead of rejected 场景。 |
| `test_import_runner.py` | `contract` | 覆盖 job transitions are explicit、runner keeps per record errors and continues、sync slices reuse the same source listing 等 22 个场景。 |
| `test_import_skip_batching.py` | `contract` | 覆盖 batch limit and tail use fewer commits、batched skips keep their order against the next write、flushes before every nonmatching event 等 14 个场景。 |
| `test_import_snapshots.py` | `contract` | Atomic snapshot publication plus frozen manifest, path, mode, and per-file checksum guards. |
@@ -388,4 +389,4 @@
| `test_workspace_recall.py` | `contract` | Workspace memories were unreachable from every host; these pin the way in. |
| `test_write_queue_lease_renewal.py` | `contract` | Rule 8: a lease must outlive the work it covers, and be renewed while it runs. |
-共 383 个测试文件。
+共 384 个测试文件。
diff --git a/memhub/apps/core/tests/core_test_classification.py b/memhub/apps/core/tests/core_test_classification.py
index 8e07749..dd44d20 100644
--- a/memhub/apps/core/tests/core_test_classification.py
+++ b/memhub/apps/core/tests/core_test_classification.py
@@ -14,8 +14,8 @@ from pathlib import Path
import pytest
PRIMARY_MARKERS = ("unit", "contract", "integration", "performance", "live")
-EXPECTED_TEST_FILE_COUNT = 383
-EXPECTED_TEST_FILES_SHA256 = "3c1a3d3febaf478e1ba005fbcd22ad23d76fc5925b32b8d8740e4e65c554b922"
+EXPECTED_TEST_FILE_COUNT = 384
+EXPECTED_TEST_FILES_SHA256 = "33cef58fc5a4eb505da9d519149e0444bf581a336232d47a3d70f3aade0ef8ed"
# Pure transformations, validation models, parity checks and state machines.
# This list is intentionally small: uncertain files default to contract.
diff --git a/memhub/apps/core/tests/test_import_oversized_verbatim.py b/memhub/apps/core/tests/test_import_oversized_verbatim.py
new file mode 100644
index 0000000..b74d44e
--- /dev/null
+++ b/memhub/apps/core/tests/test_import_oversized_verbatim.py
@@ -0,0 +1,52 @@
+from pathlib import Path
+
+from one_memory.imports.access import AuthorizedSourceFile
+from one_memory.imports.models import MAX_VERBATIM_CHARACTERS, WrittenBy
+from one_memory.imports.sources.common import ParsedMessage, to_evidence
+
+
+def _source_file(tmp_path: Path) -> AuthorizedSourceFile:
+ return AuthorizedSourceFile(
+ source_id="claude-code",
+ root=tmp_path,
+ path=tmp_path / "session.jsonl",
+ relative_path=Path("session.jsonl"),
+ max_bytes=400_000_000,
+ file_identity="identity",
+ size=1,
+ modified_ns=1,
+ changed_ns=1,
+ )
+
+
+def _evidence(tmp_path: Path, verbatim: str):
+ return to_evidence(
+ source_id="claude-code",
+ source_file=_source_file(tmp_path),
+ parsed=ParsedMessage(
+ conversation_id="conversation",
+ message_id="message",
+ role="user",
+ written_by=WrittenBy.PERSON,
+ verbatim=verbatim,
+ raw_verbatim=verbatim,
+ ),
+ ordinal="1",
+ source_version=None,
+ )
+
+
+def test_oversized_verbatim_is_truncated_instead_of_rejected(tmp_path):
+ oversized = "x" * (MAX_VERBATIM_CHARACTERS + 5)
+ evidence = _evidence(tmp_path, oversized)
+
+ assert len(evidence.verbatim) <= MAX_VERBATIM_CHARACTERS
+ assert "truncated: original 1000005 chars" in evidence.verbatim
+ assert len(evidence.raw_verbatim) <= MAX_VERBATIM_CHARACTERS
+ assert "truncated: original 1000005 chars" in evidence.raw_verbatim
+
+
+def test_normal_verbatim_is_unchanged(tmp_path):
+ evidence = _evidence(tmp_path, "a normal message")
+ assert evidence.verbatim == "a normal message"
+ assert evidence.raw_verbatim == "a normal message"