Skip to content
Python

fix(import): report oversized source files as file_too_large (#108)

u7-u74e44f5902672
diff --git a/memhub/apps/core/src/one_memory/imports/adapters.py b/memhub/apps/core/src/one_memory/imports/adapters.py
index 0580197..a0fb277 100644
--- a/memhub/apps/core/src/one_memory/imports/adapters.py
+++ b/memhub/apps/core/src/one_memory/imports/adapters.py
@@ -34,10 +34,11 @@ class ScanIssueCode(StrEnum):
SOURCE_BUSY = "source_busy"
UNSUPPORTED_SCHEMA = "unsupported_schema"
BUDGET_EXCEEDED = "budget_exceeded"
- #: The file is bigger than the import's own per-file ceiling. Skipped with
- #: a reason rather than read, because one oversized history file can hold
- #: the import open long enough to take the process with it. Distinct from
- #: the access layer's `FILE_TOO_LARGE`, which is a per-root limit.
+ #: The file is bigger than a per-file ceiling: either the import's own
+ #: ceiling or the access gate's per-root `max_file_bytes`. Both are skipped
+ #: with this reason rather than read, because one oversized history file
+ #: can hold the import open long enough to take the process with it, and
+ #: the user-facing cause is the same size limit.
FILE_TOO_LARGE = "file_too_large"
HISTORY_WINDOW_EXCLUDED = "history_window_excluded"
diff --git a/memhub/apps/core/src/one_memory/imports/sources/common.py b/memhub/apps/core/src/one_memory/imports/sources/common.py
index 1429d86..1370f57 100644
--- a/memhub/apps/core/src/one_memory/imports/sources/common.py
+++ b/memhub/apps/core/src/one_memory/imports/sources/common.py
@@ -1120,6 +1120,11 @@ def discovery_failure_code(
) -> ScanIssueCode:
if failure.reason_code is SourceAccessErrorCode.BUDGET_EXCEEDED:
return ScanIssueCode.BUDGET_EXCEEDED
+ if failure.reason_code is SourceAccessErrorCode.FILE_TOO_LARGE:
+ # The file is readable and its format is fine; only its size is over
+ # the access gate. Reporting that as `unsupported_schema` sends the
+ # user looking for a parser problem that does not exist.
+ return ScanIssueCode.FILE_TOO_LARGE
return ScanIssueCode.UNSUPPORTED_SCHEMA
@@ -1127,6 +1132,8 @@ def _scan_failure_code(code: SourceAccessErrorCode) -> ScanIssueCode:
"""Map a per-file open failure onto a machine code, never onto free text."""
if code is SourceAccessErrorCode.BUDGET_EXCEEDED:
return ScanIssueCode.BUDGET_EXCEEDED
+ if code is SourceAccessErrorCode.FILE_TOO_LARGE:
+ return ScanIssueCode.FILE_TOO_LARGE
if code is SourceAccessErrorCode.SYMLINK_NOT_ALLOWED:
return ScanIssueCode.UNSUPPORTED_SCHEMA
if code in {
diff --git a/memhub/apps/core/tests/test_source_access.py b/memhub/apps/core/tests/test_source_access.py
index 4d1b6ab..77d4170 100644
--- a/memhub/apps/core/tests/test_source_access.py
+++ b/memhub/apps/core/tests/test_source_access.py
@@ -224,6 +224,11 @@ def test_live_adapter_keeps_symlink_refusal_in_existing_issue_taxonomy() -> None
)
+def test_live_adapter_reports_an_oversized_file_as_file_too_large() -> None:
+ """The open path and the discovery path must agree on the size reason."""
+ assert _scan_failure_code(SourceAccessErrorCode.FILE_TOO_LARGE) is ScanIssueCode.FILE_TOO_LARGE
+
+
def test_live_adapter_runner_persists_symlink_refusal_as_unsupported_schema(
tmp_path, monkeypatch
) -> None:
diff --git a/memhub/apps/core/tests/test_source_discovery.py b/memhub/apps/core/tests/test_source_discovery.py
index b91ea60..040d820 100644
--- a/memhub/apps/core/tests/test_source_discovery.py
+++ b/memhub/apps/core/tests/test_source_discovery.py
@@ -3,10 +3,12 @@ from one_memory.imports.access import (
AuthorizedSourceRoot,
SourceAccessErrorCode,
)
+from one_memory.imports.adapters import ScanIssueCode
from one_memory.imports.discovery import (
SourceFileDiscoveryFailure,
discover_source_files,
)
+from one_memory.imports.sources.common import discovery_failure_code
def grant(root, home):
@@ -60,3 +62,35 @@ def test_discovery_does_not_follow_directory_or_file_symlinks(tmp_path) -> None:
assert len(events) == 1
assert isinstance(events[0], SourceFileDiscoveryFailure)
assert "private" not in events[0].file_key
+
+
+def test_discovery_reports_an_oversized_file_as_file_too_large(tmp_path) -> None:
+ """A size-gate refusal is a size problem, not a schema problem.
+
+ The Codex report in #102 shows the access gate refusing a file over the
+ per-root limit while the UI said `unsupported_schema`, which sends the user
+ looking for a parser that is working fine.
+ """
+ root = tmp_path / "source"
+ root.mkdir()
+ (root / "big.jsonl").write_text('{"content":"too big"}\n')
+ granted = AuthorizedSourceRoot.grant(
+ source_id="fixture",
+ root=root,
+ user_confirmed=True,
+ home=tmp_path / "home",
+ max_file_bytes=8,
+ )
+
+ events = list(
+ discover_source_files(
+ granted,
+ suffixes=frozenset({".jsonl"}),
+ max_targets=10,
+ )
+ )
+
+ assert len(events) == 1
+ assert isinstance(events[0], SourceFileDiscoveryFailure)
+ assert events[0].reason_code is SourceAccessErrorCode.FILE_TOO_LARGE
+ assert discovery_failure_code(events[0]) is ScanIssueCode.FILE_TOO_LARGE