Skip to content

Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai

Fix length-in-range verify check for CJK reports (#64)

Matt Sakamaki15010c514224
diff --git a/src/hyperresearch/core/profiles.py b/src/hyperresearch/core/profiles.py
index 71d388f..89fac0c 100644
--- a/src/hyperresearch/core/profiles.py
+++ b/src/hyperresearch/core/profiles.py
@@ -137,6 +137,15 @@ class Profile(BaseModel):
single_draft_reads: Range
must_read: dict[str, Range]
word_targets: dict[str, Range]
+ # Character-count targets for response_formats whose script doesn't
+ # delimit words with ASCII whitespace (see runs.py:_lacks_word_boundaries).
+ # An editorial constant per format, exactly like word_targets itself --
+ # not derived from any word-per-character formula. Shipped values are
+ # tuned for CJK, the only such script family this project has real usage
+ # data for; deployments serving a different one (Thai, Lao, Khmer, ...)
+ # override this per profile in .hyperresearch/config.toml, the same way
+ # they'd override word_targets or any other tunable.
+ char_targets_no_word_boundary: dict[str, Range] = Field(default_factory=dict)
citation_density_min: float
citation_totals: dict[str, Range]
@@ -226,6 +235,7 @@ _FULL: dict = {
"single_draft_reads": (8, 15),
"must_read": {"argumentative": (35, 50), "structured": (25, 40), "short": (20, 30)},
"word_targets": {"short": (500, 2000), "structured": (2000, 5000), "argumentative": (5000, 10000)},
+ "char_targets_no_word_boundary": {"short": (1500, 6000), "structured": (6000, 15000), "argumentative": (20000, 25000)},
"citation_density_min": 2.0,
"citation_totals": {"argumentative": (80, 150), "structured": (40, 80), "short": (15, 30)},
"critic_finding_caps": {"dialectic": 12, "depth": 12, "width": 10, "instruction": 15},
diff --git a/src/hyperresearch/core/runs.py b/src/hyperresearch/core/runs.py
index 987d74e..683148c 100644
--- a/src/hyperresearch/core/runs.py
+++ b/src/hyperresearch/core/runs.py
@@ -28,6 +28,21 @@ from pathlib import Path
MANIFEST_VERSION = 1
MANIFEST_NAME = "run.json"
+
+# Avg. chars per ASCII-whitespace token above which str.split() word counts
+# are no longer trustworthy (see _lacks_word_boundaries).
+_NO_WORD_BOUNDARY_AVG_TOKEN_CHARS = 15.0
+
+
+def _lacks_word_boundaries(text: str) -> bool:
+ """True if ASCII-whitespace doesn't meaningfully segment `text` into words."""
+ tokens = text.split()
+ if not tokens:
+ return False
+ avg_token_chars = sum(len(t) for t in tokens) / len(tokens)
+ return avg_token_chars >= _NO_WORD_BOUNDARY_AVG_TOKEN_CHARS
+
+
EVENTS_NAME = "events.jsonl"
RUN_STATUSES = ("running", "paused", "blocked", "done", "failed", "aborted")
@@ -386,13 +401,30 @@ def verify_run(vault, vault_tag: str) -> dict:
)
if response_format and response_format in profile.word_targets:
- low, high = profile.word_targets[response_format]
- wc = len(report_text.split())
- check(
- "length-in-range",
- low * 0.8 <= wc <= high * 1.2,
- f"{wc} words vs target {low}-{high} ({response_format}; ±20% tolerance)",
- )
+ if _lacks_word_boundaries(report_text):
+ # char_targets_no_word_boundary is profile-configurable per
+ # response_format; falls back to word_target * 3 (a CJK-shaped
+ # guess) if a format has no explicit target.
+ char_targets = getattr(profile, "char_targets_no_word_boundary", None) or {}
+ if response_format in char_targets:
+ low, high = char_targets[response_format]
+ else:
+ low_w, high_w = profile.word_targets[response_format]
+ low, high = low_w * 3, high_w * 3
+ count = len(report_text)
+ check(
+ "length-in-range",
+ low * 0.8 <= count <= high * 1.2,
+ f"{count} chars vs target {low}-{high} ({response_format}; char-based; ±20% tolerance)",
+ )
+ else:
+ low, high = profile.word_targets[response_format]
+ wc = len(report_text.split())
+ check(
+ "length-in-range",
+ low * 0.8 <= wc <= high * 1.2,
+ f"{wc} words vs target {low}-{high} ({response_format}; ±20% tolerance)",
+ )
missing = [h for h in required_headings if h not in report_text]
check(
diff --git a/tests/test_core/test_profiles.py b/tests/test_core/test_profiles.py
index 02c6e17..496350f 100644
--- a/tests/test_core/test_profiles.py
+++ b/tests/test_core/test_profiles.py
@@ -167,6 +167,26 @@ class TestUserOverlay:
assert p.source_target == (100, 150)
assert p.depth_budget_brackets == ((35, 20), (0, 5))
+ def test_char_targets_no_word_boundary_overridable_for_non_cjk_scripts(
+ self, tmp_path: Path
+ ):
+ """The shipped char_targets_no_word_boundary values are calibrated
+ for CJK -- the only non-word-boundary script this project has real
+ usage data for. A deployment serving a different such script (Thai,
+ Lao, Khmer, ...) must be able to supply its own calibration through
+ the same profile-overlay mechanism as every other tunable, with no
+ code changes."""
+ cfg = self._write(
+ tmp_path,
+ "[profile.full]\n"
+ "char_targets_no_word_boundary = "
+ "{ argumentative = [30000, 45000] }\n",
+ )
+ p = resolve_profile("full", cfg)
+ assert p.char_targets_no_word_boundary == {"argumentative": (30000, 45000)}
+ # Unrelated keys are untouched by the override.
+ assert p.word_targets["argumentative"] == (5000, 10000)
+
def test_listing_includes_user_profiles(self, tmp_path: Path):
cfg = self._write(tmp_path, "[profile.dissertation]\nsource_min = 250\n")
assert list_profiles(cfg) == ["light", "full", "premier", "dissertation"]
diff --git a/tests/test_core/test_verification.py b/tests/test_core/test_verification.py
index d7d4d08..0d6f36d 100644
--- a/tests/test_core/test_verification.py
+++ b/tests/test_core/test_verification.py
@@ -239,6 +239,123 @@ class TestIndependence:
assert any(c["kind"] == "url" for c in result["clusters"])
+class TestCJKLengthCheck:
+ """`str.split()` word counts collapse to near-zero for prose in scripts
+ that don't delimit words with ASCII whitespace (Chinese, Japanese,
+ Korean, Thai, ...), so a report that is squarely within its documented
+ character-count target must not fail length-in-range on a spurious
+ word count. The detector is a structural heuristic (average
+ whitespace-delimited token length), not a hardcoded script allowlist --
+ it must generalize to any such script without code changes."""
+
+ def test_lacks_word_boundaries(self):
+ from hyperresearch.core.runs import _lacks_word_boundaries
+
+ # Japanese (kana + kanji) and Chinese (hanzi) prose is written with
+ # no spaces between words at all.
+ assert _lacks_word_boundaries("この文章はほぼ全て日本語の文字で構成されています。")
+ assert _lacks_word_boundaries("这是一段中文测试文本内容和证据。")
+ # Structural-only proof of generality to a non-CJK, non-Latin
+ # script family (Thai) -- no claim about grammaticality, just that
+ # a long run of a spaceless script's characters trips the same
+ # heuristic without any script-specific code.
+ assert _lacks_word_boundaries("คำ" * 30)
+ # Korean is the counter-example that justifies a *structural*
+ # heuristic over a hardcoded script list: unlike Chinese/Japanese,
+ # modern Korean orthography IS space-delimited, so str.split()
+ # word counts are meaningful for it and it must NOT trip this
+ # branch. A script-identity allowlist would get this wrong.
+ assert not _lacks_word_boundaries(
+ "이것은 한국어 문장으로 구성된 테스트입니다 그리고 띄어쓰기를 사용합니다."
+ )
+ assert not _lacks_word_boundaries(
+ "This paragraph is entirely English prose with no CJK content at all."
+ )
+ # A handful of long wikilink citation keys inside a mostly-English
+ # report should not trip the branch -- many short real words still
+ # dominate the average token length.
+ assert not _lacks_word_boundaries(
+ "This is an English report with real citations. "
+ "[[dagitty-a-graphical-tool-for-analyzing-causal-diagrams-semantic-scholar]] " * 15
+ )
+
+ def test_verify_passes_cjk_report_within_char_target_despite_low_word_count(
+ self, tmp_vault
+ ):
+ init_run(tmp_vault, "cjk-01", profile="light")
+ run_dir = tmp_vault.run_dir("cjk-01")
+ (run_dir / "prompt-decomposition.json").write_text(json.dumps({
+ "response_format": "short",
+ "required_section_headings": ["## 結果"],
+ }), encoding="utf-8")
+ (run_dir / "polish-log.json").write_text('{"applied": []}', encoding="utf-8")
+ # light/short CJK char target is 1500-6000; this Japanese sentence
+ # has almost no ASCII whitespace, so str.split() would count only a
+ # handful of "words" despite being solidly in-range by characters.
+ sentence = "これは実質的な証拠を伴う文章である[[src-note]]。"
+ report = tmp_vault.root / "research" / "notes" / "final_report_cjk-01.md"
+ body = "## 結果\n\n" + (sentence * 90)
+ report.write_text(body, encoding="utf-8")
+
+ word_count = len(body.split())
+ assert word_count < 500 # would fail the English word-count target
+
+ result = verify_run(tmp_vault, "cjk-01")
+ by_name = {c["name"]: c for c in result["checks"]}
+ assert by_name["length-in-range"]["ok"] is True
+ assert "chars" in by_name["length-in-range"]["detail"]
+ assert result["passed"] is True
+
+ def test_verify_still_fails_cjk_report_far_outside_char_target(self, tmp_vault):
+ init_run(tmp_vault, "cjk-02", profile="light")
+ run_dir = tmp_vault.run_dir("cjk-02")
+ (run_dir / "prompt-decomposition.json").write_text(json.dumps({
+ "response_format": "short",
+ "required_section_headings": ["## 結果"],
+ }), encoding="utf-8")
+ # Well under the 1500-char floor (even with the 20% tolerance) --
+ # the CJK branch must still be a real length check, not a bypass.
+ report = tmp_vault.root / "research" / "notes" / "final_report_cjk-02.md"
+ report.write_text("## 結果\n\n短い。", encoding="utf-8")
+
+ result = verify_run(tmp_vault, "cjk-02")
+ by_name = {c["name"]: c for c in result["checks"]}
+ assert by_name["length-in-range"]["ok"] is False
+ assert result["passed"] is False
+
+ def test_verify_respects_profile_override_for_non_cjk_script(self, tmp_vault):
+ """End-to-end proof that a deployment serving a non-CJK,
+ non-word-boundary script (simulated here with a spaceless Thai-script
+ blob) is not stuck with CJK-calibrated numbers: it configures its own
+ char_targets_no_word_boundary via the standard profile-overlay
+ mechanism, and verify_run honors it -- no code change needed to
+ support a script this project has no real usage data for."""
+ cfg_dir = tmp_vault.root / ".hyperresearch"
+ cfg_dir.mkdir(parents=True, exist_ok=True)
+ (cfg_dir / "config.toml").write_text(
+ "[profile.light]\n"
+ "char_targets_no_word_boundary = { short = [300, 900] }\n",
+ encoding="utf-8",
+ )
+ init_run(tmp_vault, "th-01", profile="light")
+ run_dir = tmp_vault.run_dir("th-01")
+ (run_dir / "prompt-decomposition.json").write_text(json.dumps({
+ "response_format": "short",
+ "required_section_headings": ["## Results"],
+ }), encoding="utf-8")
+ (run_dir / "polish-log.json").write_text('{"applied": []}', encoding="utf-8")
+ # A spaceless Thai-script blob: within the overridden 300-900 char
+ # target, but far below the shipped CJK default's 1500-char floor --
+ # this only passes if the override is actually being read.
+ report = tmp_vault.root / "research" / "notes" / "final_report_th-01.md"
+ report.write_text("## Results\n\n" + ("คำ" * 250), encoding="utf-8")
+
+ result = verify_run(tmp_vault, "th-01")
+ by_name = {c["name"]: c for c in result["checks"]}
+ assert by_name["length-in-range"]["ok"] is True
+ assert "300-900" in by_name["length-in-range"]["detail"]
+
+
class TestTelemetryAndVerify:
def test_run_report_rollup(self, tmp_vault):
init_run(tmp_vault, "tel-01", profile="light")