Skip to content

Turn any technical book PDF into a Claude Code skill — ready to study, reference, and use while you work.

feat(extractor): detect Roman-numeral chapter headings + harden discovery_tax (#28)

Virgilio Junior61e40e498d4a
diff --git a/scripts/extractor/utils.py b/scripts/extractor/utils.py
index 7cfd7c7..b6a5fd1 100644
--- a/scripts/extractor/utils.py
+++ b/scripts/extractor/utils.py
@@ -59,18 +59,57 @@ _EXPLICIT_CHAPTER = re.compile(
# "Chapter 8 are relevant...") is prose / a cross-reference, not a heading.
_HEADING_TAIL = re.compile(r"^\s*$|^\s*[.:\-—–]|^\s+[A-ZÀ-Ú0-9\"“(]")
+# Roman-numeral chapter heading: "I: Loomings", "II. The Carpet-Bag".
+# Requires a separator (":" or ".") and a Capitalized title after it, so a bare
+# "I" or "V." (a page divider / list marker) is not mistaken for a chapter.
+_ROMAN_HEAD = re.compile(r"^\s*([IVXLCDM]+)\s*[:.]\s+[A-ZÀ-Ú\"“(]")
+_ROMAN_VALUES = {"I": 1, "V": 5, "X": 10, "L": 50, "C": 100, "D": 500, "M": 1000}
+
+
+def _int_to_roman(n: int) -> str:
+ table = [(1000, "M"), (900, "CM"), (500, "D"), (400, "CD"), (100, "C"),
+ (90, "XC"), (50, "L"), (40, "XL"), (10, "X"), (9, "IX"),
+ (5, "V"), (4, "IV"), (1, "I")]
+ out = []
+ for val, sym in table:
+ while n >= val:
+ out.append(sym)
+ n -= val
+ return "".join(out)
+
+
+def _roman_to_int(s: str) -> int | None:
+ """Convert a Roman numeral to int, returning None if it isn't canonical."""
+ s = s.upper()
+ total = prev = 0
+ for ch in reversed(s):
+ v = _ROMAN_VALUES.get(ch)
+ if v is None:
+ return None
+ total += -v if v < prev else v
+ prev = max(prev, v)
+ if total == 0 or total > 200:
+ return None
+ # Reject non-canonical forms ("IIII", "VV") by round-tripping.
+ return total if _int_to_roman(total) == s else None
+
def _chapter_number(line: str) -> int | None:
- """Return the chapter number if the line is a genuine chapter heading."""
+ """Return the chapter number if the line is a genuine chapter heading.
+
+ Handles both Arabic ("Chapter 5", "Capítulo 5: ...") and Roman-numeral
+ ("I: Loomings", "II. The Carpet-Bag") heading styles.
+ """
s = line.strip()
if len(s) > 80:
return None
m = _EXPLICIT_CHAPTER.match(s)
- if not m:
- return None
- if not _HEADING_TAIL.match(m.group("rest")):
- return None
- return int(m.group(1))
+ if m and _HEADING_TAIL.match(m.group("rest")):
+ return int(m.group(1))
+ rm = _ROMAN_HEAD.match(s)
+ if rm:
+ return _roman_to_int(rm.group(1))
+ return None
def detect_structure(text: str) -> dict:
diff --git a/tests/test_discovery_tax.py b/tests/test_discovery_tax.py
index 766c126..cfbcca1 100644
--- a/tests/test_discovery_tax.py
+++ b/tests/test_discovery_tax.py
@@ -44,10 +44,16 @@ class TestSplitChapters:
def test_detects_three_chapters(self):
segs = dt.split_chapters(SYNTHETIC_BOOK)
chapters = segs[1:]
- assert len(chapters) == 3
- # tuple shape is (number, heading, body)
- assert chapters[0][0] == 1
- assert chapters[0][1].strip().startswith("Capítulo 1")
+ # ToC entries + body headings both segment now; count DISTINCT numbers.
+ assert {c[0] for c in chapters} == {1, 2, 3}
+
+ def test_best_chapter_picks_largest_body_over_toc_line(self):
+ # A ToC line and the real body share "Capítulo 2"; the body has more text.
+ text = ("Sumário\nCapítulo 2: Recrutamento\n"
+ "Capítulo 2\n" + ("conteudo real " * 50) + "\n")
+ chapters = dt.split_chapters(text)[1:]
+ heading, body_tok = dt.best_chapter(chapters, 2, dt.count_tokens)
+ assert body_tok > 20 # picked the real body, not the 1-line ToC entry
def test_cross_reference_does_not_split(self):
text = "Capítulo 1\nbody\nComo vimos no Capítulo 2, isso importa.\nmore body\n"
diff --git a/tests/test_extractor.py b/tests/test_extractor.py
index 7fb4edd..af7c7bf 100644
--- a/tests/test_extractor.py
+++ b/tests/test_extractor.py
@@ -585,6 +585,18 @@ class TestDetectStructure:
text = "Capítulo 1: Alicerces\n...\nCapítulo 1\nbody of chapter one\n"
assert detect_structure(text)["chapters_detected"] == 1
+ def test_roman_numeral_chapters(self):
+ text = "I: Loomings\nbody\nII: The Carpet-Bag\nbody\nIII: The Spouter-Inn\nbody\n"
+ assert detect_structure(text)["chapters_detected"] == 3
+
+ def test_roman_requires_title_after_separator(self):
+ # bare "V." (page divider) or "I" alone is not a chapter
+ assert detect_structure("V.\nI\nII\n")["chapters_detected"] == 0
+
+ def test_roman_rejects_non_canonical(self):
+ # "IIII"/"VV" are not valid roman numerals
+ assert detect_structure("IIII: Bad\nVV: Also bad\n")["chapters_detected"] == 0
+
def test_scans_full_text_not_just_head(self):
# A chapter heading far past the old 50k-char window must still be found.
text = "Capítulo 1\n" + ("filler word " * 6000) + "\nCapítulo 2\n"
diff --git a/tools/discovery_tax.py b/tools/discovery_tax.py
index fcd64b9..96ec016 100644
--- a/tools/discovery_tax.py
+++ b/tools/discovery_tax.py
@@ -33,28 +33,16 @@ import re
import sys
from pathlib import Path
-# Robust chapter-heading detection, aligned with the extractor's detect_structure:
-# accept "Chapter N", "Capítulo N", "Chapter N. Title"; reject prose
-# cross-references ("Chapter 6 explores...") via a lowercase tail, and bound the
-# number to 1..99 so years ("2025.") are not chapters.
-_CHAPTER_HEAD = re.compile(r"^\s*(?:chapter|cap[ií]tulo|ch\.?)\s*(\d{1,2})\b(?P<rest>.*)$",
- re.IGNORECASE)
-_HEADING_TAIL = re.compile(r"^\s*$|^\s*[.:\-—–]|^\s+[A-ZÀ-Ú0-9\"“(]")
+# Reuse the extractor's hardened chapter detection instead of duplicating it, so
+# discovery_tax and the pipeline always agree on what a chapter is (Arabic +
+# Roman headings, prose/cross-reference rejection, list-item rejection).
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
+from extractor.utils import _chapter_number as chapter_number # noqa: E402
+
TOC_RE = re.compile(r"^\s*(?:sum[áa]rio|table of contents|contents|[íi]ndice)\s*$",
re.IGNORECASE | re.MULTILINE)
-def chapter_number(line: str) -> int | None:
- """Return the chapter number if the line is a genuine chapter heading, else None."""
- s = line.strip()
- if len(s) > 80:
- return None
- m = _CHAPTER_HEAD.match(s)
- if not m or not _HEADING_TAIL.match(m.group("rest")):
- return None
- return int(m.group(1))
-
-
def count_tokens(text: str) -> int:
"""Real BPE count via tiktoken if available; else words/0.75 heuristic."""
try:
@@ -74,23 +62,30 @@ def token_method() -> str:
def split_chapters(text: str) -> list[tuple[int | None, str, str]]:
- """Return [(number, heading, body)]. Splits on genuine chapter headings.
+ """Return [(number, heading, body)], one segment per heading occurrence.
+
The text before the first heading is the leading 'front matter / ToC'
- segment (number=None). A new segment only starts on the FIRST occurrence of
- each chapter number, so repeated cross-references to an already-seen chapter
- don't fragment the body."""
+ segment (number=None). A chapter number may appear more than once — a ToC
+ entry and the real body share a heading format — so callers should pick the
+ LARGEST-body occurrence as the real chapter (see best_chapter)."""
lines = text.splitlines()
segments: list[tuple[int | None, str, list[str]]] = [(None, "__front__", [])]
- seen: set[int] = set()
for line in lines:
num = chapter_number(line)
- if num is not None and num not in seen:
- seen.add(num)
+ if num is not None:
segments.append((num, line.strip(), []))
segments[-1][2].append(line)
return [(n, h, "\n".join(b)) for n, h, b in segments]
+def best_chapter(chapters: list[tuple[int | None, str, str]], n: int,
+ tok) -> tuple[str, int] | None:
+ """Return (heading, body_tokens) for chapter number `n`, choosing the
+ occurrence with the largest body — the real chapter, not a ToC line."""
+ cands = [(h, tok(b)) for num, h, b in chapters if num == n]
+ return max(cands, key=lambda x: x[1]) if cands else None
+
+
def extract_toc(front_matter: str) -> str:
"""Best-effort slice of the ToC block from the front matter."""
m = TOC_RE.search(front_matter)
@@ -126,16 +121,19 @@ def main() -> int:
toc = extract_toc(front)
toc_tok = count_tokens(toc)
- # Select the target by chapter NUMBER (robust to extra cross-ref segments);
- # fall back to positional if that number isn't present.
+ # Distinct chapter numbers present (a number can recur: ToC entry + body).
+ distinct = sorted({num for num, _, _ in chapters if num is not None})
+
+ # Select the target by chapter NUMBER, taking the largest-body occurrence so
+ # a ToC line isn't mistaken for the chapter. Fall back to positional.
n = args.target_chapter
- idx = next((i for i, (num, _, _) in enumerate(chapters) if num == n), None)
- if idx is None:
- idx = max(0, min(n - 1, len(chapters) - 1))
- n = chapters[idx][0] or n
- target_raw = count_tokens(chapters[idx][2])
- prior_raw = count_tokens(chapters[idx - 1][2]) if idx >= 1 else 0
- target_heading = chapters[idx][1]
+ best = best_chapter(chapters, n, count_tokens)
+ if best is None:
+ n = distinct[min(n - 1, len(distinct) - 1)] if distinct else n
+ best = best_chapter(chapters, n, count_tokens)
+ target_heading, target_raw = best
+ prior = best_chapter(chapters, n - 1, count_tokens)
+ prior_raw = prior[1] if prior else 0
# book-to-skill resident cost
if args.skill_dir:
@@ -169,7 +167,7 @@ def main() -> int:
print("Discovery Loop Tax — measured on a real book\n")
print(f" token method : {token_method()}")
print(f" source : {Path(args.full_text).name}")
- print(f" chapters : {len(chapters)} detected")
+ print(f" chapters : {len(distinct)} detected")
print(f" target : chapter {n} ({target_heading[:60]})")
print(f" book total : {total:,} tokens\n")