Turn any technical book PDF into a Claude Code skill — ready to study, reference, and use while you work.
Python agent-skillsai-agentsbook-to-skillcontext-engineeringdocument-processingedtechknowledge-baseknowledge-managementllmpdf-to-markdownragself-studystudy-tools
feat(extractor): detect Roman-numeral chapter headings + harden discovery_tax (#28)
Virgilio Junior61e40e498d4a
diff --git a/scripts/extractor/utils.py b/scripts/extractor/utils.pyindex 7cfd7c7..b6a5fd1 100644--- a/scripts/extractor/utils.py+++ b/scripts/extractor/utils.py@@ -59,18 +59,57 @@ _EXPLICIT_CHAPTER = re.compile(# "Chapter 8 are relevant...") is prose / a cross-reference, not a heading._HEADING_TAIL = re.compile(r"^\s*$|^\s*[.:\-—–]|^\s+[A-ZÀ-Ú0-9\"“(]")+# Roman-numeral chapter heading: "I: Loomings", "II. The Carpet-Bag".+# Requires a separator (":" or ".") and a Capitalized title after it, so a bare+# "I" or "V." (a page divider / list marker) is not mistaken for a chapter.+_ROMAN_HEAD = re.compile(r"^\s*([IVXLCDM]+)\s*[:.]\s+[A-ZÀ-Ú\"“(]")+_ROMAN_VALUES = {"I": 1, "V": 5, "X": 10, "L": 50, "C": 100, "D": 500, "M": 1000}+++def _int_to_roman(n: int) -> str:+ table = [(1000, "M"), (900, "CM"), (500, "D"), (400, "CD"), (100, "C"),+ (90, "XC"), (50, "L"), (40, "XL"), (10, "X"), (9, "IX"),+ (5, "V"), (4, "IV"), (1, "I")]+ out = []+ for val, sym in table:+ while n >= val:+ out.append(sym)+ n -= val+ return "".join(out)+++def _roman_to_int(s: str) -> int | None:+ """Convert a Roman numeral to int, returning None if it isn't canonical."""+ s = s.upper()+ total = prev = 0+ for ch in reversed(s):+ v = _ROMAN_VALUES.get(ch)+ if v is None:+ return None+ total += -v if v < prev else v+ prev = max(prev, v)+ if total == 0 or total > 200:+ return None+ # Reject non-canonical forms ("IIII", "VV") by round-tripping.+ return total if _int_to_roman(total) == s else None+def _chapter_number(line: str) -> int | None:- """Return the chapter number if the line is a genuine chapter heading."""+ """Return the chapter number if the line is a genuine chapter heading.++ Handles both Arabic ("Chapter 5", "Capítulo 5: ...") and Roman-numeral+ ("I: Loomings", "II. The Carpet-Bag") heading styles.+ """s = line.strip()if len(s) > 80:return Nonem = _EXPLICIT_CHAPTER.match(s)- if not m:- return None- if not _HEADING_TAIL.match(m.group("rest")):- return None- return int(m.group(1))+ if m and _HEADING_TAIL.match(m.group("rest")):+ return int(m.group(1))+ rm = _ROMAN_HEAD.match(s)+ if rm:+ return _roman_to_int(rm.group(1))+ return Nonedef detect_structure(text: str) -> dict:diff --git a/tests/test_discovery_tax.py b/tests/test_discovery_tax.pyindex 766c126..cfbcca1 100644--- a/tests/test_discovery_tax.py+++ b/tests/test_discovery_tax.py@@ -44,10 +44,16 @@ class TestSplitChapters:def test_detects_three_chapters(self):segs = dt.split_chapters(SYNTHETIC_BOOK)chapters = segs[1:]- assert len(chapters) == 3- # tuple shape is (number, heading, body)- assert chapters[0][0] == 1- assert chapters[0][1].strip().startswith("Capítulo 1")+ # ToC entries + body headings both segment now; count DISTINCT numbers.+ assert {c[0] for c in chapters} == {1, 2, 3}++ def test_best_chapter_picks_largest_body_over_toc_line(self):+ # A ToC line and the real body share "Capítulo 2"; the body has more text.+ text = ("Sumário\nCapítulo 2: Recrutamento\n"+ "Capítulo 2\n" + ("conteudo real " * 50) + "\n")+ chapters = dt.split_chapters(text)[1:]+ heading, body_tok = dt.best_chapter(chapters, 2, dt.count_tokens)+ assert body_tok > 20 # picked the real body, not the 1-line ToC entrydef test_cross_reference_does_not_split(self):text = "Capítulo 1\nbody\nComo vimos no Capítulo 2, isso importa.\nmore body\n"diff --git a/tests/test_extractor.py b/tests/test_extractor.pyindex 7fb4edd..af7c7bf 100644--- a/tests/test_extractor.py+++ b/tests/test_extractor.py@@ -585,6 +585,18 @@ class TestDetectStructure:text = "Capítulo 1: Alicerces\n...\nCapítulo 1\nbody of chapter one\n"assert detect_structure(text)["chapters_detected"] == 1+ def test_roman_numeral_chapters(self):+ text = "I: Loomings\nbody\nII: The Carpet-Bag\nbody\nIII: The Spouter-Inn\nbody\n"+ assert detect_structure(text)["chapters_detected"] == 3++ def test_roman_requires_title_after_separator(self):+ # bare "V." (page divider) or "I" alone is not a chapter+ assert detect_structure("V.\nI\nII\n")["chapters_detected"] == 0++ def test_roman_rejects_non_canonical(self):+ # "IIII"/"VV" are not valid roman numerals+ assert detect_structure("IIII: Bad\nVV: Also bad\n")["chapters_detected"] == 0+def test_scans_full_text_not_just_head(self):# A chapter heading far past the old 50k-char window must still be found.text = "Capítulo 1\n" + ("filler word " * 6000) + "\nCapítulo 2\n"diff --git a/tools/discovery_tax.py b/tools/discovery_tax.pyindex fcd64b9..96ec016 100644--- a/tools/discovery_tax.py+++ b/tools/discovery_tax.py@@ -33,28 +33,16 @@ import reimport sysfrom pathlib import Path-# Robust chapter-heading detection, aligned with the extractor's detect_structure:-# accept "Chapter N", "Capítulo N", "Chapter N. Title"; reject prose-# cross-references ("Chapter 6 explores...") via a lowercase tail, and bound the-# number to 1..99 so years ("2025.") are not chapters.-_CHAPTER_HEAD = re.compile(r"^\s*(?:chapter|cap[ií]tulo|ch\.?)\s*(\d{1,2})\b(?P<rest>.*)$",- re.IGNORECASE)-_HEADING_TAIL = re.compile(r"^\s*$|^\s*[.:\-—–]|^\s+[A-ZÀ-Ú0-9\"“(]")+# Reuse the extractor's hardened chapter detection instead of duplicating it, so+# discovery_tax and the pipeline always agree on what a chapter is (Arabic ++# Roman headings, prose/cross-reference rejection, list-item rejection).+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))+from extractor.utils import _chapter_number as chapter_number # noqa: E402+TOC_RE = re.compile(r"^\s*(?:sum[áa]rio|table of contents|contents|[íi]ndice)\s*$",re.IGNORECASE | re.MULTILINE)-def chapter_number(line: str) -> int | None:- """Return the chapter number if the line is a genuine chapter heading, else None."""- s = line.strip()- if len(s) > 80:- return None- m = _CHAPTER_HEAD.match(s)- if not m or not _HEADING_TAIL.match(m.group("rest")):- return None- return int(m.group(1))--def count_tokens(text: str) -> int:"""Real BPE count via tiktoken if available; else words/0.75 heuristic."""try:@@ -74,23 +62,30 @@ def token_method() -> str:def split_chapters(text: str) -> list[tuple[int | None, str, str]]:- """Return [(number, heading, body)]. Splits on genuine chapter headings.+ """Return [(number, heading, body)], one segment per heading occurrence.+The text before the first heading is the leading 'front matter / ToC'- segment (number=None). A new segment only starts on the FIRST occurrence of- each chapter number, so repeated cross-references to an already-seen chapter- don't fragment the body."""+ segment (number=None). A chapter number may appear more than once — a ToC+ entry and the real body share a heading format — so callers should pick the+ LARGEST-body occurrence as the real chapter (see best_chapter)."""lines = text.splitlines()segments: list[tuple[int | None, str, list[str]]] = [(None, "__front__", [])]- seen: set[int] = set()for line in lines:num = chapter_number(line)- if num is not None and num not in seen:- seen.add(num)+ if num is not None:segments.append((num, line.strip(), []))segments[-1][2].append(line)return [(n, h, "\n".join(b)) for n, h, b in segments]+def best_chapter(chapters: list[tuple[int | None, str, str]], n: int,+ tok) -> tuple[str, int] | None:+ """Return (heading, body_tokens) for chapter number `n`, choosing the+ occurrence with the largest body — the real chapter, not a ToC line."""+ cands = [(h, tok(b)) for num, h, b in chapters if num == n]+ return max(cands, key=lambda x: x[1]) if cands else None++def extract_toc(front_matter: str) -> str:"""Best-effort slice of the ToC block from the front matter."""m = TOC_RE.search(front_matter)@@ -126,16 +121,19 @@ def main() -> int:toc = extract_toc(front)toc_tok = count_tokens(toc)- # Select the target by chapter NUMBER (robust to extra cross-ref segments);- # fall back to positional if that number isn't present.+ # Distinct chapter numbers present (a number can recur: ToC entry + body).+ distinct = sorted({num for num, _, _ in chapters if num is not None})++ # Select the target by chapter NUMBER, taking the largest-body occurrence so+ # a ToC line isn't mistaken for the chapter. Fall back to positional.n = args.target_chapter- idx = next((i for i, (num, _, _) in enumerate(chapters) if num == n), None)- if idx is None:- idx = max(0, min(n - 1, len(chapters) - 1))- n = chapters[idx][0] or n- target_raw = count_tokens(chapters[idx][2])- prior_raw = count_tokens(chapters[idx - 1][2]) if idx >= 1 else 0- target_heading = chapters[idx][1]+ best = best_chapter(chapters, n, count_tokens)+ if best is None:+ n = distinct[min(n - 1, len(distinct) - 1)] if distinct else n+ best = best_chapter(chapters, n, count_tokens)+ target_heading, target_raw = best+ prior = best_chapter(chapters, n - 1, count_tokens)+ prior_raw = prior[1] if prior else 0# book-to-skill resident costif args.skill_dir:@@ -169,7 +167,7 @@ def main() -> int:print("Discovery Loop Tax — measured on a real book\n")print(f" token method : {token_method()}")print(f" source : {Path(args.full_text).name}")- print(f" chapters : {len(chapters)} detected")+ print(f" chapters : {len(distinct)} detected")print(f" target : chapter {n} ({target_heading[:60]})")print(f" book total : {total:,} tokens\n")