Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

fix(data): provide PDF and DOCX ingestion dependencies (#2096)

Nurkhan Esenbek9eea76322575
diff --git a/changelog.d/0.76.0/2096.added.md b/changelog.d/0.76.0/2096.added.md
new file mode 100644
index 00000000..55305f25
--- /dev/null
+++ b/changelog.d/0.76.0/2096.added.md
@@ -0,0 +1 @@
+- Add the `documents` extra (`pypdf>=6.19.0` and `python-docx>=1.2.0`) for PDF/DOCX ingestion, and point missing-reader errors to its install command; Markdown and TXT need no extra. (#2057; #2096 by @kok-o)
diff --git a/docs/commands.md b/docs/commands.md
index 17f17d5d..93890b6b 100644
--- a/docs/commands.md
+++ b/docs/commands.md
@@ -144,7 +144,7 @@ soup data push --input d.jsonl --hf-dataset u/n --hub modelscope|modelers Uploa
soup data registry List all registered datasets
soup data demo List bundled demo JSONL fixtures
soup data demo alpaca_demo --output ./d.jsonl Copy a bundled demo JSONL fixture
-soup data ingest <file> PDF/DOCX/MD/TXT -> JSONL (one row per page/heading)
+soup data ingest <file> PDF/DOCX/MD/TXT -> JSONL (PDF/DOCX: pip install "soup-cli[documents]")
soup data preprocess <config> AOT-tokenize and cache for reuse across runs
soup data recipe <path> Validate / execute a Data Recipe DAG
soup data mix BETA mixture-weight optimiser (proxy runs)
diff --git a/docs/data.md b/docs/data.md
index 11811db5..e8f9facf 100644
--- a/docs/data.md
+++ b/docs/data.md
@@ -1114,7 +1114,12 @@ of being silently skipped.
**Document ingestion (PDF / DOCX / MD / TXT → JSONL):**
+PDF and DOCX need the optional `documents` extra. Markdown uses the built-in heading
+splitter, and TXT uses the built-in reader; neither needs an extra. The `pdf` extra
+renders compliance reports with reportlab and does not provide document readers.
+
```bash
+pip install "soup-cli[documents]"
soup data ingest report.pdf --output report.jsonl
soup data ingest README.md
soup data ingest notes.docx
diff --git a/docs/models.md b/docs/models.md
index b3d59e00..bc8956bb 100644
--- a/docs/models.md
+++ b/docs/models.md
@@ -185,6 +185,7 @@ spelling is unchanged. See [the README's install section](../README.md#1-install
| `eval` | `pip install "soup-cli[eval]"` | Benchmark evaluation (lm-evaluation-harness) |
| `aider` | `pip install "soup-cli[aider]"` | Aider CLI; Polyglot evaluation also needs Aider's source-built Docker image |
| `data` | `pip install "soup-cli[data]"` | Deduplication (MinHash via datasketch) |
+| `documents` | `pip install "soup-cli[documents]"` | PDF / DOCX ingestion (pypdf, python-docx); Markdown and TXT need no extra |
| `data-pro` | `pip install "soup-cli[data-pro]"` | Language detection + PII (langdetect, presidio) |
| `deepspeed` | `pip install "soup-cli[deepspeed]"` | Multi-GPU training (DeepSpeed ZeRO) |
| `liger` | `pip install "soup-cli[liger]"` | Liger Kernel fused ops |
diff --git a/pyproject.toml b/pyproject.toml
index 1ee47a1b..82e0a5c3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -162,6 +162,9 @@ eval = ["lm-eval>=0.4.0"]
# docs/evaluation.md; the wheel alone cannot supply benchmark/benchmark.py.
aider = ["aider-chat>=0.86.2,<0.87"]
data = ["datasketch>=1.6.0", "scikit-learn>=1.0.0"]
+# PDF/DOCX readers for `soup data ingest`; floors checked by a manual round trip.
+# CI uses stubs; real-reader tests run when the extra is installed.
+documents = ["pypdf>=6.19.0", "python-docx>=1.2.0"]
wandb = ["wandb>=0.15.0,<0.18.0"]
# Self-references `[train]` so CI / contributors get the full training stack
# (CI runs `pip install -e ".[dev]"`; without this every test would fail at
diff --git a/src/soup_cli/commands/data.py b/src/soup_cli/commands/data.py
index 4d6636eb..bb27b235 100644
--- a/src/soup_cli/commands/data.py
+++ b/src/soup_cli/commands/data.py
@@ -2935,10 +2935,13 @@ def ingest_document(
"""Ingest a document into JSONL with one row per page / heading.
Lazy-imports the per-format extractor so missing optional deps don't
- break the rest of `soup data --help`. Supported formats:
- - .pdf → pypdf
- - .docx → python-docx
- - .md → markdown
+ break the rest of `soup data --help`.
+ Install PDF/DOCX readers with `pip install "soup-cli\\[documents]"`.
+
+ Supported formats:
+ - .pdf → pypdf (documents extra)
+ - .docx → python-docx (documents extra)
+ - .md → built-in heading splitter
- .txt → built-in
"""
import stat
@@ -3018,7 +3021,7 @@ def ingest_document(
from pypdf import PdfReader
except ImportError:
console.print(
- "[red]pypdf not installed.[/] Run: pip install pypdf"
+ r'[red]pypdf not installed.[/] Run: pip install "soup-cli\[documents]"'
)
raise typer.Exit(1) from None
reader = PdfReader(in_real)
@@ -3033,7 +3036,7 @@ def ingest_document(
from docx import Document
except ImportError:
console.print(
- "[red]python-docx not installed.[/] Run: pip install python-docx"
+ r'[red]python-docx not installed.[/] Run: pip install "soup-cli\[documents]"'
)
raise typer.Exit(1) from None
doc = Document(in_real)
diff --git a/tests/test_cli_help_markup.py b/tests/test_cli_help_markup.py
index 6186870e..efe2f031 100644
--- a/tests/test_cli_help_markup.py
+++ b/tests/test_cli_help_markup.py
@@ -6,7 +6,7 @@ without a trace unless escaped as `\\[tag]`.
This ratchet walks the live Click command tree derived from Typer and ensures:
1. No command or parameter help string has unescaped non-style markup tags dropped.
-2. The 7 fixed help strings retain their literal bracketed text in rendered `--help`.
+2. The fixed help strings retain their literal bracketed text in rendered `--help`.
"""
from __future__ import annotations
@@ -163,7 +163,7 @@ class TestMarkupScannerDetection:
class TestEscapedHelpStringsReachTerminal:
- """The 7 fixed sites must actually display their literal bracketed text in --help."""
+ """The fixed sites must actually display their literal bracketed text in --help."""
@pytest.mark.parametrize(
("argv", "expected_literal"),
@@ -175,6 +175,7 @@ class TestEscapedHelpStringsReachTerminal:
(["eval", "citation", "--help"], "[doc-N]"),
(["adapters", "sign", "--help"], "soup-cli[sigstore]"),
(["data", "brain-rot", "--help"], "[data-pro]"),
+ (["data", "ingest", "--help"], 'pip install "soup-cli[documents]"'),
],
)
def test_help_contains_literal_bracketed_text(self, argv: list[str], expected_literal: str):
diff --git a/tests/test_document_ingest_extra.py b/tests/test_document_ingest_extra.py
new file mode 100644
index 00000000..b0902b5b
--- /dev/null
+++ b/tests/test_document_ingest_extra.py
@@ -0,0 +1,189 @@
+"""#2057: document readers have an installable extra and actionable CLI errors."""
+
+from __future__ import annotations
+
+import ast
+import json
+import re
+import sys
+from pathlib import Path
+from types import ModuleType, SimpleNamespace
+
+import pytest
+from packaging.requirements import Requirement
+from rich.console import Console
+from typer.testing import CliRunner
+
+from soup_cli.cli import app
+from soup_cli.commands import data
+from tests.conftest import strip_ansi
+
+ROOT = Path(__file__).resolve().parents[1]
+INSTALL = 'pip install "soup-cli[documents]"'
+
+
[email protected](autouse=True)
+def ingest_console(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(data, "console", Console(
+ width=100, force_terminal=True, color_system="standard", no_color=False,
+ legacy_windows=False,
+ ))
+
+
+def _ingest(source: Path, output: str = "rows.jsonl") -> list[dict]:
+ result = CliRunner().invoke(app, ["data", "ingest", source.name, "--output", output])
+ assert result.exit_code == 0, (result.output, result.exception)
+ return [json.loads(line) for line in source.with_name(output).read_text("utf-8").splitlines()]
+
+
+def test_documents_extra_declares_both_reader_floors() -> None:
+ # Pin the declared requirements independently of optional reader installs.
+ text = (ROOT / "pyproject.toml").read_text("utf-8")
+ table = text.split("[project.optional-dependencies]", 1)[1].split("\n[", 1)[0]
+ match = re.search(r"^documents\s*=\s*(\[[^\]]*\])", table, re.M)
+ assert match, "no installable documents extra"
+ requirements = [Requirement(value) for value in ast.literal_eval(match.group(1))]
+ assert {str(requirement) for requirement in requirements} == {
+ "pypdf>=6.19.0", "python-docx>=1.2.0",
+ }
+
+
+def test_documents_stays_opt_in_with_the_all_extra() -> None:
+ text = (ROOT / "pyproject.toml").read_text("utf-8")
+ match = re.search(r"^all\s*=\s*(\[.*\])$", text, re.M)
+ assert match, "no all extra"
+ assert all(
+ "documents" not in Requirement(value).extras for value in ast.literal_eval(match.group(1))
+ )
+
+
[email protected]("extension,module,package", [
+ ("pdf", "pypdf", "pypdf"), ("docx", "docx", "python-docx"),
+])
+def test_missing_reader_names_package_and_installable_extra(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, extension: str, module: str, package: str,
+) -> None:
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setitem(sys.modules, module, None)
+ source = tmp_path / f"document.{extension}"
+ source.write_bytes(b"reader must not be called")
+ output = tmp_path / "rows.jsonl"
+ output.write_text("existing output\n", encoding="utf-8")
+ result = CliRunner().invoke(
+ app, ["data", "ingest", source.name, "--output", output.name], color=True,
+ )
+ assert result.exit_code == 1, (result.output, result.exception)
+ assert "\x1b[" in result.output # The real rendering console exercised colour.
+ text = " ".join(strip_ansi(result.output).split())
+ assert f"{package} not installed." in text
+ assert INSTALL in text
+ assert output.read_text("utf-8") == "existing output\n"
+
+
+def test_pdf_reader_keeps_each_page_and_empty_page(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.chdir(tmp_path)
+ source = tmp_path / "report.pdf"
+ source.write_bytes(b"stubbed PDF")
+ calls = []
+
+ def reader(path: str) -> SimpleNamespace:
+ calls.append(path)
+ return SimpleNamespace(pages=[
+ SimpleNamespace(extract_text=lambda: "First page"),
+ SimpleNamespace(extract_text=lambda: None),
+ ])
+
+ module = ModuleType("pypdf")
+ module.PdfReader = reader
+ monkeypatch.setitem(sys.modules, "pypdf", module)
+ assert _ingest(source) == [
+ {"text": "First page", "source": "report.pdf", "page": 0},
+ {"text": "", "source": "report.pdf", "page": 1},
+ ]
+ assert calls == [str(source)]
+
+
+def test_docx_reader_strips_text_but_preserves_physical_paragraph_indices(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.chdir(tmp_path)
+ source = tmp_path / "notes.docx"
+ source.write_bytes(b"stubbed DOCX")
+ calls = []
+
+ def document(path: str) -> SimpleNamespace:
+ calls.append(path)
+ return SimpleNamespace(paragraphs=[
+ SimpleNamespace(text=text) for text in (" First paragraph ", " ", "Second paragraph")
+ ])
+
+ module = ModuleType("docx")
+ module.Document = document
+ monkeypatch.setitem(sys.modules, "docx", module)
+ assert _ingest(source) == [
+ {"text": "First paragraph", "source": "notes.docx", "para": 0},
+ {"text": "Second paragraph", "source": "notes.docx", "para": 2},
+ ]
+ assert calls == [str(source)]
+
+
[email protected]("extension,text,expected", [
+ ("txt", "Hello\nWorld", [{"text": "Hello\nWorld", "source": "notes.txt"}]),
+ ("md", "# Title\n\nBody", [
+ {"text": "Body", "section": "Title", "level": 1, "source": "notes.md"},
+ ]),
+])
+def test_builtin_formats_need_no_document_readers(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, extension: str, text: str,
+ expected: list[dict],
+) -> None:
+ monkeypatch.chdir(tmp_path)
+ for module in ("pypdf", "docx", "markdown"):
+ monkeypatch.setitem(sys.modules, module, None)
+ source = tmp_path / f"notes.{extension}"
+ source.write_text(text, encoding="utf-8")
+ assert _ingest(source) == expected
+
+
+def test_real_pdf_round_trip(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ pypdf = pytest.importorskip("pypdf")
+ monkeypatch.chdir(tmp_path)
+ source = tmp_path / "report.pdf"
+ # A tiny text PDF using only the reader's dependency; no PDF renderer required.
+ writer = pypdf.PdfWriter()
+ for text in ("First page", "Second page"):
+ page = writer.add_blank_page(width=300, height=200)
+ font = pypdf.generic.DictionaryObject({
+ pypdf.generic.NameObject("/Type"): pypdf.generic.NameObject("/Font"),
+ pypdf.generic.NameObject("/Subtype"): pypdf.generic.NameObject("/Type1"),
+ pypdf.generic.NameObject("/BaseFont"): pypdf.generic.NameObject("/Helvetica"),
+ })
+ page[pypdf.generic.NameObject("/Resources")] = pypdf.generic.DictionaryObject({
+ pypdf.generic.NameObject("/Font"): pypdf.generic.DictionaryObject({
+ pypdf.generic.NameObject("/F1"): font,
+ }),
+ })
+ stream = pypdf.generic.DecodedStreamObject()
+ stream.set_data(f"BT /F1 12 Tf 20 100 Td ({text}) Tj ET".encode("ascii"))
+ page[pypdf.generic.NameObject("/Contents")] = stream
+ writer.write(source)
+ assert _ingest(source) == [
+ {"text": "First page", "source": "report.pdf", "page": 0},
+ {"text": "Second page", "source": "report.pdf", "page": 1},
+ ]
+
+
+def test_real_docx_round_trip(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ docx = pytest.importorskip("docx")
+ monkeypatch.chdir(tmp_path)
+ source = tmp_path / "notes.docx"
+ document = docx.Document()
+ for text in (" First paragraph ", "", "Второй абзац"):
+ document.add_paragraph(text)
+ document.save(source)
+ assert _ingest(source) == [
+ {"text": "First paragraph", "source": "notes.docx", "para": 0},
+ {"text": "Второй абзац", "source": "notes.docx", "para": 2},
+ ]