Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
fix(data): provide PDF and DOCX ingestion dependencies (#2096)
Nurkhan Esenbek9eea76322575
diff --git a/changelog.d/0.76.0/2096.added.md b/changelog.d/0.76.0/2096.added.mdnew file mode 100644index 00000000..55305f25--- /dev/null+++ b/changelog.d/0.76.0/2096.added.md@@ -0,0 +1 @@+- Add the `documents` extra (`pypdf>=6.19.0` and `python-docx>=1.2.0`) for PDF/DOCX ingestion, and point missing-reader errors to its install command; Markdown and TXT need no extra. (#2057; #2096 by @kok-o)diff --git a/docs/commands.md b/docs/commands.mdindex 17f17d5d..93890b6b 100644--- a/docs/commands.md+++ b/docs/commands.md@@ -144,7 +144,7 @@ soup data push --input d.jsonl --hf-dataset u/n --hub modelscope|modelers Uploasoup data registry List all registered datasetssoup data demo List bundled demo JSONL fixturessoup data demo alpaca_demo --output ./d.jsonl Copy a bundled demo JSONL fixture-soup data ingest <file> PDF/DOCX/MD/TXT -> JSONL (one row per page/heading)+soup data ingest <file> PDF/DOCX/MD/TXT -> JSONL (PDF/DOCX: pip install "soup-cli[documents]")soup data preprocess <config> AOT-tokenize and cache for reuse across runssoup data recipe <path> Validate / execute a Data Recipe DAGsoup data mix BETA mixture-weight optimiser (proxy runs)diff --git a/docs/data.md b/docs/data.mdindex 11811db5..e8f9facf 100644--- a/docs/data.md+++ b/docs/data.md@@ -1114,7 +1114,12 @@ of being silently skipped.**Document ingestion (PDF / DOCX / MD / TXT → JSONL):**+PDF and DOCX need the optional `documents` extra. Markdown uses the built-in heading+splitter, and TXT uses the built-in reader; neither needs an extra. The `pdf` extra+renders compliance reports with reportlab and does not provide document readers.+```bash+pip install "soup-cli[documents]"soup data ingest report.pdf --output report.jsonlsoup data ingest README.mdsoup data ingest notes.docxdiff --git a/docs/models.md b/docs/models.mdindex b3d59e00..bc8956bb 100644--- a/docs/models.md+++ b/docs/models.md@@ -185,6 +185,7 @@ spelling is unchanged. See [the README's install section](../README.md#1-install| `eval` | `pip install "soup-cli[eval]"` | Benchmark evaluation (lm-evaluation-harness) || `aider` | `pip install "soup-cli[aider]"` | Aider CLI; Polyglot evaluation also needs Aider's source-built Docker image || `data` | `pip install "soup-cli[data]"` | Deduplication (MinHash via datasketch) |+| `documents` | `pip install "soup-cli[documents]"` | PDF / DOCX ingestion (pypdf, python-docx); Markdown and TXT need no extra || `data-pro` | `pip install "soup-cli[data-pro]"` | Language detection + PII (langdetect, presidio) || `deepspeed` | `pip install "soup-cli[deepspeed]"` | Multi-GPU training (DeepSpeed ZeRO) || `liger` | `pip install "soup-cli[liger]"` | Liger Kernel fused ops |diff --git a/pyproject.toml b/pyproject.tomlindex 1ee47a1b..82e0a5c3 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -162,6 +162,9 @@ eval = ["lm-eval>=0.4.0"]# docs/evaluation.md; the wheel alone cannot supply benchmark/benchmark.py.aider = ["aider-chat>=0.86.2,<0.87"]data = ["datasketch>=1.6.0", "scikit-learn>=1.0.0"]+# PDF/DOCX readers for `soup data ingest`; floors checked by a manual round trip.+# CI uses stubs; real-reader tests run when the extra is installed.+documents = ["pypdf>=6.19.0", "python-docx>=1.2.0"]wandb = ["wandb>=0.15.0,<0.18.0"]# Self-references `[train]` so CI / contributors get the full training stack# (CI runs `pip install -e ".[dev]"`; without this every test would fail atdiff --git a/src/soup_cli/commands/data.py b/src/soup_cli/commands/data.pyindex 4d6636eb..bb27b235 100644--- a/src/soup_cli/commands/data.py+++ b/src/soup_cli/commands/data.py@@ -2935,10 +2935,13 @@ def ingest_document("""Ingest a document into JSONL with one row per page / heading.Lazy-imports the per-format extractor so missing optional deps don't- break the rest of `soup data --help`. Supported formats:- - .pdf → pypdf- - .docx → python-docx- - .md → markdown+ break the rest of `soup data --help`.+ Install PDF/DOCX readers with `pip install "soup-cli\\[documents]"`.++ Supported formats:+ - .pdf → pypdf (documents extra)+ - .docx → python-docx (documents extra)+ - .md → built-in heading splitter- .txt → built-in"""import stat@@ -3018,7 +3021,7 @@ def ingest_document(from pypdf import PdfReaderexcept ImportError:console.print(- "[red]pypdf not installed.[/] Run: pip install pypdf"+ r'[red]pypdf not installed.[/] Run: pip install "soup-cli\[documents]"')raise typer.Exit(1) from Nonereader = PdfReader(in_real)@@ -3033,7 +3036,7 @@ def ingest_document(from docx import Documentexcept ImportError:console.print(- "[red]python-docx not installed.[/] Run: pip install python-docx"+ r'[red]python-docx not installed.[/] Run: pip install "soup-cli\[documents]"')raise typer.Exit(1) from Nonedoc = Document(in_real)diff --git a/tests/test_cli_help_markup.py b/tests/test_cli_help_markup.pyindex 6186870e..efe2f031 100644--- a/tests/test_cli_help_markup.py+++ b/tests/test_cli_help_markup.py@@ -6,7 +6,7 @@ without a trace unless escaped as `\\[tag]`.This ratchet walks the live Click command tree derived from Typer and ensures:1. No command or parameter help string has unescaped non-style markup tags dropped.-2. The 7 fixed help strings retain their literal bracketed text in rendered `--help`.+2. The fixed help strings retain their literal bracketed text in rendered `--help`."""from __future__ import annotations@@ -163,7 +163,7 @@ class TestMarkupScannerDetection:class TestEscapedHelpStringsReachTerminal:- """The 7 fixed sites must actually display their literal bracketed text in --help."""+ """The fixed sites must actually display their literal bracketed text in --help."""@pytest.mark.parametrize(("argv", "expected_literal"),@@ -175,6 +175,7 @@ class TestEscapedHelpStringsReachTerminal:(["eval", "citation", "--help"], "[doc-N]"),(["adapters", "sign", "--help"], "soup-cli[sigstore]"),(["data", "brain-rot", "--help"], "[data-pro]"),+ (["data", "ingest", "--help"], 'pip install "soup-cli[documents]"'),],)def test_help_contains_literal_bracketed_text(self, argv: list[str], expected_literal: str):diff --git a/tests/test_document_ingest_extra.py b/tests/test_document_ingest_extra.pynew file mode 100644index 00000000..b0902b5b--- /dev/null+++ b/tests/test_document_ingest_extra.py@@ -0,0 +1,189 @@+"""#2057: document readers have an installable extra and actionable CLI errors."""++from __future__ import annotations++import ast+import json+import re+import sys+from pathlib import Path+from types import ModuleType, SimpleNamespace++import pytest+from packaging.requirements import Requirement+from rich.console import Console+from typer.testing import CliRunner++from soup_cli.cli import app+from soup_cli.commands import data+from tests.conftest import strip_ansi++ROOT = Path(__file__).resolve().parents[1]+INSTALL = 'pip install "soup-cli[documents]"'++[email protected](autouse=True)+def ingest_console(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(data, "console", Console(+ width=100, force_terminal=True, color_system="standard", no_color=False,+ legacy_windows=False,+ ))+++def _ingest(source: Path, output: str = "rows.jsonl") -> list[dict]:+ result = CliRunner().invoke(app, ["data", "ingest", source.name, "--output", output])+ assert result.exit_code == 0, (result.output, result.exception)+ return [json.loads(line) for line in source.with_name(output).read_text("utf-8").splitlines()]+++def test_documents_extra_declares_both_reader_floors() -> None:+ # Pin the declared requirements independently of optional reader installs.+ text = (ROOT / "pyproject.toml").read_text("utf-8")+ table = text.split("[project.optional-dependencies]", 1)[1].split("\n[", 1)[0]+ match = re.search(r"^documents\s*=\s*(\[[^\]]*\])", table, re.M)+ assert match, "no installable documents extra"+ requirements = [Requirement(value) for value in ast.literal_eval(match.group(1))]+ assert {str(requirement) for requirement in requirements} == {+ "pypdf>=6.19.0", "python-docx>=1.2.0",+ }+++def test_documents_stays_opt_in_with_the_all_extra() -> None:+ text = (ROOT / "pyproject.toml").read_text("utf-8")+ match = re.search(r"^all\s*=\s*(\[.*\])$", text, re.M)+ assert match, "no all extra"+ assert all(+ "documents" not in Requirement(value).extras for value in ast.literal_eval(match.group(1))+ )++[email protected]("extension,module,package", [+ ("pdf", "pypdf", "pypdf"), ("docx", "docx", "python-docx"),+])+def test_missing_reader_names_package_and_installable_extra(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, extension: str, module: str, package: str,+) -> None:+ monkeypatch.chdir(tmp_path)+ monkeypatch.setitem(sys.modules, module, None)+ source = tmp_path / f"document.{extension}"+ source.write_bytes(b"reader must not be called")+ output = tmp_path / "rows.jsonl"+ output.write_text("existing output\n", encoding="utf-8")+ result = CliRunner().invoke(+ app, ["data", "ingest", source.name, "--output", output.name], color=True,+ )+ assert result.exit_code == 1, (result.output, result.exception)+ assert "\x1b[" in result.output # The real rendering console exercised colour.+ text = " ".join(strip_ansi(result.output).split())+ assert f"{package} not installed." in text+ assert INSTALL in text+ assert output.read_text("utf-8") == "existing output\n"+++def test_pdf_reader_keeps_each_page_and_empty_page(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.chdir(tmp_path)+ source = tmp_path / "report.pdf"+ source.write_bytes(b"stubbed PDF")+ calls = []++ def reader(path: str) -> SimpleNamespace:+ calls.append(path)+ return SimpleNamespace(pages=[+ SimpleNamespace(extract_text=lambda: "First page"),+ SimpleNamespace(extract_text=lambda: None),+ ])++ module = ModuleType("pypdf")+ module.PdfReader = reader+ monkeypatch.setitem(sys.modules, "pypdf", module)+ assert _ingest(source) == [+ {"text": "First page", "source": "report.pdf", "page": 0},+ {"text": "", "source": "report.pdf", "page": 1},+ ]+ assert calls == [str(source)]+++def test_docx_reader_strips_text_but_preserves_physical_paragraph_indices(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.chdir(tmp_path)+ source = tmp_path / "notes.docx"+ source.write_bytes(b"stubbed DOCX")+ calls = []++ def document(path: str) -> SimpleNamespace:+ calls.append(path)+ return SimpleNamespace(paragraphs=[+ SimpleNamespace(text=text) for text in (" First paragraph ", " ", "Second paragraph")+ ])++ module = ModuleType("docx")+ module.Document = document+ monkeypatch.setitem(sys.modules, "docx", module)+ assert _ingest(source) == [+ {"text": "First paragraph", "source": "notes.docx", "para": 0},+ {"text": "Second paragraph", "source": "notes.docx", "para": 2},+ ]+ assert calls == [str(source)]++[email protected]("extension,text,expected", [+ ("txt", "Hello\nWorld", [{"text": "Hello\nWorld", "source": "notes.txt"}]),+ ("md", "# Title\n\nBody", [+ {"text": "Body", "section": "Title", "level": 1, "source": "notes.md"},+ ]),+])+def test_builtin_formats_need_no_document_readers(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, extension: str, text: str,+ expected: list[dict],+) -> None:+ monkeypatch.chdir(tmp_path)+ for module in ("pypdf", "docx", "markdown"):+ monkeypatch.setitem(sys.modules, module, None)+ source = tmp_path / f"notes.{extension}"+ source.write_text(text, encoding="utf-8")+ assert _ingest(source) == expected+++def test_real_pdf_round_trip(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ pypdf = pytest.importorskip("pypdf")+ monkeypatch.chdir(tmp_path)+ source = tmp_path / "report.pdf"+ # A tiny text PDF using only the reader's dependency; no PDF renderer required.+ writer = pypdf.PdfWriter()+ for text in ("First page", "Second page"):+ page = writer.add_blank_page(width=300, height=200)+ font = pypdf.generic.DictionaryObject({+ pypdf.generic.NameObject("/Type"): pypdf.generic.NameObject("/Font"),+ pypdf.generic.NameObject("/Subtype"): pypdf.generic.NameObject("/Type1"),+ pypdf.generic.NameObject("/BaseFont"): pypdf.generic.NameObject("/Helvetica"),+ })+ page[pypdf.generic.NameObject("/Resources")] = pypdf.generic.DictionaryObject({+ pypdf.generic.NameObject("/Font"): pypdf.generic.DictionaryObject({+ pypdf.generic.NameObject("/F1"): font,+ }),+ })+ stream = pypdf.generic.DecodedStreamObject()+ stream.set_data(f"BT /F1 12 Tf 20 100 Td ({text}) Tj ET".encode("ascii"))+ page[pypdf.generic.NameObject("/Contents")] = stream+ writer.write(source)+ assert _ingest(source) == [+ {"text": "First page", "source": "report.pdf", "page": 0},+ {"text": "Second page", "source": "report.pdf", "page": 1},+ ]+++def test_real_docx_round_trip(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ docx = pytest.importorskip("docx")+ monkeypatch.chdir(tmp_path)+ source = tmp_path / "notes.docx"+ document = docx.Document()+ for text in (" First paragraph ", "", "Второй абзац"):+ document.add_paragraph(text)+ document.save(source)+ assert _ingest(source) == [+ {"text": "First paragraph", "source": "notes.docx", "para": 0},+ {"text": "Второй абзац", "source": "notes.docx", "para": 2},+ ]