Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

9.9 KB
Raw Blame
from io import BytesIO
from pathlib import Path
from unittest.mock import Mock, patch

import pytest
from openpyxl import Workbook

from xagent.providers.pdf_parser.base import ParseResult
from xagent.providers.pdf_parser.deepdoc import DeepDocParser, _parse_xlsx_rows


# A fixture to easily access test resource files
@pytest.fixture
def resource_path() -> Path:
    """Provides the correct path to the test resources directory."""
    return Path("tests/resources/test_files")


# List of files for the smoke test
# We exclude files that deepdoc is not expected to handle, like .ppt, .jpg
SUPPORTED_FILES = [
    "test.pdf",
    "test.docx",
    "test.xlsx",
    "test.csv",
    "test.html",
    "test.json",
    "test.md",
    "test.txt",
    "gbk.txt",
]


@pytest.mark.asyncio
async def test_deepdoc_raw_output_deepdoc_engine(resource_path: Path):
    """Test DeepDoc parser with raw output enabled using DeepDoc engine."""
    test_file = resource_path / "test.pdf"
    parser = DeepDocParser(enable_raw_output=True)

    result = await parser.parse(str(test_file))

    # Check basic structure
    assert result is not None
    assert isinstance(result, ParseResult)

    # Check raw output fields
    assert result.raw_parser_output is not None
    assert result.parser_engine == "deepdoc"
    assert result.has_visualization_data

    # Check raw output structure
    raw_output = result.raw_parser_output
    assert "format" in raw_output
    assert raw_output["format"] == "deepdoc_pdf"
    assert "bboxes" in raw_output
    assert "total_elements" in raw_output
    assert "has_positions" in raw_output

    # Check visualization elements
    viz_elements = result.get_visualization_elements()
    assert isinstance(viz_elements, list)

    if viz_elements:  # Only check structure if elements exist
        element = viz_elements[0]
        required_keys = ["id", "type", "content", "bbox", "page", "metadata"]
        for key in required_keys:
            assert key in element

        # Check bbox structure
        bbox = element["bbox"]
        assert all(k in bbox for k in ["x0", "y0", "x1", "y1"])

        # Check metadata structure
        metadata = element["metadata"]
        assert "parser" in metadata
        assert "content_type" in metadata
        assert "has_image" in metadata
        assert metadata["parser"] == "deepdoc"

    # Check visualization summary
    viz_summary = result.get_visualization_summary()
    assert viz_summary["available"] is True
    assert viz_summary["parser_engine"] == "deepdoc"
    assert "total_elements" in viz_summary
    assert "elements_by_type" in viz_summary


@pytest.mark.asyncio
async def test_deepdoc_standard_mode(resource_path: Path):
    """Test DeepDoc parser in standard mode (no raw output)."""
    test_file = resource_path / "test.pdf"
    parser = DeepDocParser(enable_raw_output=False)

    result = await parser.parse(str(test_file))

    # Check basic structure
    assert result is not None
    assert isinstance(result, ParseResult)

    # Check that raw output is disabled
    assert result.raw_parser_output is None
    assert result.parser_engine is None
    assert not result.has_visualization_data

    # Check that visualization elements returns empty list
    viz_elements = result.get_visualization_elements()
    assert viz_elements == []


@pytest.mark.asyncio
@pytest.mark.parametrize("filename", SUPPORTED_FILES)
async def test_deepdoc_smoke_test(resource_path: Path, filename: str):
    """Ensures the DeepDocParser can process all supported file types without crashing."""
    test_file = resource_path / filename
    parser = DeepDocParser()

    try:
        result = await parser.parse(str(test_file))
        assert result is not None
        assert isinstance(result, ParseResult)
        # A basic check that some content was produced
        assert result.text_segments or result.figures or result.tables
    except Exception as e:
        # If gbk.txt fails with a UnicodeDecodeError, we accept that as a known limitation for now
        if "gbk" in filename and isinstance(e, UnicodeDecodeError):
            pytest.skip(f"DeepDocParser (TxtParser) does not support GBK encoding: {e}")
        else:
            pytest.fail(
                f"DeepDocParser failed on {filename} with an unexpected exception: {e}"
            )


@pytest.mark.asyncio
async def test_deepdoc_pdf_details(resource_path: Path):
    """Tests detailed, lossless extraction from a PDF file."""
    test_file = resource_path / "test.pdf"
    parser = DeepDocParser()
    # Pass a mock doc_id for the image saving path
    result = await parser.parse(str(test_file), doc_id="test_doc_id")

    assert result.text_segments, "No text segments were extracted from the PDF."
    assert result.tables, "No tables were extracted from the PDF."

    if result.figures:
        figure = result.figures[0]
        assert isinstance(figure.text, str)
        assert figure.metadata.get("type") == "figure"
        figure_image_path_str = figure.metadata.get("image_path")
        assert isinstance(figure_image_path_str, str) and figure_image_path_str
        assert Path(figure_image_path_str).exists(), (
            f"Figure image file not found at {figure_image_path_str}"
        )

    # Deep inspect the first table
    table = result.tables[0]
    assert isinstance(table.html, str) and table.html
    assert table.metadata.get("type") == "table"
    table_image_path_str = table.metadata.get("image_path")
    assert isinstance(table_image_path_str, str) and table_image_path_str
    assert Path(table_image_path_str).exists(), (
        f"Table image file not found at {table_image_path_str}"
    )

    # Deep inspect the first figure if present

    # Check that basic metadata is present (position info is now in raw_parser_output)
    first_segment = result.text_segments[0]
    assert "layout_type" in first_segment.metadata, "Layout type should be in metadata"
    assert "doc_id" in first_segment.metadata, "Doc ID should be in metadata"
    assert "page_number" in first_segment.metadata, "Page number should be in metadata"

    # Check layout metadata for text segments - accept both 'text' and 'title'
    layout_type = first_segment.metadata.get("layout_type")
    assert layout_type in ["text", "title"], (
        f"Text segments should have layout_type 'text' or 'title', got '{layout_type}'"
    )

    # Check basic metadata for tables
    if result.tables:
        table = result.tables[0]
        assert table.metadata.get("layout_type") == "table", (
            "Tables should have layout_type 'table'"
        )

        # Check basic metadata for figures
    if result.figures:
        figure = result.figures[0]
        assert figure.metadata.get("layout_type") == "figure", (
            "Figures should have layout_type 'figure'"
        )


@pytest.mark.asyncio
async def test_deepdoc_docx_details(resource_path: Path):
    """Tests detailed, lossless extraction from a DOCX file."""
    test_file = resource_path / "test.docx"
    parser = DeepDocParser()
    result = await parser.parse(str(test_file))

    assert result.text_segments, "No text segments were extracted from the DOCX."

    # Check for style metadata
    first_segment = result.text_segments[0]
    assert "style" in first_segment.metadata, "Style metadata is missing."

    # Check for tables if any
    if result.tables:
        assert result.tables[0].html is not None


@pytest.mark.asyncio
async def test_deepdoc_excel_details(resource_path: Path):
    """Tests detailed extraction from an XLSX file."""
    test_file = resource_path / "test.xlsx"
    parser = DeepDocParser()
    result = await parser.parse(str(test_file))

    assert result.text_segments, "No text segments were extracted from the XLSX."
    # Check that rows are parsed as segments
    assert len(result.text_segments) > 1
    # Check content of a known row/cell if possible (highly dependent on test file content)
    # For now, we just ensure segments are created.


@pytest.mark.asyncio
async def test_deepdoc_xlsx_parses_rows_without_repeating_title(tmp_path: Path):
    file_path = tmp_path / "structured.xlsx"
    wb = Workbook()
    ws = wb.active
    ws.title = "Sheet1"
    ws.append(["Quarterly Enrollment Review"])
    ws.append(["Track", "Candidate ID", "Name", "Status"])
    ws.append(["Direct Entry", "A-0001", "Student One", "Passed"])
    ws.append(["Direct Entry", "A-0002", "Student Two", "Passed"])
    wb.save(file_path)

    parser = DeepDocParser()
    result = await parser.parse(str(file_path))

    assert result.text_segments[0].metadata["row_type"] == "title"
    assert result.text_segments[0].text == "Quarterly Enrollment Review"
    assert result.text_segments[1].metadata["row_type"] == "header"
    assert result.text_segments[1].text == "Track | Candidate ID | Name | Status"

    first_data_row = result.text_segments[2]
    assert first_data_row.metadata["row_type"] == "data"
    assert "Quarterly Enrollment Review" not in first_data_row.text
    assert (
        first_data_row.text
        == "Track: Direct Entry | Candidate ID: A-0001 | Name: Student One | Status: Passed"
    )


def test_parse_xlsx_rows_closes_workbook() -> None:
    workbook = Mock()
    workbook.sheetnames = ["Sheet1"]
    worksheet = Mock()
    worksheet.title = "Sheet1"
    worksheet.iter_rows.return_value = iter(
        [
            ("Quarterly Enrollment Review", None),
            ("Track", "Status"),
            ("Direct Entry", "Passed"),
        ]
    )
    workbook.worksheets = [worksheet]

    with patch(
        "xagent.providers.pdf_parser.deepdoc.load_workbook", return_value=workbook
    ):
        result = _parse_xlsx_rows("structured.xlsx")

    assert result.text_segments
    workbook.close.assert_called_once()


@pytest.mark.asyncio
async def test_deepdoc_invalid_xlsx_raises_parse_error() -> None:
    parser = DeepDocParser()

    with pytest.raises(ValueError, match="Failed to parse spreadsheet rows"):
        await parser.parse(BytesIO(b"not-a-valid-xlsx"), file_ext=".xlsx")