The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
unstructured[patch]: add to integration tests (#26666)
ccurme7d49ee974141
diff --git a/libs/partners/unstructured/tests/integration_tests/test_document_loaders.py b/libs/partners/unstructured/tests/integration_tests/test_document_loaders.pyindex f4fa4e7f9a..f27ddf7186 100644--- a/libs/partners/unstructured/tests/integration_tests/test_document_loaders.py+++ b/libs/partners/unstructured/tests/integration_tests/test_document_loaders.py@@ -1,8 +1,9 @@import osfrom pathlib import Path-from typing import Callable+from typing import Callable, Listimport pytest+from langchain_core.documents import Documentfrom langchain_unstructured import UnstructuredLoader@@ -13,6 +14,51 @@ EXAMPLE_DOCS_DIRECTORY = str(UNSTRUCTURED_API_KEY = os.getenv("UNSTRUCTURED_API_KEY")+def _check_docs_content(docs: List[Document]) -> None:+ assert all(+ doc.metadata.get("filename") == "layout-parser-paper.pdf" for doc in docs+ )+ assert (+ sum(doc.metadata.get("category") == "PageBreak" for doc in docs) == 16+ ) # 16 page doc++ expected_metadata_keys = [+ "source",+ "languages",+ "page_number",+ "category",+ "coordinates",+ "element_id",+ ]+ for doc in docs:+ if doc.page_content:+ for key in expected_metadata_keys:+ assert key in doc.metadata+ else:+ assert doc.metadata.get("category") == "PageBreak"++ page_numbers = []+ for doc in docs:+ if page_number := doc.metadata.get("page_number"):+ page_numbers.append(page_number)++ assert set(page_numbers) == set(range(1, 17))+ assert len(docs) >= 32 # (16 pages * (>=1 element per page) + 16 page breaks)++ page_1_content = ""+ for doc in docs:+ if doc.metadata.get("page_number") == 1:+ page_1_content += f" {doc.page_content}"+ assert (+ "LayoutParser: A Unified Toolkit for Deep Learning "+ "Based Document Image Analysis"+ ) in page_1_content++ categories = set(doc.metadata.get("category") for doc in docs)+ assert "NarrativeText" in categories+ assert "Title" in categories++# -- Local partition --@@ -27,10 +73,24 @@ def test_loader_partitions_locally() -> None:include_page_breaks=True,).load()- assert all(- doc.metadata.get("filename") == "layout-parser-paper.pdf" for doc in docs+ _check_docs_content(docs)+++async def test_loader_partitions_locally_async_lazy() -> None:+ file_path = os.path.join(EXAMPLE_DOCS_DIRECTORY, "layout-parser-paper.pdf")++ loader = UnstructuredLoader(+ file_path=file_path,+ # Unstructured kwargs+ strategy="fast",+ include_page_breaks=True,)- assert any(doc.metadata.get("category") == "PageBreak" for doc in docs)+ docs = []+ async for doc in loader.alazy_load():+ docs.append(doc)++ _check_docs_content(docs)@pytest.mark.local@@ -79,16 +139,30 @@ def test_loader_partitions_via_api() -> None:# Unstructured kwargsstrategy="fast",include_page_breaks=True,+ coordinates=True,)docs = loader.load()- assert len(docs) > 1- assert any(doc.metadata.get("category") == "PageBreak" for doc in docs)- assert all(- doc.metadata.get("filename") == "layout-parser-paper.pdf" for doc in docs+ _check_docs_content(docs)+++async def test_loader_partitions_via_api_async_lazy() -> None:+ file_path = os.path.join(EXAMPLE_DOCS_DIRECTORY, "layout-parser-paper.pdf")+ loader = UnstructuredLoader(+ file_path=file_path,+ partition_via_api=True,+ # Unstructured kwargs+ strategy="fast",+ include_page_breaks=True,+ coordinates=True,)- assert docs[0].metadata.get("element_id") is not None++ docs = []+ async for doc in loader.alazy_load():+ docs.append(doc)++ _check_docs_content(docs)def test_loader_partitions_multiple_via_api() -> None:@@ -124,6 +198,22 @@ def test_loader_partition_via_api_raises_TypeError_with_invalid_arg() -> None:loader.load()+def test_loader_partitions_via_api_hi_res() -> None:+ file_path = os.path.join(EXAMPLE_DOCS_DIRECTORY, "layout-parser-paper.pdf")+ loader = UnstructuredLoader(+ file_path=file_path,+ partition_via_api=True,+ # Unstructured kwargs+ strategy="hi_res",+ )++ docs = loader.load()++ categories = set(doc.metadata.get("category") for doc in docs)+ assert "Table" in categories+ assert "Image" in categories++# -- fixtures ---