The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
unstructured[patch]: support loading URLs (#26670)
ccurmeeef18dec442e
diff --git a/docs/docs/integrations/document_loaders/unstructured_file.ipynb b/docs/docs/integrations/document_loaders/unstructured_file.ipynbindex b376721eb8..02ff6a7c79 100644--- a/docs/docs/integrations/document_loaders/unstructured_file.ipynb+++ b/docs/docs/integrations/document_loaders/unstructured_file.ipynb@@ -16,7 +16,7 @@"\n","| Class | Package | Local | Serializable | [JS support](https://js.langchain.com/docs/integrations/document_loaders/file_loaders/unstructured/)|\n","| :--- | :--- | :---: | :---: | :---: |\n",- "| [UnstructuredLoader](https://python.langchain.com/api_reference/unstructured/document_loaders/langchain_unstructured.document_loaders.UnstructuredLoader.html) | [langchain_community](https://python.langchain.com/api_reference/unstructured/index.html) | ✅ | ❌ | ✅ | \n",+ "| [UnstructuredLoader](https://python.langchain.com/api_reference/unstructured/document_loaders/langchain_unstructured.document_loaders.UnstructuredLoader.html) | [langchain_unstructured](https://python.langchain.com/api_reference/unstructured/index.html) | ✅ | ❌ | ✅ | \n","### Loader features\n","| Source | Document Lazy Loading | Native Async Support\n","| :---: | :---: | :---: | \n",@@ -519,6 +519,47 @@"print(\"Length of text in the document:\", len(docs[0].page_content))"]},+ {+ "cell_type": "markdown",+ "id": "3ec3c22d-02cd-498b-921f-b839d1404f32",+ "metadata": {},+ "source": [+ "## Loading web pages\n",+ "\n",+ "`UnstructuredLoader` accepts a `web_url` kwarg when run locally that populates the `url` parameter of the underlying Unstructured [partition](https://docs.unstructured.io/open-source/core-functionality/partitioning). This allows for the parsing of remotely hosted documents, such as HTML web pages.\n",+ "\n",+ "Example usage:"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 6,+ "id": "bf9a8546-659d-4861-bff2-fdf1ad93ac65",+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "page_content='Example Domain' metadata={'category_depth': 0, 'languages': ['eng'], 'filetype': 'text/html', 'url': 'https://www.example.com', 'category': 'Title', 'element_id': 'fdaa78d856f9d143aeeed85bf23f58f8'}\n",+ "\n",+ "page_content='This domain is for use in illustrative examples in documents. You may use this domain in literature without prior coordination or asking for permission.' metadata={'languages': ['eng'], 'parent_id': 'fdaa78d856f9d143aeeed85bf23f58f8', 'filetype': 'text/html', 'url': 'https://www.example.com', 'category': 'NarrativeText', 'element_id': '3652b8458b0688639f973fe36253c992'}\n",+ "\n",+ "page_content='More information...' metadata={'category_depth': 0, 'link_texts': ['More information...'], 'link_urls': ['https://www.iana.org/domains/example'], 'languages': ['eng'], 'filetype': 'text/html', 'url': 'https://www.example.com', 'category': 'Title', 'element_id': '793ab98565d6f6d6f3a6d614e3ace2a9'}\n",+ "\n"+ ]+ }+ ],+ "source": [+ "from langchain_unstructured import UnstructuredLoader\n",+ "\n",+ "loader = UnstructuredLoader(web_url=\"https://www.example.com\")\n",+ "docs = loader.load()\n",+ "\n",+ "for doc in docs:\n",+ " print(f\"{doc}\\n\")"+ ]+ },{"cell_type": "markdown","id": "ce01aa40",@@ -546,7 +587,7 @@"name": "python","nbconvert_exporter": "python","pygments_lexer": "ipython3",- "version": "3.10.13"+ "version": "3.10.4"}},"nbformat": 4,diff --git a/libs/partners/unstructured/langchain_unstructured/document_loaders.py b/libs/partners/unstructured/langchain_unstructured/document_loaders.pyindex bd3bcd6dbb..aef5135f2e 100644--- a/libs/partners/unstructured/langchain_unstructured/document_loaders.py+++ b/libs/partners/unstructured/langchain_unstructured/document_loaders.py@@ -76,6 +76,25 @@ class UnstructuredLoader(BaseLoader):{'source': './example_data/layout-parser-paper.pdf', 'coordinates': {'points': ((16.34, 213.36), (16.34, 253.36), (36.34, 253.36), (36.34, 213.36)), 'system': 'PixelSpace', 'layout_width': 612, 'layout_height': 792}, 'file_directory': './example_data', 'filename': 'layout-parser-paper.pdf', 'languages': ['eng'], 'last_modified': '2024-07-25T21:28:58', 'page_number': 1, 'filetype': 'application/pdf', 'category': 'UncategorizedText', 'element_id': 'd3ce55f220dfb75891b4394a18bcb973'}+ Load URL:+ .. code-block:: python++ loader = UnstructuredLoader(web_url="https://www.example.com/")+ print(docs[0])++ .. code-block:: none++ page_content='Example Domain' metadata={'category_depth': 0, 'languages': ['eng'], 'filetype': 'text/html', 'url': 'https://www.example.com/', 'category': 'Title', 'element_id': 'fdaa78d856f9d143aeeed85bf23f58f8'}++ .. code-block:: python++ print(docs[1])++ .. code-block:: none++ page_content='This domain is for use in illustrative examples in documents. You may use this domain in literature without prior coordination or asking for permission.' metadata={'languages': ['eng'], 'parent_id': 'fdaa78d856f9d143aeeed85bf23f58f8', 'filetype': 'text/html', 'url': 'https://www.example.com/', 'category': 'NarrativeText', 'element_id': '3652b8458b0688639f973fe36253c992'}++References----------https://docs.unstructured.io/api-reference/api-services/sdk@@ -95,6 +114,7 @@ class UnstructuredLoader(BaseLoader):api_key: Optional[str] = None,client: Optional[UnstructuredClient] = None,url: Optional[str] = None,+ web_url: Optional[str] = None,**kwargs: Any,):"""Initialize loader."""@@ -124,6 +144,8 @@ class UnstructuredLoader(BaseLoader):self.partition_via_api = partition_via_apiself.post_processors = post_processorsself.unstructured_kwargs = kwargs+ if web_url:+ self.unstructured_kwargs["url"] = web_urldef lazy_load(self) -> Iterator[Document]:"""Load file(s) to the _UnstructuredBaseLoader."""diff --git a/libs/partners/unstructured/tests/integration_tests/test_document_loaders.py b/libs/partners/unstructured/tests/integration_tests/test_document_loaders.pyindex f27ddf7186..3b1824f3b4 100644--- a/libs/partners/unstructured/tests/integration_tests/test_document_loaders.py+++ b/libs/partners/unstructured/tests/integration_tests/test_document_loaders.py@@ -128,6 +128,17 @@ def test_loader_partitions_locally_and_applies_post_processors(assert docs[0].page_content.endswith("THE END!")+def test_url_loader() -> None:+ docs = UnstructuredLoader(web_url="https://www.example.com/").load()++ for doc in docs:+ assert doc.page_content+ assert doc.metadata["filetype"] == "text/html"+ assert doc.metadata["url"] == "https://www.example.com/"+ assert doc.metadata["category"]++# -- API partition --