Skip to content

The agent engineering platform.

community: PlaywrightURLLoader should wait for page load event before attempting to extract data (#30043)

Daniel Rauber186cd7f1a1b7
diff --git a/libs/community/langchain_community/document_loaders/url_playwright.py b/libs/community/langchain_community/document_loaders/url_playwright.py
index 3f35061540..8e06ce4be0 100644
--- a/libs/community/langchain_community/document_loaders/url_playwright.py
+++ b/libs/community/langchain_community/document_loaders/url_playwright.py
@@ -177,6 +177,8 @@ class PlaywrightURLLoader(BaseLoader):
if response is None:
raise ValueError(f"page.goto() returned None for url {url}")
+ page.wait_for_load_state("load")
+
text = self.evaluator.evaluate(page, browser, response)
metadata = {"source": url}
yield Document(page_content=text, metadata=metadata)
@@ -216,6 +218,8 @@ class PlaywrightURLLoader(BaseLoader):
if response is None:
raise ValueError(f"page.goto() returned None for url {url}")
+ await page.wait_for_load_state("load")
+
text = await self.evaluator.evaluate_async(page, browser, response)
metadata = {"source": url}
yield Document(page_content=text, metadata=metadata)