Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix: add max retry limit to _get_page_content to prevent infinite loop

haosenwang1018d3c251c1ab3a
diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py
index 1a8d799..1860153 100644
--- a/scrapling/engines/toolbelt/convertor.py
+++ b/scrapling/engines/toolbelt/convertor.py
@@ -187,34 +187,34 @@ class ResponseFactory:
return history
@classmethod
- def _get_page_content(cls, page: SyncPage) -> str:
+ def _get_page_content(cls, page: SyncPage, max_retries: int = 10) -> str:
"""
A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108
:param page: The page to extract content from.
+ :param max_retries: Maximum number of retry attempts before returning empty string.
:return:
"""
- while True:
+ for _ in range(max_retries):
try:
return page.content() or ""
except PlaywrightError:
page.wait_for_timeout(500)
- continue
- return "" # pyright: ignore
+ return ""
@classmethod
- async def _get_async_page_content(cls, page: AsyncPage) -> str:
+ async def _get_async_page_content(cls, page: AsyncPage, max_retries: int = 10) -> str:
"""
A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108
:param page: The page to extract content from.
+ :param max_retries: Maximum number of retry attempts before returning empty string.
:return:
"""
- while True:
+ for _ in range(max_retries):
try:
return (await page.content()) or ""
except PlaywrightError:
await page.wait_for_timeout(500)
- continue
- return "" # pyright: ignore
+ return ""
@classmethod
async def from_async_playwright_response(