🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix: add max retry limit to _get_page_content to prevent infinite loop
haosenwang1018d3c251c1ab3a
diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.pyindex 1a8d799..1860153 100644--- a/scrapling/engines/toolbelt/convertor.py+++ b/scrapling/engines/toolbelt/convertor.py@@ -187,34 +187,34 @@ class ResponseFactory:return history@classmethod- def _get_page_content(cls, page: SyncPage) -> str:+ def _get_page_content(cls, page: SyncPage, max_retries: int = 10) -> str:"""A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108:param page: The page to extract content from.+ :param max_retries: Maximum number of retry attempts before returning empty string.:return:"""- while True:+ for _ in range(max_retries):try:return page.content() or ""except PlaywrightError:page.wait_for_timeout(500)- continue- return "" # pyright: ignore+ return ""@classmethod- async def _get_async_page_content(cls, page: AsyncPage) -> str:+ async def _get_async_page_content(cls, page: AsyncPage, max_retries: int = 10) -> str:"""A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108:param page: The page to extract content from.+ :param max_retries: Maximum number of retry attempts before returning empty string.:return:"""- while True:+ for _ in range(max_retries):try:return (await page.content()) or ""except PlaywrightError:await page.wait_for_timeout(500)- continue- return "" # pyright: ignore+ return ""@classmethodasync def from_async_playwright_response(