Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(playwright): silence transient errors in `enqueueLinksByClickingElements` (#3732)
Gil Cohen7b98cc350f8e
diff --git a/packages/playwright-crawler/src/internals/enqueue-links/click-elements.ts b/packages/playwright-crawler/src/internals/enqueue-links/click-elements.tsindex bd5acf278..78e7c4059 100644--- a/packages/playwright-crawler/src/internals/enqueue-links/click-elements.ts+++ b/packages/playwright-crawler/src/internals/enqueue-links/click-elements.ts@@ -352,7 +352,7 @@ export async function clickElementsAndInterceptNavigationRequests(// browser.off(BrowserEmittedEvents.TargetCreated, onTargetCreated);page.off('framenavigated', onFrameNavigated);- await context.unroute('*', onInterceptedRequest);+ await context.unroute('**', onInterceptedRequest);const serializedRequests = Array.from(uniqueRequests);return serializedRequests.map((r) => JSON.parse(r));@@ -405,7 +405,15 @@ function createTargetCreatedHandler(requests: Set<string>): (popup: Page) => Pro* @ignore*/function isTopFrameNavigationRequest(page: Page, req: Request): boolean {- return req.isNavigationRequest() && req.frame() === page.mainFrame();+ try {+ return req.isNavigationRequest() && req.frame() === page.mainFrame();+ } catch {+ // `req.frame()` throws when the owning frame is unavailable - e.g. the request was+ // issued by a service worker, or before/after its frame existed (see #3216). Such a+ // request is not a top-frame navigation, so swallow the throw and let it pass through+ // instead of crashing the route handler (which would leave the route unhandled).+ return false;+ }}/**