Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: Fix rebase conflict resolution errors
Jan Buchar94603bbf97ec
diff --git a/packages/memory-storage/package.json b/packages/memory-storage/package.jsonindex 40b5ee42b..705f15b73 100644--- a/packages/memory-storage/package.json+++ b/packages/memory-storage/package.json@@ -51,6 +51,7 @@"json5": "^2.2.3","mime-types": "^3.0.1","proper-lockfile": "^4.1.2",+ "p-limit": "^6.2.0","tslib": "^2.8.1"}}diff --git a/packages/utils/src/internals/sitemap.ts b/packages/utils/src/internals/sitemap.tsindex 9067e66c5..b2e5f04a1 100644--- a/packages/utils/src/internals/sitemap.ts+++ b/packages/utils/src/internals/sitemap.ts@@ -617,6 +617,7 @@ export async function* discoverValidSitemaps(proxyUrl,timeoutMillis: requestTimeoutMillis,signal,+ httpClient,});// Surface all referenced sitemaps, including cross-host; scoping happens at load time.for (const sitemapUrl of robotsFile.getSitemaps({ enqueueStrategy: 'all' })) {