Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: using GET for candidate checks - sitemaps (#3464)
nikitachapovskii-devf82eb04673b6
diff --git a/packages/utils/src/internals/sitemap.ts b/packages/utils/src/internals/sitemap.tsindex cc74ca776..d74eda264 100644--- a/packages/utils/src/internals/sitemap.ts+++ b/packages/utils/src/internals/sitemap.ts@@ -546,19 +546,22 @@ export async function* discoverValidSitemaps(} else {const firstUrl = new URL(domainUrls[0]);const possibleSitemapPathnames = ['/sitemap.xml', '/sitemap.txt', '/sitemap_index.xml'];- for (const pathname of possibleSitemapPathnames) {+ const candidateSitemapUrls = possibleSitemapPathnames.map((pathname) => {firstUrl.pathname = pathname;- const candidateSitemapUrl = firstUrl.toString();+ return firstUrl.toString();+ });+ const candidateResults = await Promise.allSettled(candidateSitemapUrls.map(urlExists));- try {- if (await urlExists(candidateSitemapUrl)) {- if (addSitemapUrl(candidateSitemapUrl)) {- yield candidateSitemapUrl;- }+ for (const [index, result] of candidateResults.entries()) {+ const candidateSitemapUrl = candidateSitemapUrls[index];++ if (result.status === 'fulfilled') {+ if (result.value && addSitemapUrl(candidateSitemapUrl)) {+ yield candidateSitemapUrl;}- } catch (err) {+ } else {log.debug(`Failed to check sitemap candidate ${candidateSitemapUrl} for ${hostname}`, {- error: err,+ error: result.reason,});}}diff --git a/packages/utils/test/sitemap.test.ts b/packages/utils/test/sitemap.test.tsindex 52ad9fdeb..2fbc54928 100644--- a/packages/utils/test/sitemap.test.ts+++ b/packages/utils/test/sitemap.test.ts@@ -581,12 +581,14 @@ describe('discoverValidSitemaps', () => {urls.push(url);}- expect(urls).toEqual([- 'http://domain-a.com/sitemap.xml',- 'http://domain-b.com/sitemap.xml',- 'http://domain-a.com/sitemap.txt',- 'http://domain-b.com/sitemap.txt',- ]);+ expect(urls.slice().sort()).toEqual(+ [+ 'http://domain-a.com/sitemap.xml',+ 'http://domain-b.com/sitemap.xml',+ 'http://domain-a.com/sitemap.txt',+ 'http://domain-b.com/sitemap.txt',+ ].sort(),+ );});it('aborts when timeoutMillis elapses', async () => {