Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(utils): drop query string and fragment from guessed sitemap urls (#4132)
Suliman Abdulrazzaq91291b2b443e
diff --git a/packages/utils/src/internals/sitemap.ts b/packages/utils/src/internals/sitemap.tsindex b63349470..6117c657d 100644--- a/packages/utils/src/internals/sitemap.ts+++ b/packages/utils/src/internals/sitemap.ts@@ -653,6 +653,11 @@ export async function* discoverValidSitemaps(}} else {const firstUrl = new URL(domainUrls[0]);+ // The guessed candidates are well-known paths on the same origin, so the query string+ // and fragment of the input URL are not part of them - carrying them over probes (and+ // yields) URLs like `/sitemap.xml?page=2`, which servers may well answer with a 404.+ firstUrl.search = '';+ firstUrl.hash = '';const possibleSitemapPathnames = ['/sitemap.xml', '/sitemap.txt', '/sitemap_index.xml'];const candidateSitemapUrls = possibleSitemapPathnames.map((pathname) => {firstUrl.pathname = pathname;diff --git a/packages/utils/test/sitemap.test.ts b/packages/utils/test/sitemap.test.tsindex fbdd006e5..70cef14f2 100644--- a/packages/utils/test/sitemap.test.ts+++ b/packages/utils/test/sitemap.test.ts@@ -767,6 +767,25 @@ describe('discoverValidSitemaps', () => {expect(urls).toEqual(['http://sitemap-discovery.com/sitemap.xml']);});+ it('probes well-known paths without the query string and fragment of the input url', async () => {+ nock('http://sitemap-discovery.com')+ .get('/robots.txt')+ .reply(404)+ .head('/sitemap.xml')+ .reply(200, '')+ .head('/sitemap.txt')+ .reply(404, '')+ .head('/sitemap_index.xml')+ .reply(404, '');++ const urls = [];+ for await (const url of discoverValidSitemaps(['http://sitemap-discovery.com/products?page=2#top'])) {+ urls.push(url);+ }++ expect(urls).toEqual(['http://sitemap-discovery.com/sitemap.xml']);+ });+it('extracts sitemaps from multiple domains with mixed order', async () => {nock('http://domain-a.com').get('/robots.txt')