Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: add Sitemap.tryCommonNames to check well known sitemap locations (#2311)
Jan Buchar85589f167196
diff --git a/packages/utils/src/internals/sitemap.ts b/packages/utils/src/internals/sitemap.tsindex e13c147bd..f652e70e1 100644--- a/packages/utils/src/internals/sitemap.ts+++ b/packages/utils/src/internals/sitemap.ts@@ -114,6 +114,27 @@ export class Sitemap {return parser;}+ /**+ * Try to load sitemap from the most common locations - `/sitemap.xml` and `/sitemap.txt`.+ * For loading based on `Sitemap` entries in `robots.txt`, the {@apilink RobotsFile} class should be used.+ * @param url The domain URL to fetch the sitemap for.+ * @param proxyUrl A proxy to be used for fetching the sitemap file.+ */+ static async tryCommonNames(url: string, proxyUrl?: string): Promise<Sitemap> {+ const sitemapUrls: string[] = [];++ const sitemapUrl = new URL(url);+ sitemapUrl.search = '';++ sitemapUrl.pathname = '/sitemap.xml';+ sitemapUrls.push(sitemapUrl.toString());++ sitemapUrl.pathname = '/sitemap.txt';+ sitemapUrls.push(sitemapUrl.toString());++ return Sitemap.load(sitemapUrls, proxyUrl);+ }+/*** Fetch sitemap content from given URL or URLs and return URLs of referenced pages.* @param urls sitemap URL(s)diff --git a/packages/utils/test/sitemap.test.ts b/packages/utils/test/sitemap.test.tsindex 0a6bea8f2..3e21fcbe9 100644--- a/packages/utils/test/sitemap.test.ts+++ b/packages/utils/test/sitemap.test.ts@@ -65,6 +65,20 @@ describe('Sitemap', () => {'<A HREF="https://ads.google.com/home/">here</A>.','</BODY></HTML>',].join('\n'))+ .get('/sitemap.xml')+ .reply(200, [+ '<?xml version="1.0" encoding="UTF-8"?>',+ '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',+ '<url>',+ '<loc>http://not-exists.com/catalog?item=80&desc=vacation_turkey</loc>',+ '<lastmod>2004-11-23</lastmod>',+ '</url>',+ '<url>',+ '<loc>http://not-exists.com/catalog?item=81&desc=vacation_maledives</loc>',+ '<lastmod>2004-11-23</lastmod>',+ '</url>',+ '</urlset>',+ ].join('\n')).get('/sitemap.txt').reply(200, ['http://not-exists.com/catalog?item=78&desc=vacation_crete',@@ -117,6 +131,16 @@ describe('Sitemap', () => {expect(sitemap.urls).toEqual([]);});+ it('autodetects sitemaps', async () => {+ const sitemap = await Sitemap.tryCommonNames('http://not-exists.com/arbitrary_url?search=xyz');+ expect(new Set(sitemap.urls)).toEqual(new Set([+ 'http://not-exists.com/catalog?item=80&desc=vacation_turkey',+ 'http://not-exists.com/catalog?item=81&desc=vacation_maledives',+ 'http://not-exists.com/catalog?item=78&desc=vacation_crete',+ 'http://not-exists.com/catalog?item=79&desc=vacation_somalia',+ ]));+ });+it('handles sitemap.txt correctly', async () => {const sitemap = await Sitemap.load('http://not-exists.com/sitemap.txt');expect(new Set(sitemap.urls)).toEqual(new Set([