Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(utils): hash raw sitemap content once per source, not per url (#4381)

Robert Sälld2d2ad569afb
diff --git a/packages/utils/src/internals/sitemap.ts b/packages/utils/src/internals/sitemap.ts
index 0959372da..510c7a5e6 100644
--- a/packages/utils/src/internals/sitemap.ts
+++ b/packages/utils/src/internals/sitemap.ts
@@ -384,6 +384,12 @@ export async function* parseSitemap<T extends ParseSitemapOptions>(
// the loop (per-entry warnings could flood the log; individual drops are logged at debug level).
let droppedUrlEntries = 0;
+ // The same for every entry of this source, so compute it once.
+ const originSitemapUrl =
+ source.type === 'url'
+ ? source.url
+ : `raw://${createHash('sha256').update(source.content).digest('base64')}`;
+
const iterator = items[Symbol.asyncIterator]();
try {
while (true) {
@@ -436,10 +442,7 @@ export async function* parseSitemap<T extends ParseSitemapOptions>(
yield {
...item,
- originSitemapUrl:
- source.type === 'url'
- ? source.url
- : `raw://${createHash('sha256').update(source.content).digest('base64')}`,
+ originSitemapUrl,
};
}
}
diff --git a/test/utils/sitemap.test.ts b/test/utils/sitemap.test.ts
index 70c82c6b3..43c401dd3 100644
--- a/test/utils/sitemap.test.ts
+++ b/test/utils/sitemap.test.ts
@@ -1,8 +1,14 @@
+import { createHash } from 'node:crypto';
import { gzipSync } from 'node:zlib';
import { FetchHttpClient } from '@crawlee/http-client';
import nock from 'nock';
-import { beforeEach, describe, expect, it } from 'vitest';
+import { beforeEach, describe, expect, it, vitest } from 'vitest';
+
+vitest.mock('node:crypto', async (importActual) => {
+ const actual = await importActual<typeof import('node:crypto')>();
+ return { ...actual, createHash: vitest.fn(actual.createHash) };
+});
import type { SitemapUrl } from '../../packages/utils/src/internals/sitemap.js';
import { discoverValidSitemaps, parseSitemap, Sitemap } from '../../packages/utils/src/internals/sitemap.js';
@@ -640,6 +646,31 @@ describe('Sitemap', () => {
expect(items[1].priority).toBeUndefined();
});
+ it('hashes raw sitemap content once, not once per url', async () => {
+ vitest.mocked(createHash).mockClear();
+ const locs = Array.from({ length: 50 }, (_, i) => `http://not-exists.com/page-${i}`);
+
+ const items: SitemapUrl[] = [];
+ for await (const item of parseSitemap([
+ {
+ type: 'raw',
+ content: [
+ '<?xml version="1.0" encoding="UTF-8"?>',
+ '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+ ...locs.map((loc) => `<url><loc>${loc}</loc></url>`),
+ '</urlset>',
+ ].join('\n'),
+ },
+ ])) {
+ items.push(item);
+ }
+
+ expect(items.map((item) => item.loc)).toEqual(locs);
+ expect(new Set(items.map((item) => item.originSitemapUrl)).size).toBe(1);
+ expect(items[0].originSitemapUrl).toMatch(/^raw:\/\//);
+ expect(createHash).toHaveBeenCalledTimes(1);
+ });
+
it('loads sitemaps that reference other sitemaps from string', async () => {
const sitemap = await Sitemap.fromXmlString(
[