Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: Sitemap-based request list implementation (#2498)
Jan Buchar7bf8f0bcd4cc
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 61b10b743..5d6f12c25 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -14,9 +14,9 @@ import type {DatasetExportOptions,FinalStatistics,GetUserDataFromRequest,+ IRequestList,ProxyInfo,Request,- RequestList,RequestOptions,RequestProvider,RouterHandler,@@ -171,7 +171,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw* > Alternatively, `requests` parameter of {@apilink BasicCrawler.run|`crawler.run()`} could be used to enqueue the initial requests -* it is a shortcut for running `crawler.addRequests()` before the `crawler.run()`.*/- requestList?: RequestList;+ requestList?: IRequestList;/*** Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.@@ -445,7 +445,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext* A reference to the underlying {@apilink RequestList} class that manages the crawler's {@apilink Request|requests}.* Only available if used by the crawler.*/- requestList?: RequestList;+ requestList?: IRequestList;/*** Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.@@ -1170,7 +1170,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext* adding it back to the queue after the timeout passes. Returns `true` if the request* should be ignored and will be reclaimed to the queue once ready.*/- protected delayRequest(request: Request, source: RequestList | RequestProvider) {+ protected delayRequest(request: Request, source: IRequestList | RequestProvider) {const domain = getDomain(request.url);if (!domain || !request) {@@ -1415,7 +1415,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextprotected async _requestFunctionErrorHandler(error: Error,crawlingContext: Context,- source: RequestList | RequestProvider,+ source: IRequestList | RequestProvider,): Promise<void> {const { request } = crawlingContext;request.pushErrorMessage(error);diff --git a/packages/core/src/storages/index.ts b/packages/core/src/storages/index.tsindex fd8260f73..60ef3bc48 100644--- a/packages/core/src/storages/index.ts+++ b/packages/core/src/storages/index.ts@@ -8,3 +8,4 @@ export { RequestQueue as RequestQueueV2 } from './request_queue_v2';export * from './storage_manager';export * from './utils';export * from './access_checking';+export * from './sitemap_request_list';diff --git a/packages/core/src/storages/request_list.ts b/packages/core/src/storages/request_list.tsindex ea85ac37c..5f4b90806 100644--- a/packages/core/src/storages/request_list.ts+++ b/packages/core/src/storages/request_list.ts@@ -20,6 +20,80 @@ export const REQUESTS_PERSISTENCE_KEY = 'REQUEST_LIST_REQUESTS';const CONTENT_TYPE_BINARY = 'application/octet-stream';+/**+ * Represents a static list of URLs to crawl.+ */+export interface IRequestList {+ /**+ * Returns the total number of unique requests present in the list.+ */+ length(): number;++ /**+ * Returns `true` if all requests were already handled and there are no more left.+ */+ isFinished(): Promise<boolean>;++ /**+ * Resolves to `true` if the next call to {@apilink IRequestList.fetchNextRequest} function+ * would return `null`, otherwise it resolves to `false`.+ * Note that even if the list is empty, there might be some pending requests currently being processed.+ */+ isEmpty(): Promise<boolean>;++ /**+ * Returns number of handled requests.+ */+ handledCount(): number;++ /**+ * Persists the current state of the `IRequestList` into the default {@apilink KeyValueStore}.+ * The state is persisted automatically in regular intervals, but calling this method manually+ * is useful in cases where you want to have the most current state available after you pause+ * or stop fetching its requests. For example after you pause or abort a crawl. Or just before+ * a server migration.+ */+ persistState(): Promise<void>;++ /**+ * Gets the next {@apilink Request} to process. First, the function gets a request previously reclaimed+ * using the {@apilink RequestList.reclaimRequest} function, if there is any.+ * Otherwise it gets the next request from sources.+ *+ * The function's `Promise` resolves to `null` if there are no more+ * requests to process.+ */+ fetchNextRequest(): Promise<Request | null>;++ /**+ * Gets the next {@apilink Request} to process. First, the function gets a request previously reclaimed+ * using the {@apilink RequestList.reclaimRequest} function, if there is any.+ * Otherwise it gets the next request from sources.+ *+ * The function resolves to `null` if there are no more requests to process.+ *+ * Can be used to iterate over the `RequestList` instance in a `for await .. of` loop.+ * Provides an alternative for the repeated use of `fetchNextRequest`.+ */+ [Symbol.asyncIterator](): AsyncGenerator<Request>;++ /**+ * Reclaims request to the list if its processing failed.+ * The request will become available in the next `this.fetchNextRequest()`.+ */+ reclaimRequest(request: Request): Promise<void>;++ /**+ * Marks request as handled after successful processing.+ */+ markRequestHandled(request: Request): Promise<void>;++ /**+ * @internal+ */+ inProgress: Set<string>;+}+export interface RequestListOptions {/*** An array of sources of URLs for the {@apilink RequestList}. It can be either an array of strings,@@ -229,7 +303,7 @@ export interface RequestListOptions {* ```* @category Sources*/-export class RequestList {+export class RequestList implements IRequestList {private log = log.child({ prefix: 'RequestList' });/**@@ -431,11 +505,7 @@ export class RequestList {}/**- * Persists the current state of the `RequestList` into the default {@apilink KeyValueStore}.- * The state is persisted automatically in regular intervals, but calling this method manually- * is useful in cases where you want to have the most current state available after you pause- * or stop fetching its requests. For example after you pause or abort a crawl. Or just before- * a server migration.+ * @inheritDoc*/async persistState(): Promise<void> {if (!this.persistStateKey) {@@ -570,9 +640,7 @@ export class RequestList {}/**- * Resolves to `true` if the next call to {@apilink RequestList.fetchNextRequest} function- * would return `null`, otherwise it resolves to `false`.- * Note that even if the list is empty, there might be some pending requests currently being processed.+ * @inheritDoc*/async isEmpty(): Promise<boolean> {this._ensureIsInitialized();@@ -581,7 +649,7 @@ export class RequestList {}/**- * Returns `true` if all requests were already handled and there are no more left.+ * @inheritDoc*/async isFinished(): Promise<boolean> {this._ensureIsInitialized();@@ -590,12 +658,7 @@ export class RequestList {}/**- * Gets the next {@apilink Request} to process. First, the function gets a request previously reclaimed- * using the {@apilink RequestList.reclaimRequest} function, if there is any.- * Otherwise it gets the next request from sources.- *- * The function's `Promise` resolves to `null` if there are no more- * requests to process.+ * @inheritDoc*/async fetchNextRequest(): Promise<Request | null> {this._ensureIsInitialized();@@ -621,6 +684,17 @@ export class RequestList {return null;}+ /**+ * @inheritDoc+ */+ async *[Symbol.asyncIterator]() {+ while (true) {+ const req = await this.fetchNextRequest();+ if (!req) break;+ yield req;+ }+ }+private ensureRequest(requestLike: Request | RequestOptions, index: number): Request {if (requestLike instanceof Request) {return requestLike;@@ -631,7 +705,7 @@ export class RequestList {}/**- * Marks request as handled after successful processing.+ * @inheritDoc*/async markRequestHandled(request: Request): Promise<void> {const { uniqueKey } = request;@@ -645,8 +719,7 @@ export class RequestList {}/**- * Reclaims request to the list if its processing failed.- * The request will become available in the next `this.fetchNextRequest()`.+ * @inheritDoc*/async reclaimRequest(request: Request): Promise<void> {const { uniqueKey } = request;@@ -798,7 +871,7 @@ export class RequestList {}/**- * Returns number of handled requests.+ * @inheritDoc*/handledCount(): number {this._ensureIsInitialized();diff --git a/packages/core/src/storages/sitemap_request_list.ts b/packages/core/src/storages/sitemap_request_list.tsnew file mode 100644index 000000000..b7f85900f--- /dev/null+++ b/packages/core/src/storages/sitemap_request_list.ts@@ -0,0 +1,430 @@+import { Transform } from 'node:stream';++import defaultLog from '@apify/log';+import { parseSitemap } from '@crawlee/utils';+import ow from 'ow';++import { KeyValueStore } from './key_value_store';+import type { IRequestList } from './request_list';+import { purgeDefaultStorages } from './utils';+import { Request } from '../request';++export interface SitemapRequestListOptions {+ /**+ * List of sitemap URLs to parse.+ */+ sitemapUrls: string[];+ /**+ * Proxy URL to be used for sitemap loading.+ */+ proxyUrl?: string;+ /**+ * Key for persisting the state of the request list in the `KeyValueStore`.+ */+ persistStateKey?: string;+ /**+ * Abort signal to be used for sitemap loading.+ */+ signal?: AbortSignal;+ /**+ * Timeout for sitemap loading in milliseconds. If both `signal` and `timeoutMillis` are provided, either of them can abort the loading.+ */+ timeoutMillis?: number;+ /**+ * Maximum number of buffered URLs for the sitemap loading stream.+ * If the buffer is full, the stream will pause until the buffer is drained.+ *+ * @default 200+ */+ maxBufferSize?: number;+}++interface SitemapParsingProgress {+ inProgressSitemapUrl: string | null;+ inProgressEntries: Set<string>;+ pendingSitemapUrls: Set<string>;+}++interface SitemapRequestListState {+ urlQueue: string[];+ reclaimed: string[];+ sitemapParsingProgress: Record<keyof SitemapParsingProgress, any>;+ abortLoading: boolean;+ requestData: [string, Request][];+}++/**+ * A list of URLs to crawl parsed from a sitemap.+ *+ * The loading of the sitemap is performed in the background so that crawling can start before the sitemap is fully loaded.+ */+export class SitemapRequestList implements IRequestList {+ /**+ * Set of URLs that were returned by `fetchNextRequest()` and not marked as handled yet.+ * @internal+ */+ inProgress = new Set<string>();++ /** Set of URLs for which `reclaimRequest()` was called. */+ private reclaimed = new Set<string>();++ /**+ * Map of returned Request objects that have not been marked as handled yet.+ *+ * We use this to persist custom user fields on the in-progress (or reclaimed) requests.+ */+ private requestData = new Map<string, Request>();++ /**+ * Object for keeping track of the sitemap parsing progress.+ */+ private sitemapParsingProgress: SitemapParsingProgress = {+ /**+ * URL of the sitemap that is currently being parsed. `null` if no sitemap is being parsed.+ */+ inProgressSitemapUrl: null,+ /**+ * Buffer for URLs from the currently parsed sitemap. Used for tracking partially loaded sitemaps across migrations.+ */+ inProgressEntries: new Set<string>(),+ /**+ * Set of sitemap URLs that have not been fully parsed yet. If the set is empty and `inProgressSitemapUrl` is `null`, the sitemap loading is finished.+ */+ pendingSitemapUrls: new Set<string>(),+ };++ /**+ * Object stream of URLs parsed from the sitemaps.+ * Using `highWaterMark`, this can manage the speed of the sitemap loading.+ *+ * Fetch the next URL to be processed using `fetchNextRequest()`.+ */+ private urlQueueStream: Transform;++ /**+ * Indicates whether the request list sitemap loading was aborted.+ *+ * If the loading was aborted before the sitemaps were fully loaded, the request list might be missing some URLs.+ * The `isSitemapFullyLoaded` method can be used to check if the sitemaps were fully loaded.+ *+ * If the loading is aborted and all the requests are handled, `isFinished()` will return `true`.+ */+ private abortLoading: boolean = false;++ /** Number of URLs that were marked as handled */+ private handledUrlCount = 0;++ private persistStateKey?: string;++ private store?: KeyValueStore;++ /**+ * Proxy URL to be used for sitemap loading.+ */+ private proxyUrl: string | undefined;++ /**+ * Logger instance.+ */+ private log = defaultLog.child({ prefix: 'SitemapRequestList' });++ /** @internal */+ private constructor(options: SitemapRequestListOptions) {+ ow(+ options,+ ow.object.exactShape({+ sitemapUrls: ow.array.ofType(ow.string),+ proxyUrl: ow.optional.string,+ persistStateKey: ow.optional.string,+ signal: ow.optional.any(),+ timeoutMillis: ow.optional.number,+ maxBufferSize: ow.optional.number,+ }),+ );++ this.persistStateKey = options.persistStateKey;+ this.proxyUrl = options.proxyUrl;++ this.urlQueueStream = new Transform({+ objectMode: true,+ highWaterMark: options.maxBufferSize ?? 200,+ });+ this.urlQueueStream.pause();++ this.sitemapParsingProgress.pendingSitemapUrls = new Set(options.sitemapUrls);+ }++ /**+ * Adds a URL to the queue of parsed URLs.+ *+ * Blocks if the stream is full until it is drained.+ */+ private async pushNextUrl(url: string | null) {+ return new Promise<void>((resolve) => {+ if (!this.urlQueueStream.push(url)) {+ // This doesn't work with the 'drain' event (it's not emitted for some reason).+ this.urlQueueStream.once('readdata', () => {+ resolve();+ });+ } else {+ resolve();+ }+ });+ }++ /**+ * Reads the next URL from the queue of parsed URLs.+ *+ * If the stream is empty, blocks until a new URL is pushed.+ * @returns The next URL from the queue or `null` if we have read all URLs.+ */+ private async readNextUrl(): Promise<string | null> {+ return new Promise((resolve) => {+ const result = this.urlQueueStream.read();++ if (!result && !this.isSitemapFullyLoaded()) {+ this.urlQueueStream.once('readable', () => {+ const nextUrl = this.urlQueueStream.read();+ resolve(nextUrl);+ });+ } else {+ resolve(result);+ }+ this.urlQueueStream.emit('readdata');+ });+ }++ /**+ * Indicates whether the background processing of sitemap contents has successfully finished.+ *+ * If this is `false`, the background processing is either still in progress or was aborted.+ */+ isSitemapFullyLoaded(): boolean {+ return (+ this.sitemapParsingProgress.inProgressSitemapUrl === null &&+ this.sitemapParsingProgress.pendingSitemapUrls.size === 0+ );+ }++ /**+ * Start processing the sitemaps and loading the URLs.+ *+ * Resolves once all the sitemaps URLs have been fully loaded (sets `isSitemapFullyLoaded` to `true`).+ */+ private async load(): Promise<void> {+ while (!this.isSitemapFullyLoaded() && !this.abortLoading) {+ const sitemapUrl =+ this.sitemapParsingProgress.inProgressSitemapUrl ??+ this.sitemapParsingProgress.pendingSitemapUrls.values().next().value;++ try {+ for await (const item of parseSitemap([{ type: 'url', url: sitemapUrl }], this.proxyUrl, {+ maxDepth: 0,+ emitNestedSitemaps: true,+ })) {+ if (!item.originSitemapUrl) {+ // This is a nested sitemap+ this.sitemapParsingProgress.pendingSitemapUrls.add(item.loc);+ continue;+ }++ if (!this.sitemapParsingProgress.inProgressEntries.has(item.loc)) {+ await this.pushNextUrl(item.loc);+ this.sitemapParsingProgress.inProgressEntries.add(item.loc);+ }+ }+ } catch (e: any) {+ this.log.error('Error loading sitemap contents:', e);+ }++ this.sitemapParsingProgress.pendingSitemapUrls.delete(sitemapUrl);+ this.sitemapParsingProgress.inProgressEntries.clear();+ this.sitemapParsingProgress.inProgressSitemapUrl = null;+ }++ await this.pushNextUrl(null);+ }++ /**+ * Open a sitemap and start processing it.+ *+ * Resolves to a new instance of `SitemapRequestList`, which **might not be fully loaded yet** - i.e. the sitemap might still be loading in the background.+ *+ * Track the loading progress using the `isSitemapFullyLoaded` property.+ */+ static async open(options: SitemapRequestListOptions): Promise<SitemapRequestList> {+ const requestList = new SitemapRequestList(options);+ await requestList.restoreState();+ void requestList.load();++ options?.signal?.addEventListener('abort', () => {+ requestList.abortLoading = true;+ });++ if (options.timeoutMillis) {+ setTimeout(() => {+ requestList.abortLoading = true;+ }, options.timeoutMillis);+ }++ return requestList;+ }++ /**+ * @inheritDoc+ */+ length(): number {+ return this.urlQueueStream.readableLength + this.handledUrlCount - this.inProgress.size - this.reclaimed.size;+ }++ /**+ * @inheritDoc+ */+ async isFinished(): Promise<boolean> {+ return (+ (await this.isEmpty()) && this.inProgress.size === 0 && (this.isSitemapFullyLoaded() || this.abortLoading)+ );+ }++ /**+ * @inheritDoc+ */+ async isEmpty(): Promise<boolean> {+ return this.reclaimed.size === 0 && this.urlQueueStream.readableLength === 0;+ }++ /**+ * @inheritDoc+ */+ handledCount(): number {+ return this.handledUrlCount;+ }++ /**+ * @inheritDoc+ */+ async persistState(): Promise<void> {+ if (this.persistStateKey === undefined) {+ return;+ }++ this.store ??= await KeyValueStore.open();++ const urlQueue = [];++ while (this.urlQueueStream.readableLength > 0) {+ const url = this.urlQueueStream.read();+ if (url === null) {+ break;+ }+ urlQueue.push(url);+ }++ for (const url of urlQueue) {+ this.urlQueueStream.push(url);+ }++ await this.store.setValue(this.persistStateKey, {+ sitemapParsingProgress: {+ pendingSitemapUrls: Array.from(this.sitemapParsingProgress.pendingSitemapUrls),+ inProgressSitemapUrl: this.sitemapParsingProgress.inProgressSitemapUrl,+ inProgressEntries: Array.from(this.sitemapParsingProgress.inProgressEntries),+ },+ urlQueue,+ reclaimed: [...this.inProgress, ...this.reclaimed], // In-progress and reclaimed requests will be both retried if state is restored+ requestData: Array.from(this.requestData.entries()),+ abortLoading: this.abortLoading,+ } satisfies SitemapRequestListState);+ }++ private async restoreState(): Promise<void> {+ await purgeDefaultStorages({ onlyPurgeOnce: true });++ if (this.persistStateKey === undefined) {+ return;+ }++ this.store ??= await KeyValueStore.open();+ const state = await this.store.getValue<SitemapRequestListState>(this.persistStateKey);++ if (state === null) {+ return;+ }++ this.reclaimed = new Set(state.reclaimed);+ this.sitemapParsingProgress = {+ pendingSitemapUrls: new Set(state.sitemapParsingProgress.pendingSitemapUrls),+ inProgressSitemapUrl: state.sitemapParsingProgress.inProgressSitemapUrl,+ inProgressEntries: new Set(state.sitemapParsingProgress.inProgressEntries),+ };++ this.requestData = new Map(state.requestData ?? []);++ for (const url of state.urlQueue) {+ this.urlQueueStream.push(url);+ }++ this.abortLoading = state.abortLoading;+ }++ /**+ * @inheritDoc+ */+ async fetchNextRequest(): Promise<Request | null> {+ // Try to return a reclaimed request first+ let nextUrl: string | null = this.reclaimed.values().next().value;+ if (nextUrl) {+ this.reclaimed.delete(nextUrl);+ } else {+ // Otherwise read next url from the stream+ nextUrl = await this.readNextUrl();+ if (!nextUrl) {+ return null;+ }+ this.requestData.set(nextUrl, new Request({ url: nextUrl }));+ }++ this.inProgress.add(nextUrl);+ return this.requestData.get(nextUrl)!;+ }++ /**+ * @inheritDoc+ */+ async *[Symbol.asyncIterator]() {+ while ((!this.isSitemapFullyLoaded() && !this.abortLoading) || !(await this.isEmpty())) {+ const request = await this.fetchNextRequest();+ if (!request) break;++ yield request;+ }+ }++ /**+ * @inheritDoc+ */+ async reclaimRequest(request: Request): Promise<void> {+ this.ensureInProgressAndNotReclaimed(request.url);+ this.reclaimed.add(request.url);+ this.inProgress.delete(request.url);+ }++ /**+ * @inheritDoc+ */+ async markRequestHandled(request: Request): Promise<void> {+ this.handledUrlCount += 1;+ this.ensureInProgressAndNotReclaimed(request.url);+ this.inProgress.delete(request.url);+ this.requestData.delete(request.url);+ }++ private ensureInProgressAndNotReclaimed(url: string): void {+ if (!this.inProgress.has(url)) {+ throw new Error(`The request is not being processed (url: ${url})`);+ }+ if (this.reclaimed.has(url)) {+ throw new Error(`The request was already reclaimed (url: ${url})`);+ }+ }+}diff --git a/packages/utils/package.json b/packages/utils/package.jsonindex a4d4fcb43..f672ebdba 100644--- a/packages/utils/package.json+++ b/packages/utils/package.json@@ -56,7 +56,7 @@"got-scraping": "^4.0.3","ow": "^0.28.1","robots-parser": "^3.0.1",- "sax": "^1.3.0",+ "sax": "^1.4.1","tslib": "^2.4.0","whatwg-mimetype": "^4.0.0"},diff --git a/packages/utils/src/internals/sitemap.ts b/packages/utils/src/internals/sitemap.tsindex c3b68db61..eee26c869 100644--- a/packages/utils/src/internals/sitemap.ts+++ b/packages/utils/src/internals/sitemap.ts@@ -1,36 +1,48 @@-import { Readable, Writable } from 'node:stream';+import { createHash } from 'node:crypto';+import type { Duplex } from 'node:stream';+import { PassThrough, Readable, Transform, pipeline } from 'node:stream';import { StringDecoder } from 'node:string_decoder';import { createGunzip } from 'node:zlib';import log from '@apify/log';-import type { SAXStream } from 'sax';import sax from 'sax';import MIMEType from 'whatwg-mimetype';-type SitemapSource = { type: 'url'; url: string } | { type: 'raw'; content: string };+interface SitemapUrlData {+ loc: string;+ lastmod?: Date;+ changefreq?: 'always' | 'hourly' | 'daily' | 'weekly' | 'monthly' | 'yearly' | 'never';+ priority?: number;+}-class ParsingState {- sources: SitemapSource[] = [];- urls: string[] = [];- visitedSitemapUrls: string[] = [];- context?: 'sitemapindex' | 'urlset';- loc = false;+export type SitemapUrl = SitemapUrlData & {+ originSitemapUrl: string;+};- resetContext() {- this.context = undefined;- this.loc = false;- }+interface NestedSitemap {+ loc: string;+ originSitemapUrl: null;}-class SitemapTxtParser extends Writable {+type SitemapSource = ({ type: 'url'; url: string } | { type: 'raw'; content: string }) & { depth?: number };+type SitemapItem = ({ type: 'url' } & SitemapUrlData) | { type: 'sitemapUrl'; url: string };++class SitemapTxtParser extends Transform {private decoder: StringDecoder = new StringDecoder('utf8');private buffer: string = '';- constructor(- private parsingState: ParsingState,- private onEnd: () => void,- ) {- super();+ constructor() {+ super({+ readableObjectMode: true,+ transform: (chunk, _encoding, callback) => {+ this.processBuffer(this.decoder.write(chunk), false);+ callback();+ },+ flush: (callback) => {+ this.processBuffer(this.decoder.end(), true);+ callback();+ },+ });}private processBuffer(input: string, finalize: boolean): void {@@ -43,24 +55,249 @@ class SitemapTxtParser extends Writable {.filter((part) => part.length > 0);if (finalize) {- this.parsingState.urls.push(...parts);+ for (const url of parts) {+ this.push({ type: 'url', loc: url } satisfies SitemapItem);+ }+this.buffer = '';} else if (parts.length > 0) {- this.parsingState.urls.push(...parts.slice(0, -1));+ for (const url of parts.slice(0, -1)) {+ this.push({ type: 'url', loc: url } satisfies SitemapItem);+ }+this.buffer = parts.at(-1)!;}}}+}- override _write(chunk: any, _encoding: BufferEncoding, callback: (error?: Error | null | undefined) => void): void {- this.processBuffer(this.decoder.write(chunk), false);- callback();+class SitemapXmlParser extends Transform {+ private decoder: StringDecoder = new StringDecoder('utf8');+ private parser = new sax.SAXParser(true);++ private rootTagName?: 'sitemapindex' | 'urlset';+ private currentTag?: 'loc' | 'lastmod' | 'changefreq' | 'priority' = undefined;+ private url: Partial<SitemapUrl> = {};++ constructor() {+ super({+ readableObjectMode: true,+ transform: (chunk, _encoding, callback) => {+ this.parser.write(this.decoder.write(chunk));+ callback();+ },+ flush: (callback) => {+ const rest = this.decoder.end();+ if (rest.length > 0) {+ this.parser.write(rest);+ }++ this.parser.end();+ callback();+ },+ });++ this.parser.onopentag = this.onOpenTag.bind(this);+ this.parser.onclosetag = this.onCloseTag.bind(this);++ this.parser.ontext = this.onText.bind(this);+ this.parser.oncdata = this.onText.bind(this);++ this.parser.onerror = this.destroy.bind(this);}- override _final(callback: (error?: Error | null | undefined) => void): void {- this.processBuffer(this.decoder.end(), true);- callback();- this.onEnd();+ private onOpenTag(node: sax.Tag | sax.QualifiedTag) {+ if (this.rootTagName !== undefined) {+ if (+ node.name === 'loc' ||+ node.name === 'lastmod' ||+ node.name === 'priority' ||+ node.name === 'changefreq'+ ) {+ this.currentTag = node.name;+ }+ }+ if (node.name === 'urlset') {+ this.rootTagName = 'urlset';+ }+ if (node.name === 'sitemapindex') {+ this.rootTagName = 'sitemapindex';+ }+ }++ private onCloseTag(name: string) {+ if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {+ this.currentTag = undefined;+ }++ if (name === 'url' && this.url.loc !== undefined) {+ this.push({ type: 'url', ...this.url, loc: this.url.loc } satisfies SitemapItem);+ this.url = {};+ }+ }++ private onText(text: string) {+ if (this.currentTag === 'loc') {+ if (this.rootTagName === 'sitemapindex') {+ this.push({ type: 'sitemapUrl', url: text } satisfies SitemapItem);+ }++ if (this.rootTagName === 'urlset') {+ this.url ??= {};+ this.url.loc = text;+ }+ }++ text = text.trim();++ if (this.currentTag === 'lastmod') {+ this.url.lastmod = new Date(text);+ }++ if (this.currentTag === 'priority') {+ this.url.priority = Number(text);+ }++ if (this.currentTag === 'changefreq') {+ if (['always', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'never'].includes(text)) {+ this.url.changefreq = text as SitemapUrl['changefreq'];+ }+ }+ }+}++interface ParseSitemapOptions {+ /**+ * If set to `true`, elements reffering to other sitemaps will be emitted as special objects with a `bouba` property.+ */+ emitNestedSitemaps?: true | false;+ /**+ * Maximum depth of nested sitemaps to follow.+ */+ maxDepth?: number;+}++export async function* parseSitemap<T extends ParseSitemapOptions>(+ initialSources: SitemapSource[],+ proxyUrl?: string,+ options?: T,+): AsyncIterable<T['emitNestedSitemaps'] extends true ? SitemapUrl | NestedSitemap : SitemapUrl> {+ const { gotScraping } = await import('got-scraping');+ const { fileTypeStream } = await import('file-type');++ const sources = [...initialSources];+ const visitedSitemapUrls = new Set<string>();++ const createParser = (contentType: string = '', url?: URL): Duplex => {+ let mimeType: MIMEType | null;++ try {+ mimeType = new MIMEType(contentType);+ } catch (e) {+ mimeType = null;+ }++ if (mimeType?.isXML() || url?.pathname.endsWith('.xml')) {+ return new SitemapXmlParser();+ }++ if (mimeType?.essence === 'text/plain' || url?.pathname.endsWith('.txt')) {+ return new SitemapTxtParser();+ }++ throw new Error(`Unsupported sitemap content type (contentType = ${contentType}, url = ${url?.toString()})`);+ };++ while (sources.length > 0) {+ const source = sources.shift()!;++ if ((source?.depth ?? 0) > (options?.maxDepth ?? Infinity)) {+ log.debug(+ `Skipping sitemap ${source.type === 'url' ? source.url : ''} because it reached max depth ${options!.maxDepth!}.`,+ );+ continue;+ }++ let items: AsyncIterable<SitemapItem> | null = null;++ if (source.type === 'url') {+ const sitemapUrl = new URL(source.url);+ visitedSitemapUrls.add(sitemapUrl.toString());++ try {+ const sitemapStream = await new Promise<ReturnType<typeof gotScraping.stream>>((resolve, reject) => {+ const request = gotScraping.stream({ url: sitemapUrl, proxyUrl, method: 'GET' });+ request.on('response', () => resolve(request));+ request.on('error', reject);+ });++ if (sitemapStream.response!.statusCode === 200) {+ let contentType = sitemapStream.response!.headers['content-type'];++ const streamWithType = await fileTypeStream(sitemapStream);+ if (streamWithType.fileType !== undefined) {+ contentType = streamWithType.fileType.mime;+ }++ let isGzipped = false;++ if (+ contentType !== undefined+ ? contentType === 'application/gzip'+ : sitemapUrl.pathname.endsWith('.gz')+ ) {+ isGzipped = true;++ if (sitemapUrl.pathname.endsWith('.gz')) {+ sitemapUrl.pathname = sitemapUrl.pathname.substring(0, sitemapUrl.pathname.length - 3);+ }+ }++ items = pipeline(+ streamWithType,+ isGzipped ? createGunzip() : new PassThrough(),+ createParser(contentType, sitemapUrl),+ (error) => {+ if (error !== undefined) {+ log.warning(`Malformed sitemap content: ${sitemapUrl}, ${error}`);+ }+ },+ );+ }+ } catch (e) {+ log.warning(`Malformed sitemap content: ${sitemapUrl}, ${e}`);+ }+ } else if (source.type === 'raw') {+ items = pipeline(Readable.from([source.content]), createParser('text/xml'), (error) => {+ if (error !== undefined) {+ log.warning(`Malformed sitemap content: ${error}`);+ }+ });+ }++ if (items === null) {+ continue;+ }++ for await (const item of items) {+ if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {+ sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });+ if (options?.emitNestedSitemaps) {+ // @ts-ignore+ yield { loc: item.url, originSitemapUrl: null };+ }+ }++ if (item.type === 'url') {+ yield {+ ...item,+ originSitemapUrl:+ source.type === 'url'+ ? source.url+ : `raw://${createHash('sha256').update(source.content).digest('base64')}`,+ };+ }+ }}}@@ -79,52 +316,6 @@ class SitemapTxtParser extends Writable {export class Sitemap {constructor(readonly urls: string[]) {}- protected static createXmlParser(- parsingState: ParsingState,- onEnd: () => void,- onError: (error: Error) => void,- ): SAXStream {- const parser = sax.createStream(true);-- parser.on('opentag', (node) => {- if (node.name === 'loc' && parsingState.context !== undefined) {- parsingState.loc = true;- }- if (node.name === 'urlset') {- parsingState.context = 'urlset';- }- if (node.name === 'sitemapindex') {- parsingState.context = 'sitemapindex';- }- });-- parser.on('closetag', (name) => {- if (name === 'loc') {- parsingState.loc = false;- }- });-- const onText = (text: string) => {- if (parsingState.loc) {- if (parsingState.context === 'sitemapindex') {- if (!parsingState.visitedSitemapUrls.includes(text)) {- parsingState.sources.push({ type: 'url', url: text });- }- }- if (parsingState.context === 'urlset') {- parsingState.urls.push(text);- }- }- };- parser.on('text', onText);- parser.on('cdata', onText);-- parser.on('end', onEnd);- parser.on('error', onError);-- return parser;- }-/*** Try to load sitemap from the most common locations - `/sitemap.xml` and `/sitemap.txt`.* For loading based on `Sitemap` entries in `robots.txt`, the {@apilink RobotsFile} class should be used.@@ -152,10 +343,10 @@ export class Sitemap {* @param proxyUrl URL of a proxy to be used for fetching sitemap contents*/static async load(urls: string | string[], proxyUrl?: string): Promise<Sitemap> {- const parsingState = new ParsingState();- parsingState.sources = (Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url }));-- return await this.parse(parsingState, proxyUrl);+ return await this.parse(+ (Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })),+ proxyUrl,+ );}/**@@ -164,100 +355,20 @@ export class Sitemap {* @param proxyUrl URL of a proxy to be used for fetching sitemap contents*/static async fromXmlString(content: string, proxyUrl?: string): Promise<Sitemap> {- const parsingState = new ParsingState();- parsingState.sources = [{ type: 'raw', content }];-- return await this.parse(parsingState, proxyUrl);- }-- protected static async parse(parsingState: ParsingState, proxyUrl?: string): Promise<Sitemap> {- const { gotScraping } = await import('got-scraping');- const { fileTypeStream } = await import('file-type');-- while (parsingState.sources.length > 0) {- const source = parsingState.sources.pop()!;- parsingState.resetContext();-- if (source.type === 'url') {- const sitemapUrl = new URL(source.url);- parsingState.visitedSitemapUrls.push(sitemapUrl.toString());-- try {- const sitemapStream = await new Promise<ReturnType<typeof gotScraping.stream>>(- (resolve, reject) => {- const request = gotScraping.stream({ url: sitemapUrl, proxyUrl, method: 'GET' });- request.on('response', () => resolve(request));- request.on('error', reject);- },- );-- if (sitemapStream.response!.statusCode === 200) {- let contentType = sitemapStream.response!.headers['content-type'];-- const streamWithType = await fileTypeStream(sitemapStream);- if (streamWithType.fileType !== undefined) {- contentType = streamWithType.fileType.mime;- }-- await new Promise((resolve, reject) => {- let stream: Readable = streamWithType;-- if (- contentType !== undefined- ? contentType === 'application/gzip'- : sitemapUrl.pathname.endsWith('.gz')- ) {- stream = stream.pipe(createGunzip()).on('error', reject);-- if (sitemapUrl.pathname.endsWith('.gz')) {- sitemapUrl.pathname = sitemapUrl.pathname.substring(- 0,- sitemapUrl.pathname.length - 3,- );- }- }-- stream.pipe(this.createParser(resolve, reject, parsingState, contentType, sitemapUrl));- });- }- } catch (e) {- log.warning(`Malformed sitemap content: ${sitemapUrl}, ${e}`);- }- }-- if (source.type === 'raw') {- await new Promise((resolve, reject) => {- Readable.from([source.content]).pipe(this.createParser(resolve, reject, parsingState, 'text/xml'));- });- }- }-- return new Sitemap(parsingState.urls);+ return await this.parse([{ type: 'raw', content }], proxyUrl);}- protected static createParser(- resolve: (value: unknown) => void,- reject: (value: unknown) => void,- parsingState: ParsingState,- contentType: string = '',- url?: URL,- ) {- let mimeType: MIMEType | null;+ protected static async parse(sources: SitemapSource[], proxyUrl?: string): Promise<Sitemap> {+ const urls: string[] = [];try {- mimeType = new MIMEType(contentType);+ for await (const item of parseSitemap(sources, proxyUrl)) {+ urls.push(item.loc);+ }} catch (e) {- mimeType = null;- }-- if (mimeType?.isXML() || url?.pathname.endsWith('.xml')) {- return Sitemap.createXmlParser(parsingState, () => resolve(undefined), reject);+ return new Sitemap([]);}- if (mimeType?.essence === 'text/plain' || url?.pathname.endsWith('.txt')) {- return new SitemapTxtParser(parsingState, () => resolve(undefined));- }-- throw new Error(`Unsupported sitemap content type (contentType = ${contentType}, url = ${url?.toString()})`);+ return new Sitemap(urls);}}diff --git a/packages/utils/test/sitemap.test.ts b/packages/utils/test/sitemap.test.tsindex 8075ca4af..f1691e96e 100644--- a/packages/utils/test/sitemap.test.ts+++ b/packages/utils/test/sitemap.test.ts@@ -1,7 +1,8 @@import nock from 'nock';import { describe, expect, it, beforeEach } from 'vitest';-import { Sitemap } from '../src/internals/sitemap';+import type { SitemapUrl } from '../src/internals/sitemap';+import { Sitemap, parseSitemap } from '../src/internals/sitemap';describe('Sitemap', () => {beforeEach(() => {@@ -16,7 +17,7 @@ describe('Sitemap', () => {'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">','<url>','<loc>http://not-exists.com/</loc>',- '<lastmod>2005-01-01</lastmod>',+ '<lastmod>2005-02-03</lastmod>','<changefreq>monthly</changefreq>','<priority>0.8</priority>','</url>',@@ -187,6 +188,24 @@ describe('Sitemap', () => {);});+ it('extracts metadata from sitemaps', async () => {+ const items: SitemapUrl[] = [];++ for await (const item of parseSitemap([{ type: 'url', url: 'http://not-exists.com/sitemap_child.xml' }])) {+ items.push(item);+ }++ expect(items).toHaveLength(5);+ expect(items).toContainEqual(+ expect.objectContaining({+ loc: 'http://not-exists.com/',+ priority: 0.8,+ changefreq: 'monthly',+ lastmod: new Date('2005-02-03'),+ }),+ );+ });+it('extracts urls from gzipped sitemaps', async () => {const sitemap = await Sitemap.load('http://not-exists.com/sitemap_child.xml.gz');expect(new Set(sitemap.urls)).toEqual(diff --git a/test/core/request_list.test.ts b/test/core/request_list.test.tsindex 925de603d..e99dc9f2f 100644--- a/test/core/request_list.test.ts+++ b/test/core/request_list.test.ts@@ -138,6 +138,24 @@ describe('RequestList', () => {expect(await newList.isEmpty()).toBe(true);});+ test('`RequestList` is `for .. await` iterable', async () => {+ const sources = [+ 'https://example.com/1',+ 'https://example.com/2',+ 'https://example.com/3',+ 'https://example.com/4',+ 'https://example.com/5',+ 'https://example.com/6',+ 'https://example.com/7',+ 'https://example.com/8',+ ];+ const requestList = await RequestList.open(null, sources);++ for await (const request of requestList) {+ expect(request?.url).toBe(sources.shift());+ }+ });+test('should correctly load list from hosted files in correct order', async () => {const spy = vitest.spyOn(RequestList.prototype as any, '_downloadListOfUrls');const list1 = ['https://example.com', 'https://google.com', 'https://wired.com'];diff --git a/test/core/sitemap_request_list.test.ts b/test/core/sitemap_request_list.test.tsnew file mode 100644index 000000000..7f0a6d048--- /dev/null+++ b/test/core/sitemap_request_list.test.ts@@ -0,0 +1,370 @@+import type { Server } from 'http';+import type { AddressInfo } from 'net';+import { Readable } from 'stream';+import { finished } from 'stream/promises';++import { SitemapRequestList } from '@crawlee/core';+import { sleep } from '@crawlee/utils';+import express from 'express';+import { startExpressAppPromise } from 'test/shared/_helper';+import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator';++// Express server for serving sitemaps+let url = 'http://localhost';+let server: Server;++beforeAll(async () => {+ const app = express();++ server = await startExpressAppPromise(app, 0);+ url = `http://localhost:${(server.address() as AddressInfo).port}`;++ app.get('/sitemap.xml', async (req, res) => {+ res.setHeader('content-type', 'text/xml');+ res.write(+ [+ '<?xml version="1.0" encoding="UTF-8"?>',+ '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',+ '<url>',+ '<loc>http://not-exists.com/</loc>',+ '</url>',+ '<url>',+ '<loc>http://not-exists.com/catalog?item=12&desc=vacation_hawaii</loc>',+ '</url>',+ '<url>',+ '<loc>http://not-exists.com/catalog?item=73&desc=vacation_new_zealand</loc>',+ '</url>',+ '<url>',+ '<loc>http://not-exists.com/catalog?item=74&desc=vacation_newfoundland</loc>',+ '</url>',+ '<url>',+ '<loc>http://not-exists.com/catalog?item=83&desc=vacation_usa</loc>',+ '</url>',+ '</urlset>',+ ].join('\n'),+ );+ res.end();+ });++ app.get('/sitemap-stream.xml', async (req, res) => {+ async function* stream() {+ yield [+ '<?xml version="1.0" encoding="UTF-8"?>',+ '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',+ '<url>',+ '<loc>http://not-exists.com/catalog?item=80&desc=vacation_turkey</loc>',+ '</url>',+ '<url>',+ '<loc>http://not-exists.com/catalog?item=80&desc=vacation_mauritius</loc>',+ '</url>',+ ].join('\n');++ await sleep(100);++ yield [+ '<url>',+ '<loc>http://not-exists.com/catalog?item=81&desc=vacation_maledives</loc>',+ '</url>',+ '</urlset>',+ ].join('\n');+ }++ res.setHeader('content-type', 'text/xml');++ await finished(Readable.from(stream()).pipe(res));++ res.end();+ });++ app.get('/sitemap-stream-linger.xml', async (req, res) => {+ async function* stream() {+ yield [+ '<?xml version="1.0" encoding="UTF-8"?>',+ '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',+ '<url>',+ '<loc>http://not-exists.com/catalog?item=80&desc=vacation_turkey</loc>',+ '</url>',+ '<url>',+ '<loc>http://not-exists.com/catalog?item=81&desc=vacation_maledives</loc>',+ '</url>',+ ].join('\n');++ await sleep(200);++ yield '</urlset>';+ }++ res.setHeader('content-type', 'text/xml');++ await finished(Readable.from(stream()).pipe(res));++ res.end();+ });++ app.get('/sitemap-index.xml', async (req, res) => {+ res.setHeader('content-type', 'text/xml');+ res.write(+ [+ '<?xml version="1.0" encoding="UTF-8"?>',+ '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',+ '<sitemap>',+ `<loc>${url}/sitemap-stream-linger.xml</loc>`,+ '</sitemap>',+ '<sitemap>',+ `<loc>${url}/sitemap.xml</loc>`,+ '</sitemap>',+ '</sitemapindex>',+ ].join('\n'),+ );++ res.end();+ });+});++afterAll(async () => {+ server.close();+});++// Storage emulator for persistence+const emulator = new MemoryStorageEmulator();++beforeAll(async () => {+ await emulator.init();+});++afterAll(async () => {+ await emulator.destroy();+});++describe('SitemapRequestList', () => {+ test('requests are available before the sitemap is fully loaded', async () => {+ const list = await SitemapRequestList.open({ sitemapUrls: [`${url}/sitemap-stream.xml`] });++ while (await list.isEmpty()) {+ await sleep(20);+ }++ await expect(list.isFinished(), 'list should not be finished').resolves.toBe(false);+ await expect(list.isEmpty(), 'list should not be empty').resolves.toBe(false);++ const firstRequest = await list.fetchNextRequest();+ expect(firstRequest).not.toBe(null);++ const secondRequest = await list.fetchNextRequest();+ expect(secondRequest).not.toBe(null);++ const thirdRequest = await list.fetchNextRequest();+ expect(thirdRequest).not.toBe(null);+ });++ test('draining the request list between sitemaps', async () => {+ const list = await SitemapRequestList.open({ sitemapUrls: [`${url}/sitemap-index.xml`] });++ while (await list.isEmpty()) {+ await sleep(20);+ }++ const firstBatch: Request[] = [];++ while (!(await list.isEmpty())) {+ const request = await list.fetchNextRequest();+ firstBatch.push(request);+ await list.markRequestHandled(request);+ }++ expect(firstBatch).toHaveLength(2);++ while (await list.isEmpty()) {+ await sleep(20);+ }++ const secondBatch: Request[] = [];++ while (!(await list.isEmpty())) {+ const request = await list.fetchNextRequest();+ secondBatch.push(request);+ await list.markRequestHandled(request);+ }++ expect(secondBatch).toHaveLength(5);++ expect(list.isFinished()).resolves.toBe(true);+ expect(list.handledCount()).toBe(7);+ });++ test('for..await syntax works with requestIterator', async () => {+ const list = await SitemapRequestList.open({ sitemapUrls: [`${url}/sitemap-index.xml`] });++ for await (const request of list) {+ await list.markRequestHandled(request);+ }++ expect(list.isFinished()).resolves.toBe(true);+ expect(list.handledCount()).toBe(7);+ });++ test('aborting long sitemap load works', async () => {+ const controller = new AbortController();++ const list = await SitemapRequestList.open({+ sitemapUrls: [`${url}/sitemap-index.xml`],+ signal: controller.signal,+ });++ await sleep(50); // Loads the first sub-sitemap, but not the second+ controller.abort();++ for await (const request of list) {+ await list.markRequestHandled(request);+ }++ expect(list.isFinished()).resolves.toBe(true);+ expect(list.isSitemapFullyLoaded()).toBe(false);+ expect(list.handledCount()).toBe(2);+ });++ test('timeout option works', async () => {+ const list = await SitemapRequestList.open({+ sitemapUrls: [`${url}/sitemap-index.xml`],+ timeoutMillis: 50, // Loads the first sub-sitemap, but not the second+ });++ for await (const request of list) {+ await list.markRequestHandled(request);+ }++ expect(list.isFinished()).resolves.toBe(true);+ expect(list.isSitemapFullyLoaded()).toBe(false);+ expect(list.handledCount()).toBe(2);+ });++ test('resurrection does not resume aborted loading', async () => {+ const options = {+ sitemapUrls: [`${url}/sitemap-index.xml`],+ persistStateKey: 'resurrection-abort',+ timeoutMillis: 50,+ };++ {+ const list = await SitemapRequestList.open(options);++ await sleep(50);++ expect(list.isEmpty()).resolves.toBe(false);+ await list.persistState();+ }++ const newList = await SitemapRequestList.open(options);+ for await (const request of newList) {+ await newList.markRequestHandled(request);+ }++ expect(newList.handledCount()).toBe(2);+ });++ test('processing the whole list', async () => {+ const list = await SitemapRequestList.open({ sitemapUrls: [`${url}/sitemap.xml`] });+ const requests: Request[] = [];++ await expect(list.isFinished()).resolves.toBe(false);++ while (!(await list.isFinished())) {+ const request = await list.fetchNextRequest();+ await list.markRequestHandled(request);+ requests.push(request);+ }++ await expect(list.isEmpty()).resolves.toBe(true);+ expect(requests.map((it) => it.url)).toEqual([+ 'http://not-exists.com/',+ 'http://not-exists.com/catalog?item=12&desc=vacation_hawaii',+ 'http://not-exists.com/catalog?item=73&desc=vacation_new_zealand',+ 'http://not-exists.com/catalog?item=74&desc=vacation_newfoundland',+ 'http://not-exists.com/catalog?item=83&desc=vacation_usa',+ ]);++ expect(list.handledCount()).toEqual(5);+ });++ test('processing the whole list with reclaiming', async () => {+ const list = await SitemapRequestList.open({ sitemapUrls: [`${url}/sitemap.xml`] });+ const requests: Request[] = [];++ await expect(list.isFinished()).resolves.toBe(false);+ let counter = 0;++ while (!(await list.isFinished())) {+ const request = await list.fetchNextRequest();++ if (counter % 2 === 0) {+ await list.markRequestHandled(request);+ requests.push(request);+ } else {+ await list.reclaimRequest(request);+ }++ counter += 1;+ }++ await expect(list.isEmpty()).resolves.toBe(true);+ expect(new Set(requests.map((it) => it.url))).toEqual(+ new Set([+ 'http://not-exists.com/',+ 'http://not-exists.com/catalog?item=12&desc=vacation_hawaii',+ 'http://not-exists.com/catalog?item=73&desc=vacation_new_zealand',+ 'http://not-exists.com/catalog?item=74&desc=vacation_newfoundland',+ 'http://not-exists.com/catalog?item=83&desc=vacation_usa',+ ]),+ );++ expect(list.handledCount()).toEqual(5);+ });++ test('persists state', async () => {+ const options = { sitemapUrls: [`${url}/sitemap-stream.xml`], persistStateKey: 'some-key' };+ const list = await SitemapRequestList.open(options);++ const firstRequest = await list.fetchNextRequest();+ await list.markRequestHandled(firstRequest);++ await list.persistState();++ const newList = await SitemapRequestList.open(options);+ await expect(newList.isEmpty()).resolves.toBe(false);++ while (!(await newList.isFinished())) {+ const request = await newList.fetchNextRequest();+ await newList.markRequestHandled(request);+ }++ expect(list.handledCount()).toBe(1);+ expect(newList.handledCount()).toBe(2);+ });++ test('state persistence tracks user changes', async () => {+ const options = {+ sitemapUrls: [`${url}/sitemap-stream.xml`],+ persistStateKey: 'persist-user-changes',+ };++ const userDataPayload = { some: 'data' };+ let firstLoadedUrl;++ {+ const list = await SitemapRequestList.open(options);++ const firstRequest = await list.fetchNextRequest();+ firstRequest.userData = userDataPayload;+ firstLoadedUrl = firstRequest.url;++ await list.persistState();+ // simulates a migration in the middle of request processing+ }++ const newList = await SitemapRequestList.open(options);+ const restoredRequest = await newList.fetchNextRequest();++ expect(restoredRequest.url).toEqual(firstLoadedUrl);+ expect(restoredRequest.userData).toEqual(userDataPayload);+ });+});diff --git a/yarn.lock b/yarn.lockindex bd5ce6f63..06a24e7dc 100644--- a/yarn.lock+++ b/yarn.lock@@ -831,7 +831,7 @@ __metadata:got-scraping: "npm:^4.0.3"ow: "npm:^0.28.1"robots-parser: "npm:^3.0.1"- sax: "npm:^1.3.0"+ sax: "npm:^1.4.1"tslib: "npm:^2.4.0"whatwg-mimetype: "npm:^4.0.0"languageName: unknown@@ -10651,7 +10651,7 @@ __metadata:languageName: nodelinkType: hard-"sax@npm:^1.3.0":+"sax@npm:^1.4.1":version: 1.4.1resolution: "sax@npm:1.4.1"checksum: 10c0/6bf86318a254c5d898ede6bd3ded15daf68ae08a5495a2739564eb265cd13bcc64a07ab466fb204f67ce472bb534eb8612dac587435515169593f4fffa11de7c