Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: Persist SitemapRequestList state periodically (#2923)

Jan Buchare6e7a9feed5d
diff --git a/packages/core/src/storages/sitemap_request_list.ts b/packages/core/src/storages/sitemap_request_list.ts
index 97d2017e1..de9b9dbeb 100644
--- a/packages/core/src/storages/sitemap_request_list.ts
+++ b/packages/core/src/storages/sitemap_request_list.ts
@@ -3,11 +3,14 @@ import { Transform } from 'node:stream';
import { parseSitemap, type ParseSitemapOptions } from '@crawlee/utils';
import { minimatch } from 'minimatch';
import ow from 'ow';
+import type { RequiredDeep } from 'type-fest';
import defaultLog from '@apify/log';
+import { Configuration } from '../configuration';
import type { GlobInput, RegExpInput, UrlPatternObject } from '../enqueue_links';
import { constructGlobObjectsFromGlobs, constructRegExpObjectsFromRegExps } from '../enqueue_links';
+import { type EventManager, EventType } from '../events/event_manager';
import { Request } from '../request';
import { KeyValueStore } from './key_value_store';
import type { IRequestList } from './request_list';
@@ -67,6 +70,16 @@ export interface SitemapRequestListOptions extends UrlConstraints {
* Key for persisting the state of the request list in the `KeyValueStore`.
*/
persistStateKey?: string;
+ /**
+ * Persistence-related options to control how and when crawler's data gets persisted.
+ */
+ persistenceOptions?: {
+ /**
+ * Use this flag to disable or enable periodic persistence to key value store.
+ * @default true
+ */
+ enable?: boolean;
+ };
/**
* Abort signal to be used for sitemap loading.
*/
@@ -86,6 +99,10 @@ export interface SitemapRequestListOptions extends UrlConstraints {
* Advanced options for the underlying `parseSitemap` call.
*/
parseSitemapOptions?: Omit<ParseSitemapOptions, 'emitNestedSitemaps' | 'maxDepth'>;
+ /**
+ * Crawlee configuration
+ */
+ config?: Configuration;
}
interface SitemapParsingProgress {
@@ -183,6 +200,11 @@ export class SitemapRequestList implements IRequestList {
private urlExcludePatternObjects: UrlPatternObject[] = [];
private urlPatternObjects: UrlPatternObject[] = [];
+ /** EventManager used to handle persistence */
+ private events: EventManager;
+
+ private persistenceOptions: RequiredDeep<SitemapRequestListOptions['persistenceOptions']>;
+
/** @internal */
private constructor(options: SitemapRequestListOptions) {
ow(
@@ -200,10 +222,12 @@ export class SitemapRequestList implements IRequestList {
ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp')),
),
regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),
+ config: ow.optional.object,
+ persistenceOptions: ow.optional.object,
}),
);
- const { globs, exclude, regexps } = options;
+ const { globs, exclude, regexps, config = Configuration.getGlobalConfig() } = options;
if (exclude?.length) {
for (const excl of exclude) {
@@ -224,11 +248,14 @@ export class SitemapRequestList implements IRequestList {
}
this.persistStateKey = options.persistStateKey;
+ this.persistenceOptions = { enable: true, ...options.persistenceOptions };
+
this.proxyUrl = options.proxyUrl;
this.urlQueueStream = this.createNewStream(options.maxBufferSize ?? 200);
this.sitemapParsingProgress.pendingSitemapUrls = new Set(options.sitemapUrls);
+ this.events = config.getEventManager();
}
/**
@@ -390,6 +417,10 @@ export class SitemapRequestList implements IRequestList {
await requestList.restoreState();
void requestList.load({ parseSitemapOptions: options.parseSitemapOptions });
+ if (requestList.persistenceOptions.enable) {
+ requestList.events.on(EventType.PERSIST_STATE, requestList.persistState);
+ }
+
options?.signal?.addEventListener('abort', () => {
requestList.abortLoading = true;
});
@@ -562,6 +593,7 @@ export class SitemapRequestList implements IRequestList {
async teardown(): Promise<void> {
this.closed = true;
this.abortLoading = true;
+ this.events.off(EventType.PERSIST_STATE, this.persistState);
await this.persistState();
this.urlQueueStream.emit('readdata'); // unblocks the potentially waiting `pushNextUrl` call