Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(core): use short timeouts for periodic `KVS.setRecord` calls (#2962)

Martin Adámekd31d90e5288e
diff --git a/packages/core/src/crawlers/statistics.ts b/packages/core/src/crawlers/statistics.ts
index 1d81b2b88..975a65379 100644
--- a/packages/core/src/crawlers/statistics.ts
+++ b/packages/core/src/crawlers/statistics.ts
@@ -329,7 +329,17 @@ export class Statistics {
this.log.debug('Persisting state', { persistStateKey: this.persistStateKey });
- await this.keyValueStore.setValue(this.persistStateKey, this.toJSON());
+ // use half the interval of `persistState` to avoid race conditions
+ const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis')!;
+ const timeoutSecs = persistStateIntervalMillis / 2_000;
+ await this.keyValueStore
+ .setValue(this.persistStateKey, this.toJSON(), {
+ timeoutSecs,
+ doNotRetryTimeouts: true,
+ })
+ .catch((error) =>
+ this.log.warning(`Failed to persist the statistics to ${this.persistStateKey}`, { error }),
+ );
}
/**
diff --git a/packages/core/src/session_pool/session_pool.ts b/packages/core/src/session_pool/session_pool.ts
index 676e92207..22528a29e 100644
--- a/packages/core/src/session_pool/session_pool.ts
+++ b/packages/core/src/session_pool/session_pool.ts
@@ -367,7 +367,18 @@ export class SessionPool extends EventEmitter {
persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
persistStateKey: this.persistStateKey,
});
- await this.keyValueStore.setValue(this.persistStateKey, this.getState());
+
+ // use half the interval of `persistState` to avoid race conditions
+ const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis')!;
+ const timeoutSecs = persistStateIntervalMillis / 2_000;
+ await this.keyValueStore
+ .setValue(this.persistStateKey, this.getState(), {
+ timeoutSecs,
+ doNotRetryTimeouts: true,
+ })
+ .catch((error) =>
+ this.log.warning(`Failed to persist the session pool stats to ${this.persistStateKey}`, { error }),
+ );
}
/**
diff --git a/packages/core/src/storages/key_value_store.ts b/packages/core/src/storages/key_value_store.ts
index c41c6c512..c3247a944 100644
--- a/packages/core/src/storages/key_value_store.ts
+++ b/packages/core/src/storages/key_value_store.ts
@@ -6,6 +6,7 @@ import JSON5 from 'json5';
import ow, { ArgumentError } from 'ow';
import { KEY_VALUE_STORE_KEY_REGEX } from '@apify/consts';
+import log from '@apify/log';
import { jsonStringifyExtended } from '@apify/utilities';
import { Configuration } from '../configuration';
@@ -273,10 +274,23 @@ export class KeyValueStore {
return;
}
+ // use half the interval of `persistState` to avoid race conditions
+ const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis')!;
+ const timeoutSecs = persistStateIntervalMillis / 2_000;
+
this.config.getEventManager().on('persistState', async () => {
+ const promises: Promise<void>[] = [];
+
for (const [key, value] of this.cache) {
- await this.setValue(key, value);
+ promises.push(
+ this.setValue(key, value, {
+ timeoutSecs,
+ doNotRetryTimeouts: true,
+ }).catch((error) => log.warning(`Failed to persist the state value to ${key}`, { error })),
+ );
}
+
+ await Promise.all(promises);
});
this.persistStateEventStarted = true;
@@ -352,6 +366,8 @@ export class KeyValueStore {
options,
ow.object.exactShape({
contentType: ow.optional.string.nonEmpty,
+ timeoutSecs: ow.optional.number,
+ doNotRetryTimeouts: ow.optional.boolean,
}),
);
@@ -380,11 +396,17 @@ export class KeyValueStore {
value = maybeStringify(value, optionsCopy);
- return this.client.setRecord({
- key,
- value,
- contentType: optionsCopy.contentType,
- });
+ return this.client.setRecord(
+ {
+ key,
+ value,
+ contentType: optionsCopy.contentType,
+ },
+ {
+ timeoutSecs: optionsCopy.timeoutSecs,
+ doNotRetryTimeouts: optionsCopy.doNotRetryTimeouts,
+ },
+ );
}
/**
@@ -719,6 +741,16 @@ export interface RecordOptions {
* Specifies a custom MIME content type of the record.
*/
contentType?: string;
+
+ /**
+ * Specifies a custom timeout for the `set-record` API call, in seconds.
+ */
+ timeoutSecs?: number;
+
+ /**
+ * If set to `true`, the `set-record` API call will not be retried if it times out.
+ */
+ doNotRetryTimeouts?: boolean;
}
export interface KeyValueStoreIteratorOptions {
diff --git a/packages/types/src/storages.ts b/packages/types/src/storages.ts
index 799e6d139..425d55ac3 100644
--- a/packages/types/src/storages.ts
+++ b/packages/types/src/storages.ts
@@ -124,6 +124,11 @@ export interface KeyValueStoreRecord {
contentType?: string;
}
+export interface KeyValueStoreRecordOptions {
+ timeoutSecs?: number;
+ doNotRetryTimeouts?: boolean;
+}
+
export interface KeyValueStoreClientUpdateOptions {
name?: string;
}
@@ -162,7 +167,7 @@ export interface KeyValueStoreClient {
listKeys(options?: KeyValueStoreClientListOptions): Promise<KeyValueStoreClientListData>;
recordExists(key: string): Promise<boolean>;
getRecord(key: string, options?: KeyValueStoreClientGetRecordOptions): Promise<KeyValueStoreRecord | undefined>;
- setRecord(record: KeyValueStoreRecord): Promise<void>;
+ setRecord(record: KeyValueStoreRecord, options?: KeyValueStoreRecordOptions): Promise<void>;
deleteRecord(key: string): Promise<void>;
}
diff --git a/test/core/crawlers/statistics.test.ts b/test/core/crawlers/statistics.test.ts
index e2b2d7ca6..439117739 100644
--- a/test/core/crawlers/statistics.test.ts
+++ b/test/core/crawlers/statistics.test.ts
@@ -186,8 +186,15 @@ describe('Statistics', () => {
// @ts-expect-error Incorrect types?
const { retryHistogram, finished, failed, ...rest } = stats.calculate();
- // @ts-expect-error Accessing private prop
- expect(setValueSpy).toBeCalledWith(stats.persistStateKey, { ...state, ...rest });
+ expect(setValueSpy).toBeCalledWith(
+ // @ts-expect-error Accessing private prop
+ stats.persistStateKey,
+ { ...state, ...rest },
+ {
+ doNotRetryTimeouts: true,
+ timeoutSecs: 30,
+ },
+ );
}, 2000);
});
diff --git a/test/core/storages/key_value_store.test.ts b/test/core/storages/key_value_store.test.ts
index d92776048..0d8bfb97c 100644
--- a/test/core/storages/key_value_store.test.ts
+++ b/test/core/storages/key_value_store.test.ts
@@ -40,11 +40,17 @@ describe('KeyValueStore', () => {
await store.setValue('key-1', record);
expect(mockSetRecord).toBeCalledTimes(1);
- expect(mockSetRecord).toBeCalledWith({
- key: 'key-1',
- value: recordStr,
- contentType: 'application/json; charset=utf-8',
- });
+ expect(mockSetRecord).toBeCalledWith(
+ {
+ key: 'key-1',
+ value: recordStr,
+ contentType: 'application/json; charset=utf-8',
+ },
+ {
+ doNotRetryTimeouts: undefined,
+ timeoutSecs: undefined,
+ },
+ );
// Get Record
const mockGetRecord = vitest
@@ -281,11 +287,17 @@ describe('KeyValueStore', () => {
await store.setValue('key-1', 'xxxx', { contentType: 'text/plain; charset=utf-8' });
expect(mockSetRecord).toBeCalledTimes(1);
- expect(mockSetRecord).toBeCalledWith({
- key: 'key-1',
- value: 'xxxx',
- contentType: 'text/plain; charset=utf-8',
- });
+ expect(mockSetRecord).toBeCalledWith(
+ {
+ key: 'key-1',
+ value: 'xxxx',
+ contentType: 'text/plain; charset=utf-8',
+ },
+ {
+ doNotRetryTimeouts: undefined,
+ timeoutSecs: undefined,
+ },
+ );
});
test('correctly passes object values as JSON', async () => {
@@ -305,11 +317,50 @@ describe('KeyValueStore', () => {
await store.setValue('key-1', record);
expect(mockSetRecord).toBeCalledTimes(1);
- expect(mockSetRecord).toBeCalledWith({
- key: 'key-1',
- value: recordStr,
- contentType: 'application/json; charset=utf-8',
+ expect(mockSetRecord).toBeCalledWith(
+ {
+ key: 'key-1',
+ value: recordStr,
+ contentType: 'application/json; charset=utf-8',
+ },
+ {
+ doNotRetryTimeouts: undefined,
+ timeoutSecs: undefined,
+ },
+ );
+ });
+
+ test('correctly passes timeout options', async () => {
+ const store = new KeyValueStore({
+ id: 'my-store-id-1',
+ client,
+ });
+
+ const record = { foo: 'bar' };
+ const recordStr = JSON.stringify(record, null, 2);
+
+ const mockSetRecord = vitest
+ // @ts-expect-error Accessing private property
+ .spyOn(store.client, 'setRecord')
+ .mockResolvedValueOnce(undefined);
+
+ await store.setValue('key-1', record, {
+ timeoutSecs: 1,
+ doNotRetryTimeouts: true,
});
+
+ expect(mockSetRecord).toBeCalledTimes(1);
+ expect(mockSetRecord).toBeCalledWith(
+ {
+ key: 'key-1',
+ value: recordStr,
+ contentType: 'application/json; charset=utf-8',
+ },
+ {
+ doNotRetryTimeouts: true,
+ timeoutSecs: 1,
+ },
+ );
});
test('correctly passes raw string values', async () => {
@@ -326,11 +377,17 @@ describe('KeyValueStore', () => {
await store.setValue('key-1', 'xxxx', { contentType: 'text/plain; charset=utf-8' });
expect(mockSetRecord).toBeCalledTimes(1);
- expect(mockSetRecord).toBeCalledWith({
- key: 'key-1',
- value: 'xxxx',
- contentType: 'text/plain; charset=utf-8',
- });
+ expect(mockSetRecord).toBeCalledWith(
+ {
+ key: 'key-1',
+ value: 'xxxx',
+ contentType: 'text/plain; charset=utf-8',
+ },
+ {
+ doNotRetryTimeouts: undefined,
+ timeoutSecs: undefined,
+ },
+ );
});
test('correctly passes raw Buffer values', async () => {
@@ -348,11 +405,17 @@ describe('KeyValueStore', () => {
await store.setValue('key-1', value, { contentType: 'image/jpeg; charset=something' });
expect(mockSetRecord).toBeCalledTimes(1);
- expect(mockSetRecord).toBeCalledWith({
- key: 'key-1',
- value,
- contentType: 'image/jpeg; charset=something',
- });
+ expect(mockSetRecord).toBeCalledWith(
+ {
+ key: 'key-1',
+ value,
+ contentType: 'image/jpeg; charset=something',
+ },
+ {
+ doNotRetryTimeouts: undefined,
+ timeoutSecs: undefined,
+ },
+ );
});
test('correctly passes a stream', async () => {
@@ -373,11 +436,17 @@ describe('KeyValueStore', () => {
value.destroy();
expect(mockSetRecord).toHaveBeenCalledTimes(1);
- expect(mockSetRecord).toHaveBeenCalledWith({
- key: 'key-1',
- value,
- contentType: 'plain/text',
- });
+ expect(mockSetRecord).toHaveBeenCalledWith(
+ {
+ key: 'key-1',
+ value,
+ contentType: 'plain/text',
+ },
+ {
+ doNotRetryTimeouts: undefined,
+ timeoutSecs: undefined,
+ },
+ );
});
});