Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: Deflake tests under high concurrency (#4059)

Jan Buchar73ee2b87dbda
diff --git a/packages/core/src/session_pool/session.ts b/packages/core/src/session_pool/session.ts
index bdf95887c..e37e048ac 100644
--- a/packages/core/src/session_pool/session.ts
+++ b/packages/core/src/session_pool/session.ts
@@ -4,7 +4,6 @@ import { z } from 'zod';
import { cryptoRandomObjectId } from '@apify/utilities';
-import { getDefaultCookieExpirationDate } from '../cookie_utils.js';
import type { CrawleeLogger } from '../log.js';
import { serviceLocator } from '../service_locator.js';
import { parseArgument, schemas, validators } from '../validators.js';
@@ -62,7 +61,10 @@ export interface SessionOptions {
/** Date of creation. */
createdAt?: Date;
- /** Date of expiration. */
+ /**
+ * Date of expiration.
+ * @default createdAt + maxAgeSecs
+ */
expiresAt?: Date;
/**
@@ -191,7 +193,8 @@ export class Session implements ISession {
retired,
log,
fingerprint,
- expiresAt = getDefaultCookieExpirationDate(maxAgeSecs),
+ // Anchored to `createdAt` rather than to "now", so the documented `createdAt + maxAgeSecs` holds.
+ expiresAt = new Date(createdAt.getTime() + maxAgeSecs * 1000),
} = parseArgument(options, sessionOptionsSchema);
this.#log = log.child({ prefix: 'Session' });
diff --git a/packages/utils/test/robots.test.ts b/packages/utils/test/robots.test.ts
index b8f253c5f..789be7f33 100644
--- a/packages/utils/test/robots.test.ts
+++ b/packages/utils/test/robots.test.ts
@@ -6,6 +6,9 @@ import { RobotsTxtFile } from '../src/internals/robots.js';
const httpClient = new FetchHttpClient();
+// Never replies within the lifetime of a test, so only the timeout/abort can settle the promise.
+const neverReplies = 'http://never-replies.com';
+
describe('RobotsTxtFile', () => {
beforeEach(() => {
nock.disableNetConnect();
@@ -33,6 +36,8 @@ describe('RobotsTxtFile', () => {
)
.get('*')
.reply(404);
+
+ nock(neverReplies).persist().get('/robots.txt').delay(30_000).reply(200, 'User-agent: *');
});
afterEach(() => {
@@ -63,44 +68,44 @@ describe('RobotsTxtFile', () => {
});
it('respects user-set timeout', async () => {
- const start = +Date.now();
- const robots = RobotsTxtFile.find('http://not-exists.com/robots.txt', { timeoutMillis: 200 });
+ const start = Date.now();
+ const robots = RobotsTxtFile.find(`${neverReplies}/robots.txt`, { timeoutMillis: 200 });
await expect(robots).rejects.toThrow(/timeout/i);
- const end = +Date.now();
- expect(end - start).toBeGreaterThanOrEqual(200);
- expect(end - start).toBeLessThanOrEqual(500);
+ const elapsed = Date.now() - start;
+ expect(elapsed).toBeGreaterThanOrEqual(200);
+ expect(elapsed).toBeLessThan(10_000);
});
it('respects AbortSignal parameter', async () => {
const controller = new AbortController();
setTimeout(() => controller.abort(), 200);
- const start = +Date.now();
- const robots = RobotsTxtFile.find('http://not-exists.com/robots.txt', { signal: controller.signal });
+ const start = Date.now();
+ const robots = RobotsTxtFile.find(`${neverReplies}/robots.txt`, { signal: controller.signal });
await expect(robots).rejects.toThrow(/aborted/i);
- const end = +Date.now();
- expect(end - start).toBeGreaterThanOrEqual(200);
- expect(end - start).toBeLessThanOrEqual(500);
+ const elapsed = Date.now() - start;
+ expect(elapsed).toBeGreaterThanOrEqual(200);
+ expect(elapsed).toBeLessThan(10_000);
});
it('respects AbortSignal parameter and timeout together', async () => {
const controller = new AbortController();
- const start = +Date.now();
- const robots = RobotsTxtFile.find('http://not-exists.com/robots.txt', {
+ const start = Date.now();
+ const robots = RobotsTxtFile.find(`${neverReplies}/robots.txt`, {
signal: controller.signal,
timeoutMillis: 200,
});
await expect(robots).rejects.toThrow(/timeout/i);
- const end = +Date.now();
- expect(end - start).toBeGreaterThanOrEqual(200);
- expect(end - start).toBeLessThanOrEqual(500);
+ const elapsed = Date.now() - start;
+ expect(elapsed).toBeGreaterThanOrEqual(200);
+ expect(elapsed).toBeLessThan(10_000);
});
it('drops cross-host and non-http(s) sitemap directives under the default same-hostname strategy', () => {
diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.ts
index 3445eac20..58b727ce0 100644
--- a/test/core/crawlers/browser_crawler.test.ts
+++ b/test/core/crawlers/browser_crawler.test.ts
@@ -692,6 +692,9 @@ describe('BrowserCrawler', () => {
},
requestList,
saveResponseCookies: true,
+ // The handoff only happens between requests: cookies set in a handler are flushed to the
+ // session jar after it returns, so cookie-2 must not start before cookie-1 has finished.
+ maxConcurrency: 1,
sessionPool: new SessionPool({
maxPoolSize: 1,
}),
@@ -921,6 +924,9 @@ describe('BrowserCrawler', () => {
sessionPool: new SessionPool({
maxPoolSize: 1,
}),
+ // A strictly serial [0..5] is only well-defined one request at a time: two handlers running
+ // together read the same `usageCount` before either marks the session good.
+ maxConcurrency: 1,
requestHandler: async ({ session }) => {
sessionUsageHistory.push((session as Session).usageCount);
},
diff --git a/test/core/crawlers/playwright_crawler.test.ts b/test/core/crawlers/playwright_crawler.test.ts
index 0e9ab7491..d22d02cf1 100644
--- a/test/core/crawlers/playwright_crawler.test.ts
+++ b/test/core/crawlers/playwright_crawler.test.ts
@@ -2,7 +2,7 @@ import type { Server } from 'node:http';
import type { AddressInfo } from 'node:net';
import os from 'node:os';
-import type { PlaywrightCrawlingContext, PlaywrightGotoOptions, Request } from '@crawlee/playwright';
+import type { PlaywrightCrawlingContext, Request } from '@crawlee/playwright';
import { type ConcurrencySystem, MemoryStorageBackend, serviceLocator } from '@crawlee/core';
import {
createPlaywrightRouter,
@@ -169,7 +169,8 @@ describe('PlaywrightCrawler', () => {
test('should override goto timeout with navigationTimeoutSecs', async () => {
const timeoutSecs = 10;
- let options: PlaywrightGotoOptions;
+ // Captured by value: `navigate()` narrows the live `gotoOptions` down to the remaining navigation window.
+ let gotoTimeout: number | undefined;
const playwrightCrawler = new PlaywrightCrawler({
requestList,
maxRequestRetries: 0,
@@ -177,14 +178,14 @@ describe('PlaywrightCrawler', () => {
requestHandler: () => {},
preNavigationHooks: [
({ gotoOptions }) => {
- options = gotoOptions;
+ gotoTimeout = gotoOptions.timeout;
},
],
navigationTimeoutSecs: timeoutSecs,
});
await playwrightCrawler.run();
- expect(options!.timeout).toEqual(timeoutSecs * 1000);
+ expect(gotoTimeout).toEqual(timeoutSecs * 1000);
});
test('does not mutate the launchContext it was given', () => {
diff --git a/test/core/crawlers/puppeteer_crawler.test.ts b/test/core/crawlers/puppeteer_crawler.test.ts
index 293a35b83..3ddea62d1 100644
--- a/test/core/crawlers/puppeteer_crawler.test.ts
+++ b/test/core/crawlers/puppeteer_crawler.test.ts
@@ -5,7 +5,7 @@ import type { AddressInfo } from 'node:net';
import os from 'node:os';
import { promisify } from 'node:util';
-import type { PuppeteerCrawlingContext, PuppeteerGoToOptions, Request } from '@crawlee/puppeteer';
+import type { PuppeteerCrawlingContext, Request } from '@crawlee/puppeteer';
import type { Cheerio, CheerioAPI } from 'cheerio';
import type { Element } from 'domhandler';
import {
@@ -137,7 +137,8 @@ describe('PuppeteerCrawler', () => {
test('should override goto timeout with navigationTimeoutSecs', async () => {
const timeoutSecs = 10;
- let options: PuppeteerGoToOptions;
+ // Captured by value: `navigate()` narrows the live `gotoOptions` down to the remaining navigation window.
+ let gotoTimeout: number | undefined;
const puppeteerCrawler = new PuppeteerCrawler({
requestList,
maxRequestRetries: 0,
@@ -145,14 +146,14 @@ describe('PuppeteerCrawler', () => {
requestHandler: () => {},
preNavigationHooks: [
({ gotoOptions }) => {
- options = gotoOptions;
+ gotoTimeout = gotoOptions.timeout;
},
],
navigationTimeoutSecs: timeoutSecs,
});
await puppeteerCrawler.run();
- expect(options!.timeout).toEqual(timeoutSecs * 1000);
+ expect(gotoTimeout).toEqual(timeoutSecs * 1000);
});
test('should throw if launchOptions.proxyUrl is supplied', async () => {
diff --git a/test/core/sitemap_request_loader.test.ts b/test/core/sitemap_request_loader.test.ts
index 019af1ba4..e14a8cd35 100644
--- a/test/core/sitemap_request_loader.test.ts
+++ b/test/core/sitemap_request_loader.test.ts
@@ -148,7 +148,11 @@ beforeAll(async () => {
res.end();
});
+ // `?linger=<ms>` (default 200) holds the response open after the two URL entries, so a test can act
+ // while this sub-sitemap is still loading.
app.get('/sitemap-stream-linger.xml', async (req, res) => {
+ const lingerMillis = Number(req.query.linger ?? 200);
+
async function* stream() {
yield [
'<?xml version="1.0" encoding="UTF-8"?>',
@@ -161,7 +165,7 @@ beforeAll(async () => {
'</url>',
].join('\n');
- await sleep(200);
+ await sleep(lingerMillis);
yield '</urlset>';
}
@@ -173,14 +177,17 @@ beforeAll(async () => {
res.end();
});
+ // Index over the lingering sub-sitemap (2 URLs) followed by a plain one (5 URLs).
app.get('/sitemap-index.xml', async (req, res) => {
+ const linger = req.query.linger === undefined ? '' : `?linger=${Number(req.query.linger)}`;
+
res.setHeader('content-type', 'text/xml');
res.write(
[
'<?xml version="1.0" encoding="UTF-8"?>',
'<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
'<sitemap>',
- `<loc>${url}/sitemap-stream-linger.xml</loc>`,
+ `<loc>${url}/sitemap-stream-linger.xml${linger}</loc>`,
'</sitemap>',
'<sitemap>',
`<loc>${url}/sitemap.xml</loc>`,
@@ -444,12 +451,17 @@ describe('SitemapRequestLoader', () => {
const controller = new AbortController();
const list = await SitemapRequestLoader.open({
- sitemapUrls: [`${url}/sitemap-index.xml`],
+ // The first sub-sitemap stays open for 5s, so the abort below always lands mid-index.
+ sitemapUrls: [`${url}/sitemap-index.xml?linger=5000`],
signal: controller.signal,
enqueueStrategy: 'all',
});
- await sleep(50); // Loads the first sub-sitemap, but not the second
+ // Abort while the first sub-sitemap is still streaming - waiting for its URLs rather than for a fixed duration.
+ while (await list.isEmpty()) {
+ await sleep(10);
+ }
+
controller.abort();
for await (const request of list) {
@@ -463,8 +475,10 @@ describe('SitemapRequestLoader', () => {
test('timeout option works', async () => {
const list = await SitemapRequestLoader.open({
- sitemapUrls: [`${url}/sitemap-index.xml`],
- timeoutMillis: 50, // Loads the first sub-sitemap, but not the second
+ // The timeout has to fire after the first sub-sitemap streamed its URLs but before it closes;
+ // the test then runs for the whole linger, since the abort cannot interrupt an in-flight fetch.
+ sitemapUrls: [`${url}/sitemap-index.xml?linger=2000`],
+ timeoutMillis: 500,
enqueueStrategy: 'all',
});
@@ -479,21 +493,27 @@ describe('SitemapRequestLoader', () => {
test('resurrection does not resume aborted loading', async () => {
const options = {
- sitemapUrls: [`${url}/sitemap-index.xml`],
+ sitemapUrls: [`${url}/sitemap-index.xml?linger=5000`],
persistStateKey: 'resurrection-abort',
- timeoutMillis: 50,
enqueueStrategy: 'all' as const,
};
{
- const list = await SitemapRequestLoader.open(options);
+ const controller = new AbortController();
+ const list = await SitemapRequestLoader.open({ ...options, signal: controller.signal });
- await sleep(50);
+ // Abort while the first sub-sitemap is still streaming, so the state is persisted with the
+ // load aborted half-way through the index.
+ while (await list.isEmpty()) {
+ await sleep(10);
+ }
- await expect(list.isEmpty()).resolves.toBe(false);
+ controller.abort();
await list.persistState();
}
+ // Deliberately no signal and no timeout here: only the restored abort flag can stop the second
+ // sub-sitemap from being fetched.
const newList = await SitemapRequestLoader.open(options);
for await (const request of newList) {
await newList.markRequestAsHandled(request);