Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

test: fix flaky tests (#3323)

Martin Adámeka2fa3eede17e
diff --git a/package.json b/package.json
index d4fb9808b..6d81d5e9b 100644
--- a/package.json
+++ b/package.json
@@ -82,7 +82,7 @@
"@types/semver": "^7.3.12",
"@types/stream-json": "^1.7.2",
"@types/yargs": "^17.0.26",
- "@vitest/coverage-v8": "^4.0.1",
+ "@vitest/coverage-v8": "^4.0.16",
"apify": "*",
"apify-node-curl-impersonate": "^1.0.15",
"basic-auth-parser": "^0.0.2",
@@ -113,7 +113,7 @@
"turbo": "^2.1.0",
"typescript": "^5.7.3",
"typescript-eslint": "^8.28.0",
- "vitest": "^4.0.1"
+ "vitest": "^4.0.16"
},
"packageManager": "[email protected]",
"volta": {
diff --git a/test/browser-pool/browser-plugins/plugins.test.ts b/test/browser-pool/browser-plugins/plugins.test.ts
index 5ceebafa7..1c2786d38 100644
--- a/test/browser-pool/browser-plugins/plugins.test.ts
+++ b/test/browser-pool/browser-plugins/plugins.test.ts
@@ -44,26 +44,20 @@ const runPluginTest = <
Controller: C,
library: L,
) => {
- let plugin = new Plugin(library as never);
-
- describe(`${plugin.constructor.name} - ${'name' in library ? library.name!() : ''} general`, () => {
- let browser: playwright.Browser | UnwrapPromise<ReturnType<(typeof puppeteer)['launch']>> | undefined;
-
- beforeEach(() => {
- plugin = new Plugin(library as never);
- });
-
- afterEach(async () => {
- await browser?.close();
- });
-
- test('should launch browser', async () => {
- browser = await plugin.launch();
- expect(typeof browser.newPage).toBe('function');
- expect(typeof browser.close).toBe('function');
+ describe(`${library.constructor.name} - ${'name' in library ? library.name!() : ''} general`, () => {
+ test.concurrent('should launch browser', async () => {
+ const plugin = new Plugin(library as never);
+ const browser = await plugin.launch();
+ try {
+ expect(typeof browser.newPage).toBe('function');
+ expect(typeof browser.close).toBe('function');
+ } finally {
+ await browser.close();
+ }
});
- test('should create launch context', () => {
+ test.concurrent('should create launch context', () => {
+ const plugin = new Plugin(library as never);
const id = 'abc';
const launchOptions = { foo: 'bar' };
const proxyUrl = 'http://proxy.com/';
@@ -100,10 +94,10 @@ const runPluginTest = <
expect(context.useIncognitoPages).toEqual(desiredObject.useIncognitoPages);
});
- test('should get default launchContext values from plugin options', async () => {
+ test.concurrent('should get default launchContext values from plugin options', async () => {
const proxyUrl = 'http://[email protected]:8080/';
- plugin = new Plugin(library as never, {
+ const plugin = new Plugin(library as never, {
proxyUrl,
userDataDir: 'test',
useIncognitoPages: true,
@@ -116,51 +110,62 @@ const runPluginTest = <
expect(context.userDataDir).toEqual('test');
});
- test('should create browser controller', () => {
+ test.concurrent('should create browser controller', () => {
+ const plugin = new Plugin(library as never);
const browserController = plugin.createController();
expect(browserController).toBeInstanceOf(Controller);
});
- test('should work with cookies', async () => {
+ test.concurrent('should work with cookies', async () => {
+ const plugin = new Plugin(library as never);
const browserController = plugin.createController();
const context = plugin.createLaunchContext();
- browser = await plugin.launch(context as never);
-
- browserController.assignBrowser(browser as never, context as never);
- browserController.activate();
+ const browser = await plugin.launch(context as never);
- const page = await browserController.newPage();
- await browserController.setCookies(page as never, [
- { name: 'TEST', value: 'TESTER-COOKIE', url: serverAddress },
- ]);
- await page.goto(serverAddress, { waitUntil: 'domcontentloaded' });
+ try {
+ browserController.assignBrowser(browser as never, context as never);
+ browserController.activate();
- const cookies = await browserController.getCookies(page as never);
- expect(cookies[0].name).toBe('TEST');
- expect(cookies[0].value).toBe('TESTER-COOKIE');
+ const page = await browserController.newPage();
+ await browserController.setCookies(page as never, [
+ { name: 'TEST', value: 'TESTER-COOKIE', url: serverAddress },
+ ]);
+ await page.goto(serverAddress, { waitUntil: 'domcontentloaded' });
+
+ const cookies = await browserController.getCookies(page as never);
+ expect(cookies[0].name).toBe('TEST');
+ expect(cookies[0].value).toBe('TESTER-COOKIE');
+ } finally {
+ await browser.close();
+ }
});
- test('newPage options cannot be used with persistent context', async () => {
+ test.concurrent('newPage options cannot be used with persistent context', async () => {
+ const plugin = new Plugin(library as never);
const browserController = plugin.createController();
const context = plugin.createLaunchContext({
useIncognitoPages: false,
});
- browser = await plugin.launch(context as never);
- browserController.assignBrowser(browser as never, context as never);
- browserController.activate();
-
+ const browser = await plugin.launch(context as never);
try {
- const page = await browserController.newPage({});
- await page.close();
+ browserController.assignBrowser(browser as never, context as never);
+ browserController.activate();
+
+ try {
+ const page = await browserController.newPage({});
+ await page.close();
- expect(false).toBe(true);
- } catch (error: any) {
- expect(error.message).toBe(
- 'A new page can be created with provided context only when using incognito pages or experimental containers.',
- );
+ expect(false).toBe(true);
+ } catch (error: any) {
+ expect(error.message).toBe(
+ 'A new page can be created with provided context only when using incognito pages or experimental containers.',
+ );
+ }
+ } finally {
+ await browser.close();
}
});
});
@@ -192,13 +197,7 @@ describe('Plugins', () => {
});
describe('Puppeteer specifics', () => {
- let browser: Browser;
-
- afterEach(async () => {
- await browser.close();
- });
-
- test('should work with non authenticated proxyUrl', async () => {
+ test.concurrent('should work with non authenticated proxyUrl', async () => {
const proxyUrl = `http://127.0.0.2:${unprotectedProxy.port}`;
const plugin = new PuppeteerPlugin(puppeteer);
@@ -214,19 +213,23 @@ describe('Plugins', () => {
},
});
- browser = await plugin.launch(context);
+ const browser = await plugin.launch(context);
- const page = await browser.newPage();
- const response = await page.goto(`http://127.0.0.1:${(target.address() as AddressInfo).port}`);
+ try {
+ const page = await browser.newPage();
+ const response = await page.goto(`http://127.0.0.1:${(target.address() as AddressInfo).port}`);
- const text = await response!.text();
+ const text = await response!.text();
- expect(text).toBe('127.0.0.2');
+ expect(text).toBe('127.0.0.2');
- await page.close();
+ await page.close();
+ } finally {
+ await browser.close();
+ }
});
- test('should work with authenticated proxyUrl', async () => {
+ test.concurrent('should work with authenticated proxyUrl', async () => {
const proxyUrl = `http://foo:[email protected]:${protectedProxy.port}`;
const plugin = new PuppeteerPlugin(puppeteer);
@@ -242,53 +245,63 @@ describe('Plugins', () => {
},
});
- browser = await plugin.launch(context);
-
- const page = await browser.newPage();
- const response = await page.goto(`http://127.0.0.1:${(target.address() as AddressInfo).port}`);
+ const browser = await plugin.launch(context);
- const text = await response!.text();
+ try {
+ const page = await browser.newPage();
+ const response = await page.goto(`http://127.0.0.1:${(target.address() as AddressInfo).port}`);
- expect(text).toBe('127.0.0.3');
+ const text = await response!.text();
- await page.close();
+ expect(text).toBe('127.0.0.3');
- await browser.close();
+ await page.close();
+ } finally {
+ await browser.close();
+ }
});
- test('should use persistent context by default', async () => {
+ test.concurrent('should use persistent context by default', async () => {
const plugin = new PuppeteerPlugin(puppeteer);
const browserController = plugin.createController();
const launchContext = plugin.createLaunchContext();
- browser = await plugin.launch(launchContext);
- browserController.assignBrowser(browser, launchContext);
- browserController.activate();
+ const browser = await plugin.launch(launchContext);
+ try {
+ browserController.assignBrowser(browser, launchContext);
+ browserController.activate();
- const page = await browserController.newPage();
- const browserContext = page.browserContext();
+ const page = await browserController.newPage();
+ const browserContext = page.browserContext();
- expect(browserContext.id).toBeFalsy();
+ expect(browserContext.id).toBeFalsy();
+ } finally {
+ await browser.close();
+ }
});
- test('should use incognito pages by option', async () => {
+ test.concurrent('should use incognito pages by option', async () => {
const plugin = new PuppeteerPlugin(puppeteer);
const browserController = plugin.createController();
const launchContext = plugin.createLaunchContext({ useIncognitoPages: true });
- browser = await plugin.launch(launchContext);
- browserController.assignBrowser(browser, launchContext);
- browserController.activate();
+ const browser = await plugin.launch(launchContext);
+ try {
+ browserController.assignBrowser(browser, launchContext);
+ browserController.activate();
- const page = await browserController.newPage();
- const browserContext = page.browserContext();
+ const page = await browserController.newPage();
+ const browserContext = page.browserContext();
- expect(browserContext.id).toBeTruthy();
+ expect(browserContext.id).toBeTruthy();
+ } finally {
+ await browser.close();
+ }
});
- test('should pass launch options to browser', async () => {
+ test.concurrent('should pass launch options to browser', async () => {
const plugin = new PuppeteerPlugin(puppeteer);
const userAgent = 'HelloWorld';
@@ -298,12 +311,16 @@ describe('Plugins', () => {
};
const launchContext = plugin.createLaunchContext({ launchOptions });
- browser = await plugin.launch(launchContext);
+ const browser = await plugin.launch(launchContext);
- expect(await browser.userAgent()).toBe(userAgent);
+ try {
+ expect(await browser.userAgent()).toBe(userAgent);
+ } finally {
+ await browser.close();
+ }
});
- test('proxyUsername and proxyPassword as newPage options', async () => {
+ test.concurrent('proxyUsername and proxyPassword as newPage options', async () => {
const plugin = new PuppeteerPlugin(puppeteer);
const browserController = new PuppeteerController(plugin);
@@ -311,23 +328,27 @@ describe('Plugins', () => {
useIncognitoPages: true,
});
- browser = await plugin.launch(launchContext);
- browserController.assignBrowser(browser, launchContext);
- browserController.activate();
+ const browser = await plugin.launch(launchContext);
+ try {
+ browserController.assignBrowser(browser, launchContext);
+ browserController.activate();
- const page = await browserController.newPage({
- proxyServer: `http://127.0.0.3:${protectedProxy.port}`,
- proxyUsername: 'foo',
- proxyPassword: 'bar',
- proxyBypassList: ['<-loopback>'],
- });
+ const page = await browserController.newPage({
+ proxyServer: `http://127.0.0.3:${protectedProxy.port}`,
+ proxyUsername: 'foo',
+ proxyPassword: 'bar',
+ proxyBypassList: ['<-loopback>'],
+ });
- const response = await page.goto(`http://127.0.0.1:${(target.address() as AddressInfo).port}`);
- const text = await response!.text();
+ const response = await page.goto(`http://127.0.0.1:${(target.address() as AddressInfo).port}`);
+ const text = await response!.text();
- expect(text).toBe('127.0.0.3');
+ expect(text).toBe('127.0.0.3');
- await page.close();
+ await page.close();
+ } finally {
+ await browser.close();
+ }
});
});
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index 5ab04a286..871286742 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -956,20 +956,28 @@ describe('BasicCrawler', () => {
const request1 = new Request({ url: 'http://example.com/1' });
vitest.spyOn(requestQueue, 'handledCount').mockReturnValue(Promise.resolve() as any);
- const markRequestHandled = vitest
- .spyOn(requestQueue, 'markRequestHandled')
- .mockReturnValue(Promise.resolve() as any);
+
+ let handledCount = 0;
+ const markRequestHandled = vitest.spyOn(requestQueue, 'markRequestHandled').mockImplementation(async () => {
+ handledCount++;
+ // Only set isFinished after both requests have been handled
+ if (handledCount >= 2) {
+ // Small delay to ensure the test can verify everything
+ setTimeout(() => {
+ isFinished = true;
+ }, 50);
+ }
+ return Promise.resolve() as any;
+ });
const isFinishedOrig = vitest.spyOn(requestQueue, 'isFinished');
requestQueue.fetchNextRequest = async () => queue.pop()!;
requestQueue.isEmpty = async () => Promise.resolve(!queue.length);
- setTimeout(() => queue.push(request0), 10);
- setTimeout(() => queue.push(request1), 100);
- setTimeout(() => {
- isFinished = true;
- }, 150);
+ // Add requests with buffer time for crawler startup
+ setTimeout(() => queue.push(request0), 150);
+ setTimeout(() => queue.push(request1), 350);
await basicCrawler.run();
@@ -1016,11 +1024,11 @@ describe('BasicCrawler', () => {
requestQueue.fetchNextRequest = async () => Promise.resolve(queue.pop()!);
requestQueue.isEmpty = async () => Promise.resolve(!queue.length);
- setTimeout(() => queue.push(request0), 10);
- setTimeout(() => queue.push(request1), 100);
+ setTimeout(() => queue.push(request0), 100);
+ setTimeout(() => queue.push(request1), 250);
setTimeout(() => {
void basicCrawler.teardown();
- }, 300);
+ }, 650);
await basicCrawler.run();
diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.ts
index 6075c1c2c..ab5b51ca0 100644
--- a/test/core/crawlers/browser_crawler.test.ts
+++ b/test/core/crawlers/browser_crawler.test.ts
@@ -26,8 +26,6 @@ import { BrowserCrawlerTest } from './basic_browser_crawler';
describe('BrowserCrawler', () => {
let prevEnvHeadless: string;
let logLevel: number;
- const localStorageEmulator = new MemoryStorageEmulator();
- let puppeteerPlugin: PuppeteerPlugin;
let serverAddress = 'http://localhost:';
let port: number;
@@ -43,383 +41,484 @@ describe('BrowserCrawler', () => {
serverAddress += port;
});
- beforeEach(async () => {
- await localStorageEmulator.init();
- puppeteerPlugin = new PuppeteerPlugin(puppeteer);
- });
-
- afterEach(async () => {
- puppeteerPlugin = null as any;
- });
-
afterAll(async () => {
- await localStorageEmulator.destroy();
log.setLevel(logLevel);
process.env.CRAWLEE_HEADLESS = prevEnvHeadless;
server.close();
});
- test('should work', async () => {
- const sources = [
- { url: `${serverAddress}/?q=1` },
- { url: `${serverAddress}/?q=2` },
- { url: `${serverAddress}/?q=3` },
- { url: `${serverAddress}/?q=4` },
- { url: `${serverAddress}/?q=5` },
- { url: `${serverAddress}/?q=6` },
- ];
- const sourcesCopy = JSON.parse(JSON.stringify(sources));
- const processed: Request[] = [];
- const failed: Request[] = [];
- const requestList = await RequestList.open(null, sources);
- const requestHandler: PuppeteerRequestHandler = async ({ page, request, response }) => {
- await page.waitForSelector('title');
-
- expect(response!.status()).toBe(200);
- request.userData.title = await page.title();
- processed.push(request);
- };
+ test.concurrent('should work', async () => {
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
+
+ try {
+ const sources = [
+ { url: `${serverAddress}/?q=1` },
+ { url: `${serverAddress}/?q=2` },
+ { url: `${serverAddress}/?q=3` },
+ { url: `${serverAddress}/?q=4` },
+ { url: `${serverAddress}/?q=5` },
+ { url: `${serverAddress}/?q=6` },
+ ];
+ const sourcesCopy = JSON.parse(JSON.stringify(sources));
+ const processed: Request[] = [];
+ const failed: Request[] = [];
+ const requestList = await RequestList.open(null, sources);
+ const requestHandler: PuppeteerRequestHandler = async ({ page, request, response }) => {
+ await page.waitForSelector('title');
+
+ expect(response!.status()).toBe(200);
+ request.userData.title = await page.title();
+ processed.push(request);
+ };
- const browserCrawler = new BrowserCrawlerTest({
- browserPoolOptions: {
- browserPlugins: [puppeteerPlugin],
- },
- requestList,
- minConcurrency: 1,
- maxConcurrency: 1,
- requestHandler,
- failedRequestHandler: async ({ request }) => {
- failed.push(request);
- },
- });
+ const browserCrawler = new BrowserCrawlerTest({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ },
+ requestList,
+ minConcurrency: 1,
+ maxConcurrency: 1,
+ requestHandler,
+ failedRequestHandler: async ({ request }) => {
+ failed.push(request);
+ },
+ });
- await browserCrawler.run();
+ await browserCrawler.run();
- expect(browserCrawler.autoscaledPool!.minConcurrency).toBe(1);
- expect(processed).toHaveLength(6);
- expect(failed).toHaveLength(0);
+ expect(browserCrawler.autoscaledPool!.minConcurrency).toBe(1);
+ expect(processed).toHaveLength(6);
+ expect(failed).toHaveLength(0);
- processed.forEach((request, id) => {
- expect(request.url).toEqual(sourcesCopy[id].url);
- expect(request.userData.title).toBe('Example Domain');
- });
+ processed.forEach((request, id) => {
+ expect(request.url).toEqual(sourcesCopy[id].url);
+ expect(request.userData.title).toBe('Example Domain');
+ });
+ } finally {
+ await localStorageEmulator.destroy();
+ }
});
- test('should teardown browser pool', async () => {
- const requestList = await RequestList.open({
- sources: [{ url: 'http://example.com/?q=1' }],
- });
- const browserCrawler = new BrowserCrawlerTest({
- browserPoolOptions: {
- browserPlugins: [puppeteerPlugin],
- },
- requestList,
- useSessionPool: true,
- requestHandler: async () => {},
- maxRequestRetries: 1,
- });
- vitest.spyOn(browserCrawler.browserPool, 'destroy');
+ test.concurrent('should teardown browser pool', async () => {
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
- await browserCrawler.run();
- expect(browserCrawler.browserPool.destroy).toBeCalled();
+ try {
+ const requestList = await RequestList.open({
+ sources: [{ url: 'http://example.com/?q=1' }],
+ });
+ const browserCrawler = new BrowserCrawlerTest({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ },
+ requestList,
+ useSessionPool: true,
+ requestHandler: async () => {},
+ maxRequestRetries: 1,
+ });
+
+ // Spy on destroy and track if it was called
+ let destroyCalled = false;
+ const originalDestroy = browserCrawler.browserPool.destroy.bind(browserCrawler.browserPool);
+ browserCrawler.browserPool.destroy = async () => {
+ destroyCalled = true;
+ return originalDestroy();
+ };
+
+ await browserCrawler.run();
+ expect(destroyCalled).toBe(true);
+ } finally {
+ await localStorageEmulator.destroy();
+ }
});
- test('should retire session after TimeoutError', async () => {
- const requestList = await RequestList.open({
- sources: [{ url: 'http://example.com/?q=1' }],
- });
- class TimeoutError extends Error {}
- let sessionGoto!: Session;
- const browserCrawler = new (class extends BrowserCrawlerTest {
- protected override async _navigationHandler(
- ctx: PuppeteerCrawlingContext,
- ): Promise<HTTPResponse | null | undefined> {
- vitest.spyOn(ctx.session!, 'markBad');
- sessionGoto = ctx.session!;
- throw new TimeoutError();
- }
- })({
- browserPoolOptions: {
- browserPlugins: [puppeteerPlugin],
- },
- requestList,
- useSessionPool: true,
- requestHandler: async () => {},
- maxRequestRetries: 1,
- });
+ test.concurrent('should retire session after TimeoutError', async () => {
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
- await browserCrawler.run();
- expect(sessionGoto.markBad).toBeCalled();
+ try {
+ const requestList = await RequestList.open({
+ sources: [{ url: 'http://example.com/?q=1' }],
+ });
+ class TimeoutError extends Error {}
+ let markBadCalled = false;
+ let sessionGoto!: Session;
+ const browserCrawler = new (class extends BrowserCrawlerTest {
+ protected override async _navigationHandler(
+ ctx: PuppeteerCrawlingContext,
+ ): Promise<HTTPResponse | null | undefined> {
+ sessionGoto = ctx.session!;
+ const originalMarkBad = sessionGoto.markBad.bind(sessionGoto);
+ sessionGoto.markBad = () => {
+ markBadCalled = true;
+ return originalMarkBad();
+ };
+ throw new TimeoutError();
+ }
+ })({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ },
+ requestList,
+ useSessionPool: true,
+ requestHandler: async () => {},
+ maxRequestRetries: 1,
+ });
+
+ await browserCrawler.run();
+ expect(markBadCalled).toBe(true);
+ } finally {
+ await localStorageEmulator.destroy();
+ }
});
- test('should evaluate preNavigationHooks', async () => {
- const requestList = await RequestList.open({
- sources: [{ url: 'http://example.com/?q=1' }],
- });
- let isEvaluated = false;
-
- const browserCrawler = new (class extends BrowserCrawlerTest {
- protected override async _navigationHandler(
- ctx: PuppeteerCrawlingContext,
- gotoOptions: PuppeteerGoToOptions,
- ): Promise<HTTPResponse | null | undefined> {
- isEvaluated = ctx.hookFinished as boolean;
- return ctx.page.goto(ctx.request.url, gotoOptions);
- }
- })({
- browserPoolOptions: {
- browserPlugins: [puppeteerPlugin],
- },
- requestList,
- useSessionPool: true,
- requestHandler: async () => {},
- maxRequestRetries: 0,
- preNavigationHooks: [
- async (crawlingContext) => {
- await sleep(10);
- crawlingContext.hookFinished = true;
+ test.concurrent('should evaluate preNavigationHooks', async () => {
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
+
+ try {
+ const requestList = await RequestList.open({
+ sources: [{ url: 'http://example.com/?q=1' }],
+ });
+ let isEvaluated = false;
+
+ const browserCrawler = new (class extends BrowserCrawlerTest {
+ protected override async _navigationHandler(
+ ctx: PuppeteerCrawlingContext,
+ gotoOptions: PuppeteerGoToOptions,
+ ): Promise<HTTPResponse | null | undefined> {
+ isEvaluated = ctx.hookFinished as boolean;
+ return ctx.page.goto(ctx.request.url, gotoOptions);
+ }
+ })({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
},
- ],
- });
+ requestList,
+ useSessionPool: true,
+ requestHandler: async () => {},
+ maxRequestRetries: 0,
+ preNavigationHooks: [
+ async (crawlingContext) => {
+ await sleep(10);
+ crawlingContext.hookFinished = true;
+ },
+ ],
+ });
- await browserCrawler.run();
+ await browserCrawler.run();
- expect(isEvaluated).toBeTruthy();
+ expect(isEvaluated).toBeTruthy();
+ } finally {
+ await localStorageEmulator.destroy();
+ }
});
- test('should evaluate postNavigationHooks', async () => {
- const requestList = await RequestList.open({
- sources: [{ url: `${serverAddress}/?q=1` }],
- });
- let isEvaluated = false;
+ test.concurrent('should evaluate postNavigationHooks', async () => {
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
- const browserCrawler = new BrowserCrawlerTest({
- browserPoolOptions: {
- browserPlugins: [puppeteerPlugin],
- },
- requestList,
- useSessionPool: true,
- requestHandler: async ({ hookFinished }) => {
- isEvaluated = hookFinished as boolean;
- },
- maxRequestRetries: 0,
- postNavigationHooks: [
- async (crawlingContext) => {
- await sleep(10);
- crawlingContext.hookFinished = true;
+ try {
+ const requestList = await RequestList.open({
+ sources: [{ url: `${serverAddress}/?q=1` }],
+ });
+ let isEvaluated = false;
+
+ const browserCrawler = new BrowserCrawlerTest({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
},
- ],
- });
+ requestList,
+ useSessionPool: true,
+ requestHandler: async ({ hookFinished }) => {
+ isEvaluated = hookFinished as boolean;
+ },
+ maxRequestRetries: 0,
+ postNavigationHooks: [
+ async (crawlingContext) => {
+ await sleep(10);
+ crawlingContext.hookFinished = true;
+ },
+ ],
+ });
- await browserCrawler.run();
+ await browserCrawler.run();
- expect(isEvaluated).toBeTruthy();
+ expect(isEvaluated).toBeTruthy();
+ } finally {
+ await localStorageEmulator.destroy();
+ }
});
- test('errorHandler has open page', async () => {
- const requestList = await RequestList.open({
- sources: [{ url: `${serverAddress}/?q=1` }],
- });
+ test.concurrent('errorHandler has open page', async () => {
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
- const result: string[] = [];
+ try {
+ const requestList = await RequestList.open({
+ sources: [{ url: `${serverAddress}/?q=1` }],
+ });
- const browserCrawler = new BrowserCrawlerTest({
- browserPoolOptions: {
- browserPlugins: [puppeteerPlugin],
- },
- requestList,
- requestHandler: async (ctx) => {
- throw new Error('Test error');
- },
- maxRequestRetries: 1,
- errorHandler: async (ctx, error) => {
- result.push(await ctx.page.evaluate(() => window.location.origin));
- },
- });
+ const result: string[] = [];
- await browserCrawler.run();
+ const browserCrawler = new BrowserCrawlerTest({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ },
+ requestList,
+ requestHandler: async (ctx) => {
+ throw new Error('Test error');
+ },
+ maxRequestRetries: 1,
+ errorHandler: async (ctx, error) => {
+ result.push(await ctx.page.evaluate(() => window.location.origin));
+ },
+ });
+
+ await browserCrawler.run();
- expect(result.length).toBe(1);
- expect(result[0]).toBe(serverAddress);
+ expect(result.length).toBe(1);
+ expect(result[0]).toBe(serverAddress);
+ } finally {
+ await localStorageEmulator.destroy();
+ }
});
- test('should correctly track request.state', async () => {
- const sources = [{ url: `${serverAddress}/?q=1` }];
- const requestList = await RequestList.open(null, sources);
- const requestStates: RequestState[] = [];
+ test.concurrent('should correctly track request.state', async () => {
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
- const browserCrawler = new BrowserCrawlerTest({
- browserPoolOptions: {
- browserPlugins: [puppeteerPlugin],
- },
- requestList,
- preNavigationHooks: [
- async ({ request }) => {
+ try {
+ const sources = [{ url: `${serverAddress}/?q=1` }];
+ const requestList = await RequestList.open(null, sources);
+ const requestStates: RequestState[] = [];
+
+ const browserCrawler = new BrowserCrawlerTest({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ },
+ requestList,
+ preNavigationHooks: [
+ async ({ request }) => {
+ requestStates.push(request.state);
+ },
+ ],
+ postNavigationHooks: [
+ async ({ request }) => {
+ requestStates.push(request.state);
+ },
+ ],
+ requestHandler: async ({ request }) => {
requestStates.push(request.state);
+ throw new Error('Error');
},
- ],
- postNavigationHooks: [
- async ({ request }) => {
+ maxRequestRetries: 1,
+ errorHandler: async ({ request }) => {
requestStates.push(request.state);
},
- ],
- requestHandler: async ({ request }) => {
- requestStates.push(request.state);
- throw new Error('Error');
- },
- maxRequestRetries: 1,
- errorHandler: async ({ request }) => {
- requestStates.push(request.state);
- },
- });
+ });
- await browserCrawler.run();
+ await browserCrawler.run();
- expect(requestStates).toEqual([
- RequestState.BEFORE_NAV,
- RequestState.AFTER_NAV,
- RequestState.REQUEST_HANDLER,
- RequestState.ERROR_HANDLER,
- RequestState.BEFORE_NAV,
- RequestState.AFTER_NAV,
- RequestState.REQUEST_HANDLER,
- ]);
+ expect(requestStates).toEqual([
+ RequestState.BEFORE_NAV,
+ RequestState.AFTER_NAV,
+ RequestState.REQUEST_HANDLER,
+ RequestState.ERROR_HANDLER,
+ RequestState.BEFORE_NAV,
+ RequestState.AFTER_NAV,
+ RequestState.REQUEST_HANDLER,
+ ]);
+ } finally {
+ await localStorageEmulator.destroy();
+ }
});
- test('should allow modifying gotoOptions by pre navigation hooks', async () => {
- const requestList = await RequestList.open({
- sources: [{ url: `${serverAddress}/?q=1` }],
- });
- let optionsGoto: PuppeteerGoToOptions;
- const browserCrawler = new (class extends BrowserCrawlerTest {
- protected override async _navigationHandler(
- ctx: PuppeteerCrawlingContext,
- gotoOptions: PuppeteerGoToOptions,
- ): Promise<HTTPResponse | null | undefined> {
- optionsGoto = gotoOptions;
- return ctx.page.goto(ctx.request.url, gotoOptions);
- }
- })({
- browserPoolOptions: {
- browserPlugins: [puppeteerPlugin],
- },
- requestList,
- useSessionPool: true,
- requestHandler: async () => {},
- maxRequestRetries: 0,
- preNavigationHooks: [
- async (_crawlingContext, gotoOptions) => {
- gotoOptions!.timeout = 60000;
+ test.concurrent('should allow modifying gotoOptions by pre navigation hooks', async () => {
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
+
+ try {
+ const requestList = await RequestList.open({
+ sources: [{ url: `${serverAddress}/?q=1` }],
+ });
+ let optionsGoto: PuppeteerGoToOptions;
+ const browserCrawler = new (class extends BrowserCrawlerTest {
+ protected override async _navigationHandler(
+ ctx: PuppeteerCrawlingContext,
+ gotoOptions: PuppeteerGoToOptions,
+ ): Promise<HTTPResponse | null | undefined> {
+ optionsGoto = gotoOptions;
+ return ctx.page.goto(ctx.request.url, gotoOptions);
+ }
+ })({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
},
- ],
- });
+ requestList,
+ useSessionPool: true,
+ requestHandler: async () => {},
+ maxRequestRetries: 0,
+ preNavigationHooks: [
+ async (_crawlingContext, gotoOptions) => {
+ gotoOptions!.timeout = 60000;
+ },
+ ],
+ });
- await browserCrawler.run();
+ await browserCrawler.run();
- expect(optionsGoto!.timeout).toEqual(60000);
+ expect(optionsGoto!.timeout).toEqual(60000);
+ } finally {
+ await localStorageEmulator.destroy();
+ }
});
- test('should ignore errors in Page.close()', async () => {
- for (let i = 0; i < 2; i++) {
+ test.concurrent('should ignore errors in Page.close()', async () => {
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
+
+ try {
+ for (let i = 0; i < 2; i++) {
+ const requestList = await RequestList.open({
+ sources: [{ url: `${serverAddress}/?q=1` }],
+ });
+ let failedCalled = false;
+
+ const browserCrawler = new BrowserCrawlerTest({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ },
+ requestList,
+ requestHandler: async ({ page }) => {
+ page.close = async () => {
+ if (i === 0) {
+ throw new Error();
+ } else {
+ return Promise.reject(new Error());
+ }
+ };
+ return Promise.resolve();
+ },
+ failedRequestHandler: async () => {
+ failedCalled = true;
+ },
+ });
+ await browserCrawler.run();
+ expect(failedCalled).toBe(false);
+ }
+ } finally {
+ await localStorageEmulator.destroy();
+ }
+ });
+
+ test.concurrent('should respect the requestHandlerTimeoutSecs option', async () => {
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
+
+ try {
const requestList = await RequestList.open({
sources: [{ url: `${serverAddress}/?q=1` }],
});
- let failedCalled = false;
+
+ const callSpy = vitest.fn();
const browserCrawler = new BrowserCrawlerTest({
browserPoolOptions: {
browserPlugins: [puppeteerPlugin],
},
requestList,
- requestHandler: async ({ page }) => {
- page.close = async () => {
- if (i === 0) {
- throw new Error();
- } else {
- return Promise.reject(new Error());
- }
- };
- return Promise.resolve();
- },
- failedRequestHandler: async () => {
- failedCalled = true;
+ requestHandler: async () => {
+ setTimeout(() => callSpy('good'), 300);
+ setTimeout(() => callSpy('bad'), 1500);
+ await new Promise(() => {});
},
+ requestHandlerTimeoutSecs: 0.5,
+ maxRequestRetries: 0,
});
await browserCrawler.run();
- expect(failedCalled).toBe(false);
+
+ expect(callSpy).toBeCalledTimes(1);
+ expect(callSpy).toBeCalledWith('good');
+ } finally {
+ await localStorageEmulator.destroy();
}
});
- test('should respect the requestHandlerTimeoutSecs option', async () => {
- const requestList = await RequestList.open({
- sources: [{ url: `${serverAddress}/?q=1` }],
- });
-
- const callSpy = vitest.fn();
+ test.concurrent('should not throw without SessionPool', async () => {
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
- const browserCrawler = new BrowserCrawlerTest({
- browserPoolOptions: {
- browserPlugins: [puppeteerPlugin],
- },
- requestList,
- requestHandler: async () => {
- setTimeout(() => callSpy('good'), 300);
- setTimeout(() => callSpy('bad'), 1500);
- await new Promise(() => {});
- },
- requestHandlerTimeoutSecs: 0.5,
- maxRequestRetries: 0,
- });
- await browserCrawler.run();
+ try {
+ const requestList = await RequestList.open({
+ sources: [{ url: 'http://example.com/?q=1' }],
+ });
+ const browserCrawler = new BrowserCrawlerTest({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ },
+ requestList,
+ useSessionPool: false,
+ requestHandler: async () => {},
+ });
- expect(callSpy).toBeCalledTimes(1);
- expect(callSpy).toBeCalledWith('good');
+ expect(browserCrawler).toBeDefined();
+ } finally {
+ await localStorageEmulator.destroy();
+ }
});
- test('should not throw without SessionPool', async () => {
- const requestList = await RequestList.open({
- sources: [{ url: 'http://example.com/?q=1' }],
- });
- const browserCrawler = new BrowserCrawlerTest({
- browserPoolOptions: {
- browserPlugins: [puppeteerPlugin],
- },
- requestList,
- useSessionPool: false,
- requestHandler: async () => {},
- });
-
- expect(browserCrawler).toBeDefined();
- });
+ test.concurrent('should correctly set session pool options', async () => {
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
- test('should correctly set session pool options', async () => {
- const requestList = await RequestList.open({
- sources: [{ url: 'http://example.com/?q=1' }],
- });
+ try {
+ const requestList = await RequestList.open({
+ sources: [{ url: 'http://example.com/?q=1' }],
+ });
- const crawler = new BrowserCrawlerTest({
- requestList,
- browserPoolOptions: {
- browserPlugins: [puppeteerPlugin],
- },
- useSessionPool: true,
- persistCookiesPerSession: false,
- sessionPoolOptions: {
- sessionOptions: {
- maxUsageCount: 1,
+ const crawler = new BrowserCrawlerTest({
+ requestList,
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
},
- persistStateKeyValueStoreId: 'abc',
- },
- requestHandler: async () => {},
- });
+ useSessionPool: true,
+ persistCookiesPerSession: false,
+ sessionPoolOptions: {
+ sessionOptions: {
+ maxUsageCount: 1,
+ },
+ persistStateKeyValueStoreId: 'abc',
+ },
+ requestHandler: async () => {},
+ });
- // @ts-expect-error Accessing private prop
- expect(crawler.sessionPoolOptions.sessionOptions.maxUsageCount).toBe(1);
- // @ts-expect-error Accessing private prop
- expect(crawler.sessionPoolOptions.persistStateKeyValueStoreId).toBe('abc');
+ // @ts-expect-error Accessing private prop
+ expect(crawler.sessionPoolOptions.sessionOptions.maxUsageCount).toBe(1);
+ // @ts-expect-error Accessing private prop
+ expect(crawler.sessionPoolOptions.persistStateKeyValueStoreId).toBe('abc');
+ } finally {
+ await localStorageEmulator.destroy();
+ }
});
test.skip('should persist cookies per session', async () => {
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const name = `list-${Math.random()}`;
const requestList = await RequestList.open({
persistStateKey: name,
@@ -474,268 +573,308 @@ describe('BrowserCrawler', () => {
});
});
- test('should throw on "blocked" status codes', async () => {
- const baseUrl = 'https://example.com/';
- const sources = BLOCKED_STATUS_CODES.map((statusCode) => {
- return {
- url: baseUrl + statusCode,
- userData: { statusCode },
- };
- });
- const requestList = await RequestList.open(null, sources);
+ test.concurrent('should throw on "blocked" status codes', async () => {
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
+
+ try {
+ const baseUrl = 'https://example.com/';
+ const sources = BLOCKED_STATUS_CODES.map((statusCode) => {
+ return {
+ url: baseUrl + statusCode,
+ userData: { statusCode },
+ };
+ });
+ const requestList = await RequestList.open(null, sources);
- let called = false;
- const failedRequests: Request[] = [];
- const crawler = new BrowserCrawlerTest({
- browserPoolOptions: {
- browserPlugins: [puppeteerPlugin],
- },
- requestList,
- useSessionPool: true,
- persistCookiesPerSession: false,
- maxRequestRetries: 0,
- requestHandler: async () => {
- called = true;
- },
- failedRequestHandler: async ({ request }) => {
- failedRequests.push(request);
- },
- });
+ let called = false;
+ const failedRequests: Request[] = [];
+ const crawler = new BrowserCrawlerTest({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ },
+ requestList,
+ useSessionPool: true,
+ persistCookiesPerSession: false,
+ maxRequestRetries: 0,
+ requestHandler: async () => {
+ called = true;
+ },
+ failedRequestHandler: async ({ request }) => {
+ failedRequests.push(request);
+ },
+ });
- // @ts-expect-error Overriding protected method
- crawler._navigationHandler = async ({ request }) => {
- return { status: () => request.userData.statusCode };
- };
+ // @ts-expect-error Overriding protected method
+ crawler._navigationHandler = async ({ request }) => {
+ return { status: () => request.userData.statusCode };
+ };
- await crawler.run();
+ await crawler.run();
- expect(failedRequests.length).toBe(3);
- failedRequests.forEach((fr) => {
- const [msg] = fr.errorMessages;
- expect(msg).toContain(`Request blocked - received ${fr.userData.statusCode} status code.`);
- });
- expect(called).toBe(false);
+ expect(failedRequests.length).toBe(3);
+ failedRequests.forEach((fr) => {
+ const [msg] = fr.errorMessages;
+ expect(msg).toContain(`Request blocked - received ${fr.userData.statusCode} status code.`);
+ });
+ expect(called).toBe(false);
+ } finally {
+ await localStorageEmulator.destroy();
+ }
});
- test('retryOnBlocked should retry on Cloudflare challenge', async () => {
- const urls = [new URL('/special/cloudflareBlocking', serverAddress).href];
- const maxSessionRotations = 1;
+ test.concurrent('retryOnBlocked should retry on Cloudflare challenge', async () => {
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
- let processed = false;
- const errorMessages: string[] = [];
+ try {
+ const urls = [new URL('/special/cloudflareBlocking', serverAddress).href];
+ const maxSessionRotations = 1;
- const crawler = new BrowserCrawlerTest({
- browserPoolOptions: {
- browserPlugins: [puppeteerPlugin],
- },
- retryOnBlocked: true,
- maxSessionRotations,
- requestHandler: async ({ page, response }) => {
- processed = true;
- },
- failedRequestHandler: async ({ request }) => {
- errorMessages.push(...request.errorMessages);
- },
- });
+ let processed = false;
+ const errorMessages: string[] = [];
- await crawler.run(urls);
+ const crawler = new BrowserCrawlerTest({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ },
+ retryOnBlocked: true,
+ maxSessionRotations,
+ requestHandler: async ({ page, response }) => {
+ processed = true;
+ },
+ failedRequestHandler: async ({ request }) => {
+ errorMessages.push(...request.errorMessages);
+ },
+ });
- expect(errorMessages).toHaveLength(urls.length * (maxSessionRotations + 1));
- expect(errorMessages.every((x) => x.includes('Detected a session error, rotating session...'))).toBe(true);
- expect(processed).toBe(false);
+ await crawler.run(urls);
+
+ expect(errorMessages).toHaveLength(urls.length * (maxSessionRotations + 1));
+ expect(errorMessages.every((x) => x.includes('Detected a session error, rotating session...'))).toBe(true);
+ expect(processed).toBe(false);
+ } finally {
+ await localStorageEmulator.destroy();
+ }
});
- test('retryOnBlocked throws on "blocked" status codes', async () => {
- const baseUrl = 'https://example.com/';
- const sources = BLOCKED_STATUS_CODES.map((statusCode) => {
- return {
- url: baseUrl + statusCode,
- userData: { statusCode },
- };
- });
- const requestList = await RequestList.open(null, sources);
- const maxSessionRotations = 1;
- const errorMessages: string[] = [];
+ test.concurrent('retryOnBlocked throws on "blocked" status codes', async () => {
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
+
+ try {
+ const baseUrl = 'https://example.com/';
+ const sources = BLOCKED_STATUS_CODES.map((statusCode) => {
+ return {
+ url: baseUrl + statusCode,
+ userData: { statusCode },
+ };
+ });
+ const requestList = await RequestList.open(null, sources);
+ const maxSessionRotations = 1;
+ const errorMessages: string[] = [];
- let processed = false;
- const crawler = new BrowserCrawlerTest({
- browserPoolOptions: {
- browserPlugins: [puppeteerPlugin],
- },
- requestList,
- retryOnBlocked: true,
- maxSessionRotations,
- requestHandler: async () => {
- processed = true;
- },
- failedRequestHandler: async ({ request }) => {
- errorMessages.push(...request.errorMessages);
- },
- });
+ let processed = false;
+ const crawler = new BrowserCrawlerTest({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ },
+ requestList,
+ retryOnBlocked: true,
+ maxSessionRotations,
+ requestHandler: async () => {
+ processed = true;
+ },
+ failedRequestHandler: async ({ request }) => {
+ errorMessages.push(...request.errorMessages);
+ },
+ });
- // @ts-expect-error Overriding protected method
- crawler._navigationHandler = async ({ request }) => {
- return { status: () => request.userData.statusCode };
- };
+ // @ts-expect-error Overriding protected method
+ crawler._navigationHandler = async ({ request }) => {
+ return { status: () => request.userData.statusCode };
+ };
- await crawler.run();
+ await crawler.run();
- expect(errorMessages.length).toBe(sources.length * (maxSessionRotations + 1));
- expect(errorMessages.every((x) => x.includes('Detected a session error, rotating session...'))).toBe(true);
- expect(processed).toBe(false);
+ expect(errorMessages.length).toBe(sources.length * (maxSessionRotations + 1));
+ expect(errorMessages.every((x) => x.includes('Detected a session error, rotating session...'))).toBe(true);
+ expect(processed).toBe(false);
+ } finally {
+ await localStorageEmulator.destroy();
+ }
});
- test('should throw on "blocked" status codes (retire session)', async () => {
- const baseUrl = 'https://example.com/';
- const sources = BLOCKED_STATUS_CODES.map((statusCode) => {
- return {
- url: baseUrl + statusCode,
- userData: { statusCode },
- };
- });
- const requestList = await RequestList.open(null, sources);
+ test.concurrent('should throw on "blocked" status codes (retire session)', async () => {
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
+
+ try {
+ const baseUrl = 'https://example.com/';
+ const sources = BLOCKED_STATUS_CODES.map((statusCode) => {
+ return {
+ url: baseUrl + statusCode,
+ userData: { statusCode },
+ };
+ });
+ const requestList = await RequestList.open(null, sources);
- let called = false;
- const failedRequests: Request[] = [];
- const crawler = new BrowserCrawlerTest({
- browserPoolOptions: {
- browserPlugins: [puppeteerPlugin],
- },
- requestList,
- useSessionPool: true,
- persistCookiesPerSession: false,
- maxRequestRetries: 0,
- requestHandler: async () => {
- called = true;
- },
- failedRequestHandler: async ({ request }) => {
- failedRequests.push(request);
- },
- });
+ let called = false;
+ const failedRequests: Request[] = [];
+ const crawler = new BrowserCrawlerTest({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ },
+ requestList,
+ useSessionPool: true,
+ persistCookiesPerSession: false,
+ maxRequestRetries: 0,
+ requestHandler: async () => {
+ called = true;
+ },
+ failedRequestHandler: async ({ request }) => {
+ failedRequests.push(request);
+ },
+ });
- // @ts-expect-error Overriding protected method
- crawler._navigationHandler = async ({ request }) => {
- return { status: () => request.userData.statusCode };
- };
+ // @ts-expect-error Overriding protected method
+ crawler._navigationHandler = async ({ request }) => {
+ return { status: () => request.userData.statusCode };
+ };
- await crawler.run();
+ await crawler.run();
- expect(failedRequests.length).toBe(3);
- failedRequests.forEach((fr) => {
- const [msg] = fr.errorMessages;
- expect(msg).toContain(`Request blocked - received ${fr.userData.statusCode} status code.`);
- });
- expect(called).toBe(false);
+ expect(failedRequests.length).toBe(3);
+ failedRequests.forEach((fr) => {
+ const [msg] = fr.errorMessages;
+ expect(msg).toContain(`Request blocked - received ${fr.userData.statusCode} status code.`);
+ });
+ expect(called).toBe(false);
+ } finally {
+ await localStorageEmulator.destroy();
+ }
});
- test('should retire browser with session', async () => {
- const requestList = await RequestList.open({
- sources: [{ url: 'http://example.com/?q=1' }],
- });
- let resolve: (value?: unknown) => void;
+ test.concurrent('should retire browser with session', async () => {
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
- const retirementPromise = new Promise((r) => {
- resolve = r;
- });
- let called = false;
- const browserCrawler = new (class extends BrowserCrawlerTest {
- protected override async _navigationHandler(
- ctx: PuppeteerCrawlingContext,
- ): Promise<HTTPResponse | null | undefined> {
- ctx.crawler.browserPool.on(BROWSER_POOL_EVENTS.BROWSER_RETIRED, () => {
- resolve();
- called = true;
- });
- ctx.session!.retire();
- return ctx.page.goto(ctx.request.url);
- }
- })({
- browserPoolOptions: {
- browserPlugins: [puppeteerPlugin],
- },
- requestList,
- useSessionPool: true,
- requestHandler: async () => {
- await retirementPromise;
- },
- maxRequestRetries: 1,
- });
+ try {
+ const requestList = await RequestList.open({
+ sources: [{ url: 'http://example.com/?q=1' }],
+ });
+ let resolve: (value?: unknown) => void;
- await browserCrawler.run();
+ const retirementPromise = new Promise((r) => {
+ resolve = r;
+ });
+ let called = false;
+ const browserCrawler = new (class extends BrowserCrawlerTest {
+ protected override async _navigationHandler(
+ ctx: PuppeteerCrawlingContext,
+ ): Promise<HTTPResponse | null | undefined> {
+ ctx.crawler.browserPool.on(BROWSER_POOL_EVENTS.BROWSER_RETIRED, () => {
+ resolve();
+ called = true;
+ });
+ ctx.session!.retire();
+ return ctx.page.goto(ctx.request.url);
+ }
+ })({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ },
+ requestList,
+ useSessionPool: true,
+ requestHandler: async () => {
+ await retirementPromise;
+ },
+ maxRequestRetries: 1,
+ });
+
+ await browserCrawler.run();
- expect(called).toBeTruthy();
+ expect(called).toBeTruthy();
+ } finally {
+ await localStorageEmulator.destroy();
+ }
});
- test('should increment session usage correctly', async () => {
- const sessionUsageHistory: number[] = [];
+ test.concurrent('should increment session usage correctly', async () => {
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
- const browserCrawler = new BrowserCrawlerTest({
- browserPoolOptions: {
- browserPlugins: [puppeteerPlugin],
- },
- useSessionPool: true,
- sessionPoolOptions: {
- maxPoolSize: 1,
- },
- requestHandler: async ({ session }) => {
- sessionUsageHistory.push(session!.usageCount);
- },
- });
+ try {
+ const sessionUsageHistory: number[] = [];
- await browserCrawler.run([
- { url: `${serverAddress}/?q=1` },
- { url: `${serverAddress}/?q=2` },
- { url: `${serverAddress}/?q=3` },
- { url: `${serverAddress}/?q=4` },
- { url: `${serverAddress}/?q=5` },
- { url: `${serverAddress}/?q=6` },
- ]);
+ const browserCrawler = new BrowserCrawlerTest({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ },
+ useSessionPool: true,
+ sessionPoolOptions: {
+ maxPoolSize: 1,
+ },
+ requestHandler: async ({ session }) => {
+ sessionUsageHistory.push(session!.usageCount);
+ },
+ });
- expect(sessionUsageHistory).toEqual([0, 1, 2, 3, 4, 5]);
+ await browserCrawler.run([
+ { url: `${serverAddress}/?q=1` },
+ { url: `${serverAddress}/?q=2` },
+ { url: `${serverAddress}/?q=3` },
+ { url: `${serverAddress}/?q=4` },
+ { url: `${serverAddress}/?q=5` },
+ { url: `${serverAddress}/?q=6` },
+ ]);
+
+ expect(sessionUsageHistory).toEqual([0, 1, 2, 3, 4, 5]);
+ } finally {
+ await localStorageEmulator.destroy();
+ }
});
- test('should allow using fingerprints from browser pool', async () => {
- const requestList = await RequestList.open({
- sources: [{ url: `${serverAddress}/?q=1` }],
- });
- const browserCrawler = new BrowserCrawlerTest({
- browserPoolOptions: {
- browserPlugins: [puppeteerPlugin],
- useFingerprints: true,
- fingerprintOptions: {
- fingerprintGeneratorOptions: {
- operatingSystems: [OperatingSystemsName.windows],
+ test.concurrent('should allow using fingerprints from browser pool', async () => {
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
+
+ try {
+ const requestList = await RequestList.open({
+ sources: [{ url: `${serverAddress}/?q=1` }],
+ });
+ const browserCrawler = new BrowserCrawlerTest({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ useFingerprints: true,
+ fingerprintOptions: {
+ fingerprintGeneratorOptions: {
+ operatingSystems: [OperatingSystemsName.windows],
+ },
},
},
- },
- requestList,
- useSessionPool: false,
- requestHandler: async ({ browserController }) => {
- expect(browserController.launchContext.fingerprint).toBeDefined();
- },
- });
+ requestList,
+ useSessionPool: false,
+ requestHandler: async ({ browserController }) => {
+ expect(browserController.launchContext.fingerprint).toBeDefined();
+ },
+ });
- await browserCrawler.run();
- expect.hasAssertions();
+ await browserCrawler.run();
+ expect.hasAssertions();
+ } finally {
+ await localStorageEmulator.destroy();
+ }
});
describe('proxy', () => {
- let requestList: RequestList | undefined;
- beforeEach(async () => {
- requestList = await RequestList.open({
- sources: [
- { url: 'http://example.com/?q=1' },
- { url: 'http://example.com/?q=2' },
- { url: 'http://example.com/?q=3' },
- { url: 'http://example.com/?q=4' },
- ],
- });
- });
-
- afterEach(() => {
- requestList = undefined;
- });
-
// TODO move to actor sdk tests before splitting the repos
// test('browser should launch with correct proxyUrl', async () => {
// process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
@@ -818,272 +957,365 @@ describe('BrowserCrawler', () => {
// stub.mockClear();
// });
+ // This test manipulates environment variables, so it must NOT be run concurrently
test('browser should launch with rotated custom proxy', async () => {
- process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
+
+ try {
+ process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
+
+ const requestList = await RequestList.open({
+ sources: [
+ { url: `${serverAddress}/?q=1` },
+ { url: `${serverAddress}/?q=2` },
+ { url: `${serverAddress}/?q=3` },
+ ],
+ });
- const proxyConfiguration = new ProxyConfiguration({
- proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],
- });
+ const proxyConfiguration = new ProxyConfiguration({
+ proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],
+ });
- const browserProxies: string[] = [];
+ const browserProxies: string[] = [];
- const browserCrawler = new BrowserCrawlerTest({
- browserPoolOptions: {
- browserPlugins: [puppeteerPlugin],
- maxOpenPagesPerBrowser: 1,
- retireBrowserAfterPageCount: 1,
- },
- requestList,
- requestHandler: async () => {},
- proxyConfiguration,
- maxRequestRetries: 0,
- maxConcurrency: 1,
- });
+ const browserCrawler = new BrowserCrawlerTest({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ maxOpenPagesPerBrowser: 1,
+ retireBrowserAfterPageCount: 1,
+ },
+ requestList,
+ requestHandler: async () => {},
+ proxyConfiguration,
+ maxRequestRetries: 0,
+ maxConcurrency: 1,
+ });
- browserCrawler.browserPool.postLaunchHooks.push((_pageId, browserController) => {
- browserProxies.push(browserController.launchContext.proxyUrl!);
- });
+ browserCrawler.browserPool.postLaunchHooks.push((_pageId, browserController) => {
+ browserProxies.push(browserController.launchContext.proxyUrl!);
+ });
- await browserCrawler.run();
+ await browserCrawler.run();
- // @ts-expect-error Accessing private property
- const proxiesToUse = proxyConfiguration.proxyUrls!;
- for (const proxyUrl of proxiesToUse) {
- expect(browserProxies.includes(new URL(proxyUrl!).href.slice(0, -1))).toBeTruthy();
- }
+ // @ts-expect-error Accessing private property
+ const proxiesToUse = proxyConfiguration.proxyUrls!;
+ for (const proxyUrl of proxiesToUse) {
+ expect(browserProxies.includes(new URL(proxyUrl!).href.slice(0, -1))).toBeTruthy();
+ }
- delete process.env[ENV_VARS.PROXY_PASSWORD];
+ delete process.env[ENV_VARS.PROXY_PASSWORD];
+ } finally {
+ await localStorageEmulator.destroy();
+ }
});
- test('proxy rotation on error works as expected', async () => {
- const goodProxyUrl = 'http://good.proxy';
- const proxyConfiguration = new ProxyConfiguration({
- proxyUrls: ['http://localhost', 'http://localhost:1234', goodProxyUrl],
- });
- const requestHandler = vitest.fn();
-
- const browserCrawler = new (class extends BrowserCrawlerTest {
- protected override async _navigationHandler(
- ctx: PuppeteerCrawlingContext,
- ): Promise<HTTPResponse | null | undefined> {
- const { session } = ctx;
- const proxyInfo = await this.proxyConfiguration!.newProxyInfo(session?.id);
+ test.concurrent('proxy rotation on error works as expected', async () => {
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
+
+ try {
+ const requestList = await RequestList.open({
+ sources: [
+ { url: 'http://example.com/?q=1' },
+ { url: 'http://example.com/?q=2' },
+ { url: 'http://example.com/?q=3' },
+ { url: 'http://example.com/?q=4' },
+ ],
+ });
- if (proxyInfo!.url !== goodProxyUrl) {
- throw new Error('ERR_PROXY_CONNECTION_FAILED');
- }
+ const goodProxyUrl = 'http://good.proxy';
+ const proxyConfiguration = new ProxyConfiguration({
+ proxyUrls: ['http://localhost', 'http://localhost:1234', goodProxyUrl],
+ });
+ const requestHandler = vitest.fn();
- return null;
- }
- })({
- browserPoolOptions: {
- browserPlugins: [puppeteerPlugin],
- },
- requestList,
- maxRequestRetries: 0,
- maxConcurrency: 1,
- useSessionPool: true,
- proxyConfiguration,
- requestHandler,
- });
+ const browserCrawler = new (class extends BrowserCrawlerTest {
+ protected override async _navigationHandler(
+ ctx: PuppeteerCrawlingContext,
+ ): Promise<HTTPResponse | null | undefined> {
+ const { session } = ctx;
+ const proxyInfo = await this.proxyConfiguration!.newProxyInfo(session?.id);
- await expect(browserCrawler.run()).resolves.not.toThrow();
- expect(requestHandler).toHaveBeenCalledTimes(requestList!.length());
- });
+ if (proxyInfo!.url !== goodProxyUrl) {
+ throw new Error('ERR_PROXY_CONNECTION_FAILED');
+ }
- test('proxy rotation on error respects maxSessionRotations, calls failedRequestHandler', async () => {
- const proxyConfiguration = new ProxyConfiguration({
- proxyUrls: ['http://localhost', 'http://localhost:1234'],
- });
- const failedRequestHandler = vitest.fn();
+ return null;
+ }
+ })({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ },
+ requestList,
+ maxRequestRetries: 0,
+ maxConcurrency: 1,
+ useSessionPool: true,
+ proxyConfiguration,
+ requestHandler,
+ });
- /**
- * The first increment is the base case when the proxy is retrieved for the first time.
- */
- let numberOfRotations = -requestList!.length();
- const browserCrawler = new (class extends BrowserCrawlerTest {
- protected override async _navigationHandler(
- ctx: PuppeteerCrawlingContext,
- ): Promise<HTTPResponse | null | undefined> {
- const { session } = ctx;
- const proxyInfo = await this.proxyConfiguration!.newProxyInfo(session?.id);
+ await expect(browserCrawler.run()).resolves.not.toThrow();
+ expect(requestHandler).toHaveBeenCalledTimes(requestList!.length());
+ } finally {
+ await localStorageEmulator.destroy();
+ }
+ });
- numberOfRotations++;
+ test.concurrent(
+ 'proxy rotation on error respects maxSessionRotations, calls failedRequestHandler',
+ async () => {
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
+
+ try {
+ const requestList = await RequestList.open({
+ sources: [
+ { url: 'http://example.com/?q=1' },
+ { url: 'http://example.com/?q=2' },
+ { url: 'http://example.com/?q=3' },
+ { url: 'http://example.com/?q=4' },
+ ],
+ });
- if (proxyInfo!.url.includes('localhost')) {
- throw new Error('ERR_PROXY_CONNECTION_FAILED');
- }
+ const proxyConfiguration = new ProxyConfiguration({
+ proxyUrls: ['http://localhost', 'http://localhost:1234'],
+ });
+ const failedRequestHandler = vitest.fn();
+
+ /**
+ * The first increment is the base case when the proxy is retrieved for the first time.
+ */
+ let numberOfRotations = -requestList!.length();
+ const browserCrawler = new (class extends BrowserCrawlerTest {
+ protected override async _navigationHandler(
+ ctx: PuppeteerCrawlingContext,
+ ): Promise<HTTPResponse | null | undefined> {
+ const { session } = ctx;
+ const proxyInfo = await this.proxyConfiguration!.newProxyInfo(session?.id);
+
+ numberOfRotations++;
+
+ if (proxyInfo!.url.includes('localhost')) {
+ throw new Error('ERR_PROXY_CONNECTION_FAILED');
+ }
+
+ return null;
+ }
+ })({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ },
+ requestList,
+ maxSessionRotations: 5,
+ maxConcurrency: 1,
+ proxyConfiguration,
+ requestHandler: async () => {},
+ failedRequestHandler,
+ });
- return null;
+ await browserCrawler.run();
+ expect(failedRequestHandler).toBeCalledTimes(requestList!.length());
+ expect(numberOfRotations).toBe(requestList!.length() * 5);
+ } finally {
+ await localStorageEmulator.destroy();
}
- })({
- browserPoolOptions: {
- browserPlugins: [puppeteerPlugin],
- },
- requestList,
- maxSessionRotations: 5,
- maxConcurrency: 1,
- proxyConfiguration,
- requestHandler: async () => {},
- failedRequestHandler,
- });
+ },
+ );
+
+ test.concurrent('proxy rotation logs the original proxy error', async () => {
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
+
+ try {
+ const requestList = await RequestList.open({
+ sources: [
+ { url: 'http://example.com/?q=1' },
+ { url: 'http://example.com/?q=2' },
+ { url: 'http://example.com/?q=3' },
+ { url: 'http://example.com/?q=4' },
+ ],
+ });
- await browserCrawler.run();
- expect(failedRequestHandler).toBeCalledTimes(requestList!.length());
- expect(numberOfRotations).toBe(requestList!.length() * 5);
- });
+ const proxyConfiguration = new ProxyConfiguration({ proxyUrls: ['http://localhost:1234'] });
- test('proxy rotation logs the original proxy error', async () => {
- const proxyConfiguration = new ProxyConfiguration({ proxyUrls: ['http://localhost:1234'] });
+ const proxyError =
+ 'Proxy responded with 400 - Bad request. Also, this error message contains some useful payload.';
- const proxyError =
- 'Proxy responded with 400 - Bad request. Also, this error message contains some useful payload.';
+ const crawler = new (class extends BrowserCrawlerTest {
+ protected override async _navigationHandler(
+ ctx: PuppeteerCrawlingContext,
+ ): Promise<HTTPResponse | null | undefined> {
+ const { session } = ctx;
+ const proxyInfo = await this.proxyConfiguration!.newProxyInfo(session?.id);
- const crawler = new (class extends BrowserCrawlerTest {
- protected override async _navigationHandler(
- ctx: PuppeteerCrawlingContext,
- ): Promise<HTTPResponse | null | undefined> {
- const { session } = ctx;
- const proxyInfo = await this.proxyConfiguration!.newProxyInfo(session?.id);
+ if (proxyInfo!.url.includes('localhost')) {
+ throw new Error(proxyError);
+ }
- if (proxyInfo!.url.includes('localhost')) {
- throw new Error(proxyError);
+ return null;
}
+ })({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ },
+ requestList,
+ maxSessionRotations: 1,
+ maxConcurrency: 1,
+ proxyConfiguration,
+ requestHandler: async () => {},
+ });
- return null;
- }
- })({
- browserPoolOptions: {
- browserPlugins: [puppeteerPlugin],
- },
- requestList,
- maxSessionRotations: 1,
- maxConcurrency: 1,
- proxyConfiguration,
- requestHandler: async () => {},
- });
-
- const spy = vitest.spyOn((crawler as any).log, 'warning' as any).mockImplementation(() => {});
+ const spy = vitest.spyOn((crawler as any).log, 'warning' as any).mockImplementation(() => {});
- await crawler.run([serverAddress]);
+ await crawler.run([serverAddress]);
- expect(spy).toBeCalled();
- expect(spy.mock.calls[0][0]).toEqual(
- 'When using RequestList and RequestQueue at the same time, you should instantiate both explicitly and provide them in the crawler options, to ensure correctly handled restarts of the crawler.',
- );
- expect(spy.mock.calls[1][0]).toEqual(expect.stringContaining(proxyError));
+ expect(spy).toBeCalled();
+ expect(spy.mock.calls[0][0]).toEqual(
+ 'When using RequestList and RequestQueue at the same time, you should instantiate both explicitly and provide them in the crawler options, to ensure correctly handled restarts of the crawler.',
+ );
+ expect(spy.mock.calls[1][0]).toEqual(expect.stringContaining(proxyError));
+ } finally {
+ await localStorageEmulator.destroy();
+ }
});
});
describe('Crawling context', () => {
- const sources = ['http://example.com/'];
- let requestList: RequestList;
- let actualLogLevel: number;
- beforeEach(async () => {
- actualLogLevel = log.getLevel();
- log.setLevel(log.LEVELS.OFF);
- requestList = await RequestList.open(null, sources.slice());
- });
-
- afterAll(() => {
- log.setLevel(actualLogLevel);
- });
+ // This describe block manipulates log levels (global state), so tests must NOT be concurrent
test('uses correct crawling context', async () => {
- let prepareCrawlingContext: PuppeteerCrawlingContext;
-
- const gotoFunction = async (crawlingContext: PuppeteerCrawlingContext) => {
- prepareCrawlingContext = crawlingContext;
- expect(crawlingContext.request).toBeInstanceOf(Request);
- expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool);
- expect(crawlingContext.session).toBeInstanceOf(Session);
- expect(typeof crawlingContext.page).toBe('object');
- };
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
- const requestHandler = async (crawlingContext: PuppeteerCrawlingContext) => {
- expect(crawlingContext === prepareCrawlingContext).toEqual(true);
- expect(crawlingContext.request).toBeInstanceOf(Request);
- expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool);
- expect(crawlingContext.session).toBeInstanceOf(Session);
- expect(typeof crawlingContext.page).toBe('object');
- expect(crawlingContext.crawler).toBeInstanceOf(BrowserCrawlerTest);
- expect(Object.hasOwn(crawlingContext, 'response')).toBe(true);
+ const actualLogLevel = log.getLevel();
+ log.setLevel(log.LEVELS.OFF);
- throw new Error('some error');
- };
+ try {
+ const sources = ['http://example.com/'];
+ const requestList = await RequestList.open(null, sources.slice());
+
+ let prepareCrawlingContext: PuppeteerCrawlingContext;
+
+ const gotoFunction = async (crawlingContext: PuppeteerCrawlingContext) => {
+ prepareCrawlingContext = crawlingContext;
+ expect(crawlingContext.request).toBeInstanceOf(Request);
+ expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool);
+ expect(crawlingContext.session).toBeInstanceOf(Session);
+ expect(typeof crawlingContext.page).toBe('object');
+ };
+
+ const requestHandler = async (crawlingContext: PuppeteerCrawlingContext) => {
+ expect(crawlingContext === prepareCrawlingContext).toEqual(true);
+ expect(crawlingContext.request).toBeInstanceOf(Request);
+ expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool);
+ expect(crawlingContext.session).toBeInstanceOf(Session);
+ expect(typeof crawlingContext.page).toBe('object');
+ expect(crawlingContext.crawler).toBeInstanceOf(BrowserCrawlerTest);
+ expect(Object.hasOwn(crawlingContext, 'response')).toBe(true);
+
+ throw new Error('some error');
+ };
+
+ const failedRequestHandler = async (crawlingContext: PuppeteerCrawlingContext, error: Error) => {
+ expect(crawlingContext).toBe(prepareCrawlingContext);
+ expect(crawlingContext.request).toBeInstanceOf(Request);
+ expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool);
+ expect(crawlingContext.session).toBeInstanceOf(Session);
+ expect(typeof crawlingContext.page).toBe('object');
+ expect(crawlingContext.crawler).toBeInstanceOf(BrowserCrawlerTest);
+ expect(crawlingContext.crawler.browserPool).toBeInstanceOf(BrowserPool);
+ expect(Object.hasOwn(crawlingContext, 'response')).toBe(true);
+
+ expect(crawlingContext.error).toBeInstanceOf(Error);
+ expect(error).toBeInstanceOf(Error);
+ expect(error.message).toEqual('some error');
+ };
+
+ const browserCrawler = new BrowserCrawlerTest({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ },
+ requestList,
+ maxRequestRetries: 0,
+ maxConcurrency: 1,
+ useSessionPool: true,
+ requestHandler,
+ failedRequestHandler,
+ });
+ // @ts-expect-error Overriding protected method
+ browserCrawler._navigationHandler = gotoFunction;
- const failedRequestHandler = async (crawlingContext: PuppeteerCrawlingContext, error: Error) => {
- expect(crawlingContext).toBe(prepareCrawlingContext);
- expect(crawlingContext.request).toBeInstanceOf(Request);
- expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool);
- expect(crawlingContext.session).toBeInstanceOf(Session);
- expect(typeof crawlingContext.page).toBe('object');
- expect(crawlingContext.crawler).toBeInstanceOf(BrowserCrawlerTest);
- expect(crawlingContext.crawler.browserPool).toBeInstanceOf(BrowserPool);
- expect(Object.hasOwn(crawlingContext, 'response')).toBe(true);
-
- expect(crawlingContext.error).toBeInstanceOf(Error);
- expect(error).toBeInstanceOf(Error);
- expect(error.message).toEqual('some error');
- };
+ await browserCrawler.run();
+ } finally {
+ log.setLevel(actualLogLevel);
+ await localStorageEmulator.destroy();
+ }
+ });
+ });
- const browserCrawler = new BrowserCrawlerTest({
+ // These tests cannot run concurrently because they use crawler.run([urls])
+ // which creates internal request queues that can conflict
+ test("enqueueLinks() should skip links that don't match the strategy post redirect", async () => {
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
+
+ try {
+ const succeeded: string[] = [];
+
+ const crawler = new BrowserCrawlerTest({
browserPoolOptions: {
browserPlugins: [puppeteerPlugin],
},
- requestList,
- maxRequestRetries: 0,
maxConcurrency: 1,
- useSessionPool: true,
- requestHandler,
- failedRequestHandler,
+ maxRequestRetries: 0,
+ requestHandler: async ({ page, enqueueLinks }) => {
+ succeeded.push(await page.title());
+ await enqueueLinks({ strategy: EnqueueStrategy.SameOrigin });
+ },
});
- // @ts-expect-error Overriding protected method
- browserCrawler._navigationHandler = gotoFunction;
-
- await browserCrawler.run();
- });
- });
-
- test("enqueueLinks() should skip links that don't match the strategy post redirect", async () => {
- const succeeded: string[] = [];
-
- const crawler = new BrowserCrawlerTest({
- browserPoolOptions: {
- browserPlugins: [puppeteerPlugin],
- },
- maxConcurrency: 1,
- maxRequestRetries: 0,
- requestHandler: async ({ page, enqueueLinks }) => {
- succeeded.push(await page.title());
- await enqueueLinks({ strategy: EnqueueStrategy.SameOrigin });
- },
- });
- await crawler.run([`${serverAddress}/special/redirect`]);
+ await crawler.run([`${serverAddress}/special/redirect`]);
- expect(succeeded).toHaveLength(1);
- expect(succeeded[0]).toEqual('Redirecting outside');
+ expect(succeeded).toHaveLength(1);
+ expect(succeeded[0]).toEqual('Redirecting outside');
+ } finally {
+ await localStorageEmulator.destroy();
+ }
});
test('enqueueLinks should respect maxCrawlDepth', async () => {
- const succeeded: string[] = [];
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
- const crawler = new BrowserCrawlerTest({
- browserPoolOptions: {
- browserPlugins: [puppeteerPlugin],
- },
- maxCrawlDepth: 1,
- maxRequestsPerCrawl: 10, // avoiding accidental runaway
- requestHandler: async ({ page, enqueueLinks }) => {
- succeeded.push(await page.title());
- await enqueueLinks({ strategy: EnqueueStrategy.All });
- },
- });
+ try {
+ const succeeded: string[] = [];
+
+ const crawler = new BrowserCrawlerTest({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ },
+ maxCrawlDepth: 1,
+ maxRequestsPerCrawl: 10, // avoiding accidental runaway
+ requestHandler: async ({ page, enqueueLinks }) => {
+ succeeded.push(await page.title());
+ await enqueueLinks({ strategy: EnqueueStrategy.All });
+ },
+ });
- await crawler.run([`${serverAddress}/special/html-type`]);
+ await crawler.run([`${serverAddress}/special/html-type`]);
- expect(succeeded).toHaveLength(2);
- expect(succeeded).toEqual(['Example Domain', 'Example Domains']);
+ expect(succeeded).toHaveLength(2);
+ expect(succeeded).toEqual(['Example Domain', 'Example Domains']);
+ } finally {
+ await localStorageEmulator.destroy();
+ }
});
});
diff --git a/test/core/crawlers/file_download.test.ts b/test/core/crawlers/file_download.test.ts
index 123432ece..d772950e2 100644
--- a/test/core/crawlers/file_download.test.ts
+++ b/test/core/crawlers/file_download.test.ts
@@ -163,6 +163,17 @@ test('crawler with streamHandler waits for the stream to finish', async () => {
const fileUrl = new URL(`/file?size=${5 * 1024}&seed=789&throttle=1000`, url).toString();
await crawler.run([fileUrl]);
+ // Wait for the stream to finish (pipeline is async)
+ await new Promise<void>((resolve) => {
+ if (bufferingStream.writableFinished) {
+ resolve(undefined);
+ } else {
+ bufferingStream.once('finish', resolve);
+ // Timeout fallback
+ void setTimeout(1000).then(resolve);
+ }
+ });
+
// the stream should be finished once the crawler finishes.
expect(bufferingStream.writableFinished).toBe(true);
diff --git a/test/core/puppeteer_utils.test.ts b/test/core/puppeteer_utils.test.ts
index 50c157827..adca1bada 100644
--- a/test/core/puppeteer_utils.test.ts
+++ b/test/core/puppeteer_utils.test.ts
@@ -242,7 +242,7 @@ describe('puppeteerUtils', () => {
urlPatterns: ['.css'],
});
page.on('response', (response) => loadedUrls.push(response.url()));
- await page.goto(`${serverAddress}/special/resources`, { waitUntil: 'load' });
+ await page.goto(`${serverAddress}/special/resources`, { waitUntil: 'networkidle0' });
expect(loadedUrls).toEqual(
expect.arrayContaining([
diff --git a/test/utils/fixtures/child.js b/test/utils/fixtures/child.js
index fb2b7d031..f47b12e79 100644
--- a/test/utils/fixtures/child.js
+++ b/test/utils/fixtures/child.js
@@ -1,4 +1,4 @@
// dummy script for testing ps-tree.ts
setTimeout(() => {
/* Does nothing, but prevents exit */
-}, 1000);
+}, 5000);
diff --git a/test/utils/psTree.test.ts b/test/utils/psTree.test.ts
index 8c7f3079a..8040f8663 100644
--- a/test/utils/psTree.test.ts
+++ b/test/utils/psTree.test.ts
@@ -8,19 +8,34 @@ const scripts = {
child: path.join(__dirname, 'fixtures', 'child.js'),
};
+// Helper to poll for a condition on process tree
+async function waitForCondition(
+ pid: number,
+ condition: (children: any[]) => boolean,
+ maxWaitMs = 3000,
+): Promise<any[]> {
+ const startTime = Date.now();
+ while (Date.now() - startTime < maxWaitMs) {
+ const children = await psTree(pid);
+ if (condition(children)) {
+ return children;
+ }
+ await new Promise((resolve) => setTimeout(resolve, 100));
+ }
+ return psTree(pid); // Final attempt
+}
+
describe('psTree()', () => {
test('Spawn a Parent process which has Ten Child Processes', async () => {
const parent = exec(`node ${scripts.parent}`);
- // Wait for the child process(es) to spawn.
- await new Promise((resolve) => setTimeout(resolve, 500));
-
- const children = await psTree(parent.pid!);
+ // Poll until child processes are detected
+ const children = await waitForCondition(parent.pid!, (c) => c.length > 0);
expect(children.length).toBeGreaterThan(0);
- // Allow time for the processes to be terminated.
- await new Promise((resolve) => setTimeout(resolve, 2000));
+ // Poll until processes terminate
+ await waitForCondition(parent.pid!, (c) => c.length === 0, 10000);
const postKillChildren = await psTree(parent.pid!);
@@ -30,17 +45,17 @@ describe('psTree()', () => {
test('Includes itself if includeRoot is true', async () => {
const parent = exec(`node ${scripts.parent}`);
- // Wait for the child process(es) to spawn.
- await new Promise((resolve) => setTimeout(resolve, 500));
-
- const processes = await psTree(parent.pid!, true);
+ // Poll until processes are detected (parent + at least one child)
+ await waitForCondition(parent.pid!, (c) => c.length > 1, 3000);
- const parentProcess = processes.find((process) => Number.parseInt(process.PID, 10) === parent.pid!);
+ // Get processes with includeRoot after waiting
+ const processesWithRoot = await psTree(parent.pid!, true);
+ const parentProcess = processesWithRoot.find((process) => Number.parseInt(process.PID, 10) === parent.pid!);
expect(parentProcess).toBeDefined();
- // Allow time for the processes to be terminated.
- await new Promise((resolve) => setTimeout(resolve, 2000));
+ // Poll until processes terminate
+ await waitForCondition(parent.pid!, (c) => c.length === 0, 10000);
const postKillChildren = await psTree(parent.pid!);
diff --git a/yarn.lock b/yarn.lock
index 599b0efff..027c54a00 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -762,7 +762,7 @@ __metadata:
"@types/semver": "npm:^7.3.12"
"@types/stream-json": "npm:^1.7.2"
"@types/yargs": "npm:^17.0.26"
- "@vitest/coverage-v8": "npm:^4.0.1"
+ "@vitest/coverage-v8": "npm:^4.0.16"
apify: "npm:*"
apify-node-curl-impersonate: "npm:^1.0.15"
basic-auth-parser: "npm:^0.0.2"
@@ -793,7 +793,7 @@ __metadata:
turbo: "npm:^2.1.0"
typescript: "npm:^5.7.3"
typescript-eslint: "npm:^8.28.0"
- vitest: "npm:^4.0.1"
+ vitest: "npm:^4.0.16"
languageName: unknown
linkType: soft
@@ -3224,50 +3224,50 @@ __metadata:
languageName: node
linkType: hard
-"@vitest/coverage-v8@npm:^4.0.1":
- version: 4.0.8
- resolution: "@vitest/coverage-v8@npm:4.0.8"
+"@vitest/coverage-v8@npm:^4.0.16":
+ version: 4.0.16
+ resolution: "@vitest/coverage-v8@npm:4.0.16"
dependencies:
"@bcoe/v8-coverage": "npm:^1.0.2"
- "@vitest/utils": "npm:4.0.8"
+ "@vitest/utils": "npm:4.0.16"
ast-v8-to-istanbul: "npm:^0.3.8"
- debug: "npm:^4.4.3"
istanbul-lib-coverage: "npm:^3.2.2"
istanbul-lib-report: "npm:^3.0.1"
istanbul-lib-source-maps: "npm:^5.0.6"
istanbul-reports: "npm:^3.2.0"
magicast: "npm:^0.5.1"
+ obug: "npm:^2.1.1"
std-env: "npm:^3.10.0"
tinyrainbow: "npm:^3.0.3"
peerDependencies:
- "@vitest/browser": 4.0.8
- vitest: 4.0.8
+ "@vitest/browser": 4.0.16
+ vitest: 4.0.16
peerDependenciesMeta:
"@vitest/browser":
optional: true
- checksum: 10c0/e1bd3f261a10042c2033bb07a4cffe8c3a059d367e9c32f092d7808dfab2736775bf105ff610d274cc7519d3c94f797ac8e2498f70c36391547f98a1715f8a08
+ checksum: 10c0/3edd18dc994949d5180a3fbd9c1af4ca4756735e82cffb73b3c0918ad23a4c71521287a205cc61a39b63453448e9bfd207f82b2d472fd757dfbb47987dbe99a8
languageName: node
linkType: hard
-"@vitest/expect@npm:4.0.8":
- version: 4.0.8
- resolution: "@vitest/expect@npm:4.0.8"
+"@vitest/expect@npm:4.0.16":
+ version: 4.0.16
+ resolution: "@vitest/expect@npm:4.0.16"
dependencies:
"@standard-schema/spec": "npm:^1.0.0"
"@types/chai": "npm:^5.2.2"
- "@vitest/spy": "npm:4.0.8"
- "@vitest/utils": "npm:4.0.8"
- chai: "npm:^6.2.0"
+ "@vitest/spy": "npm:4.0.16"
+ "@vitest/utils": "npm:4.0.16"
+ chai: "npm:^6.2.1"
tinyrainbow: "npm:^3.0.3"
- checksum: 10c0/0d80695c9cfdae33eafbb39bd6bac99baa117127191e50b907544a3dc7e52c8d7d57ff7f24c88960097c71c07bf7d0babefd0f8dd8706adcfb70cdecf1128f79
+ checksum: 10c0/add4dde3548b6f65b6d7d364607713f9db258642add248a23805fa1172e48d76a7822080249efb882120b408772684569b2e78581ed3d5f282e215d7f21be183
languageName: node
linkType: hard
-"@vitest/mocker@npm:4.0.8":
- version: 4.0.8
- resolution: "@vitest/mocker@npm:4.0.8"
+"@vitest/mocker@npm:4.0.16":
+ version: 4.0.16
+ resolution: "@vitest/mocker@npm:4.0.16"
dependencies:
- "@vitest/spy": "npm:4.0.8"
+ "@vitest/spy": "npm:4.0.16"
estree-walker: "npm:^3.0.3"
magic-string: "npm:^0.30.21"
peerDependencies:
@@ -3278,54 +3278,54 @@ __metadata:
optional: true
vite:
optional: true
- checksum: 10c0/a73a3e801cd3a57efada45603abd3982aa3b22bd5011be9255a28f4e690509ea09a323120e7f6b993eb32d4eb7f7411a466eba53f1f3f2462ee908552ea0a395
+ checksum: 10c0/cf4469a4745e3cdd46e8ee4e20aa04369e7f985d40175d974d3a6f6d331bf9d8610f9638c5a18f7ff59a30ff04b19f4b823457b4c79142186fe463fa4cee80c5
languageName: node
linkType: hard
-"@vitest/pretty-format@npm:4.0.8":
- version: 4.0.8
- resolution: "@vitest/pretty-format@npm:4.0.8"
+"@vitest/pretty-format@npm:4.0.16":
+ version: 4.0.16
+ resolution: "@vitest/pretty-format@npm:4.0.16"
dependencies:
tinyrainbow: "npm:^3.0.3"
- checksum: 10c0/04df23f459f30026ea3e99940459d21bd8db3d5fa2cf111a8125ba29af847de9f13094ee1b35f241bb5ac9cb7a683cee584849b6d966996445e1e57c5f81c96c
+ checksum: 10c0/11243e9c2d2d011ae23825c6b7464a4385a4a4efc4ceb28b7854bb9d73491f440b89d12f62c5c9737d26375cf9585b11bc20183d4dea4e983e79d5e162407eb9
languageName: node
linkType: hard
-"@vitest/runner@npm:4.0.8":
- version: 4.0.8
- resolution: "@vitest/runner@npm:4.0.8"
+"@vitest/runner@npm:4.0.16":
+ version: 4.0.16
+ resolution: "@vitest/runner@npm:4.0.16"
dependencies:
- "@vitest/utils": "npm:4.0.8"
+ "@vitest/utils": "npm:4.0.16"
pathe: "npm:^2.0.3"
- checksum: 10c0/db4d51aee7a5bada9f97a0c8fc40b2ed0f301212ab2be28a024fcee1fa442393a933df820311d96bb42763a33ef1873e8ced470377dfea3af6304eed59f09d02
+ checksum: 10c0/7f4614a9fe5e9f3683d30fb82d1489796c669df45fbc0beb22d39539e4b12ebef462062705545ca04391a0406af62088cbf1d613a812ecc9ea753a0edbfd5d26
languageName: node
linkType: hard
-"@vitest/snapshot@npm:4.0.8":
- version: 4.0.8
- resolution: "@vitest/snapshot@npm:4.0.8"
+"@vitest/snapshot@npm:4.0.16":
+ version: 4.0.16
+ resolution: "@vitest/snapshot@npm:4.0.16"
dependencies:
- "@vitest/pretty-format": "npm:4.0.8"
+ "@vitest/pretty-format": "npm:4.0.16"
magic-string: "npm:^0.30.21"
pathe: "npm:^2.0.3"
- checksum: 10c0/1764d0e5aeab755710f4dc9e29e80dcaef310a7be9b48f6fde6344b3af34a1107bcab0a57ef1e1ae3e963e4b89affb5b9752618bec83b44033e8659152b664ce
+ checksum: 10c0/4fa63ffa4f30c909078210a1edcb059dbfa3ec3deaebb8f93637f65a7efae9a2d7714129bae0cf615512a683e925cf31f281fc4cb02f1fdc4c72f68ce21ca11f
languageName: node
linkType: hard
-"@vitest/spy@npm:4.0.8":
- version: 4.0.8
- resolution: "@vitest/spy@npm:4.0.8"
- checksum: 10c0/357b3ebc10421d9de34a3c20ff898fb13e1df6e484671c3043949e83ea4263f2442bc636f9b6eb5e44395229422242ec4bc62fd277a1de5b346c01ab79a95d4a
+"@vitest/spy@npm:4.0.16":
+ version: 4.0.16
+ resolution: "@vitest/spy@npm:4.0.16"
+ checksum: 10c0/2502918e703d60ef64854d0fa83ebf94da64b80e81b80c319568feee3d86069fd46e24880a768edba06c8caba13801e44005e17a0f16d9b389486f24d539f0bf
languageName: node
linkType: hard
-"@vitest/utils@npm:4.0.8":
- version: 4.0.8
- resolution: "@vitest/utils@npm:4.0.8"
+"@vitest/utils@npm:4.0.16":
+ version: 4.0.16
+ resolution: "@vitest/utils@npm:4.0.16"
dependencies:
- "@vitest/pretty-format": "npm:4.0.8"
+ "@vitest/pretty-format": "npm:4.0.16"
tinyrainbow: "npm:^3.0.3"
- checksum: 10c0/384e5db47a89e63143c335bf644d9be6e0a7f7555ed368837b9497dda20e080fcaa0c5b1c9bd8a9b49478d2b8dcfeb31be2bfb9fe7a5590f1453cbf372906436
+ checksum: 10c0/bba35b4e102be03e106ced227809437573aa5c5f64d512301ca8de127dcb91cbedc11a2e823305f8ba82528c909c10510ec8c7e3d92b3d6d1c1aec33e143572a
languageName: node
linkType: hard
@@ -4346,10 +4346,10 @@ __metadata:
languageName: node
linkType: hard
-"chai@npm:^6.2.0":
- version: 6.2.0
- resolution: "chai@npm:6.2.0"
- checksum: 10c0/a4b7d7f5907187e09f1847afa838d6d1608adc7d822031b7900813c4ed5d9702911ac2468bf290676f22fddb3d727b1be90b57c1d0a69b902534ee29cdc6ff8a
+"chai@npm:^6.2.1":
+ version: 6.2.2
+ resolution: "chai@npm:6.2.2"
+ checksum: 10c0/e6c69e5f0c11dffe6ea13d0290936ebb68fcc1ad688b8e952e131df6a6d5797d5e860bc55cef1aca2e950c3e1f96daf79e9d5a70fb7dbaab4e46355e2635ed53
languageName: node
linkType: hard
@@ -10147,6 +10147,13 @@ __metadata:
languageName: node
linkType: hard
+"obug@npm:^2.1.1":
+ version: 2.1.1
+ resolution: "obug@npm:2.1.1"
+ checksum: 10c0/59dccd7de72a047e08f8649e94c1015ec72f94eefb6ddb57fb4812c4b425a813bc7e7cd30c9aca20db3c59abc3c85cc7a62bb656a968741d770f4e8e02bc2e78
+ languageName: node
+ linkType: hard
+
"on-finished@npm:2.4.1, on-finished@npm:^2.4.1":
version: 2.4.1
resolution: "on-finished@npm:2.4.1"
@@ -12577,14 +12584,7 @@ __metadata:
languageName: node
linkType: hard
-"tinyexec@npm:^0.3.2":
- version: 0.3.2
- resolution: "tinyexec@npm:0.3.2"
- checksum: 10c0/3efbf791a911be0bf0821eab37a3445c2ba07acc1522b1fa84ae1e55f10425076f1290f680286345ed919549ad67527d07281f1c19d584df3b74326909eb1f90
- languageName: node
- linkType: hard
-
-"tinyexec@npm:^1.0.0":
+"tinyexec@npm:^1.0.0, tinyexec@npm:^1.0.2":
version: 1.0.2
resolution: "tinyexec@npm:1.0.2"
checksum: 10c0/1261a8e34c9b539a9aae3b7f0bb5372045ff28ee1eba035a2a059e532198fe1a182ec61ac60fa0b4a4129f0c4c4b1d2d57355b5cb9aa2d17ac9454ecace502ee
@@ -13339,44 +13339,44 @@ __metadata:
languageName: node
linkType: hard
-"vitest@npm:^4.0.1":
- version: 4.0.8
- resolution: "vitest@npm:4.0.8"
- dependencies:
- "@vitest/expect": "npm:4.0.8"
- "@vitest/mocker": "npm:4.0.8"
- "@vitest/pretty-format": "npm:4.0.8"
- "@vitest/runner": "npm:4.0.8"
- "@vitest/snapshot": "npm:4.0.8"
- "@vitest/spy": "npm:4.0.8"
- "@vitest/utils": "npm:4.0.8"
- debug: "npm:^4.4.3"
+"vitest@npm:^4.0.16":
+ version: 4.0.16
+ resolution: "vitest@npm:4.0.16"
+ dependencies:
+ "@vitest/expect": "npm:4.0.16"
+ "@vitest/mocker": "npm:4.0.16"
+ "@vitest/pretty-format": "npm:4.0.16"
+ "@vitest/runner": "npm:4.0.16"
+ "@vitest/snapshot": "npm:4.0.16"
+ "@vitest/spy": "npm:4.0.16"
+ "@vitest/utils": "npm:4.0.16"
es-module-lexer: "npm:^1.7.0"
expect-type: "npm:^1.2.2"
magic-string: "npm:^0.30.21"
+ obug: "npm:^2.1.1"
pathe: "npm:^2.0.3"
picomatch: "npm:^4.0.3"
std-env: "npm:^3.10.0"
tinybench: "npm:^2.9.0"
- tinyexec: "npm:^0.3.2"
+ tinyexec: "npm:^1.0.2"
tinyglobby: "npm:^0.2.15"
tinyrainbow: "npm:^3.0.3"
vite: "npm:^6.0.0 || ^7.0.0"
why-is-node-running: "npm:^2.3.0"
peerDependencies:
"@edge-runtime/vm": "*"
- "@types/debug": ^4.1.12
+ "@opentelemetry/api": ^1.9.0
"@types/node": ^20.0.0 || ^22.0.0 || >=24.0.0
- "@vitest/browser-playwright": 4.0.8
- "@vitest/browser-preview": 4.0.8
- "@vitest/browser-webdriverio": 4.0.8
- "@vitest/ui": 4.0.8
+ "@vitest/browser-playwright": 4.0.16
+ "@vitest/browser-preview": 4.0.16
+ "@vitest/browser-webdriverio": 4.0.16
+ "@vitest/ui": 4.0.16
happy-dom: "*"
jsdom: "*"
peerDependenciesMeta:
"@edge-runtime/vm":
optional: true
- "@types/debug":
+ "@opentelemetry/api":
optional: true
"@types/node":
optional: true
@@ -13394,7 +13394,7 @@ __metadata:
optional: true
bin:
vitest: vitest.mjs
- checksum: 10c0/9fa05e70168ef7098a4a441775024231faa12db2374429eeb1967e8338bd5a6a4cd25e555ac991d95d040544b42395a7425839324bb4ab124eaa80e5cf39db63
+ checksum: 10c0/b195c272198f7957c11186eb70ee78e2ec0f4524b4b5306ca8f05e41b3d84c6a4a15d02fca58d82f2b32ba61f610ae8a2a23d463a8336d7323e4832db5eef223
languageName: node
linkType: hard