Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

test: fix storage pollution in browser crawler tests (#3861)

Martin Adámek92a356fa1cd3
diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.ts
index d7f207fd7..1bbd4080c 100644
--- a/test/core/crawlers/browser_crawler.test.ts
+++ b/test/core/crawlers/browser_crawler.test.ts
@@ -8,7 +8,14 @@ import {
PuppeteerPlugin,
RemoteBrowserPool,
} from '@crawlee/browser-pool';
-import { BLOCKED_STATUS_CODES, MemoryStorageBackend, serviceLocator, SessionPool } from '@crawlee/core';
+import {
+ bindMethodsToServiceLocator,
+ BLOCKED_STATUS_CODES,
+ MemoryStorageBackend,
+ serviceLocator,
+ ServiceLocator,
+ SessionPool,
+} from '@crawlee/core';
import type { PuppeteerGoToOptions } from '@crawlee/puppeteer';
import { EnqueueStrategy, ProxyConfiguration, Request, RequestList, RequestState, Session } from '@crawlee/puppeteer';
import { sleep } from '@crawlee/utils';
@@ -49,11 +56,20 @@ describe('BrowserCrawler', () => {
server.close();
});
+ aroundEach(async (t) => {
+ const scopedServiceLocator = new ServiceLocator();
+ const { run } = bindMethodsToServiceLocator(scopedServiceLocator, {});
+
+ await run(t);
+ });
+
+ // Runs inside the `aroundEach` scope, after the global setup's `serviceLocator.reset()`,
+ // so it can't be clobbered by it.
beforeEach(() => {
serviceLocator.setStorageBackend(new MemoryStorageBackend());
});
- test('should work', async () => {
+ test.concurrent('should work', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const sources = [
@@ -101,7 +117,7 @@ describe('BrowserCrawler', () => {
});
});
- test('should teardown browser pool', async () => {
+ test.concurrent('should teardown browser pool', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const requestList = await RequestList.open({
@@ -130,7 +146,7 @@ describe('BrowserCrawler', () => {
expect(destroyCalled).toBe(true);
});
- test('should not tear down a user-supplied browser pool', async () => {
+ test.concurrent('should not tear down a user-supplied browser pool', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const externalPool = new BrowserPoolClass({ browserPlugins: [puppeteerPlugin] });
@@ -161,7 +177,7 @@ describe('BrowserCrawler', () => {
}
});
- test('builds and owns a RemoteBrowserPool from the remoteBrowser option', async () => {
+ test.concurrent('builds and owns a RemoteBrowserPool from the remoteBrowser option', async () => {
const crawler = new BrowserCrawlerTest({
remoteBrowser: { endpoint: 'ws://remote:9222', maxOpenBrowsers: 2 },
browserPoolOptions: { browserPlugins: [new PuppeteerPlugin(puppeteer)] },
@@ -174,7 +190,7 @@ describe('BrowserCrawler', () => {
await (crawler.browserPool as RemoteBrowserPool).destroy();
});
- test('uses browserPool and ignores remoteBrowser when both are set', async () => {
+ test.concurrent('uses browserPool and ignores remoteBrowser when both are set', async () => {
const externalPool = new BrowserPoolClass({ browserPlugins: [new PuppeteerPlugin(puppeteer)] });
try {
@@ -190,7 +206,7 @@ describe('BrowserCrawler', () => {
}
});
- test('should retire session after TimeoutError', async () => {
+ test.concurrent('should retire session after TimeoutError', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const requestList = await RequestList.open({
@@ -225,7 +241,7 @@ describe('BrowserCrawler', () => {
expect(markBadCalled).toBe(true);
});
- test('should evaluate preNavigationHooks', async () => {
+ test.concurrent('should evaluate preNavigationHooks', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const requestList = await RequestList.open({
@@ -252,7 +268,7 @@ describe('BrowserCrawler', () => {
expect(hook).toHaveBeenCalled();
});
- test('should evaluate postNavigationHooks', async () => {
+ test.concurrent('should evaluate postNavigationHooks', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const requestList = await RequestList.open({
@@ -279,7 +295,7 @@ describe('BrowserCrawler', () => {
expect(hook).toHaveBeenCalled();
});
- test('postNavigationHooks can override response, observed downstream', async () => {
+ test.concurrent('postNavigationHooks can override response, observed downstream', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const requestList = await RequestList.open({
@@ -319,7 +335,7 @@ describe('BrowserCrawler', () => {
expect(observed.fromHandler).toBe(fakeStatus);
});
- test('errorHandler has open page', async () => {
+ test.concurrent('errorHandler has open page', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const requestList = await RequestList.open({
@@ -349,7 +365,7 @@ describe('BrowserCrawler', () => {
});
// see https://github.com/apify/crawlee/issues/3873
- test('errorHandler has open page after non-timeout navigation error', async () => {
+ test.concurrent('errorHandler has open page after non-timeout navigation error', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const requestList = await RequestList.open({
@@ -380,7 +396,7 @@ describe('BrowserCrawler', () => {
expect(pageClosedStates[0]).toBe(false);
});
- test('should correctly track request.state', async () => {
+ test.concurrent('should correctly track request.state', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const sources = [{ url: `${serverAddress}/?q=1` }];
@@ -425,7 +441,7 @@ describe('BrowserCrawler', () => {
]);
});
- test('should allow modifying gotoOptions by pre navigation hooks', async () => {
+ test.concurrent('should allow modifying gotoOptions by pre navigation hooks', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const requestList = await RequestList.open({
@@ -460,7 +476,7 @@ describe('BrowserCrawler', () => {
expect(optionsGoto!.timeout).toEqual(60000);
});
- test('should ignore errors in Page.close()', async () => {
+ test.concurrent('should ignore errors in Page.close()', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
for (let i = 0; i < 2; i++) {
@@ -493,7 +509,7 @@ describe('BrowserCrawler', () => {
}
});
- test('should respect the requestHandlerTimeoutSecs option', async () => {
+ test.concurrent('should respect the requestHandlerTimeoutSecs option', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const requestList = await RequestList.open({
@@ -523,7 +539,7 @@ describe('BrowserCrawler', () => {
expect(callSpy).toBeCalledWith('good');
});
- test('should not throw without SessionPool', async () => {
+ test.concurrent('should not throw without SessionPool', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const requestList = await RequestList.open({
@@ -541,7 +557,7 @@ describe('BrowserCrawler', () => {
expect(browserCrawler).toBeDefined();
});
- test('should correctly set session pool options', async () => {
+ test.concurrent('should correctly set session pool options', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const requestList = await RequestList.open({
@@ -625,7 +641,7 @@ describe('BrowserCrawler', () => {
});
});
- test('should throw on "blocked" status codes', async () => {
+ test.concurrent('should throw on "blocked" status codes', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const baseUrl = 'https://example.com/';
@@ -670,7 +686,7 @@ describe('BrowserCrawler', () => {
expect(called).toBe(false);
});
- test('retryOnBlocked should retry on Cloudflare challenge', async () => {
+ test.concurrent('retryOnBlocked should retry on Cloudflare challenge', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const urls = [new URL('/special/cloudflareBlocking', serverAddress).href];
@@ -700,7 +716,7 @@ describe('BrowserCrawler', () => {
expect(processed).toBe(false);
});
- test('retryOnBlocked throws on "blocked" status codes', async () => {
+ test.concurrent('retryOnBlocked throws on "blocked" status codes', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const baseUrl = 'https://example.com/';
@@ -742,7 +758,7 @@ describe('BrowserCrawler', () => {
expect(processed).toBe(false);
});
- test('should throw on "blocked" status codes (retire session)', async () => {
+ test.concurrent('should throw on "blocked" status codes (retire session)', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const baseUrl = 'https://example.com/';
@@ -787,7 +803,7 @@ describe('BrowserCrawler', () => {
expect(called).toBe(false);
});
- test('should retire browser with session', async () => {
+ test.concurrent('should retire browser with session', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const requestList = await RequestList.open({
@@ -814,7 +830,7 @@ describe('BrowserCrawler', () => {
expect(retiredBrowserCount).toBeGreaterThan(0);
});
- test('should increment session usage correctly', async () => {
+ test.concurrent('should increment session usage correctly', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const sessionUsageHistory: number[] = [];
@@ -843,7 +859,7 @@ describe('BrowserCrawler', () => {
expect(sessionUsageHistory).toEqual([0, 1, 2, 3, 4, 5]);
});
- test('should allow using fingerprints from browser pool', async () => {
+ test.concurrent('should allow using fingerprints from browser pool', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const pool = new BrowserPoolClass({
@@ -926,7 +942,7 @@ describe('BrowserCrawler', () => {
delete process.env[ENV_VARS.PROXY_PASSWORD];
});
- test('proxy rotation on error works as expected', async () => {
+ test.concurrent('proxy rotation on error works as expected', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const requestList = await RequestList.open({
@@ -973,7 +989,7 @@ describe('BrowserCrawler', () => {
expect(requestHandler).toHaveBeenCalledTimes(4);
});
- test('proxy rotation on error respects maxRequestRetries, calls failedRequestHandler', async () => {
+ test.concurrent('proxy rotation on error respects maxRequestRetries, calls failedRequestHandler', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const requestList = await RequestList.open({
@@ -1025,7 +1041,7 @@ describe('BrowserCrawler', () => {
expect(numberOfRotations).toBe(4 * 5);
});
- test('proxy rotation logs the original proxy error', async () => {
+ test.concurrent('proxy rotation logs the original proxy error', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const requestList = await RequestList.open({