Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: log original error message on session rotation (#2022)
Jindřich Bär8a11ffbdaef6
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex c0d5be868..d88106684 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -626,7 +626,7 @@ export abstract class BrowserCrawler<*/protected _throwIfProxyError(error: Error) {if (this.isProxyError(error)) {- throw new SessionError(error.message);+ throw new SessionError(this._getMessageFromError(error) as string);}}diff --git a/packages/core/src/errors.ts b/packages/core/src/errors.tsindex 26631745e..f8a0e1499 100644--- a/packages/core/src/errors.ts+++ b/packages/core/src/errors.ts@@ -31,6 +31,6 @@ export class RetryRequestError extends Error {*/export class SessionError extends RetryRequestError {constructor(message?: string) {- super(message ?? 'Detected a session error, rotating session...');+ super(`Detected a session error, rotating session... ${message ? `\n${message}` : ''}`);}}diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex 64c74c407..1de29fca8 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -597,7 +597,7 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, H}if (this.isProxyError(e as Error)) {- throw new SessionError();+ throw new SessionError(this._getMessageFromError(e as Error) as string);} else {throw e;}diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.tsindex a26457eae..3483740d2 100644--- a/test/core/crawlers/browser_crawler.test.ts+++ b/test/core/crawlers/browser_crawler.test.ts@@ -851,6 +851,41 @@ describe('BrowserCrawler', () => {await expect(browserCrawler.run()).rejects.toThrow();expect(numberOfRotations).toBe(5);});++ test('proxy rotation logs the original proxy error', async () => {+ const proxyConfiguration = new ProxyConfiguration({ proxyUrls: ['http://localhost:1234'] });++ const proxyError = 'Proxy responded with 400 - Bad request. Also, this error message contains some useful payload.';++ const crawler = new class extends BrowserCrawlerTest {+ protected override async _navigationHandler(ctx: PuppeteerCrawlingContext): Promise<HTTPResponse | null | undefined> {+ const { session } = ctx;+ const proxyInfo = await this.proxyConfiguration.newProxyInfo(session?.id);++ if (proxyInfo.url.includes('localhost')) {+ throw new Error(proxyError);+ }++ return null;+ }+ }({+ browserPoolOptions: {+ browserPlugins: [puppeteerPlugin],+ },+ requestList,+ maxSessionRotations: 1,+ maxConcurrency: 1,+ proxyConfiguration,+ requestHandler: async () => {},+ });++ const spy = jest.spyOn((crawler as any).log, 'warning' as any).mockImplementation(() => {});++ await expect(crawler.run([serverAddress])).rejects.toThrow();++ expect(spy).toBeCalled();+ expect(spy.mock.calls[0][0]).toEqual(expect.stringContaining(proxyError));+ });});describe('Crawling context', () => {diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.tsindex 93e52365f..e77f8ec35 100644--- a/test/core/crawlers/cheerio_crawler.test.ts+++ b/test/core/crawlers/cheerio_crawler.test.ts@@ -743,6 +743,33 @@ describe('CheerioCrawler', () => {await expect(crawler.run([serverAddress])).rejects.toThrow();expect(numberOfRotations).toBe(5);});++ test('proxy rotation logs the original proxy error', async () => {+ const proxyConfiguration = new ProxyConfiguration({ proxyUrls: ['http://localhost:1234'] });++ const proxyError = 'Proxy responded with 400 - Bad request. Also, this error message contains some useful payload.';++ const crawler = new CheerioCrawler({+ proxyConfiguration,+ maxSessionRotations: 1,+ requestHandler: async () => {},+ });++ jest.spyOn(crawler, '_requestAsBrowser' as any).mockImplementation(async ({ proxyUrl }: any) => {+ if (proxyUrl.includes('localhost')) {+ throw new Error(proxyError);+ }++ return null;+ });++ const spy = jest.spyOn((crawler as any).log, 'warning' as any).mockImplementation(() => {});++ await expect(crawler.run([serverAddress])).rejects.toThrow();++ expect(spy).toBeCalled();+ expect(spy.mock.calls[0][0]).toEqual(expect.stringContaining(proxyError));+ });});describe('SessionPool', () => {