Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: log original error message on session rotation (#2022)

Jindřich Bär8a11ffbdaef6
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index c0d5be868..d88106684 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -626,7 +626,7 @@ export abstract class BrowserCrawler<
*/
protected _throwIfProxyError(error: Error) {
if (this.isProxyError(error)) {
- throw new SessionError(error.message);
+ throw new SessionError(this._getMessageFromError(error) as string);
}
}
diff --git a/packages/core/src/errors.ts b/packages/core/src/errors.ts
index 26631745e..f8a0e1499 100644
--- a/packages/core/src/errors.ts
+++ b/packages/core/src/errors.ts
@@ -31,6 +31,6 @@ export class RetryRequestError extends Error {
*/
export class SessionError extends RetryRequestError {
constructor(message?: string) {
- super(message ?? 'Detected a session error, rotating session...');
+ super(`Detected a session error, rotating session... ${message ? `\n${message}` : ''}`);
}
}
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
index 64c74c407..1de29fca8 100644
--- a/packages/http-crawler/src/internals/http-crawler.ts
+++ b/packages/http-crawler/src/internals/http-crawler.ts
@@ -597,7 +597,7 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, H
}
if (this.isProxyError(e as Error)) {
- throw new SessionError();
+ throw new SessionError(this._getMessageFromError(e as Error) as string);
} else {
throw e;
}
diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.ts
index a26457eae..3483740d2 100644
--- a/test/core/crawlers/browser_crawler.test.ts
+++ b/test/core/crawlers/browser_crawler.test.ts
@@ -851,6 +851,41 @@ describe('BrowserCrawler', () => {
await expect(browserCrawler.run()).rejects.toThrow();
expect(numberOfRotations).toBe(5);
});
+
+ test('proxy rotation logs the original proxy error', async () => {
+ const proxyConfiguration = new ProxyConfiguration({ proxyUrls: ['http://localhost:1234'] });
+
+ const proxyError = 'Proxy responded with 400 - Bad request. Also, this error message contains some useful payload.';
+
+ const crawler = new class extends BrowserCrawlerTest {
+ protected override async _navigationHandler(ctx: PuppeteerCrawlingContext): Promise<HTTPResponse | null | undefined> {
+ const { session } = ctx;
+ const proxyInfo = await this.proxyConfiguration.newProxyInfo(session?.id);
+
+ if (proxyInfo.url.includes('localhost')) {
+ throw new Error(proxyError);
+ }
+
+ return null;
+ }
+ }({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ },
+ requestList,
+ maxSessionRotations: 1,
+ maxConcurrency: 1,
+ proxyConfiguration,
+ requestHandler: async () => {},
+ });
+
+ const spy = jest.spyOn((crawler as any).log, 'warning' as any).mockImplementation(() => {});
+
+ await expect(crawler.run([serverAddress])).rejects.toThrow();
+
+ expect(spy).toBeCalled();
+ expect(spy.mock.calls[0][0]).toEqual(expect.stringContaining(proxyError));
+ });
});
describe('Crawling context', () => {
diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.ts
index 93e52365f..e77f8ec35 100644
--- a/test/core/crawlers/cheerio_crawler.test.ts
+++ b/test/core/crawlers/cheerio_crawler.test.ts
@@ -743,6 +743,33 @@ describe('CheerioCrawler', () => {
await expect(crawler.run([serverAddress])).rejects.toThrow();
expect(numberOfRotations).toBe(5);
});
+
+ test('proxy rotation logs the original proxy error', async () => {
+ const proxyConfiguration = new ProxyConfiguration({ proxyUrls: ['http://localhost:1234'] });
+
+ const proxyError = 'Proxy responded with 400 - Bad request. Also, this error message contains some useful payload.';
+
+ const crawler = new CheerioCrawler({
+ proxyConfiguration,
+ maxSessionRotations: 1,
+ requestHandler: async () => {},
+ });
+
+ jest.spyOn(crawler, '_requestAsBrowser' as any).mockImplementation(async ({ proxyUrl }: any) => {
+ if (proxyUrl.includes('localhost')) {
+ throw new Error(proxyError);
+ }
+
+ return null;
+ });
+
+ const spy = jest.spyOn((crawler as any).log, 'warning' as any).mockImplementation(() => {});
+
+ await expect(crawler.run([serverAddress])).rejects.toThrow();
+
+ expect(spy).toBeCalled();
+ expect(spy.mock.calls[0][0]).toEqual(expect.stringContaining(proxyError));
+ });
});
describe('SessionPool', () => {