Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: retry incorrect Content-Type when response has blocked status code (#2176)
Matěj Volfb54fb8bb7bc3
diff --git a/packages/core/src/session_pool/session.ts b/packages/core/src/session_pool/session.tsindex dfe97cef7..e242d0269 100644--- a/packages/core/src/session_pool/session.ts+++ b/packages/core/src/session_pool/session.ts@@ -282,8 +282,8 @@ export class Session {retireOnBlockedStatusCodes(statusCode: number, additionalBlockedStatusCodes?: number[]): boolean;retireOnBlockedStatusCodes(statusCode: number, additionalBlockedStatusCodes: number[] = []): boolean {- // @ts-expect-error- const isBlocked = this.sessionPool.blockedStatusCodes.concat(additionalBlockedStatusCodes).includes(statusCode);+ // eslint-disable-next-line dot-notation -- accessing private property+ const isBlocked = this.sessionPool['blockedStatusCodes'].concat(additionalBlockedStatusCodes).includes(statusCode);if (isBlocked) {this.retire();}diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex 169fc4cf0..fa2fefc8f 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -795,7 +795,12 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, Hthrow new Error(`Resource ${request.url} is not available in the format requested by the Accept header. Skipping resource.`);}- if (!this.supportedMimeTypes.has(type) && !this.supportedMimeTypes.has('*/*') && statusCode! < 500) {+ // eslint-disable-next-line dot-notation -- accessing private property+ const blockedStatusCodes = this.sessionPool ? this.sessionPool['blockedStatusCodes'] : [];+ // if we retry the request, can the Content-Type change?+ const isTransientContentType = statusCode! >= 500 || blockedStatusCodes.includes(statusCode!);++ if (!this.supportedMimeTypes.has(type) && !this.supportedMimeTypes.has('*/*') && !isTransientContentType) {request.noRetry = true;throw new Error(`Resource ${request.url} served Content-Type ${type}, `+ `but only ${Array.from(this.supportedMimeTypes).join(', ')} are allowed. Skipping resource.`);diff --git a/test/core/crawlers/http_crawler.test.ts b/test/core/crawlers/http_crawler.test.tsindex aeb4e2169..b8678713e 100644--- a/test/core/crawlers/http_crawler.test.ts+++ b/test/core/crawlers/http_crawler.test.ts@@ -52,6 +52,12 @@ router.set('/500Error', (req, res) => {res.end();});+router.set('/403-with-octet-stream', (req, res) => {+ res.setHeader('content-type', 'application/octet-stream');+ res.statusCode = 403;+ res.end();+});+let server: http.Server;let url: string;@@ -338,3 +344,29 @@ test('should work with delete requests', async () => {expect(failed).toHaveLength(0);});++test('should retry on 403 even with disallowed content-type', async () => {+ const succeeded: any[] = [];++ const crawler = new HttpCrawler({+ maxConcurrency: 1,+ maxRequestRetries: 1,+ preNavigationHooks: [async ({ request }) => {+ // mock 403 response with octet stream on first request attempt, but not on+ // subsequent retries, so the request should eventually succeed+ if (request.retryCount === 0) {+ request.url = `${url}/403-with-octet-stream`;+ } else {+ request.url = url;+ }+ }],+ requestHandler: async ({ request }) => {+ succeeded.push(request);+ },+ });++ await crawler.run([url]);++ expect(succeeded).toHaveLength(1);+ expect(succeeded[0].retryCount).toBe(1);+});