Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: treat `406` as other `4xx` status codes in `HttpCrawler` (#2907)
Jindřich Bärb0e6f6d3fc44
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex 8d723e6db..00b64008f 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -899,13 +899,6 @@ export class HttpCrawler<const { statusCode } = response;const { type } = parseContentTypeFromResponse(response);- if (statusCode === 406) {- request.noRetry = true;- throw new Error(- `Resource ${request.url} is not available in the format requested by the Accept header. Skipping resource.`,- );- }-// eslint-disable-next-line dot-notation -- accessing private propertyconst blockedStatusCodes = this.sessionPool ? this.sessionPool['blockedStatusCodes'] : [];// if we retry the request, can the Content-Type change?diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.tsindex 13948c018..29e3379f3 100644--- a/test/core/crawlers/cheerio_crawler.test.ts+++ b/test/core/crawlers/cheerio_crawler.test.ts@@ -502,34 +502,6 @@ describe('CheerioCrawler', () => {expect(errorMessages).toHaveLength(8);errorMessages.forEach((msg) => expect(msg).toMatch('CUSTOM_ERROR'));});-- test('when 406 is received', async () => {- // Mock Request to respond with a 406.- crawler = new CheerioCrawler({- requestList: await getRequestListForMock({- headers: {- 'content-type': 'text/plain',- },- statusCode: 406,- }),- maxRequestRetries: 1,- requestHandler: () => {- handlePageInvocationCount++;- },- failedRequestHandler: ({ request }) => {- errorMessages = [...errorMessages, ...request.errorMessages];- },- });- await crawler.run();-- expect(handlePageInvocationCount).toBe(0);- expect(errorMessages).toHaveLength(4);- errorMessages.forEach((msg) => {- expect(msg).toMatch(- 'is not available in the format requested by the Accept header. Skipping resource.',- );- });- });});});