Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: Add options for custom HTTP error status codes (#2035)
Hamza Alwanb50ef1ad51d6
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex 1de29fca8..9f37a035d 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -154,6 +154,18 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext* It passes the "Cookie" header to the request with the session cookies.*/persistCookiesPerSession?: boolean;++ /**+ * An array of HTTP response [Status Codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) to be excluded from error consideration.+ * By default, status codes >= 500 trigger errors.+ */+ ignoreHttpErrorStatusCodes?: number[];++ /**+ * An array of additional HTTP response [Status Codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) to be treated as errors.+ * By default, status codes >= 500 trigger errors.+ */+ additionalHttpErrorStatusCodes?: number[];}/**@@ -289,6 +301,8 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, Hprotected ignoreSslErrors: boolean;protected suggestResponseEncoding?: string;protected forceResponseEncoding?: string;+ protected additionalHttpErrorStatusCodes: Set<number>;+ protected ignoreHttpErrorStatusCodes: Set<number>;protected readonly supportedMimeTypes: Set<string>;protected static override optionsShape = {@@ -303,6 +317,9 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, HproxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),persistCookiesPerSession: ow.optional.boolean,+ additionalHttpErrorStatusCodes: ow.optional.array.ofType(ow.number),+ ignoreHttpErrorStatusCodes: ow.optional.array.ofType(ow.number),+preNavigationHooks: ow.optional.array,postNavigationHooks: ow.optional.array,};@@ -327,6 +344,8 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, HpersistCookiesPerSession,preNavigationHooks = [],postNavigationHooks = [],+ additionalHttpErrorStatusCodes = [],+ ignoreHttpErrorStatusCodes = [],// IgnoredhandleRequestFunction,@@ -375,6 +394,8 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, Hthis.ignoreSslErrors = ignoreSslErrors;this.suggestResponseEncoding = suggestResponseEncoding;this.forceResponseEncoding = forceResponseEncoding;+ this.additionalHttpErrorStatusCodes = new Set([...additionalHttpErrorStatusCodes]);+ this.ignoreHttpErrorStatusCodes = new Set([...ignoreHttpErrorStatusCodes]);this.proxyConfiguration = proxyConfiguration;this.preNavigationHooks = preNavigationHooks;this.postNavigationHooks = [@@ -617,7 +638,10 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, Hthis.stats.registerStatusCode(statusCode!);}- if (statusCode! >= 500) {+ const excludeError = this.ignoreHttpErrorStatusCodes.has(statusCode!);+ const includeError = this.additionalHttpErrorStatusCodes.has(statusCode!);++ if ((statusCode! >= 500 && !excludeError) || includeError) {const body = await readStreamToString(response, encoding);// Errors are often sent as JSON, so attempt to parse them,@@ -629,6 +653,10 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, Hthrow new Error(`${statusCode} - ${message}`);}+ if (includeError) {+ throw new Error(`${statusCode} - Error status code was set by user.`);+ }+// It's not a JSON, so it's probably some text. Get the first 100 chars of it.throw new Error(`${statusCode} - Internal Server Error: ${body.slice(0, 100)}`);} else if (HTML_AND_XML_MIME_TYPES.includes(type)) {diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.tsindex 6d1097a9d..0a83bc553 100644--- a/test/core/crawlers/cheerio_crawler.test.ts+++ b/test/core/crawlers/cheerio_crawler.test.ts@@ -507,6 +507,54 @@ describe('CheerioCrawler', () => {});});+ test('should ignore non http error status codes set by user', async () => {+ const requestList = await getRequestListForMock({+ headers: {+ 'content-type': 'text/plain',+ },+ statusCode: 500,+ });++ const failed: Request[] = [];++ const cheerioCrawler = new CheerioCrawler({+ requestList,+ minConcurrency: 2,+ maxConcurrency: 2,+ ignoreHttpErrorStatusCodes: [500],+ requestHandler: () => {},+ failedRequestHandler: ({ request }) => {+ failed.push(request);+ },+ });++ await cheerioCrawler.run();++ expect(cheerioCrawler.autoscaledPool.minConcurrency).toBe(2);+ expect(failed).toHaveLength(0);+ });++ test('should throw and error on http error status codes set by user', async () => {+ const requestList = await getRequestListForMirror();+ const failed: Request[] = [];++ const cheerioCrawler = new CheerioCrawler({+ requestList,+ minConcurrency: 2,+ maxConcurrency: 2,+ additionalHttpErrorStatusCodes: [200],+ requestHandler: () => {},+ failedRequestHandler: ({ request }) => {+ failed.push(request);+ },+ });++ await cheerioCrawler.run();++ expect(cheerioCrawler.autoscaledPool.minConcurrency).toBe(2);+ expect(failed).toHaveLength(4);+ });+test('should work with all defaults content types', async () => {let handledRequests = 0;const contentTypes = ['text/html', 'application/xhtml+xml', 'text/xml', 'application/xml', 'application/json'];diff --git a/test/core/crawlers/http_crawler.test.ts b/test/core/crawlers/http_crawler.test.tsindex 586b69d38..b6e2671d1 100644--- a/test/core/crawlers/http_crawler.test.ts+++ b/test/core/crawlers/http_crawler.test.ts@@ -47,6 +47,11 @@ router.set('/echo', (req, res) => {req.pipe(res);});+router.set('/500Error', (req, res) => {+ res.statusCode = 500;+ res.end();+});+let server: http.Server;let url: string;@@ -273,3 +278,41 @@ test('POST with undefined (empty) payload', async () => {expect(results).toStrictEqual(['']);});++test('should ignore non http error status codes set by user', async () => {+ const failed: any[] = [];++ const crawler = new HttpCrawler({+ minConcurrency: 2,+ maxConcurrency: 2,+ ignoreHttpErrorStatusCodes: [500],+ requestHandler: () => {},+ failedRequestHandler: ({ request }) => {+ failed.push(request);+ },+ });++ await crawler.run([`${url}/500Error`]);++ expect(crawler.autoscaledPool.minConcurrency).toBe(2);+ expect(failed).toHaveLength(0);+});++test('should throw and error on http error status codes set by user', async () => {+ const failed: any[] = [];++ const crawler = new HttpCrawler({+ minConcurrency: 2,+ maxConcurrency: 2,+ additionalHttpErrorStatusCodes: [200],+ requestHandler: () => {},+ failedRequestHandler: ({ request }) => {+ failed.push(request);+ },+ });++ await crawler.run([`${url}/hello.html`]);++ expect(crawler.autoscaledPool.minConcurrency).toBe(2);+ expect(failed).toHaveLength(1);+});