Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
refactor: remove `additionalBlockedStatusCodes` parameter of `Session.retireOnBlockedStatusCodes`
Martin Adámek9e8ad43800dc
diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.mdindex a98b680e0..3e8232966 100644--- a/docs/upgrading/upgrading_v4.md+++ b/docs/upgrading/upgrading_v4.md@@ -39,3 +39,7 @@ The crawling context no longer includes the `Error` object for failed requests.## Crawling context is strictly typedPreviously, the crawling context extended a `Record` type, allowing to access any property. This was changed to a strict type, which means that you can only access properties that are defined in the context.++## `additionalBlockedStatusCodes` parameter is removed++`additionalBlockedStatusCodes` parameter of `Session.retireOnBlockedStatusCodes` method is removed. Use the `blockedStatusCodes` crawler option instead.diff --git a/packages/core/src/cookie_utils.ts b/packages/core/src/cookie_utils.tsindex 768f1e124..71f753ecb 100644--- a/packages/core/src/cookie_utils.ts+++ b/packages/core/src/cookie_utils.ts@@ -123,7 +123,7 @@ export function mergeCookies(url: string, sourceCookies: string[]): string {});if (similarKeyCookie) {- log.deprecated(+ log.warningOnce(`Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`,);}diff --git a/packages/core/src/request.ts b/packages/core/src/request.tsindex 3ee5fed82..1b95de778 100644--- a/packages/core/src/request.ts+++ b/packages/core/src/request.ts@@ -426,8 +426,7 @@ export class Request<UserData extends Dictionary = Dictionary> {const normalizedUrl = normalizeUrl(url, keepUrlFragment) || url; // It returns null when url is invalid, causing weird errors.if (!useExtendedUniqueKey) {if (normalizedMethod !== 'GET' && payload) {- // Using log.deprecated to log only once. We should add log.once or some such.- log.deprecated(+ log.warningOnce(`We've encountered a ${normalizedMethod} Request with a payload. ` +'This is fine. Just letting you know that if your requests point to the same URL ' +'and differ only in method and payload, you should see the "useExtendedUniqueKey" option of Request constructor.',diff --git a/packages/core/src/session_pool/session.ts b/packages/core/src/session_pool/session.tsindex 8c078119e..8bb998a0d 100644--- a/packages/core/src/session_pool/session.ts+++ b/packages/core/src/session_pool/session.ts@@ -298,33 +298,14 @@ export class Session {/*** With certain status codes: `401`, `403` or `429` we can be certain* that the target website is blocking us. This function helps to do this conveniently- * by retiring the session when such code is received. Optionally the default status+ * by retiring the session when such code is received. Optionally, the default status* codes can be extended in the second parameter.* @param statusCode HTTP status code.* @returns Whether the session was retired.*/- retireOnBlockedStatusCodes(statusCode: number): boolean;-- /**- * With certain status codes: `401`, `403` or `429` we can be certain- * that the target website is blocking us. This function helps to do this conveniently- * by retiring the session when such code is received. Optionally the default status- * codes can be extended in the second parameter.- * @param statusCode HTTP status code.- * @param [additionalBlockedStatusCodes]- * Custom HTTP status codes that means blocking on particular website.- *- * **This parameter is deprecated and will be removed in next major version.**- * @returns Whether the session was retired.- * @deprecated The parameter `additionalBlockedStatusCodes` is deprecated and will be removed in next major version.- */- retireOnBlockedStatusCodes(statusCode: number, additionalBlockedStatusCodes?: number[]): boolean;-- retireOnBlockedStatusCodes(statusCode: number, additionalBlockedStatusCodes: number[] = []): boolean {+ retireOnBlockedStatusCodes(statusCode: number): boolean {// eslint-disable-next-line dot-notation -- accessing private property- const isBlocked = this.sessionPool['blockedStatusCodes']- .concat(additionalBlockedStatusCodes)- .includes(statusCode);+ const isBlocked = this.sessionPool['blockedStatusCodes'].includes(statusCode);if (isBlocked) {this.retire();}diff --git a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.tsindex fd13b7518..be627ad1d 100644--- a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts+++ b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts@@ -236,7 +236,6 @@ export async function parseWithCheerio(? null: ((await page.evaluate(`(${expandShadowRoots.toString()})(document)`)) as string);const pageContent = html || (await page.content());- console.log(ignoreShadowRoots, pageContent);return cheerio.load(pageContent);}diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.tsindex c31b9cada..1eb746dfb 100644--- a/test/core/crawlers/cheerio_crawler.test.ts+++ b/test/core/crawlers/cheerio_crawler.test.ts@@ -1128,14 +1128,14 @@ describe('CheerioCrawler', () => {});test('mergeCookies()', async () => {- const deprecatedSpy = vitest.spyOn(Log.prototype, 'deprecated');+ const warningSpy = vitest.spyOn(Log.prototype, 'warningOnce');const cookie1 = mergeCookies('https://example.com', ['foo=bar1; other=cookie1 ; coo=kie','foo=bar2; baz=123','other=cookie2;foo=bar3',]);expect(cookie1).toBe('foo=bar3; other=cookie2; coo=kie; baz=123');- expect(deprecatedSpy).not.toBeCalled();+ expect(warningSpy).not.toBeCalled();const cookie2 = mergeCookies('https://example.com', ['Foo=bar1; other=cookie1 ; coo=kie',@@ -1143,14 +1143,14 @@ describe('CheerioCrawler', () => {'Other=cookie2;foo=bar3',]);expect(cookie2).toBe('Foo=bar1; other=cookie1; coo=kie; foo=bar3; baz=123; Other=cookie2');- expect(deprecatedSpy).toBeCalledTimes(3);- expect(deprecatedSpy).toBeCalledWith(+ expect(warningSpy).toBeCalledTimes(3);+ expect(warningSpy).toBeCalledWith(`Found cookies with similar name during cookie merging: 'foo' and 'Foo'`,);- expect(deprecatedSpy).toBeCalledWith(+ expect(warningSpy).toBeCalledWith(`Found cookies with similar name during cookie merging: 'Other' and 'other'`,);- deprecatedSpy.mockClear();+ warningSpy.mockClear();const cookie3 = mergeCookies('https://example.com', ['foo=bar1; Other=cookie1 ; Coo=kie',@@ -1158,11 +1158,11 @@ describe('CheerioCrawler', () => {'Other=cookie2;Foo=bar3;coo=kee',]);expect(cookie3).toBe('foo=bar2; Other=cookie2; Coo=kie; baz=123; Foo=bar3; coo=kee');- expect(deprecatedSpy).toBeCalledTimes(2);- expect(deprecatedSpy).toBeCalledWith(+ expect(warningSpy).toBeCalledTimes(2);+ expect(warningSpy).toBeCalledWith(`Found cookies with similar name during cookie merging: 'Foo' and 'foo'`,);- expect(deprecatedSpy).toBeCalledWith(+ expect(warningSpy).toBeCalledWith(`Found cookies with similar name during cookie merging: 'coo' and 'Coo'`,);});diff --git a/test/core/session_pool/session.test.ts b/test/core/session_pool/session.test.tsindex a56a97368..09439221f 100644--- a/test/core/session_pool/session.test.ts+++ b/test/core/session_pool/session.test.ts@@ -185,17 +185,6 @@ describe('Session - testing session behaviour ', () => {});});- test('should checkStatus work with custom codes', () => {- session = new Session({ sessionPool });- const customStatusCodes = [100, 202, 300];- expect(session.retireOnBlockedStatusCodes(100, customStatusCodes)).toBeTruthy();- expect(session.retireOnBlockedStatusCodes(101, customStatusCodes)).toBeFalsy();- expect(session.retireOnBlockedStatusCodes(200, customStatusCodes)).toBeFalsy();- expect(session.retireOnBlockedStatusCodes(202, customStatusCodes)).toBeTruthy();- expect(session.retireOnBlockedStatusCodes(300, customStatusCodes)).toBeTruthy();- expect(session.retireOnBlockedStatusCodes(400, customStatusCodes)).toBeFalsy();- });-test('setCookies should work', () => {const url = 'https://example.com';const cookies = [