Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: got-scraping v4 (#2110)
Vlad Frangu2f05ed22b203
diff --git a/.eslintrc.json b/.eslintrc.jsonindex 0a78e0638..87173102a 100644--- a/.eslintrc.json+++ b/.eslintrc.json@@ -8,7 +8,7 @@"extends": "@apify/eslint-config-ts","parserOptions": {"project": "./tsconfig.eslint.json",- "ecmaVersion": 2020+ "ecmaVersion": 2022},"ignorePatterns": ["node_modules",diff --git a/package.json b/package.jsonindex 1a0224af5..b64a36759 100644--- a/package.json+++ b/package.json@@ -102,7 +102,7 @@"rimraf": "^5.0.0","ts-node": "^10.9.1","turbo": "1.10.16",- "typescript": "^5.0.0",+ "typescript": "^5.2.2","vite-tsconfig-paths": "^4.2.1","vitest": "^0.34.6"},diff --git a/packages/basic-crawler/package.json b/packages/basic-crawler/package.jsonindex f64c7eb2f..95416fad8 100644--- a/packages/basic-crawler/package.json+++ b/packages/basic-crawler/package.json@@ -51,7 +51,7 @@"@crawlee/core": "3.5.8","@crawlee/types": "3.5.8","@crawlee/utils": "3.5.8",- "got-scraping": "^3.2.15",+ "got-scraping": "^4.0.0","ow": "^0.28.1","tldts": "^6.0.0","tslib": "^2.4.0",diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 83c31ddc0..f240c187a 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -48,11 +48,11 @@ import {validators,} from '@crawlee/core';import type { Awaitable, BatchAddRequestsResult, Dictionary, SetStatusMessageOptions } from '@crawlee/types';-import { ROTATE_PROXY_ERRORS } from '@crawlee/utils';+import { ROTATE_PROXY_ERRORS, gotScraping } from '@crawlee/utils';import { stringify } from 'csv-stringify/sync';import { ensureDir, writeFile, writeJSON } from 'fs-extra';-import type { Method, OptionsInit } from 'got-scraping';-import { gotScraping } from 'got-scraping';+// @ts-expect-error This throws a compilation error due to got-scraping being ESM only but we only import types, so its alllll gooooood+import type { OptionsInit, Method } from 'got-scraping';import ow, { ArgumentError } from 'ow';import { getDomain } from 'tldts';import type { SetRequired } from 'type-fest';diff --git a/packages/core/src/crawlers/crawler_commons.ts b/packages/core/src/crawlers/crawler_commons.tsindex 4a9e9c129..049c807df 100644--- a/packages/core/src/crawlers/crawler_commons.ts+++ b/packages/core/src/crawlers/crawler_commons.ts@@ -1,4 +1,5 @@import type { Dictionary, BatchAddRequestsResult } from '@crawlee/types';+// @ts-expect-error This throws a compilation error due to got-scraping being ESM only but we only import types, so its alllll goooooodimport type { Response as GotResponse, OptionsInit } from 'got-scraping';import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links';diff --git a/packages/http-crawler/package.json b/packages/http-crawler/package.jsonindex e90a71ac0..a42623c97 100644--- a/packages/http-crawler/package.json+++ b/packages/http-crawler/package.json@@ -61,7 +61,7 @@"@types/content-type": "^1.1.5","cheerio": "^1.0.0-rc.12","content-type": "^1.0.4",- "got-scraping": "^3.2.15",+ "got-scraping": "^4.0.0","iconv-lite": "^0.6.3","mime-types": "^2.1.35","ow": "^0.28.1",diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex fdcfaa2c3..169fc4cf0 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -28,17 +28,19 @@ import {SessionError,} from '@crawlee/basic';import type { Awaitable, Dictionary } from '@crawlee/types';-import { RETRY_CSS_SELECTORS } from '@crawlee/utils';+import { RETRY_CSS_SELECTORS, gotScraping } from '@crawlee/utils';import * as cheerio from 'cheerio';import type { RequestLike, ResponseLike } from 'content-type';import contentTypeParser from 'content-type';-import type { OptionsInit, Method, Request as GotRequest, Options, PlainResponse } from 'got-scraping';-import { gotScraping, TimeoutError } from 'got-scraping';+// @ts-expect-error This throws a compilation error due to got-scraping being ESM only but we only import types, so its alllll gooooood+import type { OptionsInit, Method, Request as GotRequest, Options, PlainResponse, TimeoutError as TimeoutErrorClass } from 'got-scraping';import iconv from 'iconv-lite';import mime from 'mime-types';import ow from 'ow';import type { JsonValue } from 'type-fest';+let TimeoutError: typeof TimeoutErrorClass;+/*** Default mime types, which HttpScraper supports.*/@@ -607,6 +609,10 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, H* received content type matches text/html, application/xml, application/xhtml+xml.*/protected async _requestFunction({ request, session, proxyUrl, gotOptions }: RequestFunctionOptions): Promise<PlainResponse> {+ if (!TimeoutError) {+ ({ TimeoutError } = await import('got-scraping'));+ }+const opts = this._getRequestOptions(request, session, proxyUrl, gotOptions);try {@@ -800,8 +806,11 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, H* @internal wraps public utility for mocking purposes*/private _requestAsBrowser = async (options: OptionsInit & { isStream: true }, session?: Session) => {- return new Promise<PlainResponse>((resolve, reject) => {- const stream = gotScraping(options);+ // eslint-disable-next-line no-async-promise-executor+ return new Promise<PlainResponse>(async (resolve, reject) => {+ // This await may not be needed after the initial call, but this is needed to actually get got-scraping loaded+ // eslint-disable-next-line @typescript-eslint/await-thenable+ const stream = await gotScraping(options);stream.on('redirect', (updatedOptions: Options, redirectResponse: PlainResponse) => {if (this.persistCookiesPerSession) {diff --git a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts b/packages/linkedom-crawler/src/internals/linkedom-crawler.tsindex 9b170070a..b59931ca4 100644--- a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts+++ b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts@@ -15,6 +15,7 @@ import type {import { HttpCrawler, enqueueLinks, Router, resolveBaseUrlForEnqueueLinksFiltering, tryAbsoluteURL } from '@crawlee/http';import type { Dictionary } from '@crawlee/types';import type * as cheerio from 'cheerio';+// @ts-expect-error This throws a compilation error due to TypeScript not inferring the module has CJS versions tooimport { DOMParser } from 'linkedom/cached';export type LinkeDOMErrorHandler<diff --git a/packages/utils/package.json b/packages/utils/package.jsonindex 8980dfabd..14b4b72ac 100644--- a/packages/utils/package.json+++ b/packages/utils/package.json@@ -51,7 +51,7 @@"@apify/ps-tree": "^1.2.0","@crawlee/types": "3.5.8","cheerio": "^1.0.0-rc.12",- "got-scraping": "^3.2.15",+ "got-scraping": "^4.0.0","ow": "^0.28.1","tslib": "^2.4.0"}diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.tsindex d8b5dc476..9e9024e5b 100644--- a/packages/utils/src/index.ts+++ b/packages/utils/src/index.ts@@ -9,4 +9,6 @@ export * as social from './internals/social';export * from './internals/typedefs';export * from './internals/error_tracker';export * from './internals/open_graph_parser';+export * from './internals/gotScraping';+export { Dictionary, Awaitable, Constructor } from '@crawlee/types';diff --git a/packages/utils/src/internals/extract-urls.ts b/packages/utils/src/internals/extract-urls.tsindex 8e3402186..385c7eb0a 100644--- a/packages/utils/src/internals/extract-urls.ts+++ b/packages/utils/src/internals/extract-urls.ts@@ -1,7 +1,7 @@-import { gotScraping } from 'got-scraping';import ow from 'ow';import { URL_NO_COMMAS_REGEX } from './general';+import { gotScraping } from './gotScraping';export interface DownloadListOfUrlsOptions {/**diff --git a/packages/utils/src/internals/gotScraping.ts b/packages/utils/src/internals/gotScraping.tsnew file mode 100644index 000000000..179ffeb2d--- /dev/null+++ b/packages/utils/src/internals/gotScraping.ts@@ -0,0 +1,11 @@+// @ts-expect-error This throws a compilation error due to got-scraping being ESM only but we only import types, so its alllll gooooood+import type { GotScraping } from 'got-scraping';++// eslint-disable-next-line import/no-mutable-exports -- Borrowing a book from NodeJS's code, we override the method with the imported one once the method is called+let gotScraping = (async (...args: Parameters<GotScraping>) => {+ ({ gotScraping } = await import('got-scraping'));++ return gotScraping(...args);+}) as GotScraping;++export { gotScraping };diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.tsindex 1cbb75f2c..9abca64e3 100644--- a/test/core/crawlers/browser_crawler.test.ts+++ b/test/core/crawlers/browser_crawler.test.ts@@ -17,7 +17,6 @@ import {Session,} from '@crawlee/puppeteer';import { sleep } from '@crawlee/utils';-import { gotScraping } from 'got-scraping';import puppeteer from 'puppeteer';import type { HTTPResponse } from 'puppeteer';import { runExampleComServer } from 'test/shared/_helper';@@ -25,14 +24,6 @@ import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator';import { BrowserCrawlerTest } from './basic_browser_crawler';-vitest.mock('got-scraping', async () => {- const original: typeof import('got-scraping') = await vitest.importActual('got-scraping');- return {- ...original,- gotScraping: vitest.fn(),- };-});-describe('BrowserCrawler', () => {let prevEnvHeadless: string;let logLevel: number;diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.tsindex 9284c4712..909d3ae77 100644--- a/test/core/crawlers/cheerio_crawler.test.ts+++ b/test/core/crawlers/cheerio_crawler.test.ts@@ -21,6 +21,7 @@ import {} from '@crawlee/cheerio';import { sleep } from '@crawlee/utils';import type { Dictionary } from '@crawlee/utils';+// @ts-expect-error This throws a compilation error due to got-scraping being ESM only but we only import types, so its alllll gooooood\import type { OptionsInit } from 'got-scraping';import iconv from 'iconv-lite';import { runExampleComServer, responseSamples } from 'test/shared/_helper';diff --git a/test/core/request_list.test.ts b/test/core/request_list.test.tsindex ab39db0d5..539695b88 100644--- a/test/core/request_list.test.ts+++ b/test/core/request_list.test.ts@@ -1,7 +1,6 @@import log from '@apify/log';import { Configuration, deserializeArray, EventType, KeyValueStore, ProxyConfiguration, Request, RequestList } from '@crawlee/core';import { sleep } from '@crawlee/utils';-import { gotScraping } from 'got-scraping';import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator';/**@@ -16,14 +15,14 @@ function shuffle(array: unknown[]) : unknown[] {return out;}-vitest.mock('got-scraping', async () => {- const original: typeof import('got-scraping') = await vitest.importActual('got-scraping');+vitest.mock('@crawlee/utils/src/internals/gotScraping', async () => {return {- ...original,- gotScraping: vitest.fn(original.gotScraping),+ gotScraping: vitest.fn(),};});+const { gotScraping } = await import('@crawlee/utils/src/internals/gotScraping');+const gotScrapingSpy = vitest.mocked(gotScraping);describe('RequestList', () => {diff --git a/test/core/storages/request_queue.test.ts b/test/core/storages/request_queue.test.tsindex 9fb39b825..8ad854d9e 100644--- a/test/core/storages/request_queue.test.ts+++ b/test/core/storages/request_queue.test.ts@@ -10,18 +10,17 @@ import {ProxyConfiguration,} from '@crawlee/core';import { sleep } from '@crawlee/utils';-import { gotScraping } from 'got-scraping';import { MemoryStorageEmulator } from '../../shared/MemoryStorageEmulator';-vitest.mock('got-scraping', async () => {- const original: typeof import('got-scraping') = await vitest.importActual('got-scraping');+vitest.mock('@crawlee/utils/src/internals/gotScraping', async () => {return {- ...original,- gotScraping: vitest.fn(original.gotScraping),+ gotScraping: vitest.fn(),};});+const { gotScraping } = await import('@crawlee/utils/src/internals/gotScraping');+const gotScrapingSpy = vitest.mocked(gotScraping);describe('RequestQueue remote', () => {diff --git a/test/tsconfig.json b/test/tsconfig.jsonindex 8316ed8f2..1124d5a22 100644--- a/test/tsconfig.json+++ b/test/tsconfig.json@@ -9,6 +9,11 @@"noUnusedLocals": false,"noUnusedParameters": false,"strictNullChecks": false,- "types": ["vitest/globals"]+ "types": ["vitest/globals"],+ "module": "Node16",+ "moduleResolution": "Bundler",+ "paths": {+ "@crawlee/utils/src/*": ["packages/utils/src/*"]+ }}}diff --git a/test/utils/extract-urls.test.ts b/test/utils/extract-urls.test.tsindex 3e1aafcb6..234f2f36c 100644--- a/test/utils/extract-urls.test.ts+++ b/test/utils/extract-urls.test.ts@@ -6,23 +6,18 @@ import {extractUrls,URL_WITH_COMMAS_REGEX,} from '@crawlee/utils';-import { gotScraping } from 'got-scraping';-const baseDataPath = path.join(__dirname, '..', 'shared', 'data');--vitest.mock('got-scraping', async () => {- const original: typeof import('got-scraping') = await vitest.importActual('got-scraping');+vitest.mock('@crawlee/utils/src/internals/gotScraping', async () => {return {- ...original,gotScraping: vitest.fn(),};});-const gotScrapingSpy = vitest.mocked(gotScraping);+const { gotScraping } = await import('@crawlee/utils/src/internals/gotScraping');-afterAll(() => {- vitest.doUnmock('got-scraping');-});+const baseDataPath = path.join(__dirname, '..', 'shared', 'data');++const gotScrapingSpy = vitest.mocked(gotScraping);describe('downloadListOfUrls()', () => {test('downloads a list of URLs', async () => {diff --git a/tsconfig.build.json b/tsconfig.build.jsonindex e50f2ec74..6583b7277 100644--- a/tsconfig.build.json+++ b/tsconfig.build.json@@ -11,7 +11,9 @@"allowJs": true,"skipLibCheck": true,"resolveJsonModule": false,- "emitDecoratorMetadata": false+ "emitDecoratorMetadata": false,+ "module": "Node16",+ "moduleResolution": "Node16"},"include": ["./packages/*/src/**/*"diff --git a/vitest.config.ts b/vitest.config.tsindex 9a9daabff..aff67df59 100644--- a/vitest.config.ts+++ b/vitest.config.ts@@ -16,7 +16,7 @@ if (isCI) {export default defineConfig({plugins: [tsconfigPaths()],esbuild: {- target: 'es2021',+ target: 'es2022',keepNames: true,},test: {diff --git a/yarn.lock b/yarn.lockindex c5504c0a3..9103848a7 100644--- a/yarn.lock+++ b/yarn.lock@@ -387,7 +387,7 @@ __metadata:"@crawlee/core": "npm:3.5.8""@crawlee/types": "npm:3.5.8""@crawlee/utils": "npm:3.5.8"- got-scraping: "npm:^3.2.15"+ got-scraping: "npm:^4.0.0"ow: "npm:^0.28.1"tldts: "npm:^6.0.0"tslib: "npm:^2.4.0"@@ -508,7 +508,7 @@ __metadata:"@types/content-type": "npm:^1.1.5"cheerio: "npm:^1.0.0-rc.12"content-type: "npm:^1.0.4"- got-scraping: "npm:^3.2.15"+ got-scraping: "npm:^4.0.0"iconv-lite: "npm:^0.6.3"mime-types: "npm:^2.1.35"ow: "npm:^0.28.1"@@ -666,7 +666,7 @@ __metadata:rimraf: "npm:^5.0.0"ts-node: "npm:^10.9.1"turbo: "npm:1.10.16"- typescript: "npm:^5.0.0"+ typescript: "npm:^5.2.2"vite-tsconfig-paths: "npm:^4.2.1"vitest: "npm:^0.34.6"languageName: unknown@@ -700,7 +700,7 @@ __metadata:"@apify/ps-tree": "npm:^1.2.0""@crawlee/types": "npm:3.5.8"cheerio: "npm:^1.0.0-rc.12"- got-scraping: "npm:^3.2.15"+ got-scraping: "npm:^4.0.0"ow: "npm:^0.28.1"tslib: "npm:^2.4.0"languageName: unknown@@ -1718,29 +1718,20 @@ __metadata:languageName: nodelinkType: hard-"@sindresorhus/is@npm:4.6.0, @sindresorhus/is@npm:^4.2.0":+"@sindresorhus/is@npm:^4.2.0":version: 4.6.0resolution: "@sindresorhus/is@npm:4.6.0"checksum: 33b6fb1d0834ec8dd7689ddc0e2781c2bfd8b9c4e4bacbcb14111e0ae00621f2c264b8a7d36541799d74888b5dccdf422a891a5cb5a709ace26325eedc81e22elanguageName: nodelinkType: hard-"@sindresorhus/is@npm:^5.2.0":+"@sindresorhus/is@npm:^5.2.0, @sindresorhus/is@npm:^5.3.0":version: 5.6.0resolution: "@sindresorhus/is@npm:5.6.0"checksum: 66727344d0c92edde5760b5fd1f8092b717f2298a162a5f7f29e4953e001479927402d9d387e245fb9dc7d3b37c72e335e93ed5875edfc5203c53be8ecba1b52languageName: nodelinkType: hard-"@szmarczak/http-timer@npm:4.0.6":- version: 4.0.6- resolution: "@szmarczak/http-timer@npm:4.0.6"- dependencies:- defer-to-connect: "npm:^2.0.0"- checksum: 73946918c025339db68b09abd91fa3001e87fc749c619d2e9c2003a663039d4c3cb89836c98a96598b3d47dec2481284ba85355392644911f5ecd2336536697f- languageName: node- linkType: hard-"@szmarczak/http-timer@npm:^5.0.1":version: 5.0.1resolution: "@szmarczak/http-timer@npm:5.0.1"@@ -2117,15 +2108,6 @@ __metadata:languageName: nodelinkType: hard-"@types/responselike@npm:1.0.0":- version: 1.0.0- resolution: "@types/responselike@npm:1.0.0"- dependencies:- "@types/node": "npm:*"- checksum: 474ac2402e6d43c007eee25f50d01eb1f67255ca83dd8e036877292bbe8dd5d2d1e50b54b408e233b50a8c38e681ff3ebeaf22f18b478056eddb65536abb003a- languageName: node- linkType: hard-"@types/retry@npm:*":version: 0.12.5resolution: "@types/retry@npm:0.12.5"@@ -3311,13 +3293,6 @@ __metadata:languageName: nodelinkType: hard-"cacheable-lookup@npm:6.1.0":- version: 6.1.0- resolution: "cacheable-lookup@npm:6.1.0"- checksum: fe922b24e9868ac65cbd3b4ccd7449063d572431471aab71cbca49a2b33839c7c888b237b0922ae6b8f4ddf25d61debe204e473195d2e77a835099b8953aeb0a- languageName: node- linkType: hard-"cacheable-lookup@npm:^7.0.0":version: 7.0.0resolution: "cacheable-lookup@npm:7.0.0"@@ -3325,21 +3300,6 @@ __metadata:languageName: nodelinkType: hard-"cacheable-request@npm:7.0.2":- version: 7.0.2- resolution: "cacheable-request@npm:7.0.2"- dependencies:- clone-response: "npm:^1.0.2"- get-stream: "npm:^5.1.0"- http-cache-semantics: "npm:^4.0.0"- keyv: "npm:^4.0.0"- lowercase-keys: "npm:^2.0.0"- normalize-url: "npm:^6.0.1"- responselike: "npm:^2.0.0"- checksum: 681bad13691d0d5d10652d409374747a2ce8676f854b0d454ee8fc65e0a10a52ea83cd1f6c367ada08572fd4982f2aa2582dc38983d4e958e053e181c433765e- languageName: node- linkType: hard-"cacheable-request@npm:^10.2.8":version: 10.2.14resolution: "cacheable-request@npm:10.2.14"@@ -3373,6 +3333,13 @@ __metadata:languageName: nodelinkType: hard+"callsites@npm:^4.0.0":+ version: 4.1.0+ resolution: "callsites@npm:4.1.0"+ checksum: 91700844127a6dcd4792d231a12dd8e9ec10525eb9962180a8558417d7e3f443e52a4f14746ad2838eaf14f79431ee1539d13bd188da280f720a06a91bd1157a+ languageName: node+ linkType: hard+"camelcase-keys@npm:^6.2.2":version: 6.2.2resolution: "camelcase-keys@npm:6.2.2"@@ -3638,15 +3605,6 @@ __metadata:languageName: nodelinkType: hard-"clone-response@npm:^1.0.2":- version: 1.0.3- resolution: "clone-response@npm:1.0.3"- dependencies:- mimic-response: "npm:^1.0.0"- checksum: 06a2b611824efb128810708baee3bd169ec9a1bf5976a5258cd7eb3f7db25f00166c6eee5961f075c7e38e194f373d4fdf86b8166ad5b9c7e82bbd2e333a6087- languageName: node- linkType: hard-"clone@npm:^1.0.2":version: 1.0.4resolution: "clone@npm:1.0.4"@@ -4269,7 +4227,7 @@ __metadata:languageName: nodelinkType: hard-"defer-to-connect@npm:^2.0.0, defer-to-connect@npm:^2.0.1":+"defer-to-connect@npm:^2.0.1":version: 2.0.1resolution: "defer-to-connect@npm:2.0.1"checksum: 625ce28e1b5ad10cf77057b9a6a727bf84780c17660f6644dab61dd34c23de3001f03cedc401f7d30a4ed9965c2e8a7336e220a329146f2cf85d4eddea429782@@ -4501,6 +4459,15 @@ __metadata:languageName: nodelinkType: hard+"dot-prop@npm:^7.2.0":+ version: 7.2.0+ resolution: "dot-prop@npm:7.2.0"+ dependencies:+ type-fest: "npm:^2.11.2"+ checksum: 2621702a01e7a47730e3a8e2938a406afc79b62fbb77bd1394e786ff13776673904bf0a4fc6b812eb9849ec71034e9fc1019a9e0bbe91f84010d8a8088cd41a9+ languageName: node+ linkType: hard+"dotenv-expand@npm:~10.0.0":version: 10.0.0resolution: "dotenv-expand@npm:10.0.0"@@ -5597,13 +5564,6 @@ __metadata:languageName: nodelinkType: hard-"form-data-encoder@npm:1.7.2":- version: 1.7.2- resolution: "form-data-encoder@npm:1.7.2"- checksum: 56553768037b6d55d9de524f97fe70555f0e415e781cb56fc457a68263de3d40fadea2304d4beef2d40b1a851269bd7854e42c362107071892cb5238debe9464- languageName: node- linkType: hard-"form-data-encoder@npm:^2.1.2":version: 2.1.4resolution: "form-data-encoder@npm:2.1.4"@@ -6140,38 +6100,18 @@ __metadata:languageName: nodelinkType: hard-"got-cjs@npm:12.5.4":- version: 12.5.4- resolution: "got-cjs@npm:12.5.4"- dependencies:- "@sindresorhus/is": "npm:4.6.0"- "@szmarczak/http-timer": "npm:4.0.6"- "@types/responselike": "npm:1.0.0"- cacheable-lookup: "npm:6.1.0"- cacheable-request: "npm:7.0.2"- decompress-response: "npm:^6.0.0"- form-data-encoder: "npm:1.7.2"- get-stream: "npm:^6.0.1"- http2-wrapper: "npm:^2.1.10"- lowercase-keys: "npm:2.0.0"- p-cancelable: "npm:2.1.1"- responselike: "npm:2.0.1"- checksum: cfbee2722fdf5d23463a0bf4f259c67ffe1423cb91b3a77e683296dfaa4bdce295dd7496aee5f2a09d5da2c6fabad0bd0efd48a34783150e242e8e8517ac1bf0- languageName: node- linkType: hard--"got-scraping@npm:^3.2.15":- version: 3.2.15- resolution: "got-scraping@npm:3.2.15"+"got-scraping@npm:^4.0.0":+ version: 4.0.0+ resolution: "got-scraping@npm:4.0.0"dependencies:- got-cjs: "npm:12.5.4"- header-generator: "npm:^2.1.3"- http2-wrapper: "npm:^2.1.11"- mimic-response: "npm:^3.1.0"- ow: "npm:^0.28.1"- quick-lru: "npm:^5.1.1"- tslib: "npm:^2.4.0"- checksum: b83ed00a328a54be4d4575aed9e5fc83e6752095686d27b096505ecbee9fe02b69551a3800bc5302265c547782d05ee9f758a0f4d32bbdb6e3df04e86d851678+ got: "npm:^13.0.0"+ header-generator: "npm:^2.1.41"+ http2-wrapper: "npm:^2.2.0"+ mimic-response: "npm:^4.0.0"+ ow: "npm:^1.1.1"+ quick-lru: "npm:^7.0.0"+ tslib: "npm:^2.6.2"+ checksum: 3e2548c16ce009e39f1f7a88289213ff06f538cc9fccf658e77f44e50422e84b8a151c12dca5082ad72de1162c1f9dccdceca0eb344d7bff09b378e6acf42f60languageName: nodelinkType: hard@@ -6311,7 +6251,7 @@ __metadata:languageName: nodelinkType: hard-"header-generator@npm:^2.1.3, header-generator@npm:^2.1.43":+"header-generator@npm:^2.1.41, header-generator@npm:^2.1.43":version: 2.1.43resolution: "header-generator@npm:2.1.43"dependencies:@@ -6404,7 +6344,7 @@ __metadata:languageName: nodelinkType: hard-"http-cache-semantics@npm:^4.0.0, http-cache-semantics@npm:^4.1.0, http-cache-semantics@npm:^4.1.1":+"http-cache-semantics@npm:^4.1.0, http-cache-semantics@npm:^4.1.1":version: 4.1.1resolution: "http-cache-semantics@npm:4.1.1"checksum: ce1319b8a382eb3cbb4a37c19f6bfe14e5bb5be3d09079e885e8c513ab2d3cd9214902f8a31c9dc4e37022633ceabfc2d697405deeaf1b8f3552bb4ed996fdfc@@ -6445,7 +6385,7 @@ __metadata:languageName: nodelinkType: hard-"http2-wrapper@npm:^2.1.10, http2-wrapper@npm:^2.1.11":+"http2-wrapper@npm:^2.1.10, http2-wrapper@npm:^2.2.0":version: 2.2.1resolution: "http2-wrapper@npm:2.2.1"dependencies:@@ -7469,7 +7409,7 @@ __metadata:languageName: nodelinkType: hard-"keyv@npm:^4.0.0, keyv@npm:^4.5.3":+"keyv@npm:^4.5.3":version: 4.5.4resolution: "keyv@npm:4.5.4"dependencies:@@ -7890,13 +7830,6 @@ __metadata:languageName: nodelinkType: hard-"lowercase-keys@npm:2.0.0, lowercase-keys@npm:^2.0.0":- version: 2.0.0- resolution: "lowercase-keys@npm:2.0.0"- checksum: f82a2b3568910509da4b7906362efa40f5b54ea14c2584778ddb313226f9cbf21020a5db35f9b9a0e95847a9b781d548601f31793d736b22a2b8ae8eb9ab1082- languageName: node- linkType: hard-"lowercase-keys@npm:^3.0.0":version: 3.0.0resolution: "lowercase-keys@npm:3.0.0"@@ -8161,13 +8094,6 @@ __metadata:languageName: nodelinkType: hard-"mimic-response@npm:^1.0.0":- version: 1.0.1- resolution: "mimic-response@npm:1.0.1"- checksum: c5381a5eae997f1c3b5e90ca7f209ed58c3615caeee850e85329c598f0c000ae7bec40196580eef1781c60c709f47258131dab237cad8786f8f56750594f27fa- languageName: node- linkType: hard-"mimic-response@npm:^3.1.0":version: 3.1.0resolution: "mimic-response@npm:3.1.0"@@ -8663,13 +8589,6 @@ __metadata:languageName: nodelinkType: hard-"normalize-url@npm:^6.0.1":- version: 6.1.0- resolution: "normalize-url@npm:6.1.0"- checksum: 95d948f9bdd2cfde91aa786d1816ae40f8262946e13700bf6628105994fe0ff361662c20af3961161c38a119dc977adeb41fc0b41b1745eb77edaaf9cb22db23- languageName: node- linkType: hard-"normalize-url@npm:^8.0.0":version: 8.0.0resolution: "normalize-url@npm:8.0.0"@@ -9118,10 +9037,16 @@ __metadata:languageName: nodelinkType: hard-"p-cancelable@npm:2.1.1":- version: 2.1.1- resolution: "p-cancelable@npm:2.1.1"- checksum: 8c6dc1f8dd4154fd8b96a10e55a3a832684c4365fb9108056d89e79fbf21a2465027c04a59d0d797b5ffe10b54a61a32043af287d5c4860f1e996cbdbc847f01+"ow@npm:^1.1.1":+ version: 1.1.1+ resolution: "ow@npm:1.1.1"+ dependencies:+ "@sindresorhus/is": "npm:^5.3.0"+ callsites: "npm:^4.0.0"+ dot-prop: "npm:^7.2.0"+ lodash.isequal: "npm:^4.5.0"+ vali-date: "npm:^1.0.0"+ checksum: 3973f9d6245f2e468a0f1d614ece96f1289632f7425094e8b266b50ddbe79471f2e6cba447b80e90b54bbeb13c20e83671edfb5ef4c0b13c15546ba0710554e1languageName: nodelinkType: hard@@ -9877,6 +9802,13 @@ __metadata:languageName: nodelinkType: hard+"quick-lru@npm:^7.0.0":+ version: 7.0.0+ resolution: "quick-lru@npm:7.0.0"+ checksum: d46e3dc6d2f79f83d3dc852b32b3d9fcf5337bf5b4016122c1b729aa7bcd54dd79b2f3cb36b59a5a3f5c30443ecfbc9e7d400cfc30feedd8d58d3d4213d08a18+ languageName: node+ linkType: hard+"range-parser@npm:~1.2.1":version: 1.2.1resolution: "range-parser@npm:1.2.1"@@ -10191,15 +10123,6 @@ __metadata:languageName: nodelinkType: hard-"responselike@npm:2.0.1, responselike@npm:^2.0.0":- version: 2.0.1- resolution: "responselike@npm:2.0.1"- dependencies:- lowercase-keys: "npm:^2.0.0"- checksum: 360b6deb5f101a9f8a4174f7837c523c3ec78b7ca8a7c1d45a1062b303659308a23757e318b1e91ed8684ad1205721142dd664d94771cd63499353fd4ee732b5- languageName: node- linkType: hard-"responselike@npm:^3.0.0":version: 3.0.0resolution: "responselike@npm:3.0.0"@@ -11421,7 +11344,7 @@ __metadata:languageName: nodelinkType: hard-"tslib@npm:^2.0.1, tslib@npm:^2.1.0, tslib@npm:^2.3.0, tslib@npm:^2.3.1, tslib@npm:^2.4.0, tslib@npm:^2.5.0, tslib@npm:^2.6.0":+"tslib@npm:^2.0.1, tslib@npm:^2.1.0, tslib@npm:^2.3.0, tslib@npm:^2.3.1, tslib@npm:^2.4.0, tslib@npm:^2.5.0, tslib@npm:^2.6.0, tslib@npm:^2.6.2":version: 2.6.2resolution: "tslib@npm:2.6.2"checksum: e03a8a4271152c8b26604ed45535954c0a45296e32445b4b87f8a5abdb2421f40b59b4ca437c4346af0f28179780d604094eb64546bee2019d903d01c6c19bdb@@ -11575,6 +11498,13 @@ __metadata:languageName: nodelinkType: hard+"type-fest@npm:^2.11.2":+ version: 2.19.0+ resolution: "type-fest@npm:2.19.0"+ checksum: a5a7ecf2e654251613218c215c7493574594951c08e52ab9881c9df6a6da0aeca7528c213c622bc374b4e0cb5c443aa3ab758da4e3c959783ce884c3194e12cb+ languageName: node+ linkType: hard+"type-fest@npm:^4.0.0":version: 4.7.1resolution: "type-fest@npm:4.7.1"@@ -11646,7 +11576,7 @@ __metadata:languageName: nodelinkType: hard-"typescript@npm:>=3 < 6, typescript@npm:^5.0.0":+"typescript@npm:>=3 < 6, typescript@npm:^5.2.2":version: 5.2.2resolution: "typescript@npm:5.2.2"bin:@@ -11656,7 +11586,7 @@ __metadata:languageName: nodelinkType: hard-"typescript@patch:typescript@npm%3A>=3 < 6#optional!builtin<compat/typescript>, typescript@patch:typescript@npm%3A^5.0.0#optional!builtin<compat/typescript>":+"typescript@patch:typescript@npm%3A>=3 < 6#optional!builtin<compat/typescript>, typescript@patch:typescript@npm%3A^5.2.2#optional!builtin<compat/typescript>":version: 5.2.2resolution: "typescript@patch:typescript@npm%3A5.2.2#optional!builtin<compat/typescript>::version=5.2.2&hash=f3b441"bin: