Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
ci: delay dependency updates for 24 hours (#3190)
Martin Adámekab75b8c2dda9
diff --git a/.yarnrc.yml b/.yarnrc.ymlindex 9f52c10eb..19feceb3a 100644--- a/.yarnrc.yml+++ b/.yarnrc.yml@@ -1,2 +1,3 @@nodeLinker: node-modulesenableGlobalCache: true+npmMinimalAgeGate: 1440diff --git a/biome.json b/biome.jsonindex 7862480ba..8c23acdb0 100644--- a/biome.json+++ b/biome.json@@ -1,11 +1,12 @@{"formatter": {- "ignore": [- "website/**",- "packages/*/dist/**",- "package.json",- "lerna.json",- "scripts/actions/docker-images/state.json"+ "includes": [+ "**",+ "!**/website/**",+ "!**/packages/**/*/dist/**",+ "!**/package.json",+ "!**/lerna.json",+ "!**/scripts/actions/docker-images/state.json"],"formatWithErrors": true},diff --git a/package.json b/package.jsonindex d3d660bb2..4e166aefb 100644--- a/package.json+++ b/package.json@@ -58,7 +58,7 @@"@apify/eslint-config": "^1.0.0","@apify/log": "^2.4.0","@apify/tsconfig": "^0.1.0",- "@biomejs/biome": "^1.7.3",+ "@biomejs/biome": "^2.2.5","@commitlint/config-conventional": "^20.0.0","@playwright/browser-chromium": "1.55.1","@playwright/browser-firefox": "1.55.1",@@ -116,10 +116,10 @@"vite-tsconfig-paths": "^4.3.2","vitest": "^3.0.0"},- "packageManager": "[email protected]",+ "packageManager": "[email protected]","volta": {"node": "22.20.0",- "yarn": "4.8.1"+ "yarn": "4.10.3"},"resolutions": {"vite": "5.4.11"diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.tsindex a75e5c277..831e24f7e 100644--- a/packages/core/src/enqueue_links/shared.ts+++ b/packages/core/src/enqueue_links/shared.ts@@ -49,10 +49,7 @@ export type RegExpInput = RegExp | RegExpObject;export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'filters' | 'redirect' | 'depth';-export type SkippedRequestCallback = (args: {- url: string;- reason: SkippedRequestReason;-}) => Awaitable<void>;+export type SkippedRequestCallback = (args: { url: string; reason: SkippedRequestReason }) => Awaitable<void>;/*** @ignorediff --git a/packages/core/src/storages/request_list.ts b/packages/core/src/storages/request_list.tsindex 17abbcab7..4e6a09741 100644--- a/packages/core/src/storages/request_list.ts+++ b/packages/core/src/storages/request_list.ts@@ -962,9 +962,11 @@ export class RequestList implements IRequestList {/*** @internal wraps public utility for mocking purposes*/- private async _downloadListOfUrls(options: { url: string; urlRegExp?: RegExp; proxyUrl?: string }): Promise<- string[]- > {+ private async _downloadListOfUrls(options: {+ url: string;+ urlRegExp?: RegExp;+ proxyUrl?: string;+ }): Promise<string[]> {return downloadListOfUrls(options);}}diff --git a/packages/core/src/storages/request_provider.ts b/packages/core/src/storages/request_provider.tsindex 9c4df9712..dc1204abf 100644--- a/packages/core/src/storages/request_provider.ts+++ b/packages/core/src/storages/request_provider.ts@@ -832,9 +832,11 @@ export abstract class RequestProvider implements IStorage, IRequestManager {/*** @internal wraps public utility for mocking purposes*/- private async _downloadListOfUrls(options: { url: string; urlRegExp?: RegExp; proxyUrl?: string }): Promise<- string[]- > {+ private async _downloadListOfUrls(options: {+ url: string;+ urlRegExp?: RegExp;+ proxyUrl?: string;+ }): Promise<string[]> {return downloadListOfUrls(options);}diff --git a/packages/core/src/storages/sitemap_request_list.ts b/packages/core/src/storages/sitemap_request_list.tsindex 278d27a7c..87e06bca8 100644--- a/packages/core/src/storages/sitemap_request_list.ts+++ b/packages/core/src/storages/sitemap_request_list.ts@@ -369,7 +369,9 @@ export class SitemapRequestList implements IRequestList {*/private async load({parseSitemapOptions,- }: { parseSitemapOptions?: SitemapRequestListOptions['parseSitemapOptions'] }): Promise<void> {+ }: {+ parseSitemapOptions?: SitemapRequestListOptions['parseSitemapOptions'];+ }): Promise<void> {while (!this.isSitemapFullyLoaded() && !this.abortLoading) {const sitemapUrl =this.sitemapParsingProgress.inProgressSitemapUrl ??diff --git a/packages/http-crawler/src/internals/file-download.ts b/packages/http-crawler/src/internals/file-download.tsindex 1e5f66ef2..536a96681 100644--- a/packages/http-crawler/src/internals/file-download.ts+++ b/packages/http-crawler/src/internals/file-download.ts@@ -116,7 +116,10 @@ export function MinimumSpeedStream({export function ByteCounterStream({logTransferredBytes,loggingInterval = 5000,-}: { logTransferredBytes: (transferredBytes: number) => void; loggingInterval?: number }): Transform {+}: {+ logTransferredBytes: (transferredBytes: number) => void;+ loggingInterval?: number;+}): Transform {let transferredBytes = 0;let lastLogTime = Date.now();diff --git a/packages/impit-client/package.json b/packages/impit-client/package.jsonindex bf8955f97..17056a1b1 100644--- a/packages/impit-client/package.json+++ b/packages/impit-client/package.json@@ -62,6 +62,5 @@"@apify/datastructures": "^2.0.3","impit": "^0.5.3","tough-cookie": "^6.0.0"- },- "packageManager": "[email protected]"+ }}diff --git a/renovate.json b/renovate.jsonindex 3a7c2bc9c..3265e8a37 100644--- a/renovate.json+++ b/renovate.json@@ -24,5 +24,7 @@}],"schedule": ["every weekday"],+ "minimumReleaseAge": "1 day",+ "internalChecksFilter": "strict","ignoreDeps": ["crawlee", "cheerio", "yarn"]}diff --git a/website/package.json b/website/package.jsonindex b1bfc4dc8..b0a4d9599 100644--- a/website/package.json+++ b/website/package.json@@ -74,5 +74,5 @@"last 5 safari version"]},- "packageManager": "[email protected]"+ "packageManager": "[email protected]"}diff --git a/yarn.lock b/yarn.lockindex e5f0215f4..1ff1670f9 100644--- a/yarn.lock+++ b/yarn.lock@@ -182,18 +182,18 @@ __metadata:languageName: nodelinkType: hard-"@biomejs/biome@npm:^1.7.3":- version: 1.9.4- resolution: "@biomejs/biome@npm:1.9.4"- dependencies:- "@biomejs/cli-darwin-arm64": "npm:1.9.4"- "@biomejs/cli-darwin-x64": "npm:1.9.4"- "@biomejs/cli-linux-arm64": "npm:1.9.4"- "@biomejs/cli-linux-arm64-musl": "npm:1.9.4"- "@biomejs/cli-linux-x64": "npm:1.9.4"- "@biomejs/cli-linux-x64-musl": "npm:1.9.4"- "@biomejs/cli-win32-arm64": "npm:1.9.4"- "@biomejs/cli-win32-x64": "npm:1.9.4"+"@biomejs/biome@npm:^2.2.5":+ version: 2.2.5+ resolution: "@biomejs/biome@npm:2.2.5"+ dependencies:+ "@biomejs/cli-darwin-arm64": "npm:2.2.5"+ "@biomejs/cli-darwin-x64": "npm:2.2.5"+ "@biomejs/cli-linux-arm64": "npm:2.2.5"+ "@biomejs/cli-linux-arm64-musl": "npm:2.2.5"+ "@biomejs/cli-linux-x64": "npm:2.2.5"+ "@biomejs/cli-linux-x64-musl": "npm:2.2.5"+ "@biomejs/cli-win32-arm64": "npm:2.2.5"+ "@biomejs/cli-win32-x64": "npm:2.2.5"dependenciesMeta:"@biomejs/cli-darwin-arm64":optional: true@@ -213,62 +213,62 @@ __metadata:optional: truebin:biome: bin/biome- checksum: 10c0/b5655c5aed9a6fffe24f7d04f15ba4444389d0e891c9ed9106fab7388ac9b4be63185852cc2a937b22940dac3e550b71032a4afd306925cfea436c33e5646b3e+ checksum: 10c0/d9baff2b9d35f820e3e746fd4222ba1295d7648e04ececc4e1f25ed0c09dd4fd30505a7183731fc75252ea925728c276858c9f09751b7e1fd1c8378b05208de6languageName: nodelinkType: hard-"@biomejs/cli-darwin-arm64@npm:1.9.4":- version: 1.9.4- resolution: "@biomejs/cli-darwin-arm64@npm:1.9.4"+"@biomejs/cli-darwin-arm64@npm:2.2.5":+ version: 2.2.5+ resolution: "@biomejs/cli-darwin-arm64@npm:2.2.5"conditions: os=darwin & cpu=arm64languageName: nodelinkType: hard-"@biomejs/cli-darwin-x64@npm:1.9.4":- version: 1.9.4- resolution: "@biomejs/cli-darwin-x64@npm:1.9.4"+"@biomejs/cli-darwin-x64@npm:2.2.5":+ version: 2.2.5+ resolution: "@biomejs/cli-darwin-x64@npm:2.2.5"conditions: os=darwin & cpu=x64languageName: nodelinkType: hard-"@biomejs/cli-linux-arm64-musl@npm:1.9.4":- version: 1.9.4- resolution: "@biomejs/cli-linux-arm64-musl@npm:1.9.4"+"@biomejs/cli-linux-arm64-musl@npm:2.2.5":+ version: 2.2.5+ resolution: "@biomejs/cli-linux-arm64-musl@npm:2.2.5"conditions: os=linux & cpu=arm64 & libc=musllanguageName: nodelinkType: hard-"@biomejs/cli-linux-arm64@npm:1.9.4":- version: 1.9.4- resolution: "@biomejs/cli-linux-arm64@npm:1.9.4"+"@biomejs/cli-linux-arm64@npm:2.2.5":+ version: 2.2.5+ resolution: "@biomejs/cli-linux-arm64@npm:2.2.5"conditions: os=linux & cpu=arm64 & libc=glibclanguageName: nodelinkType: hard-"@biomejs/cli-linux-x64-musl@npm:1.9.4":- version: 1.9.4- resolution: "@biomejs/cli-linux-x64-musl@npm:1.9.4"+"@biomejs/cli-linux-x64-musl@npm:2.2.5":+ version: 2.2.5+ resolution: "@biomejs/cli-linux-x64-musl@npm:2.2.5"conditions: os=linux & cpu=x64 & libc=musllanguageName: nodelinkType: hard-"@biomejs/cli-linux-x64@npm:1.9.4":- version: 1.9.4- resolution: "@biomejs/cli-linux-x64@npm:1.9.4"+"@biomejs/cli-linux-x64@npm:2.2.5":+ version: 2.2.5+ resolution: "@biomejs/cli-linux-x64@npm:2.2.5"conditions: os=linux & cpu=x64 & libc=glibclanguageName: nodelinkType: hard-"@biomejs/cli-win32-arm64@npm:1.9.4":- version: 1.9.4- resolution: "@biomejs/cli-win32-arm64@npm:1.9.4"+"@biomejs/cli-win32-arm64@npm:2.2.5":+ version: 2.2.5+ resolution: "@biomejs/cli-win32-arm64@npm:2.2.5"conditions: os=win32 & cpu=arm64languageName: nodelinkType: hard-"@biomejs/cli-win32-x64@npm:1.9.4":- version: 1.9.4- resolution: "@biomejs/cli-win32-x64@npm:1.9.4"+"@biomejs/cli-win32-x64@npm:2.2.5":+ version: 2.2.5+ resolution: "@biomejs/cli-win32-x64@npm:2.2.5"conditions: os=win32 & cpu=x64languageName: nodelinkType: hard@@ -743,7 +743,7 @@ __metadata:"@apify/eslint-config": "npm:^1.0.0""@apify/log": "npm:^2.4.0""@apify/tsconfig": "npm:^0.1.0"- "@biomejs/biome": "npm:^1.7.3"+ "@biomejs/biome": "npm:^2.2.5""@commitlint/config-conventional": "npm:^20.0.0""@playwright/browser-chromium": "npm:1.55.1""@playwright/browser-firefox": "npm:1.55.1"