Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

ci: delay dependency updates for 24 hours (#3190)

Martin Adámekab75b8c2dda9
diff --git a/.yarnrc.yml b/.yarnrc.yml
index 9f52c10eb..19feceb3a 100644
--- a/.yarnrc.yml
+++ b/.yarnrc.yml
@@ -1,2 +1,3 @@
nodeLinker: node-modules
enableGlobalCache: true
+npmMinimalAgeGate: 1440
diff --git a/biome.json b/biome.json
index 7862480ba..8c23acdb0 100644
--- a/biome.json
+++ b/biome.json
@@ -1,11 +1,12 @@
{
"formatter": {
- "ignore": [
- "website/**",
- "packages/*/dist/**",
- "package.json",
- "lerna.json",
- "scripts/actions/docker-images/state.json"
+ "includes": [
+ "**",
+ "!**/website/**",
+ "!**/packages/**/*/dist/**",
+ "!**/package.json",
+ "!**/lerna.json",
+ "!**/scripts/actions/docker-images/state.json"
],
"formatWithErrors": true
},
diff --git a/package.json b/package.json
index d3d660bb2..4e166aefb 100644
--- a/package.json
+++ b/package.json
@@ -58,7 +58,7 @@
"@apify/eslint-config": "^1.0.0",
"@apify/log": "^2.4.0",
"@apify/tsconfig": "^0.1.0",
- "@biomejs/biome": "^1.7.3",
+ "@biomejs/biome": "^2.2.5",
"@commitlint/config-conventional": "^20.0.0",
"@playwright/browser-chromium": "1.55.1",
"@playwright/browser-firefox": "1.55.1",
@@ -116,10 +116,10 @@
"vite-tsconfig-paths": "^4.3.2",
"vitest": "^3.0.0"
},
- "packageManager": "[email protected]",
+ "packageManager": "[email protected]",
"volta": {
"node": "22.20.0",
- "yarn": "4.8.1"
+ "yarn": "4.10.3"
},
"resolutions": {
"vite": "5.4.11"
diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.ts
index a75e5c277..831e24f7e 100644
--- a/packages/core/src/enqueue_links/shared.ts
+++ b/packages/core/src/enqueue_links/shared.ts
@@ -49,10 +49,7 @@ export type RegExpInput = RegExp | RegExpObject;
export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'filters' | 'redirect' | 'depth';
-export type SkippedRequestCallback = (args: {
- url: string;
- reason: SkippedRequestReason;
-}) => Awaitable<void>;
+export type SkippedRequestCallback = (args: { url: string; reason: SkippedRequestReason }) => Awaitable<void>;
/**
* @ignore
diff --git a/packages/core/src/storages/request_list.ts b/packages/core/src/storages/request_list.ts
index 17abbcab7..4e6a09741 100644
--- a/packages/core/src/storages/request_list.ts
+++ b/packages/core/src/storages/request_list.ts
@@ -962,9 +962,11 @@ export class RequestList implements IRequestList {
/**
* @internal wraps public utility for mocking purposes
*/
- private async _downloadListOfUrls(options: { url: string; urlRegExp?: RegExp; proxyUrl?: string }): Promise<
- string[]
- > {
+ private async _downloadListOfUrls(options: {
+ url: string;
+ urlRegExp?: RegExp;
+ proxyUrl?: string;
+ }): Promise<string[]> {
return downloadListOfUrls(options);
}
}
diff --git a/packages/core/src/storages/request_provider.ts b/packages/core/src/storages/request_provider.ts
index 9c4df9712..dc1204abf 100644
--- a/packages/core/src/storages/request_provider.ts
+++ b/packages/core/src/storages/request_provider.ts
@@ -832,9 +832,11 @@ export abstract class RequestProvider implements IStorage, IRequestManager {
/**
* @internal wraps public utility for mocking purposes
*/
- private async _downloadListOfUrls(options: { url: string; urlRegExp?: RegExp; proxyUrl?: string }): Promise<
- string[]
- > {
+ private async _downloadListOfUrls(options: {
+ url: string;
+ urlRegExp?: RegExp;
+ proxyUrl?: string;
+ }): Promise<string[]> {
return downloadListOfUrls(options);
}
diff --git a/packages/core/src/storages/sitemap_request_list.ts b/packages/core/src/storages/sitemap_request_list.ts
index 278d27a7c..87e06bca8 100644
--- a/packages/core/src/storages/sitemap_request_list.ts
+++ b/packages/core/src/storages/sitemap_request_list.ts
@@ -369,7 +369,9 @@ export class SitemapRequestList implements IRequestList {
*/
private async load({
parseSitemapOptions,
- }: { parseSitemapOptions?: SitemapRequestListOptions['parseSitemapOptions'] }): Promise<void> {
+ }: {
+ parseSitemapOptions?: SitemapRequestListOptions['parseSitemapOptions'];
+ }): Promise<void> {
while (!this.isSitemapFullyLoaded() && !this.abortLoading) {
const sitemapUrl =
this.sitemapParsingProgress.inProgressSitemapUrl ??
diff --git a/packages/http-crawler/src/internals/file-download.ts b/packages/http-crawler/src/internals/file-download.ts
index 1e5f66ef2..536a96681 100644
--- a/packages/http-crawler/src/internals/file-download.ts
+++ b/packages/http-crawler/src/internals/file-download.ts
@@ -116,7 +116,10 @@ export function MinimumSpeedStream({
export function ByteCounterStream({
logTransferredBytes,
loggingInterval = 5000,
-}: { logTransferredBytes: (transferredBytes: number) => void; loggingInterval?: number }): Transform {
+}: {
+ logTransferredBytes: (transferredBytes: number) => void;
+ loggingInterval?: number;
+}): Transform {
let transferredBytes = 0;
let lastLogTime = Date.now();
diff --git a/packages/impit-client/package.json b/packages/impit-client/package.json
index bf8955f97..17056a1b1 100644
--- a/packages/impit-client/package.json
+++ b/packages/impit-client/package.json
@@ -62,6 +62,5 @@
"@apify/datastructures": "^2.0.3",
"impit": "^0.5.3",
"tough-cookie": "^6.0.0"
- },
- "packageManager": "[email protected]"
+ }
}
diff --git a/renovate.json b/renovate.json
index 3a7c2bc9c..3265e8a37 100644
--- a/renovate.json
+++ b/renovate.json
@@ -24,5 +24,7 @@
}
],
"schedule": ["every weekday"],
+ "minimumReleaseAge": "1 day",
+ "internalChecksFilter": "strict",
"ignoreDeps": ["crawlee", "cheerio", "yarn"]
}
diff --git a/website/package.json b/website/package.json
index b1bfc4dc8..b0a4d9599 100644
--- a/website/package.json
+++ b/website/package.json
@@ -74,5 +74,5 @@
"last 5 safari version"
]
},
- "packageManager": "[email protected]"
+ "packageManager": "[email protected]"
}
diff --git a/yarn.lock b/yarn.lock
index e5f0215f4..1ff1670f9 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -182,18 +182,18 @@ __metadata:
languageName: node
linkType: hard
-"@biomejs/biome@npm:^1.7.3":
- version: 1.9.4
- resolution: "@biomejs/biome@npm:1.9.4"
- dependencies:
- "@biomejs/cli-darwin-arm64": "npm:1.9.4"
- "@biomejs/cli-darwin-x64": "npm:1.9.4"
- "@biomejs/cli-linux-arm64": "npm:1.9.4"
- "@biomejs/cli-linux-arm64-musl": "npm:1.9.4"
- "@biomejs/cli-linux-x64": "npm:1.9.4"
- "@biomejs/cli-linux-x64-musl": "npm:1.9.4"
- "@biomejs/cli-win32-arm64": "npm:1.9.4"
- "@biomejs/cli-win32-x64": "npm:1.9.4"
+"@biomejs/biome@npm:^2.2.5":
+ version: 2.2.5
+ resolution: "@biomejs/biome@npm:2.2.5"
+ dependencies:
+ "@biomejs/cli-darwin-arm64": "npm:2.2.5"
+ "@biomejs/cli-darwin-x64": "npm:2.2.5"
+ "@biomejs/cli-linux-arm64": "npm:2.2.5"
+ "@biomejs/cli-linux-arm64-musl": "npm:2.2.5"
+ "@biomejs/cli-linux-x64": "npm:2.2.5"
+ "@biomejs/cli-linux-x64-musl": "npm:2.2.5"
+ "@biomejs/cli-win32-arm64": "npm:2.2.5"
+ "@biomejs/cli-win32-x64": "npm:2.2.5"
dependenciesMeta:
"@biomejs/cli-darwin-arm64":
optional: true
@@ -213,62 +213,62 @@ __metadata:
optional: true
bin:
biome: bin/biome
- checksum: 10c0/b5655c5aed9a6fffe24f7d04f15ba4444389d0e891c9ed9106fab7388ac9b4be63185852cc2a937b22940dac3e550b71032a4afd306925cfea436c33e5646b3e
+ checksum: 10c0/d9baff2b9d35f820e3e746fd4222ba1295d7648e04ececc4e1f25ed0c09dd4fd30505a7183731fc75252ea925728c276858c9f09751b7e1fd1c8378b05208de6
languageName: node
linkType: hard
-"@biomejs/cli-darwin-arm64@npm:1.9.4":
- version: 1.9.4
- resolution: "@biomejs/cli-darwin-arm64@npm:1.9.4"
+"@biomejs/cli-darwin-arm64@npm:2.2.5":
+ version: 2.2.5
+ resolution: "@biomejs/cli-darwin-arm64@npm:2.2.5"
conditions: os=darwin & cpu=arm64
languageName: node
linkType: hard
-"@biomejs/cli-darwin-x64@npm:1.9.4":
- version: 1.9.4
- resolution: "@biomejs/cli-darwin-x64@npm:1.9.4"
+"@biomejs/cli-darwin-x64@npm:2.2.5":
+ version: 2.2.5
+ resolution: "@biomejs/cli-darwin-x64@npm:2.2.5"
conditions: os=darwin & cpu=x64
languageName: node
linkType: hard
-"@biomejs/cli-linux-arm64-musl@npm:1.9.4":
- version: 1.9.4
- resolution: "@biomejs/cli-linux-arm64-musl@npm:1.9.4"
+"@biomejs/cli-linux-arm64-musl@npm:2.2.5":
+ version: 2.2.5
+ resolution: "@biomejs/cli-linux-arm64-musl@npm:2.2.5"
conditions: os=linux & cpu=arm64 & libc=musl
languageName: node
linkType: hard
-"@biomejs/cli-linux-arm64@npm:1.9.4":
- version: 1.9.4
- resolution: "@biomejs/cli-linux-arm64@npm:1.9.4"
+"@biomejs/cli-linux-arm64@npm:2.2.5":
+ version: 2.2.5
+ resolution: "@biomejs/cli-linux-arm64@npm:2.2.5"
conditions: os=linux & cpu=arm64 & libc=glibc
languageName: node
linkType: hard
-"@biomejs/cli-linux-x64-musl@npm:1.9.4":
- version: 1.9.4
- resolution: "@biomejs/cli-linux-x64-musl@npm:1.9.4"
+"@biomejs/cli-linux-x64-musl@npm:2.2.5":
+ version: 2.2.5
+ resolution: "@biomejs/cli-linux-x64-musl@npm:2.2.5"
conditions: os=linux & cpu=x64 & libc=musl
languageName: node
linkType: hard
-"@biomejs/cli-linux-x64@npm:1.9.4":
- version: 1.9.4
- resolution: "@biomejs/cli-linux-x64@npm:1.9.4"
+"@biomejs/cli-linux-x64@npm:2.2.5":
+ version: 2.2.5
+ resolution: "@biomejs/cli-linux-x64@npm:2.2.5"
conditions: os=linux & cpu=x64 & libc=glibc
languageName: node
linkType: hard
-"@biomejs/cli-win32-arm64@npm:1.9.4":
- version: 1.9.4
- resolution: "@biomejs/cli-win32-arm64@npm:1.9.4"
+"@biomejs/cli-win32-arm64@npm:2.2.5":
+ version: 2.2.5
+ resolution: "@biomejs/cli-win32-arm64@npm:2.2.5"
conditions: os=win32 & cpu=arm64
languageName: node
linkType: hard
-"@biomejs/cli-win32-x64@npm:1.9.4":
- version: 1.9.4
- resolution: "@biomejs/cli-win32-x64@npm:1.9.4"
+"@biomejs/cli-win32-x64@npm:2.2.5":
+ version: 2.2.5
+ resolution: "@biomejs/cli-win32-x64@npm:2.2.5"
conditions: os=win32 & cpu=x64
languageName: node
linkType: hard
@@ -743,7 +743,7 @@ __metadata:
"@apify/eslint-config": "npm:^1.0.0"
"@apify/log": "npm:^2.4.0"
"@apify/tsconfig": "npm:^0.1.0"
- "@biomejs/biome": "npm:^1.7.3"
+ "@biomejs/biome": "npm:^2.2.5"
"@commitlint/config-conventional": "npm:^20.0.0"
"@playwright/browser-chromium": "npm:1.55.1"
"@playwright/browser-firefox": "npm:1.55.1"