Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: session.markBad() on requestHandler error (#1709)

Jindřich Bäre87eb1f2ccd9
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 4a2719a88..91a27c8c5 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -943,7 +943,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
// reclaim session if request finishes successfully
request.state = RequestState.DONE;
- session?.markGood();
+ crawlingContext.session?.markGood();
} catch (err) {
try {
request.state = RequestState.ERROR_HANDLER;
@@ -964,7 +964,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
throw secondaryError;
}
// decrease the session score if the request fails (but the error handler did not throw)
- session?.markBad();
+ crawlingContext.session?.markBad();
} finally {
await this._cleanupContext(crawlingContext);
diff --git a/test/e2e/session-rotation/actor/.gitignore b/test/e2e/session-rotation/actor/.gitignore
new file mode 100644
index 000000000..ced7cbfc5
--- /dev/null
+++ b/test/e2e/session-rotation/actor/.gitignore
@@ -0,0 +1,7 @@
+.idea
+.DS_Store
+node_modules
+package-lock.json
+apify_storage
+crawlee_storage
+storage
diff --git a/test/e2e/session-rotation/actor/Dockerfile b/test/e2e/session-rotation/actor/Dockerfile
new file mode 100644
index 000000000..de06663ad
--- /dev/null
+++ b/test/e2e/session-rotation/actor/Dockerfile
@@ -0,0 +1,23 @@
+FROM node:16 AS builder
+
+COPY /packages ./packages
+COPY /package*.json ./
+RUN npm --quiet set progress=false \
+ && npm install --only=prod --no-optional \
+ && npm update
+
+FROM apify/actor-node-playwright-chrome:16-beta
+
+RUN rm -r node_modules
+COPY --from=builder /node_modules ./node_modules
+COPY --from=builder /packages ./packages
+COPY --from=builder /package*.json ./
+COPY /apify.json ./
+COPY /main.js ./
+
+RUN echo "Installed NPM packages:" \
+ && (npm list --only=prod --no-optional --all || true) \
+ && echo "Node.js version:" \
+ && node --version \
+ && echo "NPM version:" \
+ && npm --version
diff --git a/test/e2e/session-rotation/actor/apify.json b/test/e2e/session-rotation/actor/apify.json
new file mode 100644
index 000000000..bac48c79c
--- /dev/null
+++ b/test/e2e/session-rotation/actor/apify.json
@@ -0,0 +1,6 @@
+{
+ "name": "test-session-rotation",
+ "version": "0.0",
+ "buildTag": "latest",
+ "env": null
+}
diff --git a/test/e2e/session-rotation/actor/main.js b/test/e2e/session-rotation/actor/main.js
new file mode 100644
index 000000000..06a711315
--- /dev/null
+++ b/test/e2e/session-rotation/actor/main.js
@@ -0,0 +1,26 @@
+import { Actor } from 'apify';
+import { PlaywrightCrawler } from '@crawlee/playwright';
+import { ApifyStorageLocal } from '@apify/storage-local';
+
+const mainOptions = {
+ exit: Actor.isAtHome(),
+ storage: process.env.STORAGE_IMPLEMENTATION === 'LOCAL' ? new ApifyStorageLocal() : undefined,
+};
+
+await Actor.main(async () => {
+ const crawler = new PlaywrightCrawler({
+ maxRequestRetries: 10,
+ sessionPoolOptions: {
+ sessionOptions: {
+ maxErrorScore: 2,
+ },
+ },
+ requestHandler: async ({ session }) => {
+ const { id, usageCount, errorScore } = session;
+ Actor.pushData({ id, usageCount, errorScore });
+ throw new Error('retry');
+ },
+ });
+
+ await crawler.run(['https://crawlee.dev/']);
+}, mainOptions);
diff --git a/test/e2e/session-rotation/actor/package.json b/test/e2e/session-rotation/actor/package.json
new file mode 100644
index 000000000..56132f00f
--- /dev/null
+++ b/test/e2e/session-rotation/actor/package.json
@@ -0,0 +1,29 @@
+{
+ "name": "test-session-rotation",
+ "version": "0.0.1",
+ "description": "Session Test - Rotation",
+ "dependencies": {
+ "apify": "next",
+ "@apify/storage-local": "^2.1.0",
+ "@crawlee/basic": "file:./packages/basic-crawler",
+ "@crawlee/browser": "file:./packages/browser-crawler",
+ "@crawlee/browser-pool": "file:./packages/browser-pool",
+ "@crawlee/core": "file:./packages/core",
+ "@crawlee/memory-storage": "file:./packages/memory-storage",
+ "@crawlee/puppeteer": "file:./packages/puppeteer-crawler",
+ "@crawlee/types": "file:./packages/types",
+ "@crawlee/utils": "file:./packages/utils",
+ "puppeteer": "*"
+ },
+ "overrides": {
+ "apify": {
+ "@crawlee/core": "file:./packages/core",
+ "@crawlee/utils": "file:./packages/utils"
+ }
+ },
+ "scripts": {
+ "start": "node main.js"
+ },
+ "type": "module",
+ "license": "ISC"
+}
diff --git a/test/e2e/session-rotation/test.mjs b/test/e2e/session-rotation/test.mjs
new file mode 100644
index 000000000..5ffd7300b
--- /dev/null
+++ b/test/e2e/session-rotation/test.mjs
@@ -0,0 +1,12 @@
+import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs';
+
+const testActorDirname = getActorTestDir(import.meta.url);
+await initialize(testActorDirname);
+
+const { datasetItems } = await runActor(testActorDirname, 4096);
+
+await expect(datasetItems.length === 11, 'Retried correct number of times');
+await expect(
+ datasetItems.map(
+ (session) => datasetItems.filter((s) => s.id === session.id),
+ ).every((x) => x.length <= 2), 'No session used more than three times');