Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: session.markBad() on requestHandler error (#1709)
Jindřich Bäre87eb1f2ccd9
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 4a2719a88..91a27c8c5 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -943,7 +943,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext// reclaim session if request finishes successfullyrequest.state = RequestState.DONE;- session?.markGood();+ crawlingContext.session?.markGood();} catch (err) {try {request.state = RequestState.ERROR_HANDLER;@@ -964,7 +964,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextthrow secondaryError;}// decrease the session score if the request fails (but the error handler did not throw)- session?.markBad();+ crawlingContext.session?.markBad();} finally {await this._cleanupContext(crawlingContext);diff --git a/test/e2e/session-rotation/actor/.gitignore b/test/e2e/session-rotation/actor/.gitignorenew file mode 100644index 000000000..ced7cbfc5--- /dev/null+++ b/test/e2e/session-rotation/actor/.gitignore@@ -0,0 +1,7 @@+.idea+.DS_Store+node_modules+package-lock.json+apify_storage+crawlee_storage+storagediff --git a/test/e2e/session-rotation/actor/Dockerfile b/test/e2e/session-rotation/actor/Dockerfilenew file mode 100644index 000000000..de06663ad--- /dev/null+++ b/test/e2e/session-rotation/actor/Dockerfile@@ -0,0 +1,23 @@+FROM node:16 AS builder++COPY /packages ./packages+COPY /package*.json ./+RUN npm --quiet set progress=false \+ && npm install --only=prod --no-optional \+ && npm update++FROM apify/actor-node-playwright-chrome:16-beta++RUN rm -r node_modules+COPY --from=builder /node_modules ./node_modules+COPY --from=builder /packages ./packages+COPY --from=builder /package*.json ./+COPY /apify.json ./+COPY /main.js ./++RUN echo "Installed NPM packages:" \+ && (npm list --only=prod --no-optional --all || true) \+ && echo "Node.js version:" \+ && node --version \+ && echo "NPM version:" \+ && npm --versiondiff --git a/test/e2e/session-rotation/actor/apify.json b/test/e2e/session-rotation/actor/apify.jsonnew file mode 100644index 000000000..bac48c79c--- /dev/null+++ b/test/e2e/session-rotation/actor/apify.json@@ -0,0 +1,6 @@+{+ "name": "test-session-rotation",+ "version": "0.0",+ "buildTag": "latest",+ "env": null+}diff --git a/test/e2e/session-rotation/actor/main.js b/test/e2e/session-rotation/actor/main.jsnew file mode 100644index 000000000..06a711315--- /dev/null+++ b/test/e2e/session-rotation/actor/main.js@@ -0,0 +1,26 @@+import { Actor } from 'apify';+import { PlaywrightCrawler } from '@crawlee/playwright';+import { ApifyStorageLocal } from '@apify/storage-local';++const mainOptions = {+ exit: Actor.isAtHome(),+ storage: process.env.STORAGE_IMPLEMENTATION === 'LOCAL' ? new ApifyStorageLocal() : undefined,+};++await Actor.main(async () => {+ const crawler = new PlaywrightCrawler({+ maxRequestRetries: 10,+ sessionPoolOptions: {+ sessionOptions: {+ maxErrorScore: 2,+ },+ },+ requestHandler: async ({ session }) => {+ const { id, usageCount, errorScore } = session;+ Actor.pushData({ id, usageCount, errorScore });+ throw new Error('retry');+ },+ });++ await crawler.run(['https://crawlee.dev/']);+}, mainOptions);diff --git a/test/e2e/session-rotation/actor/package.json b/test/e2e/session-rotation/actor/package.jsonnew file mode 100644index 000000000..56132f00f--- /dev/null+++ b/test/e2e/session-rotation/actor/package.json@@ -0,0 +1,29 @@+{+ "name": "test-session-rotation",+ "version": "0.0.1",+ "description": "Session Test - Rotation",+ "dependencies": {+ "apify": "next",+ "@apify/storage-local": "^2.1.0",+ "@crawlee/basic": "file:./packages/basic-crawler",+ "@crawlee/browser": "file:./packages/browser-crawler",+ "@crawlee/browser-pool": "file:./packages/browser-pool",+ "@crawlee/core": "file:./packages/core",+ "@crawlee/memory-storage": "file:./packages/memory-storage",+ "@crawlee/puppeteer": "file:./packages/puppeteer-crawler",+ "@crawlee/types": "file:./packages/types",+ "@crawlee/utils": "file:./packages/utils",+ "puppeteer": "*"+ },+ "overrides": {+ "apify": {+ "@crawlee/core": "file:./packages/core",+ "@crawlee/utils": "file:./packages/utils"+ }+ },+ "scripts": {+ "start": "node main.js"+ },+ "type": "module",+ "license": "ISC"+}diff --git a/test/e2e/session-rotation/test.mjs b/test/e2e/session-rotation/test.mjsnew file mode 100644index 000000000..5ffd7300b--- /dev/null+++ b/test/e2e/session-rotation/test.mjs@@ -0,0 +1,12 @@+import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs';++const testActorDirname = getActorTestDir(import.meta.url);+await initialize(testActorDirname);++const { datasetItems } = await runActor(testActorDirname, 4096);++await expect(datasetItems.length === 11, 'Retried correct number of times');+await expect(+ datasetItems.map(+ (session) => datasetItems.filter((s) => s.id === session.id),+ ).every((x) => x.length <= 2), 'No session used more than three times');