Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
test: add e2e test for zero concurrency with RQ v2
Martin Adámek1bb771291304
diff --git a/test/e2e/request-queue-v2-zero-concurrency/actor/.actor/actor.json b/test/e2e/request-queue-v2-zero-concurrency/actor/.actor/actor.jsonnew file mode 100644index 000000000..959ec7eaa--- /dev/null+++ b/test/e2e/request-queue-v2-zero-concurrency/actor/.actor/actor.json@@ -0,0 +1,7 @@+{+ "actorSpecification": 1,+ "name": "test-request-queue-zero-concurrency",+ "version": "0.0",+ "buildTag": "latest",+ "env": null+}diff --git a/test/e2e/request-queue-v2-zero-concurrency/actor/.gitignore b/test/e2e/request-queue-v2-zero-concurrency/actor/.gitignorenew file mode 100644index 000000000..ced7cbfc5--- /dev/null+++ b/test/e2e/request-queue-v2-zero-concurrency/actor/.gitignore@@ -0,0 +1,7 @@+.idea+.DS_Store+node_modules+package-lock.json+apify_storage+crawlee_storage+storagediff --git a/test/e2e/request-queue-v2-zero-concurrency/actor/Dockerfile b/test/e2e/request-queue-v2-zero-concurrency/actor/Dockerfilenew file mode 100644index 000000000..36afd80b9--- /dev/null+++ b/test/e2e/request-queue-v2-zero-concurrency/actor/Dockerfile@@ -0,0 +1,16 @@+FROM apify/actor-node:20-beta++COPY packages ./packages+COPY package*.json ./++RUN npm --quiet set progress=false \+ && npm install --only=prod --no-optional --no-audit \+ && npm update --no-audit \+ && echo "Installed NPM packages:" \+ && (npm list --only=prod --no-optional --all || true) \+ && echo "Node.js version:" \+ && node --version \+ && echo "NPM version:" \+ && npm --version++COPY . ./diff --git a/test/e2e/request-queue-v2-zero-concurrency/actor/main.js b/test/e2e/request-queue-v2-zero-concurrency/actor/main.jsnew file mode 100644index 000000000..2cb940fa8--- /dev/null+++ b/test/e2e/request-queue-v2-zero-concurrency/actor/main.js@@ -0,0 +1,33 @@+import { CheerioCrawler, log, RequestQueue } from '@crawlee/cheerio';+import { Actor } from 'apify';++log.setLevel(log.LEVELS.DEBUG);++process.env.CRAWLEE_INTERNAL_TIMEOUT = '30000';++const mainOptions = {+ exit: Actor.isAtHome(),+ storage:+ process.env.STORAGE_IMPLEMENTATION === 'LOCAL'+ ? new (await import('@apify/storage-local')).ApifyStorageLocal()+ : undefined,+};++// RequestQueue auto-reset when stuck with requests in progress+await Actor.main(async () => {+ const requestQueue = await RequestQueue.open();+ await requestQueue.addRequest({ url: 'https://example.com/?q=1' });+ await requestQueue.addRequest({ url: 'https://example.com/?q=2' });+ const r3 = await requestQueue.addRequest({ url: 'https://example.com/?q=3' });+ // trigger 0 concurrency by marking one of the requests as already in progress+ requestQueue.inProgress.add(r3.requestId);++ const crawler = new CheerioCrawler({+ requestQueue,+ async requestHandler({ request }) {+ log.info(request.id);+ },+ });++ await crawler.run();+}, mainOptions);diff --git a/test/e2e/request-queue-v2-zero-concurrency/actor/package.json b/test/e2e/request-queue-v2-zero-concurrency/actor/package.jsonnew file mode 100644index 000000000..1f24f5ba2--- /dev/null+++ b/test/e2e/request-queue-v2-zero-concurrency/actor/package.json@@ -0,0 +1,28 @@+{+ "name": "test-request-queue-zero-concurrency",+ "version": "0.0.1",+ "description": "Request Queue Test - Zero Concurrency",+ "dependencies": {+ "apify": "next",+ "@apify/storage-local": "^2.1.3",+ "@crawlee/basic": "file:./packages/basic-crawler",+ "@crawlee/browser-pool": "file:./packages/browser-pool",+ "@crawlee/http": "file:./packages/http-crawler",+ "@crawlee/cheerio": "file:./packages/cheerio-crawler",+ "@crawlee/core": "file:./packages/core",+ "@crawlee/memory-storage": "file:./packages/memory-storage",+ "@crawlee/types": "file:./packages/types",+ "@crawlee/utils": "file:./packages/utils"+ },+ "overrides": {+ "apify": {+ "@crawlee/core": "file:./packages/core",+ "@crawlee/utils": "file:./packages/utils"+ }+ },+ "scripts": {+ "start": "node main.js"+ },+ "type": "module",+ "license": "ISC"+}diff --git a/test/e2e/request-queue-v2-zero-concurrency/test.mjs b/test/e2e/request-queue-v2-zero-concurrency/test.mjsnew file mode 100644index 000000000..42656d0ad--- /dev/null+++ b/test/e2e/request-queue-v2-zero-concurrency/test.mjs@@ -0,0 +1,12 @@+import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs';++const testActorDirname = getActorTestDir(import.meta.url);+await initialize(testActorDirname);++const { stats } = await runActor(testActorDirname);++await expect(stats.requestsFinished === 3, 'All requests finished');+await expect(+ stats.crawlerRuntimeMillis > 30 * 1e3 && stats.crawlerRuntimeMillis < 35 * 1e3,+ 'RequestQueue triggers auto-reset after being stuck with requests in progress',+);