Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Add ID to CrawlingContext and keep a Map of them (#889)
Ondra Urbana0deb0313f2a
diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.jsindex 7cc9ed1d5..9d5ba2b92 100644--- a/src/crawlers/basic_crawler.js+++ b/src/crawlers/basic_crawler.js@@ -1,4 +1,5 @@import { ACTOR_EVENT_NAMES } from 'apify-shared/consts';+import { cryptoRandomObjectId } from 'apify-shared/utilities';import ow, { ArgumentError } from 'ow';import * as _ from 'underscore';import AutoscaledPool from '../autoscaling/autoscaled_pool'; // eslint-disable-line import/no-duplicates@@ -18,6 +19,7 @@ import { RequestQueue } from '../storages/request_queue';import { QueueOperationInfo } from '../storages/request_queue';import { Session } from '../session_pool/session';import { SessionPoolOptions } from '../session_pool/session_pool';+/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order *//**@@ -243,6 +245,7 @@ class BasicCrawler {log,};this.useSessionPool = useSessionPool;+ this.crawlingContexts = new Map();let shouldLogMaxPagesExceeded = true;const isMaxPagesExceeded = () => maxRequestsPerCrawl && maxRequestsPerCrawl <= this.handledRequestsCount;@@ -446,7 +449,13 @@ class BasicCrawler {this.stats.startJob(statisticsId);// Shared crawling context- const crawlingContext = { request, autoscaledPool: this.autoscaledPool, session };+ const crawlingContext = {+ id: cryptoRandomObjectId(10),+ request,+ autoscaledPool: this.autoscaledPool,+ session,+ };+ this.crawlingContexts.set(crawlingContext.id, crawlingContext);try {await addTimeoutToPromise(@@ -471,6 +480,8 @@ class BasicCrawler {+ 'will make sure that the run continues where it left off, if programmed to handle restarts correctly.');throw secondaryError;}+ } finally {+ this.crawlingContexts.delete(crawlingContext.id);}}diff --git a/test/crawlers/basic_crawler.test.js b/test/crawlers/basic_crawler.test.jsindex f3e439d69..b09846d2c 100644--- a/test/crawlers/basic_crawler.test.js+++ b/test/crawlers/basic_crawler.test.js@@ -701,4 +701,47 @@ describe('BasicCrawler', () => {expect(crawler.sessionPool).toBeUndefined();});});++ describe('CrawlingContext', () => {+ test('should be kept and later deleted', async () => {+ const urls = [+ 'https://example.com/0',+ 'https://example.com/1',+ 'https://example.com/2',+ 'https://example.com/3',+ ];+ const requestList = await Apify.openRequestList(null, urls);+ let counter = 0;+ let finish;+ const allFinishedPromise = new Promise((resolve) => {+ finish = resolve;+ });+ const mainContexts = [];+ const otherContexts = [];+ const crawler = new Apify.BasicCrawler({+ requestList,+ minConcurrency: 4,+ async handleRequestFunction(crawlingContext) {+ mainContexts[counter] = crawler.crawlingContexts.get(crawlingContext.id);+ otherContexts[counter] = Array.from(crawler.crawlingContexts).map(([, v]) => v);+ counter++;+ if (counter === 4) finish();+ await allFinishedPromise;+ },+ });+ await crawler.run();++ expect(counter).toBe(4);+ expect(mainContexts).toHaveLength(4);+ expect(otherContexts).toHaveLength(4);+ expect(crawler.crawlingContexts.size).toBe(0);+ mainContexts.forEach((ctx, idx) => {+ expect(typeof ctx.id).toBe('string');+ expect(otherContexts[idx]).toContain(ctx);+ });+ otherContexts.forEach((list, idx) => {+ expect(list).toHaveLength(idx + 1);+ });+ });+ });});