Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: add `keepAlive` to crawler options (#1452)
Martin Adámek084b6b2e782c
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 94d87212e..c522b0523 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -214,6 +214,13 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw*/maxRequestsPerMinute?: number;+ /**+ * Allows to keep the crawler alive even if the {@apilink RequestQueue} gets empty.+ * By default, the `crawler.run()` will resolve once the queue is empty. With `keepAlive: true` it will keep running,+ * waiting for more requests to come. Use `crawler.teardown()` to exit the crawler.+ */+ keepAlive?: boolean;+/*** Basic crawler will initialize the {@apilink SessionPool} with the corresponding {@apilink SessionPoolOptions|`sessionPoolOptions`}.* The session instance will be than available in the {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`}.@@ -376,6 +383,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextminConcurrency: ow.optional.number,maxConcurrency: ow.optional.number,maxRequestsPerMinute: ow.optional.number.integerOrInfinite.positive.greaterThanOrEqual(1),+ keepAlive: ow.optional.boolean,// internallog: ow.optional.object,@@ -393,6 +401,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextmaxRequestRetries = 3,maxRequestsPerCrawl,autoscaledPoolOptions = {},+ keepAlive,sessionPoolOptions = {},useSessionPool = true,@@ -495,7 +504,12 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextlet shouldLogMaxPagesExceeded = true;const isMaxPagesExceeded = () => maxRequestsPerCrawl && maxRequestsPerCrawl <= this.handledRequestsCount;- const { isFinishedFunction } = autoscaledPoolOptions;+ let { isFinishedFunction } = autoscaledPoolOptions;++ // override even if `isFinishedFunction` provided by user - `keepAlive` has higher priority+ if (keepAlive) {+ isFinishedFunction = async () => false;+ }const basicCrawlerAutoscaledPoolConfiguration: Partial<AutoscaledPoolOptions> = {minConcurrency,@@ -1091,6 +1105,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextif (this._closeEvents) {await this.events.close();}++ await this.autoscaledPool?.abort();}protected _handlePropertyNameChange<New, Old>({diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 7a59026ad..3f6d90e45 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -591,6 +591,52 @@ describe('BasicCrawler', () => {jest.restoreAllMocks();});+ test('keepAlive', async () => {+ const requestQueue = new RequestQueue({ id: 'xxx', client: Configuration.getStorageClient() });+ const processed: Request[] = [];+ const queue: Request[] = [];++ const basicCrawler = new BasicCrawler({+ requestQueue,+ keepAlive: true,+ requestHandler: async ({ request }) => {+ await sleep(10);+ processed.push(request);+ },+ });++ // Speed up the test+ // @ts-expect-error Accessing private prop+ basicCrawler.autoscaledPoolOptions.maybeRunIntervalSecs = 0.05;++ const request0 = new Request({ url: 'http://example.com/0' });+ const request1 = new Request({ url: 'http://example.com/1' });++ jest.spyOn(requestQueue, 'handledCount').mockReturnValue(Promise.resolve() as any);+ const markRequestHandled = jest.spyOn(requestQueue, 'markRequestHandled')+ .mockReturnValue(Promise.resolve() as any);++ const isFinishedOrig = jest.spyOn(requestQueue, 'isFinished').mockImplementation();++ requestQueue.fetchNextRequest = () => Promise.resolve(queue.pop());+ requestQueue.isEmpty = () => Promise.resolve(!queue.length);++ setTimeout(() => queue.push(request0), 10);+ setTimeout(() => queue.push(request1), 100);+ setTimeout(() => { void basicCrawler.teardown(); }, 150);++ await basicCrawler.run();++ expect(markRequestHandled).toBeCalledWith(request0);+ expect(markRequestHandled).toBeCalledWith(request1);+ expect(isFinishedOrig).not.toBeCalled();++ // TODO: see why the request1 was passed as a second parameter to includes+ expect(processed.includes(request0)).toBe(true);++ jest.restoreAllMocks();+ });+test('should support maxRequestsPerCrawl parameter', async () => {const sources = [{ url: 'http://example.com/1' },