Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: re-apply master-side fixes over the enqueueLinks split (#4010)
Martin Adámek839a9b2d09d4
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 8198f8973..1151b0f37 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -1731,8 +1731,8 @@ export class BasicCrawler<try {await this.#autoscaledPool!.run();} finally {- await this.teardown();await this.stats.stopCapturing();+ await this.teardown();process.off('SIGINT', sigintHandler);eventManager.off(EventType.MIGRATING, boundPauseOnMigration);@@ -2154,7 +2154,11 @@ export class BasicCrawler<// Only log the limit message when an explicit `limit` was passed (not the internal// `maxRequestsPerCrawl`-derived one), and only once per call.if (options.limit !== undefined && allSkipped.some((s) => s.reason === 'limit')) {- this.log.info(`Skipping requests in this call due to the enqueueLinks limit of ${options.limit}.`);+ this.log.info(+ requestLimit === options.limit+ ? `Skipping requests in this call due to the enqueueLinks limit of ${options.limit}.`+ : `Skipping requests in this call due to the remaining maxRequestsPerCrawl budget of ${requestLimit}, which is lower than the enqueueLinks limit of ${options.limit}.`,+ );}await Promise.all(@@ -2871,14 +2875,19 @@ export class BasicCrawler<* To stop the crawler gracefully (waiting for all running requests to finish), use {@apilink BasicCrawler.stop|`crawler.stop()`} instead.*/async teardown(): Promise<void> {- serviceLocator.getEventManager().emit(EventType.PERSIST_STATE, { isMigrating: false });+ // When this crawler initialized the event manager, its close() call emits+ // the final persistence event after the crawler-specific state has been+ // saved. External event managers still need an explicit event here.+ if (!this.#closeEvents) {+ serviceLocator.getEventManager().emit(EventType.PERSIST_STATE, { isMigrating: false });+ }++ await this.#sessionPoolDep.ifOwned(async (pool) => pool.teardown({ persistState: this.#closeEvents ?? false }));if (this.#closeEvents) {await serviceLocator.getEventManager().close();}- await this.#sessionPoolDep.ifOwned((pool) => pool.teardown());-await this.#autoscaledPool?.abort();await this.#concurrencySystemDep?.ifOwned((system) => system.stop());}diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex ac672be5e..abf816d67 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -555,13 +555,13 @@ describe('BasicCrawler', () => {it.each([null, undefined, false] as const)('should skip requests when transformRequestFunction returns %s',async (returnValue) => {+ const urls = atDepth(['https://example.com/1/', 'https://example.com/2/'], 3);const transformRequestFunction = vi.fn(() => returnValue);const optionsWithTransform = { ...options, transformRequestFunction };- await crawler.exposedEnqueueLinksWithCrawlDepth(optionsWithTransform, request, requestQueue);+ await crawler.addRequests(urls, optionsWithTransform);- const requests = addRequestsBatchedMock.mock.calls[0][0];- expect(requests).toHaveLength(0);+ expect(drainedRequests).toHaveLength(0);const skippedRequests = onSkippedRequestMock.mock.calls.map((call) => call[0]);expect(skippedRequests).toHaveLength(2);@@ -3069,7 +3069,7 @@ describe('BasicCrawler', () => {crawler.stats.state.requestsFinished = 2;// e.g. `enqueueLinks({ urls, limit: config.limit })` where `config.limit` is not set- await context.enqueueLinks({ urls: requestsToAdd, limit: undefined, label: 'child' });+ await context.addRequests(requestsToAdd, { limit: undefined, label: 'child' });},});@@ -3080,7 +3080,7 @@ describe('BasicCrawler', () => {const skippedUrls = onSkippedRequest.mock.calls.map((call) => call[0])- .filter(({ reason }) => reason === 'enqueueLimit')+ .filter(({ reason }) => reason === 'limit').map(({ url }) => url).sort();@@ -3107,7 +3107,7 @@ describe('BasicCrawler', () => {crawler.stats.state.requestsFinished = 2;- await context.enqueueLinks({ urls: requestsToAdd, limit: 4, label: 'child' });+ await context.addRequests(requestsToAdd, { limit: 4, label: 'child' });},});@@ -3139,8 +3139,7 @@ describe('BasicCrawler', () => {return;}- await context.enqueueLinks({- urls: requestsToAdd,+ await context.addRequests(requestsToAdd, {limit: 1,label: 'child',onSkippedRequest: userOnSkippedRequest,@@ -3151,8 +3150,8 @@ describe('BasicCrawler', () => {await crawler.run(['http://example.com']);const skipped = [- { url: 'http://example.com/2', reason: 'enqueueLimit' },- { url: 'http://example.com/3', reason: 'enqueueLimit' },+ { url: 'http://example.com/2', reason: 'limit' },+ { url: 'http://example.com/3', reason: 'limit' },];for (const mock of [crawlerOnSkippedRequest, userOnSkippedRequest]) {@@ -3161,104 +3160,6 @@ describe('BasicCrawler', () => {);}});-- test('enqueueLinks should keep the crawler robots.txt file when passed an explicitly undefined robotsTxtFile', async () => {- const requestQueue = await RequestQueue.open();-- const crawler = new (class MockedRobotsTxtCrawler extends BasicCrawler {- override async getRobotsTxtFileForUrl(_: string) {- return RobotsTxtFile.from(- 'http://example.com/robots.txt',- `User-agent: *- Disallow: /no- `,- );- }- })({- requestQueue,- maxConcurrency: 1,- respectRobotsTxtFile: true,- requestHandler: async (context) => {- if (context.request.label) {- return;- }-- await context.enqueueLinks({- urls: ['http://example.com/yes', 'http://example.com/no'],- robotsTxtFile: undefined,- label: 'child',- });- },- });-- await crawler.run(['http://example.com/start']);-- // The disallowed URL should never make it into the queue- expect(await requestQueue.getTotalCount()).toBe(2);- });-- test('enqueueLinks should keep the crawler user-agent when passed an explicitly undefined respectRobotsTxtFile', async () => {- const requestQueue = await RequestQueue.open();- const isAllowedSpy = vitest.fn((_url: string, _userAgent?: string) => true);-- const crawler = new (class MockedRobotsTxtCrawler extends BasicCrawler {- override async getRobotsTxtFileForUrl(_: string) {- return { isAllowed: isAllowedSpy, getCrawlDelay: () => undefined } as unknown as RobotsTxtFile;- }- })({- requestQueue,- maxConcurrency: 1,- respectRobotsTxtFile: { userAgent: 'MyCrawler' },- requestHandler: async (context) => {- if (context.request.label) {- return;- }-- await context.enqueueLinks({- urls: ['http://example.com/child'],- respectRobotsTxtFile: undefined,- label: 'child',- });- },- });-- await crawler.run(['http://example.com/start']);-- expect(isAllowedSpy).toHaveBeenCalledWith('http://example.com/child', 'MyCrawler');- // the crawler user-agent must not fall back to the `*` default- expect(isAllowedSpy.mock.calls.map(([, userAgent]) => userAgent)).not.toContain('*');- });-- test('enqueueLinks should use the request queue from the options, and the crawler one when it is undefined', async () => {- const requestQueue = await RequestQueue.open();- const customQueue = await RequestQueue.open('custom-queue');-- const crawler = new BasicCrawler({- requestQueue,- requestHandler: async (context) => {- if (context.request.label) {- return;- }-- await context.enqueueLinks({- urls: ['http://example.com/custom'],- requestManager: customQueue,- label: 'child',- });-- await context.enqueueLinks({- urls: ['http://example.com/default'],- requestManager: undefined,- label: 'child',- });- },- });-- await crawler.run(['http://example.com/start']);-- expect(await customQueue.getTotalCount()).toBe(1);- expect(await requestQueue.getTotalCount()).toBe(2);- });});describe('transactional storage', () => {