Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat(basic): pass onCleanup to extendContext callbacks (#4293)

Junghwana85394e90740
diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.md
index 23a105c3b..c31e96937 100644
--- a/docs/public-api/crawlee-basic.api.md
+++ b/docs/public-api/crawlee-basic.api.md
@@ -117,7 +117,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
contextPipelineBuilder?: () => ContextPipeline<CrawlingContext, Context>;
errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
eventManager?: EventManager;
- extendContext?: (context: CrawlingContext) => Awaitable<ContextExtension>;
+ extendContext?: ContextMiddleware<CrawlingContext, ContextExtension>;
failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
httpClient?: BaseHttpClient;
id?: string;
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index ce98e5d0d..bdb6a9a91 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -72,7 +72,7 @@ import type { ConcurrencySystemOptions, IConcurrencySystem } from './autoscaling
import { ConcurrencySystem } from './autoscaling/concurrency-system.js';
import type { FinalStatistics } from './autoscaling/system-status.js';
import { mergeCookies } from './cookie-utils.js';
-import type { CleanupRegistrar } from './crawlers/context-pipeline.js';
+import type { CleanupRegistrar, ContextMiddleware } from './crawlers/context-pipeline.js';
import { ContextPipeline } from './crawlers/context-pipeline.js';
import type { CrawlingContext, TypedRequestsLike } from './crawlers/crawler-commons.js';
import type { CrawlingRequestData, SkippedRequestReason } from './crawling-request.js';
@@ -261,6 +261,10 @@ export interface BasicCrawlerOptions<
* **not** include navigation-dependent members (e.g. `page`, `response`, `$`, `body`). If you need
* those, use a `postNavigationHook` or the `requestHandler` instead.
*
+ * The second argument is the same {@apilink CleanupRegistrar|`onCleanup`} registrar used by
+ * {@apilink ContextPipeline.compose}. Use it to register teardown callbacks that run after the
+ * request handler or a pipeline failure, in reverse registration order.
+ *
* **Example usage:**
*
* ```javascript
@@ -279,7 +283,7 @@ export interface BasicCrawlerOptions<
* });
* ```
*/
- extendContext?: (context: CrawlingContext) => Awaitable<ContextExtension>;
+ extendContext?: ContextMiddleware<CrawlingContext, ContextExtension>;
/**
* *Intended for BasicCrawler subclasses*. Prepares a context pipeline that transforms the initial crawling context into the shape given by the `Context` type parameter.
@@ -905,7 +909,7 @@ export class BasicCrawler<
readonly #identity: CrawlerIdentity;
readonly #contextPipelineOptions: {
contextPipelineBuilder?: () => ContextPipeline<CrawlingContext, Context>;
- extendContext?: (context: CrawlingContext) => Awaitable<ContextExtension>;
+ extendContext?: ContextMiddleware<CrawlingContext, ContextExtension>;
};
/**
@@ -1551,7 +1555,7 @@ export class BasicCrawler<
let contextPipeline: ContextPipeline<CrawlingContext, Context>;
if (extendContext !== undefined) {
contextPipeline = ContextPipeline.create<CrawlingContext>()
- .compose(async (context) => await extendContext(context))
+ .compose(extendContext)
.chain(
subclassPipeline as unknown as ContextPipeline<
CrawlingContext & ContextExtension,
diff --git a/test/core/crawlers/basic-crawler.test.ts b/test/core/crawlers/basic-crawler.test.ts
index 650ab5c1a..6fa722bd1 100644
--- a/test/core/crawlers/basic-crawler.test.ts
+++ b/test/core/crawlers/basic-crawler.test.ts
@@ -2968,20 +2968,41 @@ describe('BasicCrawler', () => {
});
});
- test('extendContext', async () => {
- const url = 'https://example.com';
- const requestHandlerImplementation = vi.fn();
+ describe('extendContext', () => {
+ test('extends the crawling context', async () => {
+ const url = 'https://example.com';
+ const requestHandlerImplementation = vi.fn();
- const crawler = new BasicCrawler({
- extendContext: () => ({ hello: 'world' }),
- requestHandler: async ({ hello }) => {
- requestHandlerImplementation({ hello });
- },
+ const crawler = new BasicCrawler({
+ extendContext: () => ({ hello: 'world' }),
+ requestHandler: async ({ hello }) => {
+ requestHandlerImplementation({ hello });
+ },
+ });
+
+ await crawler.run([url]);
+ expect(requestHandlerImplementation).toHaveBeenCalledOnce();
+ expect(requestHandlerImplementation.mock.calls[0][0]).toMatchObject({ hello: 'world' });
});
- await crawler.run([url]);
- expect(requestHandlerImplementation).toHaveBeenCalledOnce();
- expect(requestHandlerImplementation.mock.calls[0][0]).toMatchObject({ hello: 'world' });
+ test('can register cleanups that run after the request handler', async () => {
+ const order: string[] = [];
+ const crawler = new BasicCrawler({
+ extendContext: (_, onCleanup) => {
+ onCleanup(() => {
+ order.push('cleanup');
+ });
+ return {};
+ },
+ requestHandler: async () => {
+ order.push('handler');
+ },
+ });
+
+ await crawler.run(['https://example.com']);
+
+ expect(order).toEqual(['handler', 'cleanup']);
+ });
});
test('registerDeferredCleanup runs after the error handler, even when the context pipeline itself fails', async () => {