Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore(test): finish adapting master-carried tests to v4 APIs

Martin Adámek375b08c9d546
diff --git a/packages/utils/test/sitemap.test.ts b/packages/utils/test/sitemap.test.ts
index 33273ea06..fbdd006e5 100644
--- a/packages/utils/test/sitemap.test.ts
+++ b/packages/utils/test/sitemap.test.ts
@@ -606,12 +606,14 @@ describe('Sitemap', () => {
});
it('logs one aggregated warning per sitemap for dropped URL entries', async () => {
- const spy = vi.spyOn(log, 'warning');
+ const warning = vi.fn();
- await Sitemap.load('http://not-exists.com/cross_host_urls.xml');
+ await Sitemap.load('http://not-exists.com/cross_host_urls.xml', undefined, {
+ logger: { warning, debug: vi.fn(), info: vi.fn(), error: vi.fn() } as any,
+ });
- expect(spy).toHaveBeenCalledTimes(1);
- expect(spy).toHaveBeenCalledWith(
+ expect(warning).toHaveBeenCalledTimes(1);
+ expect(warning).toHaveBeenCalledWith(
expect.stringContaining('Skipped 1 URL(s) from sitemap http://not-exists.com/cross_host_urls.xml'),
);
});
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index 02bee3361..900052b55 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -568,8 +568,8 @@ describe('BasicCrawler', () => {
const skippedRequests = onSkippedRequestMock.mock.calls.map((call) => call[0]);
expect(skippedRequests).toHaveLength(2);
- expect(skippedRequests[0]).toStrictEqual({ url: 'https://example.com/1/', reason: 'filters' });
- expect(skippedRequests[1]).toStrictEqual({ url: 'https://example.com/2/', reason: 'filters' });
+ expect(skippedRequests[0]).toStrictEqual({ url: 'https://example.com/1/', reason: 'transform' });
+ expect(skippedRequests[1]).toStrictEqual({ url: 'https://example.com/2/', reason: 'transform' });
},
);
@@ -1972,39 +1972,36 @@ describe('BasicCrawler', () => {
describe('Uses SessionPool', () => {
it('persists statistics and the session pool once when finishing', async () => {
- const persistStatistics = vitest.spyOn(Statistics.prototype, 'persistState');
- const persistSessionPool = vitest.spyOn(SessionPool.prototype, 'persistState');
+ const setValue = vitest.spyOn(KeyValueStore.prototype, 'setValue');
const crawler = new BasicCrawler({
requestHandler: async () => {
- persistStatistics.mockClear();
- persistSessionPool.mockClear();
+ setValue.mockClear();
},
});
await crawler.run(['https://example.com']);
- expect(persistStatistics).toHaveBeenCalledTimes(1);
- expect(persistSessionPool).toHaveBeenCalledTimes(1);
+ const persistedKeys = setValue.mock.calls.map(([key]) => key);
+ expect(persistedKeys.filter((key) => key.startsWith('CRAWLEE_CRAWLER_STATISTICS'))).toHaveLength(1);
+ expect(persistedKeys.filter((key) => key.startsWith('CRAWLEE_SESSION_POOL_STATE'))).toHaveLength(1);
});
it('persists the session pool once with a shared event manager', async () => {
- const persistStatistics = vitest.spyOn(Statistics.prototype, 'persistState');
- const persistSessionPool = vitest.spyOn(SessionPool.prototype, 'persistState');
await serviceLocator.getEventManager().init();
+ const setValue = vitest.spyOn(KeyValueStore.prototype, 'setValue');
const crawler = new BasicCrawler({
requestHandler: async () => {
- persistStatistics.mockClear();
- persistSessionPool.mockClear();
+ setValue.mockClear();
},
});
await crawler.run(['https://example.com']);
expect(serviceLocator.getEventManager().listenerCount(EventType.PERSIST_STATE)).toBe(0);
- expect(persistSessionPool).toHaveBeenCalledTimes(1);
await serviceLocator.getEventManager().close();
- expect(persistStatistics).toHaveBeenCalledTimes(1);
- expect(persistSessionPool).toHaveBeenCalledTimes(1);
+ const persistedKeys = setValue.mock.calls.map(([key]) => key);
+ expect(persistedKeys.filter((key) => key.startsWith('CRAWLEE_CRAWLER_STATISTICS'))).toHaveLength(1);
+ expect(persistedKeys.filter((key) => key.startsWith('CRAWLEE_SESSION_POOL_STATE'))).toHaveLength(1);
});
it('should use SessionPool', async () => {
@@ -2971,7 +2968,7 @@ describe('BasicCrawler', () => {
await crawler.run(['http://example.com']);
// 2 requests already finished and 1 is in progress, so only 2 more fit into the limit
- expect(requestQueue.getTotalCount()).toBe(3);
+ expect(await requestQueue.getTotalCount()).toBe(3);
const skippedUrls = onSkippedRequest.mock.calls
.map((call) => call[0])
@@ -3011,7 +3008,7 @@ describe('BasicCrawler', () => {
await crawler.run(['http://example.com']);
// The user limit of 4 is higher than what's left of maxRequestsPerCrawl, so only 2 are enqueued
- expect(requestQueue.getTotalCount()).toBe(3);
+ expect(await requestQueue.getTotalCount()).toBe(3);
// ...and the log message must not blame the user limit of 4 for it
expect(infoSpy).toHaveBeenCalledWith(
@@ -3089,7 +3086,7 @@ describe('BasicCrawler', () => {
await crawler.run(['http://example.com/start']);
// The disallowed URL should never make it into the queue
- expect(requestQueue.getTotalCount()).toBe(2);
+ expect(await requestQueue.getTotalCount()).toBe(2);
});
test('enqueueLinks should keep the crawler user-agent when passed an explicitly undefined respectRobotsTxtFile', async () => {
@@ -3098,7 +3095,7 @@ describe('BasicCrawler', () => {
const crawler = new (class MockedRobotsTxtCrawler extends BasicCrawler {
override async getRobotsTxtFileForUrl(_: string) {
- return { isAllowed: isAllowedSpy } as unknown as RobotsTxtFile;
+ return { isAllowed: isAllowedSpy, getCrawlDelay: () => undefined } as unknown as RobotsTxtFile;
}
})({
requestQueue,
@@ -3151,8 +3148,8 @@ describe('BasicCrawler', () => {
await crawler.run(['http://example.com/start']);
- expect(customQueue.getTotalCount()).toBe(1);
- expect(requestQueue.getTotalCount()).toBe(2);
+ expect(await customQueue.getTotalCount()).toBe(1);
+ expect(await requestQueue.getTotalCount()).toBe(2);
});
});
diff --git a/test/core/sitemap_request_loader.test.ts b/test/core/sitemap_request_loader.test.ts
index ca8b937c1..759933f85 100644
--- a/test/core/sitemap_request_loader.test.ts
+++ b/test/core/sitemap_request_loader.test.ts
@@ -576,7 +576,7 @@ describe('SitemapRequestLoader', () => {
const urls: string[] = [];
for await (const request of list) {
urls.push(request.url);
- await list.markRequestHandled(request);
+ await list.markRequestAsHandled(request);
}
return urls;
}
@@ -649,7 +649,7 @@ describe('SitemapRequestLoader', () => {
const urls = new Set<string>();
for await (const request of list) {
- await list.markRequestHandled(request);
+ await list.markRequestAsHandled(request);
urls.add(request.url);
}