Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore(test): adapt master-carried tests to v4 APIs after rebase

Martin Adámek75412fce1402
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index f0f540864..02bee3361 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -35,7 +35,7 @@ import {
ThrottlingRequestManager,
} from '@crawlee/basic';
import type { CalculatedStatistics, IConcurrencySystem, IStatistics } from '@crawlee/core';
-import { ConcurrencySystem, Dataset, MemoryStorageBackend, RequestState, SessionPool, Statistics } from '@crawlee/core';
+import { ConcurrencySystem, MemoryStorageBackend, RequestState } from '@crawlee/core';
import { BaseHttpClient } from '@crawlee/http-client';
import type { Dictionary, ISession, ProxyInfo } from '@crawlee/types';
import { RobotsTxtFile, sleep } from '@crawlee/utils';
@@ -3137,13 +3137,13 @@ describe('BasicCrawler', () => {
await context.enqueueLinks({
urls: ['http://example.com/custom'],
- requestQueue: customQueue,
+ requestManager: customQueue,
label: 'child',
});
await context.enqueueLinks({
urls: ['http://example.com/default'],
- requestQueue: undefined,
+ requestManager: undefined,
label: 'child',
});
},
diff --git a/test/core/crawlers/http_crawler.test.ts b/test/core/crawlers/http_crawler.test.ts
index 0f6f815cd..1ba725d90 100644
--- a/test/core/crawlers/http_crawler.test.ts
+++ b/test/core/crawlers/http_crawler.test.ts
@@ -342,11 +342,11 @@ test('handles cookies from redirects when the session already has cookies', asyn
const results: string[] = [];
const crawler = new HttpCrawler({
- sessionPoolOptions: {
+ sessionPool: new SessionPool({
maxPoolSize: 1,
// isolated so that cookies stored by the other tests don't leak in
- persistStateKey: 'SDK_SESSION_POOL_STATE_setCookie',
- },
+ persistStateKey: 'CRAWLEE_SESSION_POOL_STATE_setCookie',
+ }),
maxConcurrency: 1,
requestHandler: async ({ body }) => {
results.push(body.toString());
diff --git a/test/core/enqueue_links/enqueue_links.test.ts b/test/core/enqueue_links/enqueue_links.test.ts
index 11db1c9c0..c7f3a1fa2 100644
--- a/test/core/enqueue_links/enqueue_links.test.ts
+++ b/test/core/enqueue_links/enqueue_links.test.ts
@@ -745,11 +745,11 @@ describe('enqueueLinks()', () => {
});
test('ignores an explicitly undefined baseUrl and keeps the resolved one', async () => {
- const { enqueued, requestQueue } = createRequestQueueMock();
+ const { enqueued, requestQueue } = await createRequestQueueMock();
await cheerioCrawlerEnqueueLinks({
options: { strategy: EnqueueStrategy.SameDomain, baseUrl: undefined },
$,
- requestQueue,
+ requestManager: requestQueue,
originalRequestUrl: 'https://example.com',
});
@@ -769,7 +769,7 @@ describe('enqueueLinks()', () => {
options: { strategy: EnqueueStrategy.SameDomain, baseUrl: undefined },
$,
originalRequestUrl: 'https://example.com',
- enqueueLinks: async (options) => {
+ enqueueLinks: async (options?: { baseUrl?: string }) => {
forwardedBaseUrl = options?.baseUrl;
return { processedRequests: [], unprocessedRequests: [] };
},
@@ -779,11 +779,11 @@ describe('enqueueLinks()', () => {
});
test('keeps filtering by the original domain with the strategy of same-domain after an off-domain redirect', async () => {
- const { enqueued, requestQueue } = createRequestQueueMock();
+ const { enqueued, requestQueue } = await createRequestQueueMock();
await cheerioCrawlerEnqueueLinks({
options: { strategy: EnqueueStrategy.SameDomain },
$,
- requestQueue,
+ requestManager: requestQueue,
originalRequestUrl: 'https://example.com',
finalRequestUrl: 'https://another.com/',
});
diff --git a/test/core/playwright_utils.test.ts b/test/core/playwright_utils.test.ts
index 701ecafb8..2f493003e 100644
--- a/test/core/playwright_utils.test.ts
+++ b/test/core/playwright_utils.test.ts
@@ -485,21 +485,21 @@ describe('playwrightUtils', () => {
test('detects the current challenge markup', async () => {
await page.setContent(`<html><body>${currentChallengeBody}</body></html>`);
await expect(
- handleCloudflareChallenge(page, 'https://example.com', undefined, fastOptions()),
+ handleCloudflareChallenge(page, 'https://example.com', fastOptions()),
).rejects.toThrow(/Blocked by Cloudflare/);
});
test('detects the old challenge markup', async () => {
await page.setContent(`<html><body>${oldChallengeBody}</body></html>`);
await expect(
- handleCloudflareChallenge(page, 'https://example.com', undefined, fastOptions()),
+ handleCloudflareChallenge(page, 'https://example.com', fastOptions()),
).rejects.toThrow(/Blocked by Cloudflare/);
});
test('resolves without detection on a regular page', async () => {
await page.setContent('<html><body><h1>Welcome</h1></body></html>');
await expect(
- handleCloudflareChallenge(page, 'https://example.com', undefined, fastOptions()),
+ handleCloudflareChallenge(page, 'https://example.com', fastOptions()),
).resolves.toBeUndefined();
});
});
diff --git a/test/core/sitemap_request_loader.test.ts b/test/core/sitemap_request_loader.test.ts
index 236488d30..ca8b937c1 100644
--- a/test/core/sitemap_request_loader.test.ts
+++ b/test/core/sitemap_request_loader.test.ts
@@ -572,7 +572,7 @@ describe('SitemapRequestLoader', () => {
expect(restoredRequest!.userData).toMatchObject(userDataPayload);
});
- async function collectUrls(list: SitemapRequestList): Promise<string[]> {
+ async function collectUrls(list: SitemapRequestLoader): Promise<string[]> {
const urls: string[] = [];
for await (const request of list) {
urls.push(request.url);
@@ -582,13 +582,13 @@ describe('SitemapRequestLoader', () => {
}
test('default `same-hostname` strategy drops cross-host URL entries', async () => {
- const list = await SitemapRequestList.open({ sitemapUrls: [`${url}/cross-host-content.xml`] });
+ const list = await SitemapRequestLoader.open({ sitemapUrls: [`${url}/cross-host-content.xml`] });
expect(await collectUrls(list)).toEqual([`${url}/same-host-page`]);
});
test('`enqueueStrategy: all` keeps cross-host URL entries', async () => {
const cross = url.replace('localhost', '127.0.0.1');
- const list = await SitemapRequestList.open({
+ const list = await SitemapRequestLoader.open({
sitemapUrls: [`${url}/cross-host-content.xml`],
enqueueStrategy: 'all',
});
@@ -598,14 +598,14 @@ describe('SitemapRequestLoader', () => {
});
test('default `same-hostname` strategy drops cross-host nested sitemaps before fetching them', async () => {
- const list = await SitemapRequestList.open({ sitemapUrls: [`${url}/cross-host-index.xml`] });
+ const list = await SitemapRequestLoader.open({ sitemapUrls: [`${url}/cross-host-index.xml`] });
// The cross-host nested sitemap is never fetched, so its `child-page` URL is absent.
expect(await collectUrls(list)).toEqual([]);
});
test('`enqueueStrategy: all` follows cross-host nested sitemaps', async () => {
const cross = url.replace('localhost', '127.0.0.1');
- const list = await SitemapRequestList.open({
+ const list = await SitemapRequestLoader.open({
sitemapUrls: [`${url}/cross-host-index.xml`],
enqueueStrategy: 'all',
});
@@ -613,7 +613,7 @@ describe('SitemapRequestLoader', () => {
});
test('non-http(s) schemes are dropped even with `enqueueStrategy: all`', async () => {
- const list = await SitemapRequestList.open({
+ const list = await SitemapRequestLoader.open({
sitemapUrls: [`${url}/mixed-scheme.xml`],
enqueueStrategy: 'all',
});
@@ -621,7 +621,7 @@ describe('SitemapRequestLoader', () => {
});
test('the selected enqueue strategy is stamped onto emitted requests', async () => {
- const list = await SitemapRequestList.open({ sitemapUrls: [`${url}/cross-host-content.xml`] });
+ const list = await SitemapRequestLoader.open({ sitemapUrls: [`${url}/cross-host-content.xml`] });
const request = await list.fetchNextRequest();
expect(request).not.toBe(null);
@@ -633,7 +633,7 @@ describe('SitemapRequestLoader', () => {
// `maxBufferSize: 1` makes the background loader block on backpressure right after
// pushing the first URL. Persisting the state at that moment swaps the underlying stream,
// which used to orphan the pending push and hang the loading indefinitely.
- const list = await SitemapRequestList.open({
+ const list = await SitemapRequestLoader.open({
sitemapUrls: [`${url}/sitemap.xml`],
persistStateKey: 'backpressure-persist',
maxBufferSize: 1,
diff --git a/test/core/storages/utils.test.ts b/test/core/storages/utils.test.ts
index eab19ddea..f453e3012 100644
--- a/test/core/storages/utils.test.ts
+++ b/test/core/storages/utils.test.ts
@@ -71,7 +71,7 @@ describe('purgeDefaultStorages', () => {
await Promise.all(
Array.from({ length: 3 }, async () => {
- await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend: client, config });
+ await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend: client, configuration: config });
if (purging) {
purgedWhileAlreadyPurging = true;
}