Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: fix tests that were mocking RQ improperly 2

Martin Adámek554e51a54ffc
diff --git a/test/core/enqueue_links/click_elements.test.ts b/test/core/enqueue_links/click_elements.test.ts
index a7a0a1e6b..b645d3625 100644
--- a/test/core/enqueue_links/click_elements.test.ts
+++ b/test/core/enqueue_links/click_elements.test.ts
@@ -23,6 +23,21 @@ function isPlaywrightBrowser(browser: PPBrowser | PWBrowser): browser is PWBrows
return (browser as PWBrowser).browserType !== undefined;
}
+const apifyClient = Configuration.getStorageClient();
+
+function createRequestQueueMock() {
+ const enqueued: Source[] = [];
+ const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
+
+ // @ts-expect-error Override method for testing
+ requestQueue.addRequests = async function (requests) {
+ enqueued.push(...requests);
+ return { processedRequests: requests, unprocessedRequests: [] as never[] };
+ };
+
+ return { enqueued, requestQueue };
+}
+
const testCases = [
{
caseName: 'Puppeteer',
@@ -72,12 +87,7 @@ testCases.forEach(({
});
test('should work', async () => {
- const addedRequests: Source[] = [];
- const requestQueue = Object.create(RequestQueue.prototype);
- requestQueue.addRequests = async (requests: Request[]) => {
- addedRequests.push(...requests);
- return { processedRequests: requests, unprocessedRequests: [] as never[] };
- };
+ const { enqueued, requestQueue } = createRequestQueueMock();
const html = `
<html>
<body>
@@ -98,9 +108,9 @@ testCases.forEach(({
waitForPageIdleSecs: 0.025,
maxWaitForPageIdleSecs: 0.250,
});
- expect(addedRequests).toHaveLength(1);
- expect(addedRequests[0].url).toMatch(`${serverAddress}/`);
- expect(addedRequests[0].uniqueKey).toBe('key');
+ expect(enqueued).toHaveLength(1);
+ expect(enqueued[0].url).toMatch(`${serverAddress}/`);
+ expect(enqueued[0].uniqueKey).toBe('key');
expect(page.url()).toBe('about:blank');
});
diff --git a/test/core/enqueue_links/enqueue_links.test.ts b/test/core/enqueue_links/enqueue_links.test.ts
index 1fedc851b..15f7b0a9d 100644
--- a/test/core/enqueue_links/enqueue_links.test.ts
+++ b/test/core/enqueue_links/enqueue_links.test.ts
@@ -310,13 +310,7 @@ describe('enqueueLinks()', () => {
});
test('throws with RegExp pseudoUrls', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
-
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (requests) => {
- enqueued.push(...requests);
- };
+ const { enqueued, requestQueue } = createRequestQueueMock();
const pseudoUrls = [
/https:\/\/example\.com\/(\w|-|\/)*/,
@@ -461,13 +455,7 @@ describe('enqueueLinks()', () => {
});
test('correctly resolves relative URLs with the strategy of all', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (request) => {
- enqueued.push(...request);
- };
-
+ const { enqueued, requestQueue } = createRequestQueueMock();
await browserCrawlerEnqueueLinks({
options: { baseUrl: 'http://www.absolute.com/removethis/', strategy: EnqueueStrategy.All },
page,
@@ -511,13 +499,7 @@ describe('enqueueLinks()', () => {
});
test('correctly works with transformRequestFunction', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
-
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (requests) => {
- enqueued.push(...requests);
- };
+ const { enqueued, requestQueue } = createRequestQueueMock();
const pseudoUrls = [
'https://example.com/[(\\w|-|/)*]',
@@ -646,13 +628,7 @@ describe('enqueueLinks()', () => {
});
test('works with string pseudoUrls', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (request) => {
- enqueued.push(...request);
- };
- const pseudoUrls = [
+ const { enqueued, requestQueue } = createRequestQueueMock();const pseudoUrls = [
'https://example.com/[(\\w|-|/)*]',
{ purl: '[http|https]://cool.com/', method: 'POST' as const, userData: { label: 'COOL' } },
];
@@ -690,13 +666,7 @@ describe('enqueueLinks()', () => {
});
test('throws with RegExp pseudoUrls', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (request) => {
- enqueued.push(...request);
- };
- const pseudoUrls = [
+ const { enqueued, requestQueue } = createRequestQueueMock();const pseudoUrls = [
/https:\/\/example\.com\/(\w|-|\/)*/,
/(http|https):\/\/cool\.com\//,
];
@@ -711,13 +681,7 @@ describe('enqueueLinks()', () => {
});
test('works with undefined pseudoUrls[]', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (request) => {
- enqueued.push(...request);
- };
-
+ const { enqueued, requestQueue } = createRequestQueueMock();
await cheerioCrawlerEnqueueLinks({
options: { selector: '.click', strategy: EnqueueStrategy.All },
$,
@@ -745,13 +709,7 @@ describe('enqueueLinks()', () => {
});
test('throws with null pseudoUrls[]', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (request) => {
- enqueued.push(...request);
- };
-
+ const { enqueued, requestQueue } = createRequestQueueMock();
await expect(cheerioCrawlerEnqueueLinks({
options: { selector: '.click', pseudoUrls: null },
$,
@@ -761,13 +719,7 @@ describe('enqueueLinks()', () => {
});
test('works with empty pseudoUrls[]', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (request) => {
- enqueued.push(...request);
- };
-
+ const { enqueued, requestQueue } = createRequestQueueMock();
await cheerioCrawlerEnqueueLinks({
options: { selector: '.click', pseudoUrls: [], strategy: EnqueueStrategy.All },
$,
@@ -795,13 +747,7 @@ describe('enqueueLinks()', () => {
});
test('throws with sparse pseudoUrls[]', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (request) => {
- enqueued.push(...request);
- };
- const pseudoUrls = [
+ const { enqueued, requestQueue } = createRequestQueueMock();const pseudoUrls = [
'https://example.com/[(\\w|-|/)*]',
null,
'[http|https]://cool.com/',
@@ -817,13 +763,7 @@ describe('enqueueLinks()', () => {
});
test('correctly resolves relative URLs with the strategy of all', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (request) => {
- enqueued.push(...request);
- };
-
+ const { enqueued, requestQueue } = createRequestQueueMock();
await cheerioCrawlerEnqueueLinks({
options: { baseUrl: 'http://www.absolute.com/removethis/', strategy: EnqueueStrategy.All },
$,
@@ -867,13 +807,7 @@ describe('enqueueLinks()', () => {
});
test('correctly resolves relative URLs with the default strategy of same-hostname', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (request) => {
- enqueued.push(...request);
- };
-
+ const { enqueued, requestQueue } = createRequestQueueMock();
await cheerioCrawlerEnqueueLinks({
options: { baseUrl: 'http://www.absolute.com/removethis/' },
$,
@@ -917,13 +851,7 @@ describe('enqueueLinks()', () => {
});
test('correctly works with transformRequestFunction', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
-
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (requests) => {
- enqueued.push(...requests);
- };
+ const { enqueued, requestQueue } = createRequestQueueMock();
const pseudoUrls = [
'https://example.com/[(\\w|-|/)*]',
'[http|https]://cool.com/',