Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: fix tests that were mocking RQ improperly

Martin Adámek73023d1f8458
diff --git a/packages/core/test/enqueue_links/protocol-matching-based-on-strategy.test.ts b/packages/core/test/enqueue_links/protocol-matching-based-on-strategy.test.ts
index 60dcf1e1c..4e66168f3 100644
--- a/packages/core/test/enqueue_links/protocol-matching-based-on-strategy.test.ts
+++ b/packages/core/test/enqueue_links/protocol-matching-based-on-strategy.test.ts
@@ -19,14 +19,14 @@ const HTML = `
</html>
`;
-function getMockRequestQueue() {
+function createRequestQueueMock() {
const enqueued: Source[] = [];
-
const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
// @ts-expect-error Override method for testing
- requestQueue.addRequests = (requests) => {
+ requestQueue.addRequests = async function (requests) {
enqueued.push(...requests);
+ return { processedRequests: requests, unprocessedRequests: [] as never[] };
};
return { enqueued, requestQueue };
@@ -49,7 +49,7 @@ describe('enqueueLinks() - matching and ignoring http/https protocol differences
});
test('SameHostname should ignore protocol difference', async () => {
- const { enqueued, requestQueue } = getMockRequestQueue();
+ const { enqueued, requestQueue } = createRequestQueueMock();
await cheerioCrawlerEnqueueLinks({
options: { selector: 'a', strategy: EnqueueStrategy.SameHostname },
@@ -64,7 +64,7 @@ describe('enqueueLinks() - matching and ignoring http/https protocol differences
});
test('SameDomain should ignore protocol difference', async () => {
- const { enqueued, requestQueue } = getMockRequestQueue();
+ const { enqueued, requestQueue } = createRequestQueueMock();
await cheerioCrawlerEnqueueLinks({
options: { selector: 'a', strategy: EnqueueStrategy.SameDomain },
@@ -79,7 +79,7 @@ describe('enqueueLinks() - matching and ignoring http/https protocol differences
});
test('SameOrigin should respect protocol', async () => {
- const { enqueued, requestQueue } = getMockRequestQueue();
+ const { enqueued, requestQueue } = createRequestQueueMock();
await cheerioCrawlerEnqueueLinks({
options: { selector: 'a', strategy: EnqueueStrategy.SameOrigin },
diff --git a/packages/core/test/enqueue_links/user-provided-patterns-with-enqueue-strategy.test.ts b/packages/core/test/enqueue_links/user-provided-patterns-with-enqueue-strategy.test.ts
index 9c4fba7d5..c5c4f311c 100644
--- a/packages/core/test/enqueue_links/user-provided-patterns-with-enqueue-strategy.test.ts
+++ b/packages/core/test/enqueue_links/user-provided-patterns-with-enqueue-strategy.test.ts
@@ -31,14 +31,14 @@ const HTML = `
</html>
`;
-function getMockRequestQueue() {
+function createRequestQueueMock() {
const enqueued: Source[] = [];
-
const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
// @ts-expect-error Override method for testing
- requestQueue.addRequests = (requests) => {
+ requestQueue.addRequests = async function (requests) {
enqueued.push(...requests);
+ return { processedRequests: requests, unprocessedRequests: [] as never[] };
};
return { enqueued, requestQueue };
@@ -61,7 +61,7 @@ describe('enqueueLinks() - combining user patterns with enqueue strategies', ()
});
test('works with globs and same domain strategy', async () => {
- const { enqueued, requestQueue } = getMockRequestQueue();
+ const { enqueued, requestQueue } = createRequestQueueMock();
const globs = ['**/first'];
@@ -82,7 +82,7 @@ describe('enqueueLinks() - combining user patterns with enqueue strategies', ()
});
test('works with globs and all domains strategy', async () => {
- const { enqueued, requestQueue } = getMockRequestQueue();
+ const { enqueued, requestQueue } = createRequestQueueMock();
const globs = ['**/first'];
@@ -104,7 +104,7 @@ describe('enqueueLinks() - combining user patterns with enqueue strategies', ()
});
test('works with no user provided patterns but with same domain strategy', async () => {
- const { enqueued, requestQueue } = getMockRequestQueue();
+ const { enqueued, requestQueue } = createRequestQueueMock();
await cheerioCrawlerEnqueueLinks({
options: {
@@ -122,7 +122,7 @@ describe('enqueueLinks() - combining user patterns with enqueue strategies', ()
});
test('works with globs and exclude', async () => {
- const { enqueued, requestQueue } = getMockRequestQueue();
+ const { enqueued, requestQueue } = createRequestQueueMock();
const globs = ['**/first'];
const exclude = ['**/first'];
diff --git a/packages/core/test/enqueue_links/userData.test.ts b/packages/core/test/enqueue_links/userData.test.ts
index b96c07e40..722f4fbf6 100644
--- a/packages/core/test/enqueue_links/userData.test.ts
+++ b/packages/core/test/enqueue_links/userData.test.ts
@@ -20,14 +20,14 @@ const HTML = `
</html>
`;
-function getMockRequestQueue() {
+function createRequestQueueMock() {
const enqueued: Source[] = [];
-
const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
// @ts-expect-error Override method for testing
- requestQueue.addRequests = (requests) => {
+ requestQueue.addRequests = async function (requests) {
enqueued.push(...requests);
+ return { processedRequests: requests, unprocessedRequests: [] as never[] };
};
return { enqueued, requestQueue };
@@ -50,7 +50,7 @@ describe('enqueueLinks() - userData shouldn\'t be changed and outer label must t
});
test('multiple enqueues with different labels', async () => {
- const { enqueued, requestQueue } = getMockRequestQueue();
+ const { enqueued, requestQueue } = createRequestQueueMock();
const userData = { foo: 'bar' };
await cheerioCrawlerEnqueueLinks({
@@ -84,7 +84,7 @@ describe('enqueueLinks() - userData shouldn\'t be changed and outer label must t
});
test('JSON string of userData shouldn\'t change, but enqueued label should be different', async () => {
- const { enqueued, requestQueue } = getMockRequestQueue();
+ const { enqueued, requestQueue } = createRequestQueueMock();
const userData = { foo: 'bar', label: 'bogus' };
const originalUserData = JSON.stringify(userData);
diff --git a/test/core/enqueue_links/click_elements.test.ts b/test/core/enqueue_links/click_elements.test.ts
index ea9ca427a..a7a0a1e6b 100644
--- a/test/core/enqueue_links/click_elements.test.ts
+++ b/test/core/enqueue_links/click_elements.test.ts
@@ -74,7 +74,10 @@ testCases.forEach(({
test('should work', async () => {
const addedRequests: Source[] = [];
const requestQueue = Object.create(RequestQueue.prototype);
- requestQueue.addRequests = async (request: Request[]) => addedRequests.push(...request);
+ requestQueue.addRequests = async (requests: Request[]) => {
+ addedRequests.push(...requests);
+ return { processedRequests: requests, unprocessedRequests: [] as never[] };
+ };
const html = `
<html>
<body>
diff --git a/test/core/enqueue_links/enqueue_links.test.ts b/test/core/enqueue_links/enqueue_links.test.ts
index 7814917ee..1fedc851b 100644
--- a/test/core/enqueue_links/enqueue_links.test.ts
+++ b/test/core/enqueue_links/enqueue_links.test.ts
@@ -43,6 +43,19 @@ const HTML = `
</html>
`;
+function createRequestQueueMock() {
+ const enqueued: Source[] = [];
+ const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
+
+ // @ts-expect-error Override method for testing
+ requestQueue.addRequests = async function (requests) {
+ enqueued.push(...requests);
+ return { processedRequests: requests, unprocessedRequests: [] as never[] };
+ };
+
+ return { enqueued, requestQueue };
+}
+
describe('enqueueLinks()', () => {
let ll: number;
beforeAll(() => {
@@ -74,14 +87,7 @@ describe('enqueueLinks()', () => {
});
test('works with item limit', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
-
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (request) => {
- enqueued.push(...request);
- };
-
+ const { enqueued, requestQueue } = createRequestQueueMock();
await browserCrawlerEnqueueLinks({
options: { limit: 3, selector: '.click', strategy: EnqueueStrategy.All },
page,
@@ -107,13 +113,7 @@ describe('enqueueLinks()', () => {
});
test('works with globs', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
-
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (request) => {
- enqueued.push(...request);
- };
+ const { enqueued, requestQueue } = createRequestQueueMock();
const globs = [
'https://example.com/**/*',
{ glob: '?(http|https)://cool.com/', method: 'POST' as const },
@@ -152,13 +152,7 @@ describe('enqueueLinks()', () => {
});
test('works with regexps', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
-
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (request) => {
- enqueued.push(...request);
- };
+ const { enqueued, requestQueue } = createRequestQueueMock();
const regexps = [
/^https:\/\/example\.com\/(\w|\/)+/,
{ regexp: /^(http|https):\/\/cool\.com\//, method: 'POST' as const, userData: { label: 'COOL' } },
@@ -196,13 +190,7 @@ describe('enqueueLinks()', () => {
});
test('works with exclude glob', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
-
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (request) => {
- enqueued.push(...request);
- };
+ const { enqueued, requestQueue } = createRequestQueueMock();
const globs = [
'https://example.com/**/*',
{ glob: '?(http|https)://cool.com/', method: 'POST' as const },
@@ -243,13 +231,7 @@ describe('enqueueLinks()', () => {
});
test('works with exclude regexp', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
-
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (request) => {
- enqueued.push(...request);
- };
+ const { enqueued, requestQueue } = createRequestQueueMock();
const globs = [
'https://example.com/**/*',
{ glob: '?(http|https)://cool.com/', method: 'POST' as const },
@@ -290,13 +272,7 @@ describe('enqueueLinks()', () => {
});
test('works with pseudoUrls', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
-
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (request) => {
- enqueued.push(...request);
- };
+ const { enqueued, requestQueue } = createRequestQueueMock();
const pseudoUrls = [
'https://example.com/[(\\w|-|/)*]',
{ purl: '[http|https]://cool.com/', method: 'POST' as const, userData: { label: 'COOL' } },
@@ -357,13 +333,7 @@ describe('enqueueLinks()', () => {
});
test('works with undefined pseudoUrls[]', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
-
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (request) => {
- enqueued.push(...request);
- };
+ const { enqueued, requestQueue } = createRequestQueueMock();
await browserCrawlerEnqueueLinks({
options: { selector: '.click', strategy: EnqueueStrategy.All },
@@ -392,14 +362,7 @@ describe('enqueueLinks()', () => {
});
test('throws with null pseudoUrls[]', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
-
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (request) => {
- enqueued.push(...request);
- };
-
+ const { enqueued, requestQueue } = createRequestQueueMock();
await expect(browserCrawlerEnqueueLinks({
options: { selector: '.click', pseudoUrls: null },
page,
@@ -409,14 +372,7 @@ describe('enqueueLinks()', () => {
});
test('works with empty pseudoUrls[]', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
-
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (request) => {
- enqueued.push(...request);
- };
-
+ const { enqueued, requestQueue } = createRequestQueueMock();
await browserCrawlerEnqueueLinks({
options: { selector: '.click', pseudoUrls: [], strategy: EnqueueStrategy.All },
page,
@@ -444,14 +400,7 @@ describe('enqueueLinks()', () => {
});
test('throws with sparse pseudoUrls[]', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
-
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (request) => {
- enqueued.push(...request);
- };
-
+ const { enqueued, requestQueue } = createRequestQueueMock();
const pseudoUrls = [
'https://example.com/[(\\w|-|/)*]',
null,
@@ -468,14 +417,7 @@ describe('enqueueLinks()', () => {
});
test('correctly resolves relative URLs with default strategy of same-hostname', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
-
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (request) => {
- enqueued.push(...request);
- };
-
+ const { enqueued, requestQueue } = createRequestQueueMock();
await browserCrawlerEnqueueLinks({
options: { baseUrl: 'http://www.absolute.com/removethis/' },
page,
@@ -495,14 +437,7 @@ describe('enqueueLinks()', () => {
});
test('correctly resolves relative URLs with the strategy of same-domain', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
-
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (request) => {
- enqueued.push(...request);
- };
-
+ const { enqueued, requestQueue } = createRequestQueueMock();
await browserCrawlerEnqueueLinks({
options: { baseUrl: 'http://www.absolute.com/removethis/', strategy: EnqueueStrategy.SameDomain },
page,
@@ -635,13 +570,7 @@ describe('enqueueLinks()', () => {
});
test('works with globs', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
-
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (request) => {
- enqueued.push(...request);
- };
+ const { enqueued, requestQueue } = createRequestQueueMock();
const globs = [
'https://example.com/**/*',
{ glob: '?(http|https)://cool.com/', method: 'POST' as const, userData: { label: 'COOL' } },
@@ -679,13 +608,7 @@ describe('enqueueLinks()', () => {
});
test('works with RegExps', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
-
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (request) => {
- enqueued.push(...request);
- };
+ const { enqueued, requestQueue } = createRequestQueueMock();
const regexps = [
/^https:\/\/example\.com\/(\w|\/)+/,
{ regexp: /^(http|https):\/\/cool\.com\//, method: 'POST' as const, userData: { label: 'COOL' } },
@@ -970,14 +893,7 @@ describe('enqueueLinks()', () => {
});
test('correctly resolves relative URLs with the strategy of same-domain', async () => {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
-
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async (request) => {
- enqueued.push(...request);
- };
-
+ const { enqueued, requestQueue } = createRequestQueueMock();
await cheerioCrawlerEnqueueLinks({
options: { baseUrl: 'http://www.absolute.com/removethis/', strategy: EnqueueStrategy.SameDomain },
$,