Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: add `forefront` option to all `enqueueLinks` variants (#1760)
Matěj Volfa01459dffb51
diff --git a/packages/playwright-crawler/src/internals/enqueue-links/click-elements.ts b/packages/playwright-crawler/src/internals/enqueue-links/click-elements.tsindex 8f70cdc8b..6b81fe88e 100644--- a/packages/playwright-crawler/src/internals/enqueue-links/click-elements.ts+++ b/packages/playwright-crawler/src/internals/enqueue-links/click-elements.ts@@ -155,6 +155,16 @@ export interface EnqueueLinksByClickingElementsOptions {* @default 5*/maxWaitForPageIdleSecs?: number;++ /**+ * If set to `true`:+ * - while adding the request to the queue: the request will be added to the foremost position in the queue.+ * - while reclaiming the request: the request will be placed to the beginning of the queue, so that it's returned+ * in the next call to {@apilink RequestQueue.fetchNextRequest}.+ * By default, it's put to the end of the queue.+ * @default false+ */+ forefront?: boolean;}/**@@ -221,6 +231,7 @@ export async function enqueueLinksByClickingElements(options: EnqueueLinksByClicwaitForPageIdleSecs: ow.optional.number,maxWaitForPageIdleSecs: ow.optional.number,label: ow.optional.string,+ forefront: ow.optional.boolean,}));const {@@ -234,6 +245,7 @@ export async function enqueueLinksByClickingElements(options: EnqueueLinksByClictransformRequestFunction,waitForPageIdleSecs = 1,maxWaitForPageIdleSecs = 5,+ forefront,} = options;const waitForPageIdleMillis = waitForPageIdleSecs * 1000;@@ -266,7 +278,7 @@ export async function enqueueLinksByClickingElements(options: EnqueueLinksByClicrequestOptions = requestOptions.map(transformRequestFunction).filter((r) => !!r) as RequestOptions[];}const requests = createRequests(requestOptions, urlPatternObjects);- return requestQueue.addRequests(requests);+ return requestQueue.addRequests(requests, { forefront });}interface WaitForPageIdleOptions {diff --git a/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.ts b/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.tsindex b9a0e823d..748942507 100644--- a/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.ts+++ b/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.ts@@ -160,6 +160,16 @@ export interface EnqueueLinksByClickingElementsOptions {* @default 5*/maxWaitForPageIdleSecs?: number;++ /**+ * If set to `true`:+ * - while adding the request to the queue: the request will be added to the foremost position in the queue.+ * - while reclaiming the request: the request will be placed to the beginning of the queue, so that it's returned+ * in the next call to {@apilink RequestQueue.fetchNextRequest}.+ * By default, it's put to the end of the queue.+ * @default false+ */+ forefront?: boolean;}/**@@ -226,6 +236,7 @@ export async function enqueueLinksByClickingElements(options: EnqueueLinksByClicwaitForPageIdleSecs: ow.optional.number,maxWaitForPageIdleSecs: ow.optional.number,label: ow.optional.string,+ forefront: ow.optional.boolean,}));const {@@ -239,6 +250,7 @@ export async function enqueueLinksByClickingElements(options: EnqueueLinksByClictransformRequestFunction,waitForPageIdleSecs = 1,maxWaitForPageIdleSecs = 5,+ forefront,} = options;const waitForPageIdleMillis = waitForPageIdleSecs * 1000;@@ -271,7 +283,7 @@ export async function enqueueLinksByClickingElements(options: EnqueueLinksByClicrequestOptions = requestOptions.map(transformRequestFunction).filter((r) => !!r) as RequestOptions[];}const requests = createRequests(requestOptions, urlPatternObjects);- return requestQueue.addRequests(requests);+ return requestQueue.addRequests(requests, { forefront });}interface WaitForPageIdleOptions {diff --git a/test/core/enqueue_links/click_elements.test.ts b/test/core/enqueue_links/click_elements.test.tsindex 5f7dd0144..209f16a8a 100644--- a/test/core/enqueue_links/click_elements.test.ts+++ b/test/core/enqueue_links/click_elements.test.ts@@ -1,5 +1,6 @@-import type { Request, RequestOptions } from 'crawlee';+import type { Request, RequestOptions, RequestQueueOperationOptions } from 'crawlee';import {+ Configuration,RequestQueue,puppeteerClickElements,launchPuppeteer,@@ -100,6 +101,37 @@ testCases.forEach(({expect(page.url()).toBe('about:blank');});+ test('accepts forefront option', async () => {+ const addedRequests: {request: (Request | RequestOptions); options: RequestQueueOperationOptions}[] = [];+ const requestQueue = new RequestQueue({ id: 'xxx', client: Configuration.getStorageClient() });+ requestQueue.addRequests = async (requests, options) => {+ addedRequests.push(...requests.map((request) => ({ request, options })));+ return { processedRequests: [], unprocessedRequests: [] };+ };++ const html = `+<html>+ <body>+ <a href="https://www.example.com/link1">link1</div>+ <a href="https://www.example.com/link2">link2</div>+ </body>+</html>+ `;++ await page.setContent(html);+ await utils.enqueueLinksByClickingElements({+ page,+ requestQueue,+ selector: 'a',+ waitForPageIdleSecs: 0.025,+ maxWaitForPageIdleSecs: 0.250,+ forefront: true,+ });+ expect(addedRequests).toHaveLength(2);+ expect(addedRequests[0].options.forefront).toBe(true);+ expect(addedRequests[1].options.forefront).toBe(true);+ });+describe('clickElements()', () => {test('should click an element', async () => {const html = `diff --git a/test/core/enqueue_links/enqueue_links.test.ts b/test/core/enqueue_links/enqueue_links.test.tsindex 0e6d3465d..c3d3eb16a 100644--- a/test/core/enqueue_links/enqueue_links.test.ts+++ b/test/core/enqueue_links/enqueue_links.test.ts@@ -1,7 +1,7 @@import log from '@apify/log';import type { CheerioRoot } from '@crawlee/utils';import cheerio from 'cheerio';-import type { Request, RequestOptions } from '@crawlee/puppeteer';+import type { Request, RequestOptions, RequestQueueOperationOptions } from '@crawlee/puppeteer';import {browserCrawlerEnqueueLinks,Configuration,@@ -950,5 +950,32 @@ describe('enqueueLinks()', () => {expect(enqueued[2].method).toBe('GET');expect(enqueued[2].userData.foo).toBe('bar');});++ test('accepts forefront option', async () => {+ const enqueued: { request: (Request | RequestOptions); options: RequestQueueOperationOptions }[] = [];+ const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });++ requestQueue.addRequests = async (requests, options) => {+ // copy the requests to the enqueued list, along with options that were passed to addRequests,+ // so that it doesn't matter how many calls were made+ enqueued.push(...requests.map((request) => ({ request, options })));+ return { processedRequests: [], unprocessedRequests: [] };+ };++ await cheerioCrawlerEnqueueLinks({+ options: {+ forefront: true,+ },+ $,+ requestQueue,+ originalRequestUrl: 'https://example.com',+ });++ expect(enqueued).toHaveLength(5);++ for (let i = 0; i < 5; i++) {+ expect(enqueued[i].options.forefront).toBe(true);+ }+ });});});