Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Add enqueueLinks to Apify.utils

Ondra Urbandbabd569dd14
diff --git a/src/index.js b/src/index.js
index a6e58464c..8384f545c 100644
--- a/src/index.js
+++ b/src/index.js
@@ -19,6 +19,7 @@ import { apifyClient, getMemoryInfo, isAtHome, publicUtils } from './utils';
import { browse, launchWebDriver } from './webdriver';
import { puppeteerUtils } from './puppeteer_utils';
import { socialUtils } from './utils_social';
+import { enqueueLinks } from './enqueue_links';
import PseudoUrl from './pseudo_url';
/* globals module */
@@ -103,6 +104,7 @@ module.exports = {
puppeteer: puppeteerUtils,
social: socialUtils,
log,
+ enqueueLinks,
}),
};
diff --git a/test/enqueue_links.js b/test/enqueue_links.js
index d86283e2f..5b5a2fc7d 100644
--- a/test/enqueue_links.js
+++ b/test/enqueue_links.js
@@ -334,6 +334,34 @@ describe('enqueueLinks()', () => {
$ = null;
});
+ it('works from utils namespace', async () => {
+ const enqueued = [];
+ const requestQueue = new RequestQueue('xxx');
+ requestQueue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
+ const pseudoUrls = [
+ new Apify.PseudoUrl('https://example.com/[(\\w|-|/)*]', { method: 'POST' }),
+ new Apify.PseudoUrl('[http|https]://cool.com/', { userData: { foo: 'bar' } }),
+ ];
+
+ await Apify.utils.enqueueLinks({ $, selector: '.click', requestQueue, pseudoUrls });
+
+ expect(enqueued).to.have.lengthOf(3);
+
+ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
+ expect(enqueued[0].method).to.be.eql('POST');
+ expect(enqueued[0].userData).to.be.eql({});
+
+ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
+ expect(enqueued[1].method).to.be.eql('POST');
+ expect(enqueued[1].userData).to.be.eql({});
+
+ expect(enqueued[2].url).to.be.eql('http://cool.com/');
+ expect(enqueued[2].method).to.be.eql('GET');
+ expect(enqueued[2].userData.foo).to.be.eql('bar');
+ });
+
it('works with PseudoUrl instances', async () => {
const enqueued = [];
const requestQueue = new RequestQueue('xxx');
diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.js
index e18623a69..feed17122 100644
--- a/test/puppeteer_utils.js
+++ b/test/puppeteer_utils.js
@@ -1,7 +1,6 @@
import chai, { expect } from 'chai';
import chaiAsPromised from 'chai-as-promised';
import Apify from '../build/index';
-import { RequestQueue } from '../build/request_queue';
const { utils: { log } } = Apify;