Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Add enqueueLinks to Apify.utils
Ondra Urbandbabd569dd14
diff --git a/src/index.js b/src/index.jsindex a6e58464c..8384f545c 100644--- a/src/index.js+++ b/src/index.js@@ -19,6 +19,7 @@ import { apifyClient, getMemoryInfo, isAtHome, publicUtils } from './utils';import { browse, launchWebDriver } from './webdriver';import { puppeteerUtils } from './puppeteer_utils';import { socialUtils } from './utils_social';+import { enqueueLinks } from './enqueue_links';import PseudoUrl from './pseudo_url';/* globals module */@@ -103,6 +104,7 @@ module.exports = {puppeteer: puppeteerUtils,social: socialUtils,log,+ enqueueLinks,}),};diff --git a/test/enqueue_links.js b/test/enqueue_links.jsindex d86283e2f..5b5a2fc7d 100644--- a/test/enqueue_links.js+++ b/test/enqueue_links.js@@ -334,6 +334,34 @@ describe('enqueueLinks()', () => {$ = null;});+ it('works from utils namespace', async () => {+ const enqueued = [];+ const requestQueue = new RequestQueue('xxx');+ requestQueue.addRequest = async (request) => {+ enqueued.push(request);+ };+ const pseudoUrls = [+ new Apify.PseudoUrl('https://example.com/[(\\w|-|/)*]', { method: 'POST' }),+ new Apify.PseudoUrl('[http|https]://cool.com/', { userData: { foo: 'bar' } }),+ ];++ await Apify.utils.enqueueLinks({ $, selector: '.click', requestQueue, pseudoUrls });++ expect(enqueued).to.have.lengthOf(3);++ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');+ expect(enqueued[0].method).to.be.eql('POST');+ expect(enqueued[0].userData).to.be.eql({});++ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');+ expect(enqueued[1].method).to.be.eql('POST');+ expect(enqueued[1].userData).to.be.eql({});++ expect(enqueued[2].url).to.be.eql('http://cool.com/');+ expect(enqueued[2].method).to.be.eql('GET');+ expect(enqueued[2].userData.foo).to.be.eql('bar');+ });+it('works with PseudoUrl instances', async () => {const enqueued = [];const requestQueue = new RequestQueue('xxx');diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.jsindex e18623a69..feed17122 100644--- a/test/puppeteer_utils.js+++ b/test/puppeteer_utils.js@@ -1,7 +1,6 @@import chai, { expect } from 'chai';import chaiAsPromised from 'chai-as-promised';import Apify from '../build/index';-import { RequestQueue } from '../build/request_queue';const { utils: { log } } = Apify;