Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
docs: fix code examples for Adding more URLs - Finding new links and Filtering links to same domain (#1884)
Eirik Mof4b0c62b2c84
diff --git a/docs/introduction/03-filter-el.ts b/docs/introduction/03-filter-el.tsindex 00c9b0f9a..9f9c09f73 100644--- a/docs/introduction/03-filter-el.ts+++ b/docs/introduction/03-filter-el.ts@@ -5,6 +5,9 @@ const crawler = new CheerioCrawler({async requestHandler({ $, request, enqueueLinks }) {const title = $('title').text();console.log(`The title of "${request.url}" is: ${title}.`);+ // The default behavior of enqueueLinks is to stay on the same hostname,+ // so it does not require any parameters.+ // This will ensure the subdomain stays the same.await enqueueLinks();},});diff --git a/docs/introduction/03-filter-without-el.ts b/docs/introduction/03-filter-without-el.tsindex b945dd64b..de7e6e21c 100644--- a/docs/introduction/03-filter-without-el.ts+++ b/docs/introduction/03-filter-without-el.ts@@ -7,19 +7,25 @@ const crawler = new CheerioCrawler({const title = $('title').text();console.log(`The title of "${request.url}" is: ${title}.`);- // Without enqueueLinks, we first have to extract all- // the URLs from the page with Cheerio.const links = $('a[href]').map((_, el) => $(el).attr('href')).get();- // Then we need to resolve relative URLs,- // otherwise they would be unusable for crawling.- const absoluteUrls = links- .map((link) => new URL(link, request.loadedUrl).href);+ // Besides resolving the URLs, we now also need to+ // grab their hostname for filtering.+ const { hostname } = new URL(request.loadedUrl);+ const absoluteUrls = links.map(+ (link) => new URL(link, request.loadedUrl)+ );++ // We use the hostname to filter links that point+ // to a different domain, even subdomain.+ const sameHostnameLinks = absoluteUrls+ .filter((url) => url.hostname === hostname)+ .map((url) => ({ url: url.href }));// Finally, we have to add the URLs to the queue- await crawler.addRequests(absoluteUrls);+ await crawler.addRequests(sameHostnameLinks);},});diff --git a/docs/introduction/03-find-without-el.ts b/docs/introduction/03-find-without-el.tsindex 9b3aa6543..0414c504b 100644--- a/docs/introduction/03-find-without-el.ts+++ b/docs/introduction/03-find-without-el.ts@@ -9,24 +9,20 @@ const crawler = new CheerioCrawler({const title = $('title').text();console.log(`The title of "${request.url}" is: ${title}.`);+ // Without enqueueLinks, we first have to extract all+ // the URLs from the page with Cheerio.const links = $('a[href]').map((_, el) => $(el).attr('href')).get();- // Besides resolving the URLs, we now also need to- // grab their hostname for filtering.- const { hostname } = new URL(request.loadedUrl);- const absoluteUrls = links- .map((link) => new URL(link, request.loadedUrl));-- // We use the hostname to filter links that point- // to a different domain, even subdomain.- const sameHostnameLinks = absoluteUrls- .filter((url) => url.hostname === hostname)- .map((url) => ({ url: url.href }));+ // Then we need to resolve relative URLs,+ // otherwise they would be unusable for crawling.+ const absoluteUrls = links.map(+ (link) => new URL(link, request.loadedUrl).href+ );// Finally, we have to add the URLs to the queue- await crawler.addRequests(sameHostnameLinks);+ await crawler.addRequests(absoluteUrls);},});diff --git a/website/versioned_docs/version-3.3/introduction/03-filter-el.ts b/website/versioned_docs/version-3.3/introduction/03-filter-el.tsindex 00c9b0f9a..9f9c09f73 100644--- a/website/versioned_docs/version-3.3/introduction/03-filter-el.ts+++ b/website/versioned_docs/version-3.3/introduction/03-filter-el.ts@@ -5,6 +5,9 @@ const crawler = new CheerioCrawler({async requestHandler({ $, request, enqueueLinks }) {const title = $('title').text();console.log(`The title of "${request.url}" is: ${title}.`);+ // The default behavior of enqueueLinks is to stay on the same hostname,+ // so it does not require any parameters.+ // This will ensure the subdomain stays the same.await enqueueLinks();},});diff --git a/website/versioned_docs/version-3.3/introduction/03-filter-without-el.ts b/website/versioned_docs/version-3.3/introduction/03-filter-without-el.tsindex b945dd64b..de7e6e21c 100644--- a/website/versioned_docs/version-3.3/introduction/03-filter-without-el.ts+++ b/website/versioned_docs/version-3.3/introduction/03-filter-without-el.ts@@ -7,19 +7,25 @@ const crawler = new CheerioCrawler({const title = $('title').text();console.log(`The title of "${request.url}" is: ${title}.`);- // Without enqueueLinks, we first have to extract all- // the URLs from the page with Cheerio.const links = $('a[href]').map((_, el) => $(el).attr('href')).get();- // Then we need to resolve relative URLs,- // otherwise they would be unusable for crawling.- const absoluteUrls = links- .map((link) => new URL(link, request.loadedUrl).href);+ // Besides resolving the URLs, we now also need to+ // grab their hostname for filtering.+ const { hostname } = new URL(request.loadedUrl);+ const absoluteUrls = links.map(+ (link) => new URL(link, request.loadedUrl)+ );++ // We use the hostname to filter links that point+ // to a different domain, even subdomain.+ const sameHostnameLinks = absoluteUrls+ .filter((url) => url.hostname === hostname)+ .map((url) => ({ url: url.href }));// Finally, we have to add the URLs to the queue- await crawler.addRequests(absoluteUrls);+ await crawler.addRequests(sameHostnameLinks);},});diff --git a/website/versioned_docs/version-3.3/introduction/03-find-without-el.ts b/website/versioned_docs/version-3.3/introduction/03-find-without-el.tsindex 9b3aa6543..0414c504b 100644--- a/website/versioned_docs/version-3.3/introduction/03-find-without-el.ts+++ b/website/versioned_docs/version-3.3/introduction/03-find-without-el.ts@@ -9,24 +9,20 @@ const crawler = new CheerioCrawler({const title = $('title').text();console.log(`The title of "${request.url}" is: ${title}.`);+ // Without enqueueLinks, we first have to extract all+ // the URLs from the page with Cheerio.const links = $('a[href]').map((_, el) => $(el).attr('href')).get();- // Besides resolving the URLs, we now also need to- // grab their hostname for filtering.- const { hostname } = new URL(request.loadedUrl);- const absoluteUrls = links- .map((link) => new URL(link, request.loadedUrl));-- // We use the hostname to filter links that point- // to a different domain, even subdomain.- const sameHostnameLinks = absoluteUrls- .filter((url) => url.hostname === hostname)- .map((url) => ({ url: url.href }));+ // Then we need to resolve relative URLs,+ // otherwise they would be unusable for crawling.+ const absoluteUrls = links.map(+ (link) => new URL(link, request.loadedUrl).href+ );// Finally, we have to add the URLs to the queue- await crawler.addRequests(sameHostnameLinks);+ await crawler.addRequests(absoluteUrls);},});