Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: restore @ts-ignore on robotsParser calls under nodenext resolution
Martin Adámekbbd72d8ed271
diff --git a/packages/utils/src/internals/robots.ts b/packages/utils/src/internals/robots.tsindex 2e47b492f..fe62d5af3 100644--- a/packages/utils/src/internals/robots.ts+++ b/packages/utils/src/internals/robots.ts@@ -84,6 +84,7 @@ export class RobotsTxtFile {* @param [proxyUrl] a proxy to be used for fetching the robots.txt file*/static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile {+ // @ts-ignorereturn new RobotsTxtFile(url, robotsParser(url, content), proxyUrl);}@@ -128,6 +129,7 @@ export class RobotsTxtFile {);}+ // @ts-ignorereturn new RobotsTxtFile(url, robotsParser(url.toString(), await response.text()), proxyUrl, logger);}