Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: Fix more build issues
Jan Buchar51bba83ccf69
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex cf2d18825..429c1a418 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -725,9 +725,11 @@ export class HttpCrawler<if (Buffer.isEncoding(encoding)) return { response, encoding };// Try to re-encode a variety of unsupported encodings to utf-8- if (iconv.encodingExists(encoding)) {- const encodeStream = iconv.encodeStream(utf8);- const decodeStream = iconv.decodeStream(encoding).on('error', (err) => encodeStream.emit('error', err));+ if (iconv.default.encodingExists(encoding)) {+ const encodeStream = iconv.default.encodeStream(utf8);+ const decodeStream = iconv.default+ .decodeStream(encoding)+ .on('error', (err) => encodeStream.emit('error', err));const reencodedBody = response.body? Readable.toWeb(Readable.from(diff --git a/packages/utils/src/internals/robots.ts b/packages/utils/src/internals/robots.tsindex 4aedfc294..bee0e6fab 100644--- a/packages/utils/src/internals/robots.ts+++ b/packages/utils/src/internals/robots.ts@@ -77,9 +77,8 @@ export class RobotsTxtFile {): Promise<RobotsTxtFile> {const { proxyUrl, httpClient = new FetchHttpClient() } = options || {};- const response = await httpClient.sendRequest(new Request(url, { method: 'GET' }), {+ const response = await httpClient.sendRequest(new Request(url, { method: 'GET', signal: options?.signal }), {proxyUrl,- signal,timeout: options?.timeoutMillis,});