Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: always show error origin if inside the userland (#1677)
Jindřich Bärbbe9045d550f
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 0bd87c44c..cf15f2c83 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -1,6 +1,6 @@import type { Log } from '@apify/log';import defaultLog from '@apify/log';-import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';+import { addTimeoutToPromise, tryCancel, TimeoutError } from '@apify/timeout';import { cryptoRandomObjectId } from '@apify/utilities';import type { SetRequired } from 'type-fest';import type {@@ -1096,14 +1096,18 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextforceStack = true;}- // For timeout errors we want to show the stack just in case the env variable is set+ const stackLines = error?.stack ? error.stack.split('\n') : new Error().stack!.split('\n').slice(2);++ const baseDir = process.cwd();+ const userLine = stackLines.find((line) => line.includes(baseDir) && !line.includes('node_modules'));+if (error instanceof TimeoutError) {- return process.env.CRAWLEE_VERBOSE_LOG ? error.stack : error.message || error;+ return process.env.CRAWLEE_VERBOSE_LOG ? error.stack : error.message || error; // stack in timeout errors does not really help}return (process.env.CRAWLEE_VERBOSE_LOG || forceStack)- ? error.stack ?? (error.message || error)- : error.message || error;+ ? error.stack ?? ([error.message || error, ...stackLines].join('\n'))+ : [error.message || error, userLine].join('\n');}protected _canRequestBeRetried(request: Request, error: Error) {diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 4a5688a41..5535b7527 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -981,7 +981,7 @@ describe('BasicCrawler', () => {expect(typeof args[0]).toBe('string');expect(/Reclaiming failed request back to the list or queue/.test(args[0])).toBe(true);expect(/Other non-timeout error/.test(args[0])).toBe(true);- expect(/at BasicCrawler\.requestHandler/.test(args[0])).toBe(false);+ expect(args[0].split('\n').length).toBeLessThanOrEqual(2);expect(args[1]).toBeDefined();}diff --git a/test/core/crawlers/puppeteer_crawler.test.ts b/test/core/crawlers/puppeteer_crawler.test.tsindex 693867964..c8e62749e 100644--- a/test/core/crawlers/puppeteer_crawler.test.ts+++ b/test/core/crawlers/puppeteer_crawler.test.ts@@ -246,7 +246,7 @@ describe('PuppeteerCrawler', () => {await requestQueue.drop();expect(requestHandler).not.toBeCalled();- const warnings = logWarningSpy.mock.calls.map((call) => [call[0], call[1].retryCount]);+ const warnings = logWarningSpy.mock.calls.map((call) => [call[0].split('\n')[0], call[1].retryCount]);expect(warnings).toEqual([['Reclaiming failed request back to the list or queue. Navigation timed out after 0.005 seconds.',@@ -305,7 +305,7 @@ describe('PuppeteerCrawler', () => {await requestQueue.drop();expect(requestHandler).not.toBeCalled();- const warnings = logWarningSpy.mock.calls.map((call) => [call[0], call[1].retryCount]);+ const warnings = logWarningSpy.mock.calls.map((call) => [call[0].split('\n')[0], call[1].retryCount]);expect(warnings).toEqual([['Reclaiming failed request back to the list or queue. Navigation timed out after 0.005 seconds.',