Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: remove noisy implicit configuration warning from service locator (#3837)
Jindřich Bärffd1dabaad99
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 74cd36a87..07d0374d2 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -826,6 +826,9 @@ export class BasicCrawler<this.#log = serviceLocator.getLogger().child({ prefix: this.constructor.name });+ // Initialize the Configuration instance to avoid lazy loading in the components+ serviceLocator.getConfiguration();+// Store whether the user explicitly provided an IDthis.hasExplicitId = id !== undefined;// Store the user-provided ID, or generate a unique one for tracking purposes (not for state key)diff --git a/packages/core/test/core/service_locator.test.ts b/packages/core/test/core/service_locator.test.tsindex 905d90b91..2fa60f3a6 100644--- a/packages/core/test/core/service_locator.test.ts+++ b/packages/core/test/core/service_locator.test.ts@@ -128,6 +128,25 @@ describe('ServiceLocator', () => {serviceLocator.setEventManager(customEventManager);}).toThrow(/EventManager is already in use/);});++ test('warns about implicit configuration when configuration was not set beforehand', () => {+ const warningSpy = vi.fn();+ serviceLocator.setLogger(makeMockLogger({ warning: warningSpy }));++ serviceLocator.getEventManager();++ expect(warningSpy).toHaveBeenCalledWith(expect.stringMatching(/implicitly set configuration/));+ });++ test('does not warn about implicit configuration when configuration was already set', () => {+ const warningSpy = vi.fn();+ serviceLocator.setLogger(makeMockLogger({ warning: warningSpy }));+ serviceLocator.setConfiguration(new Configuration());++ serviceLocator.getEventManager();++ expect(warningSpy).not.toHaveBeenCalled();+ });});describe('StorageBackend', () => {@@ -168,6 +187,25 @@ describe('ServiceLocator', () => {serviceLocator.setStorageBackend(customStorageBackend);}).toThrow(/StorageBackend is already in use/);});++ test('warns about implicit configuration when configuration was not set beforehand', () => {+ const warningSpy = vi.fn();+ serviceLocator.setLogger(makeMockLogger({ warning: warningSpy }));++ serviceLocator.getStorageBackend();++ expect(warningSpy).toHaveBeenCalledWith(expect.stringMatching(/implicitly set configuration/));+ });++ test('does not warn about implicit configuration when configuration was already set', () => {+ const warningSpy = vi.fn();+ serviceLocator.setLogger(makeMockLogger({ warning: warningSpy }));+ serviceLocator.setConfiguration(new Configuration());++ serviceLocator.getStorageBackend();++ expect(warningSpy).not.toHaveBeenCalled();+ });});describe('Logger', () => {diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex e63d955d3..06fffa89f 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -77,6 +77,20 @@ describe('BasicCrawler', () => {server.close();});+ test('constructor eagerly resolves the configuration, avoiding a later implicit-configuration warning', () => {+ serviceLocator.reset();+ const warningSpy = vitest.spyOn(serviceLocator.getLogger(), 'warning');++ new BasicCrawler({ requestHandler: async () => {} });+ serviceLocator.getStorageBackend();++ expect(warningSpy).not.toHaveBeenCalledWith(expect.stringMatching(/implicitly set configuration/));++ // restore the shared storage backend expected by the other tests' beforeEach+ serviceLocator.reset();+ serviceLocator.setStorageBackend(new MemoryStorageBackend());+ });+test('does not leak sigint events', async () => {let count = 0;