Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

refactor: Fix rendering type detection persistence (#3011)

Jan Buchar8838d69aa716
diff --git a/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts b/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts
index 3dd7f4e89..29c342ede 100644
--- a/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts
+++ b/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts
@@ -52,7 +52,7 @@ export class RenderingTypePredictor {
this.state = new RecoverableState({
defaultState: { logreg: new LogisticRegression({ numSteps: 1000, learningRate: 0.05 }) },
serialize: (state) => JSON.stringify({ logreg: state.logreg.toJSON() }),
- deserialize: (serializedState) => ({ logreg: LogisticRegression.load(JSON.parse(serializedState)) }),
+ deserialize: (serializedState) => ({ logreg: LogisticRegression.load(JSON.parse(serializedState).logreg) }),
persistStateKey: 'rendering-type-predictor-state',
persistenceEnabled: true,
...persistenceOptions,
diff --git a/test/core/crawlers/adaptive_playwright_crawler.test.ts b/test/core/crawlers/adaptive_playwright_crawler.test.ts
index fbcf0c6b4..386b19087 100644
--- a/test/core/crawlers/adaptive_playwright_crawler.test.ts
+++ b/test/core/crawlers/adaptive_playwright_crawler.test.ts
@@ -1,9 +1,10 @@
import type { Server } from 'node:http';
import type { AddressInfo } from 'node:net';
-import { type Dictionary, KeyValueStore } from '@crawlee/core';
+import { Configuration, type Dictionary, EventType, KeyValueStore } from '@crawlee/core';
import type { AdaptivePlaywrightCrawlerOptions, Request } from '@crawlee/playwright';
-import { AdaptivePlaywrightCrawler, RequestList } from '@crawlee/playwright';
+import { AdaptivePlaywrightCrawler, RenderingTypePredictor, RequestList } from '@crawlee/playwright';
+import { sleep } from 'crawlee';
import express from 'express';
import { startExpressAppPromise } from 'test/shared/_helper';
import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator';
@@ -388,4 +389,47 @@ describe('AdaptivePlaywrightCrawler', () => {
const store = localStorageEmulator.getKeyValueStore();
expect(await store.getRecord('1')).toBeUndefined();
});
+
+ test('should persist RenderingTypePredictor state on PERSIST_STATE events', async () => {
+ const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(async ({ pushData }) => {
+ await pushData({ content: 'test data' });
+ });
+
+ const crawler = await makeOneshotCrawler(
+ {
+ requestHandler,
+ // Use a real RenderingTypePredictor instead of the mocked one
+ renderingTypePredictor: new RenderingTypePredictor({ detectionRatio: 1 }),
+ },
+ [`http://${HOSTNAME}:${port}/static`],
+ );
+
+ // Run the crawler - this will initialize the RenderingTypePredictor and potentially store results
+ await crawler.run();
+
+ // Now emit a PERSIST_STATE event to trigger state persistence
+ const events = Configuration.getEventManager();
+ events.emit(EventType.PERSIST_STATE);
+
+ // Wait a bit for the event to be processed
+ await sleep(100);
+
+ // Verify that the regression model was actually saved to the key-value store
+ const store = await KeyValueStore.open();
+ const storedState = await store.getValue<string>('rendering-type-predictor-state');
+ expect(storedState).not.toBeNull();
+
+ const parsedState = JSON.parse(storedState!);
+ expect(parsedState).toHaveProperty('logreg');
+
+ // Test that the persisted state can be successfully restored
+ // by creating a new RenderingTypePredictor and seeing if it initializes without error
+ const newPredictor = new RenderingTypePredictor({
+ detectionRatio: 0.1,
+ persistenceOptions: { persistStateKey: 'rendering-type-predictor-state' },
+ });
+
+ // This should not throw since we've persisted valid state
+ await expect(newPredictor.initialize()).resolves.not.toThrow();
+ });
});