Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Goto function deprecation (#891)

petrpatekdcce2790aa93
diff --git a/src/crawlers/browser_crawler.js b/src/crawlers/browser_crawler.js
index 61c318c53..1b9ff3b50 100644
--- a/src/crawlers/browser_crawler.js
+++ b/src/crawlers/browser_crawler.js
@@ -301,7 +301,7 @@ class BrowserCrawler extends BasicCrawler {
this.handlePageTimeoutMillis = this.handlePageTimeoutSecs * 1000;
this.gotoFunction = gotoFunction;
- this.gotoOptions = {};
+ this.defaultGotoOptions = {};
this.persistCookiesPerSession = persistCookiesPerSession;
this.proxyConfiguration = proxyConfiguration;
@@ -409,7 +409,7 @@ class BrowserCrawler extends BasicCrawler {
*/
async _handleNavigation(crawlingContext) {
// @TODO: consider deep clone
- const gotoOptions = _.clone(this.gotoOptions);
+ const gotoOptions = _.clone(this.defaultGotoOptions);
await this._executeHooks(this.preNavigationHooks, crawlingContext, gotoOptions);
crawlingContext.response = await this._navigationHandler(crawlingContext, gotoOptions);
diff --git a/src/crawlers/playwright_crawler.js b/src/crawlers/playwright_crawler.js
index e5c6ec980..5b1551fa7 100644
--- a/src/crawlers/playwright_crawler.js
+++ b/src/crawlers/playwright_crawler.js
@@ -46,7 +46,7 @@ import { apifyOptionsToLaunchOptions, getPlaywrightLauncherOrThrow } from '../pl
* let your function throw exceptions rather than catch them.
* The exceptions are logged to the request using the
* {@link Request#pushErrorMessage} function.
- * @property {number} [gotoTimeoutSecs=60]
+ * @property {number} [navigationTimeoutSecs=60]
* Timeout in which page navigation needs to finish, in seconds. When `gotoFunction()` is used and thus the default
* function is overridden, this timeout will not be used and needs to be configured in the new `gotoFunction()`.
* @property {HandleFailedRequest} [handleFailedRequestFunction]
@@ -54,7 +54,7 @@ import { apifyOptionsToLaunchOptions, getPlaywrightLauncherOrThrow } from '../pl
*
* The function receives the following object as an argument:
* ```
- * {
+ * {
* request: Request,
* response: Response,
* page: Page,
@@ -227,8 +227,8 @@ class PlaywrightCrawler extends BrowserCrawler {
browserPoolOptions: ow.optional.object,
playwrightModule: ow.optional.object,
gotoTimeoutSecs: ow.optional.number,
+ navigationTimeoutSecs: ow.optional.number,
launchOptions: ow.optional.object,
- gotoFunction: ow.undefined,
}
constructor(options = {}) {
@@ -238,6 +238,7 @@ class PlaywrightCrawler extends BrowserCrawler {
playwrightModule,
launchContext = {},
gotoTimeoutSecs,
+ navigationTimeoutSecs,
browserPoolOptions = {},
...browserCrawlerOptions
} = options;
@@ -257,6 +258,10 @@ class PlaywrightCrawler extends BrowserCrawler {
browserPoolOptions,
});
+ if (gotoTimeoutSecs) {
+ this.log.deprecated('Option "gotoTimeoutSecs" is deprecated. Use "navigationTimeoutSecs" instead.');
+ }
+
this.browserPool.postLaunchHooks.push(({ error, session }) => {
// It would be better to compare the instances,
if (error && error.constructor.name === 'TimeoutError') {
@@ -264,17 +269,20 @@ class PlaywrightCrawler extends BrowserCrawler {
}
});
- this.gotoTimeoutMillis = gotoTimeoutSecs * 1000;
-
+ this.navigationTimeoutMillis = (navigationTimeoutSecs || gotoTimeoutSecs) * 1000;
this.launchContext = launchContext;
this.playwrightModule = playwrightModule;
- this.gotoOptions = {
- timeout: this.gotoTimeoutMillis,
+ this.defaultGotoOptions = {
+ timeout: this.navigationTimeoutMillis,
};
}
async _navigationHandler(crawlingContext, gotoOptions) {
+ if (this.gotoFunction) {
+ this.log.deprecated('PlaywrightCrawler.gotoFunction is deprecated. Use "preNavigationHooks" and "postNavigationHooks" instead.');
+ return this.gotoFunction(crawlingContext, gotoOptions);
+ }
return gotoExtended(crawlingContext.page, crawlingContext.request, gotoOptions);
}
}
diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.js
index 9c6b82da6..34b38e94f 100644
--- a/src/crawlers/puppeteer_crawler.js
+++ b/src/crawlers/puppeteer_crawler.js
@@ -45,7 +45,7 @@ import applyStealthToBrowser from '../stealth/stealth';
* let your function throw exceptions rather than catch them.
* The exceptions are logged to the request using the
* {@link Request#pushErrorMessage} function.
- * @property {number} [gotoTimeoutSecs=60]
+ * @property {number} [navigationTimeoutSecs=60]
* Timeout in which page navigation needs to finish, in seconds. When `gotoFunction()` is used and thus the default
* function is overridden, this timeout will not be used and needs to be configured in the new `gotoFunction()`.
* @property {HandleFailedRequest} [handleFailedRequestFunction]
@@ -224,8 +224,8 @@ class PuppeteerCrawler extends BrowserCrawler {
...BrowserCrawler.optionsShape,
browserPoolOptions: ow.optional.object,
gotoTimeoutSecs: ow.optional.number,
+ navigationTimeoutSecs: ow.optional.number,
launchContext: ow.optional.object,
- gotoFunction: ow.undefined,
}
@@ -240,6 +240,7 @@ class PuppeteerCrawler extends BrowserCrawler {
puppeteerModule, // eslint-disable-line
launchContext = {},
gotoTimeoutSecs,
+ navigationTimeoutSecs,
browserPoolOptions = {},
proxyConfiguration,
...browserCrawlerOptions
@@ -255,6 +256,7 @@ class PuppeteerCrawler extends BrowserCrawler {
throw new Error('It is not possible to combine "options.proxyConfiguration" together with '
+ 'custom "proxyUrl" option from "options.launchPuppeteerOptions".');
}
+
browserPoolOptions.browserPlugins = [
new PuppeteerPlugin(
getPuppeteerOrThrow(puppeteerModule),
@@ -279,6 +281,10 @@ class PuppeteerCrawler extends BrowserCrawler {
}
});
+ if (gotoTimeoutSecs) {
+ this.log.deprecated('Option "gotoTimeoutSecs" is deprecated. Use "navigationTimeoutSecs" instead.');
+ }
+
if (proxyUrl) {
this.log.deprecated('options.launchPuppeteerOptions.proxyUrl is deprecated use the options.proxyConfiguration instead');
}
@@ -291,16 +297,21 @@ class PuppeteerCrawler extends BrowserCrawler {
});
}
- this.gotoTimeoutMillis = gotoTimeoutSecs * 1000;
+ this.navigationTimeoutMillis = (navigationTimeoutSecs || gotoTimeoutSecs) * 1000;
this.launchContext = launchContext;
this.puppeteerModule = puppeteerModule;
- this.gotoOptions = {
- timeout: this.gotoTimeoutMillis,
+ this.defaultGotoOptions = {
+ timeout: this.navigationTimeoutMillis,
};
}
async _navigationHandler(crawlingContext, gotoOptions) {
+ if (this.gotoFunction) {
+ this.log.deprecated('PuppeteerCrawlerOptions.gotoFunction is deprecated. Use "preNavigationHooks" and "postNavigationHooks" instead.');
+
+ return this.gotoFunction(crawlingContext, gotoOptions);
+ }
return gotoExtended(crawlingContext.page, crawlingContext.request, gotoOptions);
}
}
diff --git a/test/crawlers/playwright_crawler.test.js b/test/crawlers/playwright_crawler.test.js
index 20c7c643e..3ddc15d1c 100644
--- a/test/crawlers/playwright_crawler.test.js
+++ b/test/crawlers/playwright_crawler.test.js
@@ -73,41 +73,54 @@ describe('PlaywrightCrawler', () => {
});
});
});
- test('should throw on gotoFunction', async () => {
- try {
- const playwrightCrawler = new Apify.PlaywrightCrawler({ //eslint-disable-line
- requestList,
- maxRequestRetries: 0,
- maxConcurrency: 1,
- handlePageFunction: async () => {
- },
- gotoFunction: () => { },
- });
- } catch (e) {
- expect(e.message.includes('Expected property `gotoFunction` to be of type `undefined`')).toBeTruthy();
- }
+
+ test('should override goto timeout with gotoTimeoutSecs', async () => {
+ const timeoutSecs = 10;
+ let options;
+ const playwrightCrawler = new Apify.PlaywrightCrawler({ //eslint-disable-line
+ requestList,
+ maxRequestRetries: 0,
+ maxConcurrency: 1,
+ handlePageFunction: async () => {
+ },
+ preNavigationHooks: [(context, gotoOptions) => {
+ options = gotoOptions;
+ }],
+ gotoTimeoutSecs: timeoutSecs,
+ });
+
+ expect(playwrightCrawler.defaultGotoOptions.timeout).toEqual(timeoutSecs * 1000);
+ await playwrightCrawler.run();
+
+ expect(options.timeout).toEqual(timeoutSecs * 1000);
expect.hasAssertions();
});
+ test('should support custom gotoFunction', async () => {
+ const functions = {
+ handlePageFunction: () => { },
+ gotoFunction: ({ page, request }, options) => {
+ return page.goto(request.url, options);
+ },
+ };
+ jest.spyOn(functions, 'gotoFunction');
+ jest.spyOn(functions, 'handlePageFunction');
+ const playwrightCrawler = new Apify.PlaywrightCrawler({ //eslint-disable-line
+ requestList,
+ maxRequestRetries: 0,
+ maxConcurrency: 1,
+ handlePageFunction: functions.handlePageFunction,
+ gotoFunction: functions.gotoFunction,
+ });
- test('should launch with webkit browser', async () => {
- try {
- const playwrightCrawler = new Apify.PlaywrightCrawler({ //eslint-disable-line
- requestList,
- maxRequestRetries: 0,
- maxConcurrency: 1,
- handlePageFunction: async () => {
- },
- gotoFunction: () => { },
- });
- } catch (e) {
- expect(e.message.includes('Expected property `gotoFunction` to be of type `undefined`')).toBeTruthy();
- }
+ expect(playwrightCrawler.gotoFunction).toEqual(functions.gotoFunction);
+ await playwrightCrawler.run();
- expect.hasAssertions();
+ expect(functions.gotoFunction).toBeCalled();
+ expect(functions.handlePageFunction).toBeCalled();
});
- test('should override goto timeout ', async () => {
+ test('should override goto timeout with navigationTimeoutSecs', async () => {
const timeoutSecs = 10;
let options;
const playwrightCrawler = new Apify.PlaywrightCrawler({ //eslint-disable-line
@@ -119,14 +132,12 @@ describe('PlaywrightCrawler', () => {
preNavigationHooks: [(context, gotoOptions) => {
options = gotoOptions;
}],
- gotoTimeoutSecs: timeoutSecs,
+ navigationTimeoutSecs: timeoutSecs,
});
- expect(playwrightCrawler.gotoOptions.timeout).toEqual(timeoutSecs * 1000);
+ expect(playwrightCrawler.defaultGotoOptions.timeout).toEqual(timeoutSecs * 1000);
await playwrightCrawler.run();
expect(options.timeout).toEqual(timeoutSecs * 1000);
-
- expect.hasAssertions();
});
});
diff --git a/test/crawlers/puppeteer_crawler.test.js b/test/crawlers/puppeteer_crawler.test.js
index 42c2d8b55..706a657a0 100644
--- a/test/crawlers/puppeteer_crawler.test.js
+++ b/test/crawlers/puppeteer_crawler.test.js
@@ -72,24 +72,52 @@ describe('PuppeteerCrawler', () => {
});
});
- test('should throw on gotoFunction', async () => {
- try {
+ test('should override goto timeout with gotoTimeoutSecs ', async () => {
+ const timeoutSecs = 10;
+ let options;
const puppeteerCrawler = new Apify.PuppeteerCrawler({ //eslint-disable-line
- requestList,
- maxRequestRetries: 0,
- maxConcurrency: 1,
- handlePageFunction: async () => {
- },
- gotoFunction: () => {},
- });
- } catch (e) {
- expect(e.message.includes('Expected property `gotoFunction` to be of type `undefined`')).toBeTruthy();
- }
+ requestList,
+ maxRequestRetries: 0,
+ maxConcurrency: 1,
+ handlePageFunction: async () => {
+ },
+ preNavigationHooks: [(context, gotoOptions) => {
+ options = gotoOptions;
+ }],
+ gotoTimeoutSecs: timeoutSecs,
+ });
- expect.hasAssertions();
+ expect(puppeteerCrawler.defaultGotoOptions.timeout).toEqual(timeoutSecs * 1000);
+ await puppeteerCrawler.run();
+
+ expect(options.timeout).toEqual(timeoutSecs * 1000);
});
- test('should override goto timeout ', async () => {
+ test('should support custom gotoFunction', async () => {
+ const functions = {
+ handlePageFunction: () => {},
+ gotoFunction: ({ page, request }, options) => {
+ return page.goto(request.url, options);
+ },
+ };
+ jest.spyOn(functions, 'gotoFunction');
+ jest.spyOn(functions, 'handlePageFunction');
+ const puppeteerCrawler = new Apify.PuppeteerCrawler({ //eslint-disable-line
+ requestList,
+ maxRequestRetries: 0,
+ maxConcurrency: 1,
+ handlePageFunction: functions.handlePageFunction,
+ gotoFunction: functions.gotoFunction,
+ });
+
+ expect(puppeteerCrawler.gotoFunction).toEqual(functions.gotoFunction);
+ await puppeteerCrawler.run();
+
+ expect(functions.gotoFunction).toBeCalled();
+ expect(functions.handlePageFunction).toBeCalled();
+ });
+
+ test('should override goto timeout with navigationTimeoutSecs ', async () => {
const timeoutSecs = 10;
let options;
const puppeteerCrawler = new Apify.PuppeteerCrawler({ //eslint-disable-line
@@ -101,15 +129,13 @@ describe('PuppeteerCrawler', () => {
preNavigationHooks: [(context, gotoOptions) => {
options = gotoOptions;
}],
- gotoTimeoutSecs: timeoutSecs,
+ navigationTimeoutSecs: timeoutSecs,
});
- expect(puppeteerCrawler.gotoOptions.timeout).toEqual(timeoutSecs * 1000);
+ expect(puppeteerCrawler.defaultGotoOptions.timeout).toEqual(timeoutSecs * 1000);
await puppeteerCrawler.run();
expect(options.timeout).toEqual(timeoutSecs * 1000);
-
- expect.hasAssertions();
});
test('should throw if launchOptions.proxyUrl and proxyConfiguration is suplied', async () => {