Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fixed comments, typos and other things from PR
drobnikje6e6bad83562
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 12df03d72..3580caeec 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,6 +1,6 @@0.13.1 / 2019-03-19===================-- Added `Apify.createAdHocWebhook()`+- Added `Apify.webhook()`0.13.0 / 2019-03-14===================diff --git a/src/actor.js b/src/actor.jsindex 1e3d1cb3e..40e4f0cf1 100644--- a/src/actor.js+++ b/src/actor.js@@ -767,26 +767,29 @@ export const getApifyProxyUrl = (options = {}) => {/***- * Creates adhoc webhook for current actor run.- * Webkooks are supported only for runs on Apify cloud.+ * Creates an ad-hoc webhook for the current actor run, which lets you receive a notification when the actor run finished or failed.+ * For more information about Apify actor webhooks, please see the <a href="https://apify.com/docs/webhook" target="_blank">documentation</a>.*- * @param eventTypes {String[]} - Array of event types, which you can set for actor run, see+ * Note that webhooks are only supported for actors running on the Apify platform.+ * In local environment, the function will print a warning and have no effect.+ *+ * @param options.eventTypes {String[]} - Array of event types, which you can set for actor run, see* the <a href="https://apify.com/docs/webhooks#events-actor-run" target="_blank">actor run events</a> in the Apify doc.- * @param requestUrl {String} - URL which will be requested using HTTP POST request, when actor run will be in specific event type.+ * @param options.requestUrl {String} - URL which will be requested using HTTP POST request, when actor run will be in specific event type.*- * @return {Promise<Webhook|undefined>}+ * @return {Promise<Object|undefined>}** @memberof module:Apify* @function- * @name createAdHocWebhook+ * @name webhook*/-export const createAdHocWebhook = async (eventTypes, requestUrl) => {- checkParamOrThrow(eventTypes, 'eventTypes', 'Array');+export const webhook = async ({ eventTypes, requestUrl }) => {+ checkParamOrThrow(eventTypes, 'eventTypes', '[String]');checkParamOrThrow(requestUrl, 'requestUrl', 'String');const runId = process.env[ENV_VARS.ACTOR_RUN_ID];if (!runId) {- log.warning('Webhooks are not supported locally! We skipped createAdHocWebhook() invocation.');+ log.warning('Apify.webhook() is only supported when running on the Apify platform. The webhook will not be invoked.');return;}diff --git a/src/index.js b/src/index.jsindex 15f80709a..683cecbf5 100644--- a/src/index.js+++ b/src/index.js@@ -1,7 +1,7 @@import EventEmitter from 'events';import log from 'apify-shared/log';import { ENV_VARS } from 'apify-shared/consts';-import { main, getEnv, call, callTask, getApifyProxyUrl, metamorph, createAdHocWebhook } from './actor';+import { main, getEnv, call, callTask, getApifyProxyUrl, metamorph, webhook } from './actor';import AutoscaledPool from './autoscaling/autoscaled_pool';import BasicCrawler from './basic_crawler';import CheerioCrawler from './cheerio_crawler';@@ -58,7 +58,7 @@ module.exports = {getApifyProxyUrl,isAtHome,client: apifyClient,- createAdHocWebhook,+ webhook,// Autoscaled poolAutoscaledPool,diff --git a/test/actor.js b/test/actor.jsindex 7213cde82..da18c1c5e 100644--- a/test/actor.js+++ b/test/actor.js@@ -1142,7 +1142,7 @@ describe('Apify.getApifyProxyUrl()', () => {});-describe('Apify.createAdhocWebhook()', () => {+describe('Apify.webhook()', () => {it('works', async () => {const runId = 'my-run-id';const expectedEventTypes = ['ACTOR.RUN.SUCCEEDED'];@@ -1165,11 +1165,29 @@ describe('Apify.createAdhocWebhook()', () => {.returns(Promise.resolve());- await Apify.createAdHocWebhook(expectedEventTypes, expectedRequestUrl);+ await Apify.webhook({ eventTypes: expectedEventTypes, requestUrl: expectedRequestUrl });delete process.env[ENV_VARS.ACTOR_RUN_ID];webhooksMock.verify();webhooksMock.restore();});++ it('on local logs warning and does nothing', async () => {+ const expectedEventTypes = ['ACTOR.RUN.SUCCEEDED'];+ const expectedRequestUrl = 'http://example.com/api';++ const webhooksMock = sinon.mock(Apify.client.webhooks);+ webhooksMock.expects('createWebhook').never();++ const logMock = sinon.mock(log);+ logMock.expects('warning').once();++ await Apify.webhook({ eventTypes: expectedEventTypes, requestUrl: expectedRequestUrl });++ webhooksMock.verify();+ webhooksMock.restore();+ logMock.verify();+ logMock.restore();+ });});