Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: Add Segment analytics [internal] (#2960)
Jan Turoňea356853711c
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.ymlindex 38af5d786..ed5b432dd 100644--- a/.github/workflows/docs.yml+++ b/.github/workflows/docs.yml@@ -41,6 +41,7 @@ jobs:yarn buildenv:APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }}+ SEGMENT_TOKEN: ${{ secrets.SEGMENT_TOKEN }}- name: Set up GitHub Pagesuses: actions/configure-pages@v5diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.ymlindex af6d32a2a..68a3b39e1 100644--- a/.github/workflows/test-ci.yml+++ b/.github/workflows/test-ci.yml@@ -122,6 +122,7 @@ jobs:yarn buildenv:APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }}+ SEGMENT_TOKEN: ${{ secrets.SEGMENT_TOKEN }}lint:name: Lintdiff --git a/website/docusaurus.config.js b/website/docusaurus.config.jsindex a4a70d68d..7797a1f31 100644--- a/website/docusaurus.config.js+++ b/website/docusaurus.config.js@@ -1,4 +1,5 @@/* eslint-disable global-require */+const path = require('path');const { externalLinkProcessor } = require('./tools/utils/externalLink');const packages = [@@ -209,6 +210,13 @@ module.exports = {},};},+ [+ path.resolve(__dirname, 'src/plugins/docusaurus-plugin-segment'),+ {+ writeKey: process.env.SEGMENT_TOKEN,+ allowedInDev: false,+ },+ ],],themeConfig:/** @type {import('@docusaurus/preset-classic').ThemeConfig} */ ({diff --git a/website/src/plugins/docusaurus-plugin-segment/index.js b/website/src/plugins/docusaurus-plugin-segment/index.jsnew file mode 100644index 000000000..9a9929a52--- /dev/null+++ b/website/src/plugins/docusaurus-plugin-segment/index.js@@ -0,0 +1,39 @@+const path = require('path');++module.exports = function (context, options) {+ const { writeKey, allowedInDev = false } = options;++ return {+ name: 'docusaurus-plugin-segment',++ getClientModules() {+ return [path.resolve(__dirname, './segment')];+ },++ injectHtmlTags() {+ if (!writeKey) {+ throw new Error(+ 'You need to specify a Segment writeKey in the plugin options'+ );+ }++ if (process.env.NODE_ENV !== 'production' && !allowedInDev) {+ return {};+ }++ return {+ headTags: [+ {+ tagName: 'script',+ innerHTML: `+ !function(){var i="analytics",analytics=window[i]=window[i]||[];if(!analytics.initialize)if(analytics.invoked)window.console&&console.error&&console.error("Segment snippet included twice.");else{analytics.invoked=!0;analytics.methods=["trackSubmit","trackClick","trackLink","trackForm","pageview","identify","reset","group","track","ready","alias","debug","page","screen","once","off","on","addSourceMiddleware","addIntegrationMiddleware","setAnonymousId","addDestinationMiddleware","register"];analytics.factory=function(e){return function(){if(window[i].initialized)return window[i][e].apply(window[i],arguments);var n=Array.prototype.slice.call(arguments);if(["track","screen","alias","group","page","identify"].indexOf(e)>-1){var c=document.querySelector("link[rel='canonical']");n.push({__t:"bpc",c:c&&c.getAttribute("href")||void 0,p:location.pathname,u:location.href,s:location.search,t:document.title,r:document.referrer})}n.unshift(e);analytics.push(n);return analytics}};for(var n=0;n<analytics.methods.length;n++){var key=analytics.methods[n];analytics[key]=analytics.factory(key)}analytics.load=function(key,n){var t=document.createElement("script");t.type="text/javascript";t.async=!0;t.setAttribute("data-global-segment-analytics-key",i);t.src="https://cdn.segment.com/analytics.js/v1/" + key + "/analytics.min.js";var r=document.getElementsByTagName("script")[0];r.parentNode.insertBefore(t,r);analytics._loadOptions=n};analytics._writeKey="${writeKey}";;analytics.SNIPPET_VERSION="5.2.0";+ analytics.load("${writeKey}", { integrations: { "Segment.io": { apiHost: "analytics.apify.com/v1" } } });+ analytics.page({ app: 'crawlee', pageType: 'CRAWLEE_PAGE', page_type: 'CRAWLEE_PAGE' });+ }}();+ `,+ },+ ],+ };+ },+ };+};diff --git a/website/src/plugins/docusaurus-plugin-segment/segment.js b/website/src/plugins/docusaurus-plugin-segment/segment.jsnew file mode 100644index 000000000..8c16cd6e8--- /dev/null+++ b/website/src/plugins/docusaurus-plugin-segment/segment.js@@ -0,0 +1,14 @@+import ExecutionEnvironment from '@docusaurus/ExecutionEnvironment';++export default ExecutionEnvironment.canUseDOM ? {+ onRouteUpdate({ location }) {+ // Don't track page views on development+ if (process.env.NODE_ENV === 'production' && window.analytics) {+ window.analytics.page({+ path: location.pathname,+ url: location.href,+ search: location.search,+ });+ }+ },+} : null;