Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: Add Segment analytics [internal] (#2960)

Jan Turoňea356853711c
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 38af5d786..ed5b432dd 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -41,6 +41,7 @@ jobs:
yarn build
env:
APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }}
+ SEGMENT_TOKEN: ${{ secrets.SEGMENT_TOKEN }}
- name: Set up GitHub Pages
uses: actions/configure-pages@v5
diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.yml
index af6d32a2a..68a3b39e1 100644
--- a/.github/workflows/test-ci.yml
+++ b/.github/workflows/test-ci.yml
@@ -122,6 +122,7 @@ jobs:
yarn build
env:
APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }}
+ SEGMENT_TOKEN: ${{ secrets.SEGMENT_TOKEN }}
lint:
name: Lint
diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js
index a4a70d68d..7797a1f31 100644
--- a/website/docusaurus.config.js
+++ b/website/docusaurus.config.js
@@ -1,4 +1,5 @@
/* eslint-disable global-require */
+const path = require('path');
const { externalLinkProcessor } = require('./tools/utils/externalLink');
const packages = [
@@ -209,6 +210,13 @@ module.exports = {
},
};
},
+ [
+ path.resolve(__dirname, 'src/plugins/docusaurus-plugin-segment'),
+ {
+ writeKey: process.env.SEGMENT_TOKEN,
+ allowedInDev: false,
+ },
+ ],
],
themeConfig:
/** @type {import('@docusaurus/preset-classic').ThemeConfig} */ ({
diff --git a/website/src/plugins/docusaurus-plugin-segment/index.js b/website/src/plugins/docusaurus-plugin-segment/index.js
new file mode 100644
index 000000000..9a9929a52
--- /dev/null
+++ b/website/src/plugins/docusaurus-plugin-segment/index.js
@@ -0,0 +1,39 @@
+const path = require('path');
+
+module.exports = function (context, options) {
+ const { writeKey, allowedInDev = false } = options;
+
+ return {
+ name: 'docusaurus-plugin-segment',
+
+ getClientModules() {
+ return [path.resolve(__dirname, './segment')];
+ },
+
+ injectHtmlTags() {
+ if (!writeKey) {
+ throw new Error(
+ 'You need to specify a Segment writeKey in the plugin options'
+ );
+ }
+
+ if (process.env.NODE_ENV !== 'production' && !allowedInDev) {
+ return {};
+ }
+
+ return {
+ headTags: [
+ {
+ tagName: 'script',
+ innerHTML: `
+ !function(){var i="analytics",analytics=window[i]=window[i]||[];if(!analytics.initialize)if(analytics.invoked)window.console&&console.error&&console.error("Segment snippet included twice.");else{analytics.invoked=!0;analytics.methods=["trackSubmit","trackClick","trackLink","trackForm","pageview","identify","reset","group","track","ready","alias","debug","page","screen","once","off","on","addSourceMiddleware","addIntegrationMiddleware","setAnonymousId","addDestinationMiddleware","register"];analytics.factory=function(e){return function(){if(window[i].initialized)return window[i][e].apply(window[i],arguments);var n=Array.prototype.slice.call(arguments);if(["track","screen","alias","group","page","identify"].indexOf(e)>-1){var c=document.querySelector("link[rel='canonical']");n.push({__t:"bpc",c:c&&c.getAttribute("href")||void 0,p:location.pathname,u:location.href,s:location.search,t:document.title,r:document.referrer})}n.unshift(e);analytics.push(n);return analytics}};for(var n=0;n<analytics.methods.length;n++){var key=analytics.methods[n];analytics[key]=analytics.factory(key)}analytics.load=function(key,n){var t=document.createElement("script");t.type="text/javascript";t.async=!0;t.setAttribute("data-global-segment-analytics-key",i);t.src="https://cdn.segment.com/analytics.js/v1/" + key + "/analytics.min.js";var r=document.getElementsByTagName("script")[0];r.parentNode.insertBefore(t,r);analytics._loadOptions=n};analytics._writeKey="${writeKey}";;analytics.SNIPPET_VERSION="5.2.0";
+ analytics.load("${writeKey}", { integrations: { "Segment.io": { apiHost: "analytics.apify.com/v1" } } });
+ analytics.page({ app: 'crawlee', pageType: 'CRAWLEE_PAGE', page_type: 'CRAWLEE_PAGE' });
+ }}();
+ `,
+ },
+ ],
+ };
+ },
+ };
+};
diff --git a/website/src/plugins/docusaurus-plugin-segment/segment.js b/website/src/plugins/docusaurus-plugin-segment/segment.js
new file mode 100644
index 000000000..8c16cd6e8
--- /dev/null
+++ b/website/src/plugins/docusaurus-plugin-segment/segment.js
@@ -0,0 +1,14 @@
+import ExecutionEnvironment from '@docusaurus/ExecutionEnvironment';
+
+export default ExecutionEnvironment.canUseDOM ? {
+ onRouteUpdate({ location }) {
+ // Don't track page views on development
+ if (process.env.NODE_ENV === 'production' && window.analytics) {
+ window.analytics.page({
+ path: location.pathname,
+ url: location.href,
+ search: location.search,
+ });
+ }
+ },
+} : null;