🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Script validation (#837)
Goldflag1c655863aed8
diff --git a/client/src/api/admin/endpoints/index.ts b/client/src/api/admin/endpoints/index.tsindex 20d22e7ac..80f892feb 100644--- a/client/src/api/admin/endpoints/index.ts+++ b/client/src/api/admin/endpoints/index.ts@@ -7,8 +7,9 @@ export {fetchSitesFromOrg,fetchSiteHasData,fetchSiteIsPublic,+ verifyScript,} from "./sites";-export type { SiteResponse, GetSitesFromOrgResponse } from "./sites";+export type { SiteResponse, GetSitesFromOrgResponse, VerifyScriptResponse } from "./sites";// Organizations endpointsexport {diff --git a/client/src/api/admin/endpoints/sites.ts b/client/src/api/admin/endpoints/sites.tsindex f82c099ef..1e7736aa0 100644--- a/client/src/api/admin/endpoints/sites.ts+++ b/client/src/api/admin/endpoints/sites.ts@@ -144,3 +144,14 @@ export function fetchSiteHasData(siteId: string) {export function fetchSiteIsPublic(siteId: string | number) {return authedFetch<{ isPublic: boolean }>(`/sites/${siteId}/is-public`);}++export interface VerifyScriptResponse {+ scriptTagFound: boolean;+ scriptExecuted: boolean;+ siteIdMatch: boolean;+ issues: string[];+}++export function verifyScript(siteId: number | string) {+ return authedFetch<VerifyScriptResponse>(`/sites/${siteId}/verify-script`);+}diff --git a/client/src/api/admin/hooks/useSites.ts b/client/src/api/admin/hooks/useSites.tsindex 0130c6a09..298b947f8 100644--- a/client/src/api/admin/hooks/useSites.ts+++ b/client/src/api/admin/hooks/useSites.ts@@ -1,14 +1,15 @@-import { useQuery } from "@tanstack/react-query";+import { useMutation, useQuery } from "@tanstack/react-query";import { usePathname } from "next/navigation";-import { useStore } from "../../../lib/store";import { authClient } from "../../../lib/auth";+import { useStore } from "../../../lib/store";import {- fetchSitesFromOrg,fetchSite,fetchSiteHasData,fetchSiteIsPublic,+ fetchSitesFromOrg,GetSitesFromOrgResponse,- SiteResponse,+ verifyScript,+ VerifyScriptResponse} from "../endpoints";export function useGetSitesFromOrg(organizationId?: string) {@@ -77,6 +78,12 @@ export function useGetSiteIsPublic(siteId?: string | number) {});}+export function useVerifyScript() {+ return useMutation<VerifyScriptResponse, Error, number | string>({+ mutationFn: (siteId: number | string) => verifyScript(siteId),+ });+}+export const useCurrentSite = () => {const { data: activeOrganization } = authClient.useActiveOrganization();const { data: sites } = useGetSitesFromOrg(activeOrganization?.id);diff --git a/client/src/app/[site]/components/Header/NoData.tsx b/client/src/app/[site]/components/Header/NoData.tsxindex cc73e9f3b..054f02fee 100644--- a/client/src/app/[site]/components/Header/NoData.tsx+++ b/client/src/app/[site]/components/Header/NoData.tsx@@ -37,6 +37,7 @@ import React from "react";import { useGetSite, useSiteHasData } from "../../../../api/admin/hooks/useSites";import { CodeSnippet } from "../../../../components/CodeSnippet";import { Alert } from "../../../../components/ui/alert";+import { VerifyInstallation } from "../../../../components/VerifyInstallation";import { useStore } from "../../../../lib/store";// Custom Card Component@@ -100,6 +101,7 @@ export function NoData() {</a>{" "}if your script isn't sending traffic.</span>+ {siteMetadata?.siteId && <VerifyInstallation siteId={siteMetadata.siteId} />}{/* Framework Guide Cards */}<div className=""><h2 className="text-sm font-medium mb-4">Platform Guides</h2>diff --git a/client/src/components/SiteSettings/ScriptBuilder.tsx b/client/src/components/SiteSettings/ScriptBuilder.tsxindex 93d09e6c4..66f92a078 100644--- a/client/src/components/SiteSettings/ScriptBuilder.tsx+++ b/client/src/components/SiteSettings/ScriptBuilder.tsx@@ -1,6 +1,7 @@"use client";import { CodeSnippet } from "@/components/CodeSnippet";+import { VerifyInstallation } from "@/components/VerifyInstallation";import { Input } from "@/components/ui/input";import { Label } from "@/components/ui/label";import { Textarea } from "@/components/ui/textarea";@@ -73,21 +74,18 @@ export function ScriptBuilder({ siteId }: ScriptBuilderProps) {// Generate tracking script dynamically based on optionsconst trackingScript = `<scriptsrc="${globalThis.location.origin}/api/script.js"- data-site-id="${siteId}"${- debounceValue !== 500- ? `+ data-site-id="${siteId}"${debounceValue !== 500+ ? `data-debounce="${debounceValue}"`- : ""- }${- skipPatterns.length > 0- ? `+ : ""+ }${skipPatterns.length > 0+ ? `data-skip-patterns='${JSON.stringify(skipPatterns)}'`- : ""- }${- maskPatterns.length > 0- ? `+ : ""+ }${maskPatterns.length > 0+ ? `data-mask-patterns='${JSON.stringify(maskPatterns)}'`- : ""+ : ""}defer></script>`;@@ -103,6 +101,8 @@ export function ScriptBuilder({ siteId }: ScriptBuilderProps) {</div><CodeSnippet language="HTML" code={trackingScript} />+ <VerifyInstallation siteId={siteId} />+{/* Script Options Section */}<div className="space-y-4">{/* Skip Patterns Option */}diff --git a/client/src/components/VerifyInstallation.tsx b/client/src/components/VerifyInstallation.tsxnew file mode 100644index 000000000..4beb2ce8d--- /dev/null+++ b/client/src/components/VerifyInstallation.tsx@@ -0,0 +1,62 @@+"use client";++import { useVerifyScript } from "@/api/admin/hooks/useSites";+import { CheckCircle, AlertTriangle, XCircle, Loader2 } from "lucide-react";+import { Button } from "./ui/button";++interface VerifyInstallationProps {+ siteId: number | string;+}++export function VerifyInstallation({ siteId }: VerifyInstallationProps) {+ const { mutate: verify, data: verifyResult, isPending: isVerifying, reset: resetVerify } = useVerifyScript();++ return (+ <div className="flex flex-col gap-2">+ <Button+ onClick={() => {+ resetVerify();+ verify(siteId);+ }}+ disabled={isVerifying}+ >+ {isVerifying ? (+ <>+ <Loader2 className="w-3.5 h-3.5 animate-spin" />+ Checking installation...+ </>+ ) : (+ "Verify Installation"+ )}+ </Button>+ {verifyResult && (+ <div className="rounded-md border p-3 text-xs space-y-2">+ {verifyResult.scriptTagFound && verifyResult.scriptExecuted && verifyResult.siteIdMatch ? (+ <div className="flex text-sm items-center gap-2 text-green-600 dark:text-green-400">+ <CheckCircle className="w-4 h-4" />+ <span className="font-medium">Script is installed and running correctly.</span>+ </div>+ ) : verifyResult.scriptTagFound && !verifyResult.scriptExecuted ? (+ <div className="flex text-sm items-center gap-2 text-amber-600 dark:text-amber-400">+ <AlertTriangle className="w-4 h-4" />+ <span className="font-medium">Script tag found but not executing</span>+ </div>+ ) : (+ <div className="flex text-sm items-center gap-2 text-red-600 dark:text-red-400">+ <XCircle className="w-4 h-4" />+ <span className="font-medium">Script not detected</span>+ </div>+ )}+ {verifyResult.issues.length > 0 && (+ <ul className="list-disc pl-5 text-muted-foreground space-y-1">+ {verifyResult.issues.map((issue, i) => (+ <li key={i}>{issue}</li>+ ))}+ </ul>+ )}++ </div>+ )}+ </div>+ );+}diff --git a/docker-compose.cloud.yml b/docker-compose.cloud.ymlindex f557c91e1..86190866c 100644--- a/docker-compose.cloud.yml+++ b/docker-compose.cloud.yml@@ -119,7 +119,6 @@ services:- R2_ACCESS_KEY_ID=${R2_ACCESS_KEY_ID}- R2_SECRET_ACCESS_KEY=${R2_SECRET_ACCESS_KEY}- R2_BUCKET_NAME=${R2_BUCKET_NAME:-rybbit}- # Redis configuration- REDIS_HOST=redis- REDIS_PORT=6379- REDIS_PASSWORD=${REDIS_PASSWORD:-changeme}@@ -127,6 +126,8 @@ services:- TURNSTILE_SECRET_KEY=${TURNSTILE_SECRET_KEY}- IPAPI_KEY=${IPAPI_KEY}- GOOGLE_REDIRECT_URI=${GOOGLE_REDIRECT_URI}+ - OPENROUTER_API_KEY=${OPENROUTER_API_KEY}+ - OPENROUTER_MODEL=${OPENROUTER_MODEL}depends_on:clickhouse:condition: service_healthydiff --git a/server/src/api/sites/index.ts b/server/src/api/sites/index.tsindex 3476b3ebc..cfbd2bff7 100644--- a/server/src/api/sites/index.ts+++ b/server/src/api/sites/index.ts@@ -19,3 +19,4 @@ export { getSiteImports } from "./getSiteImports.js";export { createSiteImport } from "./createSiteImport.js";export { batchImportEvents } from "./batchImportEvents.js";export { deleteSiteImport } from "./deleteSiteImport.js";+export { verifyScript } from "./verifyScript.js";diff --git a/server/src/api/sites/verifyScript.ts b/server/src/api/sites/verifyScript.tsnew file mode 100644index 000000000..963d88aa7--- /dev/null+++ b/server/src/api/sites/verifyScript.ts@@ -0,0 +1,139 @@+import { eq } from "drizzle-orm";+import { FastifyReply, FastifyRequest } from "fastify";+import puppeteer from "puppeteer";+import { db } from "../../db/postgres/postgres.js";+import { sites } from "../../db/postgres/schema.js";++interface VerifyScriptParams {+ Params: {+ siteId: string;+ };+}++export interface VerifyScriptResponse {+ scriptTagFound: boolean;+ scriptExecuted: boolean;+ siteIdMatch: boolean;+ issues: string[];+}++export async function verifyScript(+ request: FastifyRequest<VerifyScriptParams>,+ reply: FastifyReply+) {+ const { siteId } = request.params;+ const numericSiteId = Number(siteId);++ try {+ const site = await db.query.sites.findFirst({+ where: isNaN(numericSiteId)+ ? eq(sites.id, siteId)+ : eq(sites.siteId, numericSiteId),+ });++ if (!site) {+ return reply.status(404).send({ error: "Site not found" });+ }++ const domain = site.domain;+ const url = domain.startsWith("http") ? domain : `https://${domain}`;++ const issues: string[] = [];+ let scriptTagFound = false;+ let scriptExecuted = false;+ let siteIdMatch = false;++ let browser;+ try {+ browser = await puppeteer.launch({+ headless: true,+ args: ["--no-sandbox", "--disable-setuid-sandbox"],+ });++ const page = await browser.newPage();++ // Set a reasonable timeout and user agent+ page.setDefaultNavigationTimeout(15000);+ await page.setUserAgent({+ userAgent:+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",+ });++ try {+ await page.goto(url, { waitUntil: "networkidle2", timeout: 15000 });+ } catch (navError: any) {+ issues.push(+ `Could not load ${url}: ${navError.message || "Navigation failed"}`+ );+ return reply.status(200).send({+ scriptTagFound: false,+ scriptExecuted: false,+ siteIdMatch: false,+ issues,+ } satisfies VerifyScriptResponse);+ }++ // Check for script tag in the DOM+ const scriptCheck = await page.evaluate((expectedSiteId: string) => {+ const scripts = document.querySelectorAll("script");+ let found = false;+ let siteIdCorrect = false;+ let foundSiteId: string | null = null;++ for (const script of scripts) {+ const src = script.getAttribute("src") || "";+ if (src.includes("script.js") || src.includes("rybbit")) {+ found = true;+ foundSiteId = script.getAttribute("data-site-id");+ if (foundSiteId === expectedSiteId) {+ siteIdCorrect = true;+ }+ break;+ }+ }++ return { found, siteIdCorrect, foundSiteId };+ }, site.id!);++ scriptTagFound = scriptCheck.found;+ siteIdMatch = scriptCheck.siteIdCorrect;++ if (!scriptTagFound) {+ issues.push(+ "Rybbit script tag not found in the page HTML. Make sure the script is placed in the <head> tag."+ );+ } else if (!siteIdMatch) {+ issues.push(+ `Script tag found but data-site-id is "${scriptCheck.foundSiteId}" instead of "${site.id}".`+ );+ }++ // Wait a moment for the script to execute, then check for the global+ await new Promise((resolve) => setTimeout(resolve, 2000));++ scriptExecuted = await page.evaluate(() => {+ return typeof (window as any).rybbit !== "undefined";+ });++ if (scriptTagFound && !scriptExecuted) {+ issues.push(+ "Script tag is present but the rybbit object was not found on window. The script may be blocked by a Content Security Policy, ad blocker, or the src URL may be incorrect."+ );+ }++ return reply.status(200).send({+ scriptTagFound,+ scriptExecuted,+ siteIdMatch,+ issues,+ } satisfies VerifyScriptResponse);+ } finally {+ if (browser) {+ await browser.close();+ }+ }+ } catch (error) {+ console.error("Error verifying script:", error);+ return reply.status(500).send({ error: "Internal server error" });+ }+}diff --git a/server/src/db/geolocation/geolocation.ts b/server/src/db/geolocation/geolocation.tsindex 97d503581..bf0146203 100644--- a/server/src/db/geolocation/geolocation.ts+++ b/server/src/db/geolocation/geolocation.ts@@ -133,7 +133,7 @@ async function getLocationFromIPAPI(ips: string[]): Promise<Record<string, Locatreturn results;}- logger.info(`Cache hit: ${ips.length - uncachedIps.length}/${ips.length}, fetching ${uncachedIps.length} from IPAPI`);+ // logger.info(`Cache hit: ${ips.length - uncachedIps.length}/${ips.length}, fetching ${uncachedIps.length} from IPAPI`);const localInfo = await getLocationFromLocal(uncachedIps);diff --git a/server/src/index.ts b/server/src/index.tsindex d3e80fe3c..8a8f1bbb4 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -82,6 +82,7 @@ import {getTrackingConfig,updateSiteConfig,updateSitePrivateLinkConfig,+ verifyScript,} from "./api/sites/index.js";import {createCheckoutSession,@@ -284,6 +285,7 @@ async function sitesRoutes(fastify: FastifyInstance) {fastify.get("/site/tracking-config/:siteId", getTrackingConfig); // Public - used by tracking scriptfastify.get("/sites/:siteId/excluded-ips", authSite, getSiteExcludedIPs);fastify.get("/sites/:siteId/excluded-countries", authSite, getSiteExcludedCountries);+ fastify.get("/sites/:siteId/verify-script", authSite, verifyScript);// Site Importsfastify.get("/sites/:siteId/imports", adminSite, getSiteImports);diff --git a/server/src/lib/openrouter.ts b/server/src/lib/openrouter.tsnew file mode 100644index 000000000..0da4d5e70--- /dev/null+++ b/server/src/lib/openrouter.ts@@ -0,0 +1,57 @@+const OPENROUTER_API_URL = "https://openrouter.ai/api/v1/chat/completions";++interface OpenRouterResponse {+ id: string;+ choices: Array<{+ message: {+ role: string;+ content: string;+ };+ finish_reason: string;+ }>;+}++export async function callOpenRouter(+ messages: Array<{ role: "system" | "user" | "assistant"; content: string }>,+ options?: {+ temperature?: number;+ maxTokens?: number;+ model?: string;+ }+): Promise<string> {+ const apiKey = process.env.OPENROUTER_API_KEY;+ const model = options?.model || process.env.OPENROUTER_MODEL || "moonshotai/kimi-k2.5";++ if (!apiKey) {+ throw new Error("OPENROUTER_API_KEY is not configured");+ }++ const response = await fetch(OPENROUTER_API_URL, {+ method: "POST",+ headers: {+ Authorization: `Bearer ${apiKey}`,+ "Content-Type": "application/json",+ "HTTP-Referer": "https://rybbit.com",+ "X-Title": "Rybbit Analytics",+ },+ body: JSON.stringify({+ model,+ messages,+ temperature: options?.temperature ?? 0.3,+ max_tokens: options?.maxTokens ?? 1000,+ }),+ });++ if (!response.ok) {+ const error = await response.text();+ throw new Error(`OpenRouter API error: ${response.status} - ${error}`);+ }++ const data = (await response.json()) as OpenRouterResponse;++ if (!data.choices || data.choices.length === 0) {+ throw new Error("No response from OpenRouter");+ }++ return data.choices[0].message.content;+}diff --git a/server/src/services/tracker/identifyService.ts b/server/src/services/tracker/identifyService.tsindex 49e3c2a8c..bb1940c06 100644--- a/server/src/services/tracker/identifyService.ts+++ b/server/src/services/tracker/identifyService.ts@@ -80,15 +80,15 @@ export async function handleIdentify(request: FastifyRequest, reply: FastifyRepl});} else if (existingAlias[0].userId !== user_id) {// Anonymous ID already linked to a different user - log but don't error- logger.warn(- {- siteId,- anonymousId,- existingUserId: existingAlias[0].userId,- newUserId: user_id,- },- "Anonymous ID already linked to different user"- );+ // logger.warn(+ // {+ // siteId,+ // anonymousId,+ // existingUserId: existingAlias[0].userId,+ // newUserId: user_id,+ // },+ // "Anonymous ID already linked to different user"+ // );}} catch (error) {// Handle unique constraint violation gracefully (race condition)diff --git a/server/src/services/tracker/pageviewQueue.ts b/server/src/services/tracker/pageviewQueue.tsindex e0716a5fa..6dd55cd94 100644--- a/server/src/services/tracker/pageviewQueue.ts+++ b/server/src/services/tracker/pageviewQueue.ts@@ -118,7 +118,7 @@ class PageviewQueue {};});- this.logger.info({ count: processedPageviews.length }, "Bulk insert to ClickHouse");+ // this.logger.info({ count: processedPageviews.length }, "Bulk insert to ClickHouse");// Bulk insert into databasetry {await clickhouse.insert({diff --git a/server/src/services/tracker/trackEvent.ts b/server/src/services/tracker/trackEvent.tsindex e3f469553..65ccfef7d 100644--- a/server/src/services/tracker/trackEvent.ts+++ b/server/src/services/tracker/trackEvent.ts@@ -256,7 +256,7 @@ export async function trackEvent(request: FastifyRequest, reply: FastifyReply) {// Get the site configuration to get the numeric siteIdconst siteConfiguration = await siteConfig.getConfig(validatedPayload.site_id);if (!siteConfiguration) {- logger.warn({ siteId: validatedPayload.site_id }, "Site not found");+ // logger.warn({ siteId: validatedPayload.site_id }, "Site not found");return reply.status(404).send({success: false,error: "Site not found",@@ -271,7 +271,7 @@ export async function trackEvent(request: FastifyRequest, reply: FastifyReply) {// Use custom user agent if provided, otherwise fall back to headerconst userAgent = validatedPayload.user_agent || (request.headers["user-agent"] as string);if (userAgent && isbot(userAgent)) {- logger.info({ siteId: validatedPayload.site_id, userAgent }, "Bot request filtered");+ // logger.info({ siteId: validatedPayload.site_id, userAgent }, "Bot request filtered");return reply.status(200).send({success: true,message: "Event not tracked - bot detected",@@ -281,7 +281,7 @@ export async function trackEvent(request: FastifyRequest, reply: FastifyReply) {// Check if the site has exceeded its monthly limit (using numeric siteId)if (usageService.isSiteOverLimit(siteConfiguration.siteId)) {- logger.info({ siteId: validatedPayload.site_id }, "Skipping event - site over monthly limit");+ // logger.info({ siteId: validatedPayload.site_id }, "Skipping event - site over monthly limit");return reply.status(200).send("Site over monthly limit, event not tracked");}@@ -292,7 +292,7 @@ export async function trackEvent(request: FastifyRequest, reply: FastifyReply) {if (siteConfiguration.excludedIPs && siteConfiguration.excludedIPs.length > 0) {const isExcluded = await siteConfig.isIPExcluded(requestIP, validatedPayload.site_id);if (isExcluded) {- logger.info({ siteId: validatedPayload.site_id, ip: requestIP }, "IP excluded from tracking");+ // logger.info({ siteId: validatedPayload.site_id, ip: requestIP }, "IP excluded from tracking");return reply.status(200).send({success: true,message: "Event not tracked - IP excluded",