🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
fix tracking events (#589)
Bill Yang9a970ff4ccf8
diff --git a/server/src/api/sessionReplay/recordSessionReplay.ts b/server/src/api/sessionReplay/recordSessionReplay.tsindex a094de97a..f6466a0e1 100644--- a/server/src/api/sessionReplay/recordSessionReplay.ts+++ b/server/src/api/sessionReplay/recordSessionReplay.ts@@ -36,7 +36,11 @@ export async function recordSessionReplay(reply: FastifyReply) {try {- const siteId = Number(request.params.site);+ // Get the site configuration to get the numeric siteId+ const siteId = (await siteConfig.getSiteConfig(request.params.site))?.siteId;+ if (!siteId) {+ throw new Error(`Site not found: ${request.params.site}`);+ }// Check if the site has exceeded its monthly limitif (usageService.isSiteOverLimit(Number(siteId))) {@@ -73,19 +77,6 @@ export async function recordSessionReplay(// }// }- // If no valid API key, validate origin - disabled for now- // if (!apiKeyValidation.success) {- // const originValidation = await validateOrigin(siteId, request.headers.origin as string);-- // if (!originValidation.success) {- // logger.warn(`[SessionReplay] Request rejected for site ${siteId}: ${originValidation.error}`);- // return reply.status(403).send({- // success: false,- // error: originValidation.error,- // });- // }- // }-// Check if the IP should be excluded from trackingconst requestIP = getIpAddress(request);@@ -117,6 +108,6 @@ export async function recordSessionReplay(return reply.status(400).send({ error: error.errors });}logger.error(error as Error, "Error recording session replay");- return reply.status(500).send({ error: "Internal server error" });+ return reply.status(500).send({ error });}}diff --git a/server/src/services/replay/sessionReplayIngestService.ts b/server/src/services/replay/sessionReplayIngestService.tsindex 352efa94d..b3df28302 100644--- a/server/src/services/replay/sessionReplayIngestService.ts+++ b/server/src/services/replay/sessionReplayIngestService.ts@@ -6,6 +6,7 @@ import { parseTrackingData } from "./trackingUtils.js";import { sessionsService } from "../sessions/sessionsService.js";import { userIdService } from "../userId/userIdService.js";import { r2Storage } from "../storage/r2StorageService.js";+import { siteConfig } from "../../lib/siteConfig.js";export interface RequestMetadata {userAgent: string;@@ -35,7 +36,7 @@ export class SessionReplayIngestService {// Get or create a session ID from the sessions serviceconst { sessionId } = await sessionsService.updateSession({userId,- site_id: siteId.toString(),+ siteId,});// Check if R2 storage is enabled for cloud deploymentsdiff --git a/server/src/services/sessions/sessionsService.ts b/server/src/services/sessions/sessionsService.tsindex 84b160ea9..78394062e 100644--- a/server/src/services/sessions/sessionsService.ts+++ b/server/src/services/sessions/sessionsService.ts@@ -25,20 +25,18 @@ class SessionsService {this.logger.info("Session cleanup cron initialized (runs every minute)");}- async getExistingSession(userId: string, siteId: string) {- const siteIdNumber = parseInt(siteId, 10);-+ async getExistingSession(userId: string, siteId: number) {const [existingSession] = await db.select().from(activeSessions)- .where(and(eq(activeSessions.userId, userId), eq(activeSessions.siteId, siteIdNumber)))+ .where(and(eq(activeSessions.userId, userId), eq(activeSessions.siteId, siteId))).limit(1);return existingSession || null;}- async updateSession(payload: { userId: string; site_id: string }): Promise<{ sessionId: string }> {- const existingSession = await this.getExistingSession(payload.userId, payload.site_id);+ async updateSession({ userId, siteId }: { userId: string; siteId: number }): Promise<{ sessionId: string }> {+ const existingSession = await this.getExistingSession(userId, siteId);if (existingSession) {await db@@ -50,11 +48,10 @@ class SessionsService {return { sessionId: existingSession.sessionId };}- // Insert new session with Drizzle - only include columns that exist in schemaconst insertData = {sessionId: crypto.randomUUID(),- siteId: typeof payload.site_id === "string" ? parseInt(payload.site_id, 10) : payload.site_id,- userId: payload.userId,+ siteId,+ userId,startTime: new Date(),lastActivity: new Date(),};diff --git a/server/src/services/shared/requestValidation.test.ts b/server/src/services/shared/requestValidation.test.tsindex 497047b32..ec6c93d58 100644--- a/server/src/services/shared/requestValidation.test.ts+++ b/server/src/services/shared/requestValidation.test.ts@@ -1,4 +1,4 @@-import { describe, it, expect, vi, beforeEach } from "vitest";+import { beforeEach, describe, expect, it, vi } from "vitest";// Define the expected site config type to match SiteConfigDatainterface SiteConfigData {@@ -21,7 +21,6 @@ vi.mock("../../lib/rateLimiter.js", () => ({vi.mock("../../lib/siteConfig.js", () => ({siteConfig: {-getSiteConfig: vi.fn(),},}));@@ -30,12 +29,11 @@ vi.mock("../../utils.js", () => ({normalizeOrigin: vi.fn(),}));-import { validateApiKey, validateOrigin, checkApiKeyRateLimit } from "./requestValidation.js";+import { checkApiKeyRateLimit, validateApiKey } from "./requestValidation.js";// Import mocked modulesimport { apiKeyRateLimiter } from "../../lib/rateLimiter.js";import { siteConfig } from "../../lib/siteConfig.js";-import { normalizeOrigin } from "../../utils.js";describe("validateApiKey", () => {beforeEach(() => {@@ -53,12 +51,10 @@ describe("validateApiKey", () => {});it("should return success false when site is not found", async () => {-vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(undefined);const result = await validateApiKey(1, "test-api-key");-expect(siteConfig.getSiteConfig).toHaveBeenCalledWith(1);expect(result).toEqual({ success: false, error: "Site not found" });});@@ -74,7 +70,7 @@ describe("validateApiKey", () => {blockBots: true,excludedIPs: [],};-+vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(mockSite);// Mock console.info to avoid output during tests@@ -99,7 +95,7 @@ describe("validateApiKey", () => {blockBots: true,excludedIPs: [],};-+vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(mockSite);const result = await validateApiKey(1, "invalid-api-key");@@ -118,7 +114,7 @@ describe("validateApiKey", () => {blockBots: true,excludedIPs: [],};-+vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(mockSite);await validateApiKey("123", "test-key");@@ -137,7 +133,7 @@ describe("validateApiKey", () => {blockBots: true,excludedIPs: [],}; // No apiKey property-+vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(mockSite as SiteConfigData);const result = await validateApiKey(1, "any-key");@@ -159,250 +155,6 @@ describe("validateApiKey", () => {});});-describe("validateOrigin", () => {- beforeEach(() => {- vi.clearAllMocks();- vi.unstubAllEnvs();- });-- it("should return success true when origin checking is disabled", async () => {- // Set environment variable before any imports happen- vi.stubEnv("DISABLE_ORIGIN_CHECK", "true");-- // Re-import the module to pick up the new environment variable- await vi.resetModules();-- // Re-establish the mocks after reset- vi.doMock("../../lib/siteConfig.js", () => ({- siteConfig: {-- getSiteConfig: vi.fn(),- },- }));-- vi.doMock("../../utils.js", () => ({- normalizeOrigin: vi.fn(),- }));-- const { validateOrigin } = await import("./requestValidation.js");-- const consoleSpy = vi.spyOn(console, "info").mockImplementation(() => {});-- const result = await validateOrigin(1, "https://example.com");-- expect(result).toEqual({ success: true });- expect(consoleSpy).toHaveBeenCalledWith(- "[Validation] Origin check disabled. Allowing request for site 1 from origin: https://example.com"- );-- consoleSpy.mockRestore();- vi.unstubAllEnvs();- });-- it("should return success false when site is not found", async () => {-- vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(undefined);-- const result = await validateOrigin(1, "https://example.com");-- expect(result).toEqual({ success: false, error: "Site not found" });- });-- it("should return success false when no origin header is provided", async () => {- const mockSite: SiteConfigData = {- id: "test-id",- siteId: 1,- domain: "example.com",- apiKey: "test-key",- public: true,- saltUserIds: false,- blockBots: true,- excludedIPs: [],- };-- vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(mockSite);-- const result = await validateOrigin(1);-- expect(result).toEqual({ success: false, error: "Origin header required" });- });-- it("should return success true for exact domain match", async () => {- const mockSite: SiteConfigData = {- id: "test-id",- siteId: 1,- domain: "example.com",- apiKey: "test-key",- public: true,- saltUserIds: false,- blockBots: true,- excludedIPs: [],- };-- vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(mockSite);- vi.mocked(normalizeOrigin)- .mockReturnValueOnce("example.com") // for request origin- .mockReturnValueOnce("example.com"); // for site domain-- const result = await validateOrigin(1, "https://example.com");-- expect(normalizeOrigin).toHaveBeenCalledWith("https://example.com");- expect(normalizeOrigin).toHaveBeenCalledWith("https://example.com");- expect(result).toEqual({ success: true });- });-- it("should return success true for subdomain (always allowed)", async () => {- const mockSite: SiteConfigData = {- id: "test-id",- siteId: 1,- domain: "example.com",- apiKey: "test-key",- public: true,- saltUserIds: false,- blockBots: true,- excludedIPs: [],- };-- vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(mockSite);- vi.mocked(normalizeOrigin)- .mockReturnValueOnce("sub.example.com") // for request origin- .mockReturnValueOnce("example.com"); // for site domain-- const result = await validateOrigin(1, "https://sub.example.com");-- expect(result).toEqual({ success: true });- });-- it("should return success true for nested subdomain", async () => {- const mockSite: SiteConfigData = {- id: "test-id",- siteId: 1,- domain: "example.com",- apiKey: "test-key",- public: true,- saltUserIds: false,- blockBots: true,- excludedIPs: [],- };-- vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(mockSite);- vi.mocked(normalizeOrigin)- .mockReturnValueOnce("deep.nested.sub.example.com") // for request origin- .mockReturnValueOnce("example.com"); // for site domain-- const result = await validateOrigin(1, "https://deep.nested.sub.example.com");-- expect(result).toEqual({ success: true });- });-- it("should return success false for different domain", async () => {- const mockSite: SiteConfigData = {- id: "test-id",- siteId: 1,- domain: "example.com",- apiKey: "test-key",- public: true,- saltUserIds: false,- blockBots: true,- excludedIPs: [],- };-- vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(mockSite);- vi.mocked(normalizeOrigin)- .mockReturnValueOnce("different.com") // for request origin- .mockReturnValueOnce("example.com"); // for site domain-- const result = await validateOrigin(1, "https://different.com");-- expect(result).toEqual({- success: false,- error: "Origin mismatch. Received: https://different.com",- });- });-- it("should return success false for domain that contains site domain but is not a subdomain", async () => {- const mockSite: SiteConfigData = {- id: "test-id",- siteId: 1,- domain: "example.com",- apiKey: "test-key",- public: true,- saltUserIds: false,- blockBots: true,- excludedIPs: [],- };-- vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(mockSite);- vi.mocked(normalizeOrigin)- .mockReturnValueOnce("notexample.com") // for request origin- .mockReturnValueOnce("example.com"); // for site domain-- const result = await validateOrigin(1, "https://notexample.com");-- expect(result).toEqual({- success: false,- error: "Origin mismatch. Received: https://notexample.com",- });- });-- it("should handle string siteId by converting to number", async () => {- const mockSite: SiteConfigData = {- id: "test-id",- siteId: 123,- domain: "example.com",- apiKey: "test-key",- public: true,- saltUserIds: false,- blockBots: true,- excludedIPs: [],- };-- vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(mockSite);-- await validateOrigin("123", "https://example.com");-- expect(siteConfig.getSiteConfig).toHaveBeenCalledWith(123);- });-- it("should handle invalid origin format", async () => {- const mockSite: SiteConfigData = {- id: "test-id",- siteId: 1,- domain: "example.com",- apiKey: "test-key",- public: true,- saltUserIds: false,- blockBots: true,- excludedIPs: [],- };-- vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(mockSite);- vi.mocked(normalizeOrigin).mockImplementation(() => {- throw new Error("Invalid URL");- });-- const result = await validateOrigin(1, "invalid-origin");-- expect(result).toEqual({- success: false,- error: "Invalid origin format: invalid-origin",- });- });-- it("should handle errors during validation", async () => {- vi.mocked(siteConfig.getSiteConfig).mockRejectedValue(new Error("Database error"));-- const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {});-- const result = await validateOrigin(1, "https://example.com");-- expect(result).toEqual({ success: false, error: "Internal error validating origin" });- expect(consoleSpy).toHaveBeenCalledWith("Error validating origin:", expect.any(Error));-- consoleSpy.mockRestore();- });-});-describe("checkApiKeyRateLimit", () => {beforeEach(() => {vi.clearAllMocks();diff --git a/server/src/services/shared/requestValidation.ts b/server/src/services/shared/requestValidation.tsindex c8980c546..9c29e7c35 100644--- a/server/src/services/shared/requestValidation.ts+++ b/server/src/services/shared/requestValidation.ts@@ -53,79 +53,6 @@ export async function validateApiKey(siteId: string | number, apiKey?: string):}}-/**- * Validates if the request's origin matches the registered domain for the site- * @param siteId The site ID from the tracking payload- * @param requestOrigin The origin header from the request- * @returns An object with success status and optional error message- */-export async function validateOrigin(siteId: string | number, requestOrigin?: string): Promise<OriginValidationResult> {- try {- // If origin checking is disabled, return success- if (DISABLE_ORIGIN_CHECK) {- logger.info({ siteId, origin: requestOrigin || "none" }, "Origin check disabled, allowing request");- return { success: true };- }-- // Convert siteId to number- const numericSiteId = typeof siteId === "string" ? parseInt(siteId, 10) : siteId;-- // Get the site configuration- const site = await siteConfig.getSiteConfig(numericSiteId);-- if (!site) {- return {- success: false,- error: "Site not found",- };- }-- // If no origin header, reject the request- if (!requestOrigin) {- return {- success: false,- error: "Origin header required",- };- }-- try {- // Get the domain associated with this site- const siteDomain = site.domain;-- // Normalize domains by removing all subdomain prefixes- const normalizedOriginHost = normalizeOrigin(requestOrigin);- const normalizedSiteDomain = normalizeOrigin(`https://${siteDomain}`);-- // Check for exact match first- if (normalizedOriginHost === normalizedSiteDomain) {- return { success: true };- }-- // Always allow subdomains - check if origin is a subdomain of site domain- if (normalizedOriginHost.endsWith(`.${normalizedSiteDomain}`)) {- return { success: true };- }-- // If we get here, neither exact match nor valid subdomain- return {- success: false,- error: `Origin mismatch. Received: ${requestOrigin}`,- };- } catch (error) {- return {- success: false,- error: `Invalid origin format: ${requestOrigin}`,- };- }- } catch (error) {- logger.error(error, "Error validating origin");- return {- success: false,- error: "Internal error validating origin",- };- }-}-/*** Checks if the API key rate limit has been exceeded* @param apiKey The API key to checkdiff --git a/server/src/services/tracker/trackEvent.ts b/server/src/services/tracker/trackEvent.tsindex e804d142e..636dabb0b 100644--- a/server/src/services/tracker/trackEvent.ts+++ b/server/src/services/tracker/trackEvent.ts@@ -4,7 +4,7 @@ import { z, ZodError } from "zod";import { createServiceLogger } from "../../lib/logger/logger.js";import { siteConfig } from "../../lib/siteConfig.js";import { sessionsService } from "../sessions/sessionsService.js";-import { checkApiKeyRateLimit, validateApiKey, validateOrigin } from "../shared/requestValidation.js";+import { checkApiKeyRateLimit, validateApiKey } from "../shared/requestValidation.js";import { usageService } from "../usageService.js";import { pageviewQueue } from "./pageviewQueue.js";import { createBasePayload } from "./utils.js";@@ -243,25 +243,19 @@ export async function trackEvent(request: FastifyRequest, reply: FastifyReply) {}}- // If no valid API key, validate origin - disabled for now- // if (!apiKeyValidation.success) {- // const originValidation = await validateOrigin(validatedPayload.site_id, request.headers.origin as string);-- // if (!originValidation.success) {- // logger.warn(- // { siteId: validatedPayload.site_id, error: originValidation.error },- // "Request rejected - origin validation failed"- // );- // return reply.status(403).send({- // success: false,- // error: originValidation.error,- // });- // }- // }+ // Get the site configuration to get the numeric siteId+ const siteConfiguration = await siteConfig.getSiteConfig(validatedPayload.site_id);+ if (!siteConfiguration) {+ logger.warn({ siteId: validatedPayload.site_id }, "Site not found");+ return reply.status(404).send({+ success: false,+ error: "Site not found",+ });+ }// Check if bot blocking is enabled for this site and if the request is from a bot// Skip bot check for API key authenticated requests- if (!validatedPayload.api_key && (await siteConfig.shouldBlockBots(validatedPayload.site_id))) {+ if (!validatedPayload.api_key && siteConfiguration.blockBots) {// Use custom user agent if provided, otherwise fall back to headerconst userAgent = validatedPayload.user_agent || (request.headers["user-agent"] as string);if (userAgent && isbot(userAgent)) {@@ -273,8 +267,8 @@ export async function trackEvent(request: FastifyRequest, reply: FastifyReply) {}}- // Check if the site has exceeded its monthly limit- if (usageService.isSiteOverLimit(Number(validatedPayload.site_id))) {+ // Check if the site has exceeded its monthly limit (using numeric siteId)+ if (usageService.isSiteOverLimit(siteConfiguration.siteId)) {logger.info({ siteId: validatedPayload.site_id }, "Skipping event - site over monthly limit");return reply.status(200).send("Site over monthly limit, event not tracked");}@@ -283,28 +277,36 @@ export async function trackEvent(request: FastifyRequest, reply: FastifyReply) {// Use custom IP if provided in payload, otherwise get from requestconst requestIP = validatedPayload.ip_address || request.ip || "";- if (await siteConfig.isIPExcluded(requestIP, validatedPayload.site_id)) {- logger.info({ siteId: validatedPayload.site_id, ip: requestIP }, "IP excluded from tracking");- return reply.status(200).send({- success: true,- message: "Event not tracked - IP excluded",- });+ if (siteConfiguration.excludedIPs && siteConfiguration.excludedIPs.length > 0) {+ const isExcluded = await siteConfig.isIPExcluded(requestIP, validatedPayload.site_id);+ if (isExcluded) {+ logger.info({ siteId: validatedPayload.site_id, ip: requestIP }, "IP excluded from tracking");+ return reply.status(200).send({+ success: true,+ message: "Event not tracked - IP excluded",+ });+ }}// Create base payload for the event using validated dataconst payload = await createBasePayload(request, // Pass request for IP/UAvalidatedPayload.type,- validatedPayload // Add validated payload back+ validatedPayload, // Pass original validated payload+ siteConfiguration.siteId // Pass numeric siteId separately);- // Update session++ // Update session (use numeric siteId)const { sessionId } = await sessionsService.updateSession({userId: payload.userId,- site_id: payload.site_id,+ siteId: siteConfiguration.siteId,});- // Add to queue for processing- await pageviewQueue.add({ ...payload, sessionId });+ // Add to queue for processing (payload already has numeric siteId)+ await pageviewQueue.add({+ ...payload,+ sessionId,+ });return reply.status(200).send({success: true,diff --git a/server/src/services/tracker/types.ts b/server/src/services/tracker/types.tsindex ac9efd9a1..1e773911a 100644--- a/server/src/services/tracker/types.ts+++ b/server/src/services/tracker/types.ts@@ -1,5 +1,5 @@export interface TrackingPayload {- site_id: string;+ site_id: number;hostname: string;pathname: string;querystring: string;diff --git a/server/src/services/tracker/utils.ts b/server/src/services/tracker/utils.tsindex 5d910e00b..aad7605a6 100644--- a/server/src/services/tracker/utils.ts+++ b/server/src/services/tracker/utils.ts@@ -85,21 +85,22 @@ export function clearSelfReferrer(referrer: string, hostname: string): string {export async function createBasePayload(request: FastifyRequest,eventType: "pageview" | "custom_event" | "performance" | "error" | "outbound" = "pageview",- validatedBody: ValidatedTrackingPayload+ validatedBody: ValidatedTrackingPayload,+ numericSiteId: number): Promise<TotalTrackingPayload> {// Use custom user agent if provided, otherwise fall back to headerconst userAgent = validatedBody.user_agent || request.headers["user-agent"] || "";// Override IP if provided in payloadconst ipAddress = validatedBody.ip_address || getIpAddress(request);- const siteId = validatedBody.site_id;// Use custom user ID if provided, otherwise generate oneconst userId = validatedBody.user_id? validatedBody.user_id.trim()- : await userIdService.generateUserId(ipAddress, userAgent, siteId);+ : await userIdService.generateUserId(ipAddress, userAgent, numericSiteId);return {...validatedBody,+ site_id: numericSiteId, // Use the numeric site IDhostname: validatedBody.hostname || "",pathname: validatedBody.pathname || "",querystring: validatedBody.querystring || "",diff --git a/server/src/services/userId/userIdService.ts b/server/src/services/userId/userIdService.tsindex f4630d082..5dc432e60 100644--- a/server/src/services/userId/userIdService.ts+++ b/server/src/services/userId/userIdService.ts@@ -47,7 +47,7 @@ class UserIdService {* @param siteId The site ID to check for salting configuration* @returns A sha256 hash to identify the user*/- async generateUserId(ip: string, userAgent: string, siteId?: string | number): Promise<string> {+ async generateUserId(ip: string, userAgent: string, siteId: number): Promise<string> {// Only apply salt if the site has salting enabledif (await siteConfig.shouldSaltUserIds(siteId)) {const dailySalt = this.getDailySalt(); // Get the salt for the current day