Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

fix tracking events (#589)

Bill Yang9a970ff4ccf8
diff --git a/server/src/api/sessionReplay/recordSessionReplay.ts b/server/src/api/sessionReplay/recordSessionReplay.ts
index a094de97a..f6466a0e1 100644
--- a/server/src/api/sessionReplay/recordSessionReplay.ts
+++ b/server/src/api/sessionReplay/recordSessionReplay.ts
@@ -36,7 +36,11 @@ export async function recordSessionReplay(
reply: FastifyReply
) {
try {
- const siteId = Number(request.params.site);
+ // Get the site configuration to get the numeric siteId
+ const siteId = (await siteConfig.getSiteConfig(request.params.site))?.siteId;
+ if (!siteId) {
+ throw new Error(`Site not found: ${request.params.site}`);
+ }
// Check if the site has exceeded its monthly limit
if (usageService.isSiteOverLimit(Number(siteId))) {
@@ -73,19 +77,6 @@ export async function recordSessionReplay(
// }
// }
- // If no valid API key, validate origin - disabled for now
- // if (!apiKeyValidation.success) {
- // const originValidation = await validateOrigin(siteId, request.headers.origin as string);
-
- // if (!originValidation.success) {
- // logger.warn(`[SessionReplay] Request rejected for site ${siteId}: ${originValidation.error}`);
- // return reply.status(403).send({
- // success: false,
- // error: originValidation.error,
- // });
- // }
- // }
-
// Check if the IP should be excluded from tracking
const requestIP = getIpAddress(request);
@@ -117,6 +108,6 @@ export async function recordSessionReplay(
return reply.status(400).send({ error: error.errors });
}
logger.error(error as Error, "Error recording session replay");
- return reply.status(500).send({ error: "Internal server error" });
+ return reply.status(500).send({ error });
}
}
diff --git a/server/src/services/replay/sessionReplayIngestService.ts b/server/src/services/replay/sessionReplayIngestService.ts
index 352efa94d..b3df28302 100644
--- a/server/src/services/replay/sessionReplayIngestService.ts
+++ b/server/src/services/replay/sessionReplayIngestService.ts
@@ -6,6 +6,7 @@ import { parseTrackingData } from "./trackingUtils.js";
import { sessionsService } from "../sessions/sessionsService.js";
import { userIdService } from "../userId/userIdService.js";
import { r2Storage } from "../storage/r2StorageService.js";
+import { siteConfig } from "../../lib/siteConfig.js";
export interface RequestMetadata {
userAgent: string;
@@ -35,7 +36,7 @@ export class SessionReplayIngestService {
// Get or create a session ID from the sessions service
const { sessionId } = await sessionsService.updateSession({
userId,
- site_id: siteId.toString(),
+ siteId,
});
// Check if R2 storage is enabled for cloud deployments
diff --git a/server/src/services/sessions/sessionsService.ts b/server/src/services/sessions/sessionsService.ts
index 84b160ea9..78394062e 100644
--- a/server/src/services/sessions/sessionsService.ts
+++ b/server/src/services/sessions/sessionsService.ts
@@ -25,20 +25,18 @@ class SessionsService {
this.logger.info("Session cleanup cron initialized (runs every minute)");
}
- async getExistingSession(userId: string, siteId: string) {
- const siteIdNumber = parseInt(siteId, 10);
-
+ async getExistingSession(userId: string, siteId: number) {
const [existingSession] = await db
.select()
.from(activeSessions)
- .where(and(eq(activeSessions.userId, userId), eq(activeSessions.siteId, siteIdNumber)))
+ .where(and(eq(activeSessions.userId, userId), eq(activeSessions.siteId, siteId)))
.limit(1);
return existingSession || null;
}
- async updateSession(payload: { userId: string; site_id: string }): Promise<{ sessionId: string }> {
- const existingSession = await this.getExistingSession(payload.userId, payload.site_id);
+ async updateSession({ userId, siteId }: { userId: string; siteId: number }): Promise<{ sessionId: string }> {
+ const existingSession = await this.getExistingSession(userId, siteId);
if (existingSession) {
await db
@@ -50,11 +48,10 @@ class SessionsService {
return { sessionId: existingSession.sessionId };
}
- // Insert new session with Drizzle - only include columns that exist in schema
const insertData = {
sessionId: crypto.randomUUID(),
- siteId: typeof payload.site_id === "string" ? parseInt(payload.site_id, 10) : payload.site_id,
- userId: payload.userId,
+ siteId,
+ userId,
startTime: new Date(),
lastActivity: new Date(),
};
diff --git a/server/src/services/shared/requestValidation.test.ts b/server/src/services/shared/requestValidation.test.ts
index 497047b32..ec6c93d58 100644
--- a/server/src/services/shared/requestValidation.test.ts
+++ b/server/src/services/shared/requestValidation.test.ts
@@ -1,4 +1,4 @@
-import { describe, it, expect, vi, beforeEach } from "vitest";
+import { beforeEach, describe, expect, it, vi } from "vitest";
// Define the expected site config type to match SiteConfigData
interface SiteConfigData {
@@ -21,7 +21,6 @@ vi.mock("../../lib/rateLimiter.js", () => ({
vi.mock("../../lib/siteConfig.js", () => ({
siteConfig: {
-
getSiteConfig: vi.fn(),
},
}));
@@ -30,12 +29,11 @@ vi.mock("../../utils.js", () => ({
normalizeOrigin: vi.fn(),
}));
-import { validateApiKey, validateOrigin, checkApiKeyRateLimit } from "./requestValidation.js";
+import { checkApiKeyRateLimit, validateApiKey } from "./requestValidation.js";
// Import mocked modules
import { apiKeyRateLimiter } from "../../lib/rateLimiter.js";
import { siteConfig } from "../../lib/siteConfig.js";
-import { normalizeOrigin } from "../../utils.js";
describe("validateApiKey", () => {
beforeEach(() => {
@@ -53,12 +51,10 @@ describe("validateApiKey", () => {
});
it("should return success false when site is not found", async () => {
-
vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(undefined);
const result = await validateApiKey(1, "test-api-key");
-
expect(siteConfig.getSiteConfig).toHaveBeenCalledWith(1);
expect(result).toEqual({ success: false, error: "Site not found" });
});
@@ -74,7 +70,7 @@ describe("validateApiKey", () => {
blockBots: true,
excludedIPs: [],
};
-
+
vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(mockSite);
// Mock console.info to avoid output during tests
@@ -99,7 +95,7 @@ describe("validateApiKey", () => {
blockBots: true,
excludedIPs: [],
};
-
+
vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(mockSite);
const result = await validateApiKey(1, "invalid-api-key");
@@ -118,7 +114,7 @@ describe("validateApiKey", () => {
blockBots: true,
excludedIPs: [],
};
-
+
vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(mockSite);
await validateApiKey("123", "test-key");
@@ -137,7 +133,7 @@ describe("validateApiKey", () => {
blockBots: true,
excludedIPs: [],
}; // No apiKey property
-
+
vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(mockSite as SiteConfigData);
const result = await validateApiKey(1, "any-key");
@@ -159,250 +155,6 @@ describe("validateApiKey", () => {
});
});
-describe("validateOrigin", () => {
- beforeEach(() => {
- vi.clearAllMocks();
- vi.unstubAllEnvs();
- });
-
- it("should return success true when origin checking is disabled", async () => {
- // Set environment variable before any imports happen
- vi.stubEnv("DISABLE_ORIGIN_CHECK", "true");
-
- // Re-import the module to pick up the new environment variable
- await vi.resetModules();
-
- // Re-establish the mocks after reset
- vi.doMock("../../lib/siteConfig.js", () => ({
- siteConfig: {
-
- getSiteConfig: vi.fn(),
- },
- }));
-
- vi.doMock("../../utils.js", () => ({
- normalizeOrigin: vi.fn(),
- }));
-
- const { validateOrigin } = await import("./requestValidation.js");
-
- const consoleSpy = vi.spyOn(console, "info").mockImplementation(() => {});
-
- const result = await validateOrigin(1, "https://example.com");
-
- expect(result).toEqual({ success: true });
- expect(consoleSpy).toHaveBeenCalledWith(
- "[Validation] Origin check disabled. Allowing request for site 1 from origin: https://example.com"
- );
-
- consoleSpy.mockRestore();
- vi.unstubAllEnvs();
- });
-
- it("should return success false when site is not found", async () => {
-
- vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(undefined);
-
- const result = await validateOrigin(1, "https://example.com");
-
- expect(result).toEqual({ success: false, error: "Site not found" });
- });
-
- it("should return success false when no origin header is provided", async () => {
- const mockSite: SiteConfigData = {
- id: "test-id",
- siteId: 1,
- domain: "example.com",
- apiKey: "test-key",
- public: true,
- saltUserIds: false,
- blockBots: true,
- excludedIPs: [],
- };
-
- vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(mockSite);
-
- const result = await validateOrigin(1);
-
- expect(result).toEqual({ success: false, error: "Origin header required" });
- });
-
- it("should return success true for exact domain match", async () => {
- const mockSite: SiteConfigData = {
- id: "test-id",
- siteId: 1,
- domain: "example.com",
- apiKey: "test-key",
- public: true,
- saltUserIds: false,
- blockBots: true,
- excludedIPs: [],
- };
-
- vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(mockSite);
- vi.mocked(normalizeOrigin)
- .mockReturnValueOnce("example.com") // for request origin
- .mockReturnValueOnce("example.com"); // for site domain
-
- const result = await validateOrigin(1, "https://example.com");
-
- expect(normalizeOrigin).toHaveBeenCalledWith("https://example.com");
- expect(normalizeOrigin).toHaveBeenCalledWith("https://example.com");
- expect(result).toEqual({ success: true });
- });
-
- it("should return success true for subdomain (always allowed)", async () => {
- const mockSite: SiteConfigData = {
- id: "test-id",
- siteId: 1,
- domain: "example.com",
- apiKey: "test-key",
- public: true,
- saltUserIds: false,
- blockBots: true,
- excludedIPs: [],
- };
-
- vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(mockSite);
- vi.mocked(normalizeOrigin)
- .mockReturnValueOnce("sub.example.com") // for request origin
- .mockReturnValueOnce("example.com"); // for site domain
-
- const result = await validateOrigin(1, "https://sub.example.com");
-
- expect(result).toEqual({ success: true });
- });
-
- it("should return success true for nested subdomain", async () => {
- const mockSite: SiteConfigData = {
- id: "test-id",
- siteId: 1,
- domain: "example.com",
- apiKey: "test-key",
- public: true,
- saltUserIds: false,
- blockBots: true,
- excludedIPs: [],
- };
-
- vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(mockSite);
- vi.mocked(normalizeOrigin)
- .mockReturnValueOnce("deep.nested.sub.example.com") // for request origin
- .mockReturnValueOnce("example.com"); // for site domain
-
- const result = await validateOrigin(1, "https://deep.nested.sub.example.com");
-
- expect(result).toEqual({ success: true });
- });
-
- it("should return success false for different domain", async () => {
- const mockSite: SiteConfigData = {
- id: "test-id",
- siteId: 1,
- domain: "example.com",
- apiKey: "test-key",
- public: true,
- saltUserIds: false,
- blockBots: true,
- excludedIPs: [],
- };
-
- vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(mockSite);
- vi.mocked(normalizeOrigin)
- .mockReturnValueOnce("different.com") // for request origin
- .mockReturnValueOnce("example.com"); // for site domain
-
- const result = await validateOrigin(1, "https://different.com");
-
- expect(result).toEqual({
- success: false,
- error: "Origin mismatch. Received: https://different.com",
- });
- });
-
- it("should return success false for domain that contains site domain but is not a subdomain", async () => {
- const mockSite: SiteConfigData = {
- id: "test-id",
- siteId: 1,
- domain: "example.com",
- apiKey: "test-key",
- public: true,
- saltUserIds: false,
- blockBots: true,
- excludedIPs: [],
- };
-
- vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(mockSite);
- vi.mocked(normalizeOrigin)
- .mockReturnValueOnce("notexample.com") // for request origin
- .mockReturnValueOnce("example.com"); // for site domain
-
- const result = await validateOrigin(1, "https://notexample.com");
-
- expect(result).toEqual({
- success: false,
- error: "Origin mismatch. Received: https://notexample.com",
- });
- });
-
- it("should handle string siteId by converting to number", async () => {
- const mockSite: SiteConfigData = {
- id: "test-id",
- siteId: 123,
- domain: "example.com",
- apiKey: "test-key",
- public: true,
- saltUserIds: false,
- blockBots: true,
- excludedIPs: [],
- };
-
- vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(mockSite);
-
- await validateOrigin("123", "https://example.com");
-
- expect(siteConfig.getSiteConfig).toHaveBeenCalledWith(123);
- });
-
- it("should handle invalid origin format", async () => {
- const mockSite: SiteConfigData = {
- id: "test-id",
- siteId: 1,
- domain: "example.com",
- apiKey: "test-key",
- public: true,
- saltUserIds: false,
- blockBots: true,
- excludedIPs: [],
- };
-
- vi.mocked(siteConfig.getSiteConfig).mockResolvedValue(mockSite);
- vi.mocked(normalizeOrigin).mockImplementation(() => {
- throw new Error("Invalid URL");
- });
-
- const result = await validateOrigin(1, "invalid-origin");
-
- expect(result).toEqual({
- success: false,
- error: "Invalid origin format: invalid-origin",
- });
- });
-
- it("should handle errors during validation", async () => {
- vi.mocked(siteConfig.getSiteConfig).mockRejectedValue(new Error("Database error"));
-
- const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {});
-
- const result = await validateOrigin(1, "https://example.com");
-
- expect(result).toEqual({ success: false, error: "Internal error validating origin" });
- expect(consoleSpy).toHaveBeenCalledWith("Error validating origin:", expect.any(Error));
-
- consoleSpy.mockRestore();
- });
-});
-
describe("checkApiKeyRateLimit", () => {
beforeEach(() => {
vi.clearAllMocks();
diff --git a/server/src/services/shared/requestValidation.ts b/server/src/services/shared/requestValidation.ts
index c8980c546..9c29e7c35 100644
--- a/server/src/services/shared/requestValidation.ts
+++ b/server/src/services/shared/requestValidation.ts
@@ -53,79 +53,6 @@ export async function validateApiKey(siteId: string | number, apiKey?: string):
}
}
-/**
- * Validates if the request's origin matches the registered domain for the site
- * @param siteId The site ID from the tracking payload
- * @param requestOrigin The origin header from the request
- * @returns An object with success status and optional error message
- */
-export async function validateOrigin(siteId: string | number, requestOrigin?: string): Promise<OriginValidationResult> {
- try {
- // If origin checking is disabled, return success
- if (DISABLE_ORIGIN_CHECK) {
- logger.info({ siteId, origin: requestOrigin || "none" }, "Origin check disabled, allowing request");
- return { success: true };
- }
-
- // Convert siteId to number
- const numericSiteId = typeof siteId === "string" ? parseInt(siteId, 10) : siteId;
-
- // Get the site configuration
- const site = await siteConfig.getSiteConfig(numericSiteId);
-
- if (!site) {
- return {
- success: false,
- error: "Site not found",
- };
- }
-
- // If no origin header, reject the request
- if (!requestOrigin) {
- return {
- success: false,
- error: "Origin header required",
- };
- }
-
- try {
- // Get the domain associated with this site
- const siteDomain = site.domain;
-
- // Normalize domains by removing all subdomain prefixes
- const normalizedOriginHost = normalizeOrigin(requestOrigin);
- const normalizedSiteDomain = normalizeOrigin(`https://${siteDomain}`);
-
- // Check for exact match first
- if (normalizedOriginHost === normalizedSiteDomain) {
- return { success: true };
- }
-
- // Always allow subdomains - check if origin is a subdomain of site domain
- if (normalizedOriginHost.endsWith(`.${normalizedSiteDomain}`)) {
- return { success: true };
- }
-
- // If we get here, neither exact match nor valid subdomain
- return {
- success: false,
- error: `Origin mismatch. Received: ${requestOrigin}`,
- };
- } catch (error) {
- return {
- success: false,
- error: `Invalid origin format: ${requestOrigin}`,
- };
- }
- } catch (error) {
- logger.error(error, "Error validating origin");
- return {
- success: false,
- error: "Internal error validating origin",
- };
- }
-}
-
/**
* Checks if the API key rate limit has been exceeded
* @param apiKey The API key to check
diff --git a/server/src/services/tracker/trackEvent.ts b/server/src/services/tracker/trackEvent.ts
index e804d142e..636dabb0b 100644
--- a/server/src/services/tracker/trackEvent.ts
+++ b/server/src/services/tracker/trackEvent.ts
@@ -4,7 +4,7 @@ import { z, ZodError } from "zod";
import { createServiceLogger } from "../../lib/logger/logger.js";
import { siteConfig } from "../../lib/siteConfig.js";
import { sessionsService } from "../sessions/sessionsService.js";
-import { checkApiKeyRateLimit, validateApiKey, validateOrigin } from "../shared/requestValidation.js";
+import { checkApiKeyRateLimit, validateApiKey } from "../shared/requestValidation.js";
import { usageService } from "../usageService.js";
import { pageviewQueue } from "./pageviewQueue.js";
import { createBasePayload } from "./utils.js";
@@ -243,25 +243,19 @@ export async function trackEvent(request: FastifyRequest, reply: FastifyReply) {
}
}
- // If no valid API key, validate origin - disabled for now
- // if (!apiKeyValidation.success) {
- // const originValidation = await validateOrigin(validatedPayload.site_id, request.headers.origin as string);
-
- // if (!originValidation.success) {
- // logger.warn(
- // { siteId: validatedPayload.site_id, error: originValidation.error },
- // "Request rejected - origin validation failed"
- // );
- // return reply.status(403).send({
- // success: false,
- // error: originValidation.error,
- // });
- // }
- // }
+ // Get the site configuration to get the numeric siteId
+ const siteConfiguration = await siteConfig.getSiteConfig(validatedPayload.site_id);
+ if (!siteConfiguration) {
+ logger.warn({ siteId: validatedPayload.site_id }, "Site not found");
+ return reply.status(404).send({
+ success: false,
+ error: "Site not found",
+ });
+ }
// Check if bot blocking is enabled for this site and if the request is from a bot
// Skip bot check for API key authenticated requests
- if (!validatedPayload.api_key && (await siteConfig.shouldBlockBots(validatedPayload.site_id))) {
+ if (!validatedPayload.api_key && siteConfiguration.blockBots) {
// Use custom user agent if provided, otherwise fall back to header
const userAgent = validatedPayload.user_agent || (request.headers["user-agent"] as string);
if (userAgent && isbot(userAgent)) {
@@ -273,8 +267,8 @@ export async function trackEvent(request: FastifyRequest, reply: FastifyReply) {
}
}
- // Check if the site has exceeded its monthly limit
- if (usageService.isSiteOverLimit(Number(validatedPayload.site_id))) {
+ // Check if the site has exceeded its monthly limit (using numeric siteId)
+ if (usageService.isSiteOverLimit(siteConfiguration.siteId)) {
logger.info({ siteId: validatedPayload.site_id }, "Skipping event - site over monthly limit");
return reply.status(200).send("Site over monthly limit, event not tracked");
}
@@ -283,28 +277,36 @@ export async function trackEvent(request: FastifyRequest, reply: FastifyReply) {
// Use custom IP if provided in payload, otherwise get from request
const requestIP = validatedPayload.ip_address || request.ip || "";
- if (await siteConfig.isIPExcluded(requestIP, validatedPayload.site_id)) {
- logger.info({ siteId: validatedPayload.site_id, ip: requestIP }, "IP excluded from tracking");
- return reply.status(200).send({
- success: true,
- message: "Event not tracked - IP excluded",
- });
+ if (siteConfiguration.excludedIPs && siteConfiguration.excludedIPs.length > 0) {
+ const isExcluded = await siteConfig.isIPExcluded(requestIP, validatedPayload.site_id);
+ if (isExcluded) {
+ logger.info({ siteId: validatedPayload.site_id, ip: requestIP }, "IP excluded from tracking");
+ return reply.status(200).send({
+ success: true,
+ message: "Event not tracked - IP excluded",
+ });
+ }
}
// Create base payload for the event using validated data
const payload = await createBasePayload(
request, // Pass request for IP/UA
validatedPayload.type,
- validatedPayload // Add validated payload back
+ validatedPayload, // Pass original validated payload
+ siteConfiguration.siteId // Pass numeric siteId separately
);
- // Update session
+
+ // Update session (use numeric siteId)
const { sessionId } = await sessionsService.updateSession({
userId: payload.userId,
- site_id: payload.site_id,
+ siteId: siteConfiguration.siteId,
});
- // Add to queue for processing
- await pageviewQueue.add({ ...payload, sessionId });
+ // Add to queue for processing (payload already has numeric siteId)
+ await pageviewQueue.add({
+ ...payload,
+ sessionId,
+ });
return reply.status(200).send({
success: true,
diff --git a/server/src/services/tracker/types.ts b/server/src/services/tracker/types.ts
index ac9efd9a1..1e773911a 100644
--- a/server/src/services/tracker/types.ts
+++ b/server/src/services/tracker/types.ts
@@ -1,5 +1,5 @@
export interface TrackingPayload {
- site_id: string;
+ site_id: number;
hostname: string;
pathname: string;
querystring: string;
diff --git a/server/src/services/tracker/utils.ts b/server/src/services/tracker/utils.ts
index 5d910e00b..aad7605a6 100644
--- a/server/src/services/tracker/utils.ts
+++ b/server/src/services/tracker/utils.ts
@@ -85,21 +85,22 @@ export function clearSelfReferrer(referrer: string, hostname: string): string {
export async function createBasePayload(
request: FastifyRequest,
eventType: "pageview" | "custom_event" | "performance" | "error" | "outbound" = "pageview",
- validatedBody: ValidatedTrackingPayload
+ validatedBody: ValidatedTrackingPayload,
+ numericSiteId: number
): Promise<TotalTrackingPayload> {
// Use custom user agent if provided, otherwise fall back to header
const userAgent = validatedBody.user_agent || request.headers["user-agent"] || "";
// Override IP if provided in payload
const ipAddress = validatedBody.ip_address || getIpAddress(request);
- const siteId = validatedBody.site_id;
// Use custom user ID if provided, otherwise generate one
const userId = validatedBody.user_id
? validatedBody.user_id.trim()
- : await userIdService.generateUserId(ipAddress, userAgent, siteId);
+ : await userIdService.generateUserId(ipAddress, userAgent, numericSiteId);
return {
...validatedBody,
+ site_id: numericSiteId, // Use the numeric site ID
hostname: validatedBody.hostname || "",
pathname: validatedBody.pathname || "",
querystring: validatedBody.querystring || "",
diff --git a/server/src/services/userId/userIdService.ts b/server/src/services/userId/userIdService.ts
index f4630d082..5dc432e60 100644
--- a/server/src/services/userId/userIdService.ts
+++ b/server/src/services/userId/userIdService.ts
@@ -47,7 +47,7 @@ class UserIdService {
* @param siteId The site ID to check for salting configuration
* @returns A sha256 hash to identify the user
*/
- async generateUserId(ip: string, userAgent: string, siteId?: string | number): Promise<string> {
+ async generateUserId(ip: string, userAgent: string, siteId: number): Promise<string> {
// Only apply salt if the site has salting enabled
if (await siteConfig.shouldSaltUserIds(siteId)) {
const dailySalt = this.getDailySalt(); // Get the salt for the current day