Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

refactor(sites): one module for every read of a Site's tracking configuration (#1122)

Goldflag4e9f51667a3e
diff --git a/server/src/api/sites/getSiteExcludedCountries.ts b/server/src/api/sites/getSiteExcludedCountries.ts
deleted file mode 100644
index de10c6da3..000000000
--- a/server/src/api/sites/getSiteExcludedCountries.ts
+++ /dev/null
@@ -1,54 +0,0 @@
-import { FastifyReply, FastifyRequest } from "fastify";
-import { z } from "zod";
-import { db } from "../../db/postgres/postgres.js";
-import { sites } from "../../db/postgres/schema.js";
-import { eq } from "drizzle-orm";
-
-const getSiteExcludedCountriesSchema = z.object({
- siteId: z.string().min(1),
-});
-
-export async function getSiteExcludedCountries(request: FastifyRequest, reply: FastifyReply) {
- try {
- const validationResult = getSiteExcludedCountriesSchema.safeParse(request.params);
-
- if (!validationResult.success) {
- return reply.status(400).send({
- success: false,
- error: "Invalid site ID",
- details: validationResult.error.flatten(),
- });
- }
-
- const { siteId } = validationResult.data;
- const numericSiteId = Number(siteId);
-
- const site = await db
- .select({
- excludedCountries: sites.excludedCountries,
- })
- .from(sites)
- .where(eq(sites.siteId, numericSiteId))
- .limit(1);
-
- if (site.length === 0) {
- return reply.status(404).send({
- success: false,
- error: "Site not found",
- });
- }
-
- const excludedCountries = Array.isArray(site[0].excludedCountries) ? site[0].excludedCountries : [];
-
- return reply.send({
- success: true,
- excludedCountries,
- });
- } catch (error) {
- request.log.error({ err: error }, "Error getting excluded countries");
- return reply.status(500).send({
- success: false,
- error: "Failed to get excluded countries",
- });
- }
-}
diff --git a/server/src/api/sites/getSiteExcludedIPs.ts b/server/src/api/sites/getSiteExcludedIPs.ts
deleted file mode 100644
index ef49a37ef..000000000
--- a/server/src/api/sites/getSiteExcludedIPs.ts
+++ /dev/null
@@ -1,63 +0,0 @@
-import { FastifyReply, FastifyRequest } from "fastify";
-import { z } from "zod";
-import { db } from "../../db/postgres/postgres.js";
-import { sites } from "../../db/postgres/schema.js";
-import { eq } from "drizzle-orm";
-
-const getSiteExcludedIPsSchema = z.object({
- siteId: z.string().min(1),
-});
-
-export async function getSiteExcludedIPs(request: FastifyRequest, reply: FastifyReply) {
- try {
- const validationResult = getSiteExcludedIPsSchema.safeParse(request.params);
-
- if (!validationResult.success) {
- return reply.status(400).send({
- success: false,
- error: "Invalid site ID",
- details: validationResult.error.flatten(),
- });
- }
-
- const { siteId } = validationResult.data;
- const numericSiteId = Number(siteId);
-
- // Validate that siteId is a valid integer
- if (!Number.isInteger(numericSiteId) || isNaN(numericSiteId) || numericSiteId <= 0) {
- return reply.status(400).send({
- success: false,
- error: "Invalid site ID: must be a positive integer",
- });
- }
-
- // Get the excluded IPs from the database
- const site = await db
- .select({
- excludedIPs: sites.excludedIPs,
- })
- .from(sites)
- .where(eq(sites.siteId, numericSiteId))
- .limit(1);
-
- if (site.length === 0) {
- return reply.status(404).send({
- success: false,
- error: "Site not found",
- });
- }
-
- const excludedIPs = Array.isArray(site[0].excludedIPs) ? site[0].excludedIPs : [];
-
- return reply.send({
- success: true,
- excludedIPs,
- });
- } catch (error) {
- request.log.error({ err: error }, "Error getting excluded IPs");
- return reply.status(500).send({
- success: false,
- error: "Failed to get excluded IPs",
- });
- }
-}
diff --git a/server/src/api/sites/getSiteExclusions.test.ts b/server/src/api/sites/getSiteExclusions.test.ts
new file mode 100644
index 000000000..537d14bd3
--- /dev/null
+++ b/server/src/api/sites/getSiteExclusions.test.ts
@@ -0,0 +1,113 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+ reload: vi.fn(),
+}));
+
+vi.mock("../../lib/siteConfig.js", () => ({
+ siteConfig: { reload: mocks.reload },
+}));
+
+import {
+ getSiteExcludedASNs,
+ getSiteExcludedCountries,
+ getSiteExcludedHostnames,
+ getSiteExcludedIPs,
+ getSiteExcludedPaths,
+ getSiteExcludedQueryParams,
+ getSiteExcludedUserAgents,
+} from "./getSiteExclusions.js";
+
+function createReply() {
+ const reply = {
+ statusCode: 200,
+ body: undefined as any,
+ status(code: number) {
+ reply.statusCode = code;
+ return reply;
+ },
+ send(body: unknown) {
+ reply.body = body;
+ return reply;
+ },
+ };
+ return reply;
+}
+
+function createRequest(params: unknown) {
+ return { params, log: { error: vi.fn() } } as any;
+}
+
+describe("exclusion endpoints", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ mocks.reload.mockResolvedValue({
+ siteId: 123,
+ excludedIPs: ["10.0.0.1"],
+ excludedCountries: ["DE"],
+ excludedPaths: ["/admin"],
+ excludedHostnames: ["staging.example.com"],
+ excludedUserAgents: ["curl"],
+ excludedASNs: ["AS13335"],
+ excludedQueryParams: ["fbclid"],
+ });
+ });
+
+ const endpoints = [
+ { handler: getSiteExcludedIPs, key: "excludedIPs", value: ["10.0.0.1"] },
+ { handler: getSiteExcludedCountries, key: "excludedCountries", value: ["DE"] },
+ { handler: getSiteExcludedPaths, key: "excludedPaths", value: ["/admin"] },
+ { handler: getSiteExcludedHostnames, key: "excludedHostnames", value: ["staging.example.com"] },
+ { handler: getSiteExcludedUserAgents, key: "excludedUserAgents", value: ["curl"] },
+ { handler: getSiteExcludedASNs, key: "excludedASNs", value: ["AS13335"] },
+ { handler: getSiteExcludedQueryParams, key: "excludedQueryParams", value: ["fbclid"] },
+ ];
+
+ for (const { handler, key, value } of endpoints) {
+ it(`returns ${key} under its own field name`, async () => {
+ const reply = createReply();
+ await handler(createRequest({ siteId: "123" }), reply as any);
+
+ expect(mocks.reload).toHaveBeenCalledWith(123);
+ expect(reply.statusCode).toBe(200);
+ expect(reply.body).toEqual({ success: true, [key]: value });
+ });
+ }
+
+ // getSiteExcludedCountries used to be a hand-copy that skipped this guard and
+ // passed NaN straight into the query.
+ it.each(["abc", "0", "-1", "1.5"])("rejects %s as a site id without querying", async siteId => {
+ const reply = createReply();
+ await getSiteExcludedCountries(createRequest({ siteId }), reply as any);
+
+ expect(reply.statusCode).toBe(400);
+ expect(mocks.reload).not.toHaveBeenCalled();
+ });
+
+ it("rejects a missing site id", async () => {
+ const reply = createReply();
+ await getSiteExcludedIPs(createRequest({}), reply as any);
+
+ expect(reply.statusCode).toBe(400);
+ expect(mocks.reload).not.toHaveBeenCalled();
+ });
+
+ it("404s when no site matches", async () => {
+ mocks.reload.mockResolvedValue(undefined);
+ const reply = createReply();
+ await getSiteExcludedIPs(createRequest({ siteId: "123" }), reply as any);
+
+ expect(reply.statusCode).toBe(404);
+ });
+
+ // The API playground renders the error body verbatim, so a Postgres outage
+ // has to read as one rather than as "Site not found".
+ it("500s when the configuration read throws", async () => {
+ mocks.reload.mockRejectedValue(new Error("postgres is down"));
+ const reply = createReply();
+ await getSiteExcludedIPs(createRequest({ siteId: "123" }), reply as any);
+
+ expect(reply.statusCode).toBe(500);
+ expect(reply.body).toEqual({ success: false, error: "Failed to get excluded IPs" });
+ });
+});
diff --git a/server/src/api/sites/getSiteExclusions.ts b/server/src/api/sites/getSiteExclusions.ts
index 742ec9c8b..49e313142 100644
--- a/server/src/api/sites/getSiteExclusions.ts
+++ b/server/src/api/sites/getSiteExclusions.ts
@@ -1,28 +1,44 @@
import { FastifyReply, FastifyRequest } from "fastify";
import { z } from "zod";
-import { eq } from "drizzle-orm";
-import { db } from "../../db/postgres/postgres.js";
-import { sites } from "../../db/postgres/schema.js";
+import { siteConfig, type SiteConfigData } from "../../lib/siteConfig.js";
const siteParamsSchema = z.object({
siteId: z.string().min(1),
});
/**
- * Read a single string-array exclusion column for a site and return it under `responseKey`.
- * Shared by the path / hostname / user-agent exclusion GET endpoints.
+ * The Site Configuration fields these endpoints expose. Each is returned under
+ * its own name, which is also the field name β€” the response key cannot drift
+ * from the field being read.
*/
-async function getExclusionColumn(
- request: FastifyRequest,
- reply: FastifyReply,
- column:
- | typeof sites.excludedPaths
- | typeof sites.excludedHostnames
- | typeof sites.excludedUserAgents
- | typeof sites.excludedASNs
- | typeof sites.excludedQueryParams,
- responseKey: string
-) {
+type ExclusionField = Extract<
+ keyof SiteConfigData,
+ | "excludedIPs"
+ | "excludedCountries"
+ | "excludedPaths"
+ | "excludedHostnames"
+ | "excludedUserAgents"
+ | "excludedASNs"
+ | "excludedQueryParams"
+>;
+
+// The API playground surfaces error bodies verbatim, so these read as prose.
+// Exhaustive by type β€” a new exclusion field will not compile without one.
+const FIELD_LABELS: Record<ExclusionField, string> = {
+ excludedIPs: "excluded IPs",
+ excludedCountries: "excluded countries",
+ excludedPaths: "excluded paths",
+ excludedHostnames: "excluded hostnames",
+ excludedUserAgents: "excluded user agents",
+ excludedASNs: "excluded ASNs",
+ excludedQueryParams: "excluded query params",
+};
+
+/**
+ * Read one exclusion list for a Site and return it under its own field name.
+ * Shared by all seven exclusion GET endpoints.
+ */
+async function getExclusionField(request: FastifyRequest, reply: FastifyReply, field: ExclusionField) {
try {
const validationResult = siteParamsSchema.safeParse(request.params);
@@ -42,46 +58,54 @@ async function getExclusionColumn(
});
}
- const site = await db.select({ value: column }).from(sites).where(eq(sites.siteId, numericSiteId)).limit(1);
+ // Settings screens read back the write they just made, so this must not be
+ // served from another worker's pre-write cache entry.
+ const config = await siteConfig.reload(numericSiteId);
- if (site.length === 0) {
+ if (!config) {
return reply.status(404).send({
success: false,
error: "Site not found",
});
}
- const value = Array.isArray(site[0].value) ? site[0].value : [];
-
return reply.send({
success: true,
- [responseKey]: value,
+ [field]: config[field],
});
} catch (error) {
- request.log.error({ err: error }, `Error getting ${responseKey}`);
+ request.log.error({ err: error }, `Error getting ${FIELD_LABELS[field]}`);
return reply.status(500).send({
success: false,
- error: `Failed to get ${responseKey}`,
+ error: `Failed to get ${FIELD_LABELS[field]}`,
});
}
}
+export function getSiteExcludedIPs(request: FastifyRequest, reply: FastifyReply) {
+ return getExclusionField(request, reply, "excludedIPs");
+}
+
+export function getSiteExcludedCountries(request: FastifyRequest, reply: FastifyReply) {
+ return getExclusionField(request, reply, "excludedCountries");
+}
+
export function getSiteExcludedPaths(request: FastifyRequest, reply: FastifyReply) {
- return getExclusionColumn(request, reply, sites.excludedPaths, "excludedPaths");
+ return getExclusionField(request, reply, "excludedPaths");
}
export function getSiteExcludedHostnames(request: FastifyRequest, reply: FastifyReply) {
- return getExclusionColumn(request, reply, sites.excludedHostnames, "excludedHostnames");
+ return getExclusionField(request, reply, "excludedHostnames");
}
export function getSiteExcludedUserAgents(request: FastifyRequest, reply: FastifyReply) {
- return getExclusionColumn(request, reply, sites.excludedUserAgents, "excludedUserAgents");
+ return getExclusionField(request, reply, "excludedUserAgents");
}
export function getSiteExcludedASNs(request: FastifyRequest, reply: FastifyReply) {
- return getExclusionColumn(request, reply, sites.excludedASNs, "excludedASNs");
+ return getExclusionField(request, reply, "excludedASNs");
}
export function getSiteExcludedQueryParams(request: FastifyRequest, reply: FastifyReply) {
- return getExclusionColumn(request, reply, sites.excludedQueryParams, "excludedQueryParams");
+ return getExclusionField(request, reply, "excludedQueryParams");
}
diff --git a/server/src/api/sites/getSitePrivateLinkConfig.ts b/server/src/api/sites/getSitePrivateLinkConfig.ts
index 8769109bf..1338fff98 100644
--- a/server/src/api/sites/getSitePrivateLinkConfig.ts
+++ b/server/src/api/sites/getSitePrivateLinkConfig.ts
@@ -1,37 +1,29 @@
import { FastifyRequest, FastifyReply } from "fastify";
-import { db } from "../../db/postgres/postgres.js";
-import { sites } from "../../db/postgres/schema.js";
-import { eq } from "drizzle-orm";
+import { siteConfig } from "../../lib/siteConfig.js";
export async function getSitePrivateLinkConfig(
request: FastifyRequest<{ Params: { siteId: string } }>,
reply: FastifyReply
) {
try {
- const { siteId } = request.params;
- const parsedSiteId = parseInt(siteId, 10);
+ const parsedSiteId = parseInt(request.params.siteId, 10);
if (isNaN(parsedSiteId)) {
return reply.status(400).send({ success: false, error: "Invalid site ID" });
}
- // Get site data
- const site = await db
- .select({
- privateLinkKey: sites.privateLinkKey,
- })
- .from(sites)
- .where(eq(sites.siteId, parsedSiteId))
- .limit(1);
+ // Generating or revoking the key writes through the Site Configuration
+ // module, and this screen reads that write back straight away.
+ const config = await siteConfig.reload(parsedSiteId);
- if (site.length === 0) {
+ if (!config) {
return reply.status(404).send({ success: false, error: "Site not found" });
}
return reply.send({
success: true,
data: {
- privateLinkKey: site[0].privateLinkKey,
+ privateLinkKey: config.privateLinkKey,
},
});
} catch (error) {
diff --git a/server/src/api/sites/getTrackingConfig.ts b/server/src/api/sites/getTrackingConfig.ts
index 5154be2ce..4de1ef895 100644
--- a/server/src/api/sites/getTrackingConfig.ts
+++ b/server/src/api/sites/getTrackingConfig.ts
@@ -15,26 +15,27 @@ export async function getTrackingConfig(request: FastifyRequest<{ Params: { site
// Report replay as off when the plan doesn't include it so the tracking script
// never loads the recorder (replay payloads would be dropped at ingest anyway)
const sessionReplay =
- config.type === "mobile"
- ? false
- : (config.sessionReplay && !usageService.isSiteWithoutReplay(config.siteId)) || false;
+ config.type === "mobile" ? false : config.sessionReplay && !usageService.isSiteWithoutReplay(config.siteId);
const featureFlagsEnabled = await hasFeatureFlagsForRuntime(config.siteId, "client");
// Return tracking configuration
// This endpoint is public since the analytics script needs to fetch it
+ // Every field below the plan/type overrides arrives already defaulted from
+ // the Site Configuration module β€” re-applying `?? true` / `|| false` here
+ // would only hide it if that ever stopped being true.
return reply.send({
type: config.type,
featureFlagsEnabled,
sessionReplay,
- webVitals: config.type === "mobile" ? false : config.webVitals || false,
- trackErrors: config.trackErrors || false,
- trackOutbound: config.trackOutbound ?? true,
- trackUrlParams: config.trackUrlParams ?? true,
- trackInitialPageView: config.trackInitialPageView ?? true,
- trackSpaNavigation: config.trackSpaNavigation ?? true,
- trackButtonClicks: config.trackButtonClicks || false,
- trackCopy: config.trackCopy || false,
- trackFormInteractions: config.trackFormInteractions || false,
+ webVitals: config.type === "mobile" ? false : config.webVitals,
+ trackErrors: config.trackErrors,
+ trackOutbound: config.trackOutbound,
+ trackUrlParams: config.trackUrlParams,
+ trackInitialPageView: config.trackInitialPageView,
+ trackSpaNavigation: config.trackSpaNavigation,
+ trackButtonClicks: config.trackButtonClicks,
+ trackCopy: config.trackCopy,
+ trackFormInteractions: config.trackFormInteractions,
});
} catch (error) {
request.log.error({ err: error }, "Error getting tracking config");
diff --git a/server/src/api/sites/index.ts b/server/src/api/sites/index.ts
index af28fb376..0fce68ac9 100644
--- a/server/src/api/sites/index.ts
+++ b/server/src/api/sites/index.ts
@@ -14,9 +14,9 @@ export { updateSiteConfig } from "./updateSiteConfig.js";
export { getTrackingConfig } from "./getTrackingConfig.js";
export { getSitePrivateLinkConfig } from "./getSitePrivateLinkConfig.js";
export { updateSitePrivateLinkConfig } from "./updateSitePrivateLinkConfig.js";
-export { getSiteExcludedIPs } from "./getSiteExcludedIPs.js";
-export { getSiteExcludedCountries } from "./getSiteExcludedCountries.js";
export {
+ getSiteExcludedIPs,
+ getSiteExcludedCountries,
getSiteExcludedPaths,
getSiteExcludedHostnames,
getSiteExcludedUserAgents,
diff --git a/server/src/lib/auth-middleware.test.ts b/server/src/lib/auth-middleware.test.ts
index 5042deb86..30f8089c0 100644
--- a/server/src/lib/auth-middleware.test.ts
+++ b/server/src/lib/auth-middleware.test.ts
@@ -15,7 +15,7 @@ vi.mock("./auth-utils.js", () => mocks);
vi.mock("../db/postgres/postgres.js", () => ({
db: { query: { member: { findFirst: vi.fn(async () => null) } } },
}));
-vi.mock("../utils.js", () => ({ resolveNumericSiteId: vi.fn(async () => null) }));
+vi.mock("./siteConfig.js", () => ({ siteConfig: { resolveSiteId: vi.fn(async () => null) } }));
import {
allowPublicSiteAccess,
diff --git a/server/src/lib/auth-middleware.ts b/server/src/lib/auth-middleware.ts
index 4963b3973..55ac1aa5c 100644
--- a/server/src/lib/auth-middleware.ts
+++ b/server/src/lib/auth-middleware.ts
@@ -11,7 +11,7 @@ import {
} from "./auth-utils.js";
import { getOrgMembership, isOrgAdmin } from "./access.js";
import { hasScope, scopeToString, type ScopeRequirement } from "./scopes.js";
-import { resolveNumericSiteId } from "../utils.js";
+import { siteConfig } from "./siteConfig.js";
type AuthMiddleware = (request: FastifyRequest, reply: FastifyReply) => Promise<void>;
@@ -128,12 +128,20 @@ export const resolveSiteId: AuthMiddleware = async (request, reply) => {
const params = request.params as Record<string, string>;
const siteId = getSiteIdFromParams(request);
- if (siteId && String(siteId).length > 4) {
- const numericId = await resolveNumericSiteId(siteId);
- if (!numericId) {
- return reply.status(404).send({ error: "Site not found" });
- }
+ if (!siteId || String(siteId).length <= 4) {
+ return;
+ }
+
+ const numericId = await siteConfig.resolveSiteId(siteId);
+ if (numericId !== null) {
params.siteId = String(numericId);
+ return;
+ }
+
+ // A digit-only identifier is already in the shape the handlers expect; leave
+ // it alone and let the access check below produce the 404/403, as before.
+ if (!/^\d+$/.test(siteId)) {
+ return reply.status(404).send({ error: "Site not found" });
}
};
diff --git a/server/src/lib/siteConfig.test.ts b/server/src/lib/siteConfig.test.ts
index fb711edbb..006056bc7 100644
--- a/server/src/lib/siteConfig.test.ts
+++ b/server/src/lib/siteConfig.test.ts
@@ -39,6 +39,7 @@ type DrizzleCondition = {
const dbMock = vi.hoisted(() => ({
rows: [] as SiteRow[],
queries: [] as Array<{ column: string | undefined; value: unknown }>,
+ failure: null as Error | null,
select: vi.fn(),
}));
@@ -79,6 +80,10 @@ function createSiteRow(overrides: Partial<SiteRow>): SiteRow {
};
}
+function getCache() {
+ return (siteConfig as unknown as { cache: Map<string, { data: unknown; expires: number }> }).cache;
+}
+
function getConditionDetails(condition: DrizzleCondition): { column: string | undefined; value: unknown } {
const column = condition.queryChunks?.find(chunk => chunk.name === "id" || chunk.name === "site_id")?.name;
const value = condition.queryChunks?.find(
@@ -91,13 +96,16 @@ function getConditionDetails(condition: DrizzleCondition): { column: string | un
beforeEach(() => {
dbMock.rows.length = 0;
dbMock.queries.length = 0;
+ dbMock.failure = null;
dbMock.select.mockReset();
- (siteConfig as unknown as { cache: Map<string, unknown> }).cache.clear();
+ getCache().clear();
dbMock.select.mockImplementation(() => ({
from: () => ({
where: (condition: DrizzleCondition) => ({
limit: async () => {
+ if (dbMock.failure) throw dbMock.failure;
+
const query = getConditionDetails(condition);
dbMock.queries.push(query);
@@ -179,25 +187,142 @@ describe("siteConfig.getConfig", () => {
]);
});
- it("invalidates every identifier for one site without evicting other sites", async () => {
+ it("applies the blockBots default a direct column read would miss", async () => {
+ dbMock.rows.push(createSiteRow({ siteId: 123, blockBots: undefined as unknown as boolean }));
+
+ expect((await siteConfig.getConfig(123))?.blockBots).toBe(true);
+ });
+
+ // Ingestion reads this per event and must degrade to "no configuration"
+ // rather than throw out of the tracking handler.
+ it("swallows a Postgres failure", async () => {
+ dbMock.failure = new Error("postgres is down");
+
+ expect(await siteConfig.getConfig(123)).toBeUndefined();
+ });
+
+ // Route guards resolve whatever the caller put in the URL, before auth, and a
+ // digit-only string falls back to site_id β€” so every zero-padded rendering of
+ // one site's number resolves that site under its own cache key.
+ it("caches zero-padded identifier variants separately", async () => {
+ dbMock.rows.push(createSiteRow({ id: "abcdef123456", siteId: 123 }));
+ const cache = getCache();
+
+ for (const identifier of ["123", "0123", "00123", "000123"]) {
+ expect((await siteConfig.getConfig(identifier))?.siteId).toBe(123);
+ }
+
+ expect(cache.size).toBe(4);
+ });
+
+ it("bounds the cache so those variants cannot grow it without limit", async () => {
+ dbMock.rows.push(createSiteRow({ id: "abcdef123456", siteId: 123 }));
+ const cache = getCache();
+
+ // Stand in for a cache already full of resolved variants, none expired.
+ for (let i = 0; i < 10_000; i++) {
+ cache.set(`string:variant-${i}`, { data: {} as never, expires: Date.now() + 60_000 });
+ }
+
+ expect((await siteConfig.getConfig("0000123"))?.siteId).toBe(123);
+
+ expect(cache.size).toBeLessThanOrEqual(10_000);
+ expect(cache.has("string:0000123")).toBe(true);
+ });
+
+ it("invalidates every spelling for one site without evicting other sites", async () => {
dbMock.rows.push(
createSiteRow({ id: "abcdef123456", siteId: 123, domain: "before.example" }),
createSiteRow({ id: "fedcba654321", siteId: 456, domain: "other.example" })
);
- await siteConfig.getConfig(123);
- await siteConfig.getConfig("123");
- await siteConfig.getConfig("abcdef123456");
+ // "00123" is the spelling an enumerated cache key would take β€” matching on
+ // the row's identity rather than the identifier is what catches it.
+ const spellings: Array<string | number> = [123, "123", "00123", "abcdef123456"];
+ for (const spelling of spellings) {
+ await siteConfig.getConfig(spelling);
+ }
await siteConfig.getConfig(456);
const queriesBeforeInvalidation = dbMock.queries.length;
dbMock.rows[0].domain = "after.example";
siteConfig.invalidate({ id: "abcdef123456", siteId: 123 });
+ for (const spelling of spellings) {
+ expect((await siteConfig.getConfig(spelling))?.domain).toBe("after.example");
+ }
+
+ // The other Site is untouched, and still costs nothing to read.
+ const queriesAfterRewarming = dbMock.queries.length;
+ expect(queriesAfterRewarming).toBeGreaterThan(queriesBeforeInvalidation);
+ expect((await siteConfig.getConfig(456))?.domain).toBe("other.example");
+ expect(dbMock.queries).toHaveLength(queriesAfterRewarming);
+ });
+});
+
+describe("siteConfig.reload", () => {
+ it("reads past a warm cache entry and re-seats every identifier for the site", async () => {
+ dbMock.rows.push(createSiteRow({ id: "abcdef123456", siteId: 123, domain: "before.example" }));
+
+ await siteConfig.getConfig(123);
+ await siteConfig.getConfig("abcdef123456");
+
+ // A sibling cluster worker served the write, so this process is still warm
+ // on the pre-write row.
+ dbMock.rows[0].domain = "after.example";
+
+ expect((await siteConfig.reload(123))?.domain).toBe("after.example");
+ // Both identifiers now serve the fresh row without another query.
+ const queriesAfterReload = dbMock.queries.length;
expect((await siteConfig.getConfig(123))?.domain).toBe("after.example");
- expect((await siteConfig.getConfig("123"))?.domain).toBe("after.example");
expect((await siteConfig.getConfig("abcdef123456"))?.domain).toBe("after.example");
- expect((await siteConfig.getConfig(456))?.domain).toBe("other.example");
- expect(dbMock.queries).toHaveLength(queriesBeforeInvalidation + 4);
+ expect(dbMock.queries).toHaveLength(queriesAfterReload);
+ });
+
+ it("evicts every spelling the site answered to when it is gone", async () => {
+ dbMock.rows.push(createSiteRow({ id: "abcdef123456", siteId: 123 }));
+
+ const spellings: Array<string | number> = [123, "123", "00123", "abcdef123456"];
+ for (const spelling of spellings) {
+ await siteConfig.getConfig(spelling);
+ }
+
+ // Another cluster worker deleted the Site.
+ dbMock.rows.length = 0;
+
+ expect(await siteConfig.reload(123)).toBeUndefined();
+ for (const spelling of spellings) {
+ expect(await siteConfig.getConfig(spelling)).toBeUndefined();
+ }
+ });
+
+ // A settings screen must be able to tell "no such Site" (404) from "could not
+ // ask Postgres" (500), so unlike getConfig this does not swallow.
+ it("propagates a Postgres failure", async () => {
+ dbMock.failure = new Error("postgres is down");
+
+ await expect(siteConfig.reload(123)).rejects.toThrow("postgres is down");
+ });
+});
+
+describe("siteConfig.resolveSiteId", () => {
+ it("resolves a text id to the numeric site id", async () => {
+ dbMock.rows.push(createSiteRow({ id: "abcdef123456", siteId: 123 }));
+
+ expect(await siteConfig.resolveSiteId("abcdef123456")).toBe(123);
+ });
+
+ it("returns null for an identifier no site matches", async () => {
+ expect(await siteConfig.resolveSiteId("abcdef123456")).toBeNull();
+ });
+
+ it("shares the configuration cache rather than keeping its own table", async () => {
+ dbMock.rows.push(createSiteRow({ id: "abcdef123456", siteId: 123 }));
+
+ await siteConfig.resolveSiteId("abcdef123456");
+ const queriesAfterResolve = dbMock.queries.length;
+
+ expect((await siteConfig.getConfig("abcdef123456"))?.siteId).toBe(123);
+ expect(dbMock.queries).toHaveLength(queriesAfterResolve);
});
});
diff --git a/server/src/lib/siteConfig.ts b/server/src/lib/siteConfig.ts
index 4b8b5dc52..8b251a7d6 100644
--- a/server/src/lib/siteConfig.ts
+++ b/server/src/lib/siteConfig.ts
@@ -38,6 +38,13 @@ export interface SiteConfigData {
type SiteConfigRow = typeof sites.$inferSelect;
+// A Site is reachable under several identifier spellings β€” its text id, its
+// numeric siteId, and (because a digit-only string falls back to siteId) any
+// zero-padded rendering of that number. The route guards resolve whatever the
+// caller put in the URL, before authentication, so the key space is attacker-
+// controlled and the cache has to be bounded rather than merely expiring.
+const MAX_CACHE_ENTRIES = 10_000;
+
class SiteConfig {
private cache = new Map<string, { data: SiteConfigData; expires: number }>();
private cacheTTL = 60 * 1000; // 1 minute TTL
@@ -76,62 +83,111 @@ class SiteConfig {
return this.querySiteConfig(eq(sites.siteId, Number(siteIdOrId)));
}
+ /**
+ * Read the row and apply the field defaults. Every read of a tracking-relevant
+ * Site field goes through here, so a caller can never see the raw column where
+ * this would have supplied a default (`blockBots` being the sharp one: the
+ * column is nullable, the configuration is `true`).
+ *
+ * Throws if Postgres does. `getConfig` swallows that into `undefined` because
+ * ingestion must not fail on a database blip; `reload` lets it out so an admin
+ * read answers 500 rather than claiming the Site does not exist.
+ */
+ private async loadSiteConfig(siteIdOrId: string | number): Promise<SiteConfigData | undefined> {
+ const site = await this.findSiteByIdentifier(siteIdOrId);
+
+ if (!site) {
+ return undefined;
+ }
+
+ return {
+ id: site.id,
+ siteId: site.siteId,
+ type: site.type || "web",
+ public: site.public || false,
+ embedEnabled: site.embedEnabled || false,
+ saltUserIds: site.saltUserIds || false,
+ domain: site.domain || "",
+ blockBots: site.blockBots === undefined ? true : site.blockBots,
+ firstPartyProxy: site.firstPartyProxy || false,
+ excludedIPs: Array.isArray(site.excludedIPs) ? site.excludedIPs : [],
+ excludedCountries: Array.isArray(site.excludedCountries) ? site.excludedCountries : [],
+ excludedPaths: Array.isArray(site.excludedPaths) ? site.excludedPaths : [],
+ excludedHostnames: Array.isArray(site.excludedHostnames) ? site.excludedHostnames : [],
+ excludedUserAgents: Array.isArray(site.excludedUserAgents) ? site.excludedUserAgents : [],
+ excludedASNs: Array.isArray(site.excludedASNs) ? site.excludedASNs : [],
+ excludedQueryParams: Array.isArray(site.excludedQueryParams) ? site.excludedQueryParams : [],
+ privateLinkKey: site.privateLinkKey,
+ sessionReplay: site.sessionReplay || false,
+ webVitals: site.webVitals || false,
+ trackErrors: site.trackErrors || false,
+ trackOutbound: site.trackOutbound ?? true,
+ trackUrlParams: site.trackUrlParams ?? true,
+ trackInitialPageView: site.trackInitialPageView ?? true,
+ trackSpaNavigation: site.trackSpaNavigation ?? true,
+ trackIp: site.trackIp || false,
+ trackButtonClicks: site.trackButtonClicks || false,
+ trackCopy: site.trackCopy || false,
+ trackFormInteractions: site.trackFormInteractions || false,
+ tags: Array.isArray(site.tags) ? site.tags : [],
+ };
+ }
+
+ private cacheConfig(siteIdOrId: string | number, configData: SiteConfigData): void {
+ this.cache.set(this.getCacheKey(siteIdOrId), {
+ data: configData,
+ expires: Date.now() + this.cacheTTL,
+ });
+ this.evictIfOverCapacity();
+ }
+
+ private evictIfOverCapacity(): void {
+ if (this.cache.size <= MAX_CACHE_ENTRIES) return;
+
+ const now = Date.now();
+ for (const [key, entry] of this.cache) {
+ if (entry.expires <= now) {
+ this.cache.delete(key);
+ }
+ }
+
+ // Still over after sweeping the expired: shed in insertion order.
+ for (const key of this.cache.keys()) {
+ if (this.cache.size <= MAX_CACHE_ENTRIES) break;
+ this.cache.delete(key);
+ }
+ }
+
/**
* Get site by either siteId or id
*/
private async getSiteByAnyId(siteIdOrId: string | number): Promise<SiteConfigData | undefined> {
- const cacheKey = this.getCacheKey(siteIdOrId);
- const cached = this.cache.get(cacheKey);
+ const cached = this.cache.get(this.getCacheKey(siteIdOrId));
if (cached && cached.expires > Date.now()) {
return cached.data;
}
- try {
- const site = await this.findSiteByIdentifier(siteIdOrId);
+ const configData = await this.loadSiteConfig(siteIdOrId);
- if (!site) {
- return undefined;
- }
+ if (configData) {
+ this.cacheConfig(siteIdOrId, configData);
+ }
+
+ return configData;
+ }
- const configData: SiteConfigData = {
- id: site.id,
- siteId: site.siteId,
- type: site.type || "web",
- public: site.public || false,
- embedEnabled: site.embedEnabled || false,
- saltUserIds: site.saltUserIds || false,
- domain: site.domain || "",
- blockBots: site.blockBots === undefined ? true : site.blockBots,
- firstPartyProxy: site.firstPartyProxy || false,
- excludedIPs: Array.isArray(site.excludedIPs) ? site.excludedIPs : [],
- excludedCountries: Array.isArray(site.excludedCountries) ? site.excludedCountries : [],
- excludedPaths: Array.isArray(site.excludedPaths) ? site.excludedPaths : [],
- excludedHostnames: Array.isArray(site.excludedHostnames) ? site.excludedHostnames : [],
- excludedUserAgents: Array.isArray(site.excludedUserAgents) ? site.excludedUserAgents : [],
- excludedASNs: Array.isArray(site.excludedASNs) ? site.excludedASNs : [],
- excludedQueryParams: Array.isArray(site.excludedQueryParams) ? site.excludedQueryParams : [],
- privateLinkKey: site.privateLinkKey,
- sessionReplay: site.sessionReplay || false,
- webVitals: site.webVitals || false,
- trackErrors: site.trackErrors || false,
- trackOutbound: site.trackOutbound ?? true,
- trackUrlParams: site.trackUrlParams ?? true,
- trackInitialPageView: site.trackInitialPageView ?? true,
- trackSpaNavigation: site.trackSpaNavigation ?? true,
- trackIp: site.trackIp || false,
- trackButtonClicks: site.trackButtonClicks || false,
- trackCopy: site.trackCopy || false,
- trackFormInteractions: site.trackFormInteractions || false,
- tags: Array.isArray(site.tags) ? site.tags : [],
- };
-
- this.cache.set(cacheKey, {
- data: configData,
- expires: Date.now() + this.cacheTTL,
- });
-
- return configData;
+ /**
+ * Get the full site configuration, served from the cache when it is warm.
+ *
+ * Never throws: ingestion reads this per event and must degrade to "no
+ * configuration" rather than fail on a Postgres blip.
+ */
+ async getConfig(siteIdOrId?: string | number): Promise<SiteConfigData | undefined> {
+ if (!siteIdOrId) return undefined;
+
+ try {
+ return await this.getSiteByAnyId(siteIdOrId);
} catch (error) {
logger.error(error as Error, `Error fetching site configuration for ${siteIdOrId}`);
return undefined;
@@ -139,19 +195,71 @@ class SiteConfig {
}
/**
- * Get the full site configuration
+ * Get the full site configuration, always from Postgres.
+ *
+ * The cache is invalidated on write, but only in the process that served the
+ * write β€” under `CLUSTER_WORKERS` a sibling worker can still be holding the
+ * pre-write value for up to the TTL. Settings screens read their own writes
+ * back immediately, so they use this; the ingestion path, which tolerates a
+ * minute of staleness, uses `getConfig`. Either way the read is the module's,
+ * and the fresh row repopulates the cache for everyone behind it.
+ *
+ * Throws when Postgres does, so the caller can tell "no such Site" from "could
+ * not ask".
*/
- async getConfig(siteIdOrId?: string | number): Promise<SiteConfigData | undefined> {
+ async reload(siteIdOrId?: string | number): Promise<SiteConfigData | undefined> {
if (!siteIdOrId) return undefined;
- return this.getSiteByAnyId(siteIdOrId);
+
+ const configData = await this.loadSiteConfig(siteIdOrId);
+
+ if (!configData) {
+ // The Site is gone (or was never there). Anything we were still holding
+ // for it is stale, and the cached entry is the only record of the other
+ // identifiers it answered to.
+ const cacheKey = this.getCacheKey(siteIdOrId);
+ const stale = this.cache.get(cacheKey);
+ if (stale) {
+ this.invalidate(stale.data);
+ }
+ this.cache.delete(cacheKey);
+ return undefined;
+ }
+
+ // Drop every spelling's stale entry, then re-seat the ones this row is
+ // unambiguously the answer for. `String(siteId)` is not among them: a
+ // digit-only string resolves against `sites.id` first, so another Site could
+ // legitimately own that key β€” it stays evicted and re-resolves on demand.
+ this.invalidate(configData);
+ this.cacheConfig(configData.siteId, configData);
+ if (configData.id !== null) {
+ this.cacheConfig(configData.id, configData);
+ }
+ this.cacheConfig(siteIdOrId, configData);
+
+ return configData;
}
- invalidate(site: Pick<SiteConfigData, "id" | "siteId">): void {
- const identifiers: Array<string | number | null> = [site.siteId, String(site.siteId), site.id];
+ /**
+ * Resolve any Site identifier β€” text id or legacy numeric siteId β€” to the
+ * numeric siteId. Shares the configuration cache, so the route guards and the
+ * ingestion path warm each other rather than keeping separate tables.
+ */
+ async resolveSiteId(siteIdOrId?: string | number): Promise<number | null> {
+ const config = await this.getConfig(siteIdOrId);
+ return config?.siteId ?? null;
+ }
- for (const identifier of identifiers) {
- if (identifier !== null) {
- this.cache.delete(this.getCacheKey(identifier));
+ /**
+ * Drop everything cached for one Site.
+ *
+ * Enumerating the identifier spellings would miss the zero-padded ones, so
+ * this matches on the cached row's own identity instead β€” the cache is
+ * bounded, and writes are rare next to reads.
+ */
+ invalidate(site: Pick<SiteConfigData, "id" | "siteId">): void {
+ for (const [key, entry] of this.cache) {
+ if (entry.data.siteId === site.siteId || (site.id !== null && entry.data.id === site.id)) {
+ this.cache.delete(key);
}
}
}
diff --git a/server/src/utils.ts b/server/src/utils.ts
index 465d425a6..6dc3cc04c 100644
--- a/server/src/utils.ts
+++ b/server/src/utils.ts
@@ -1,8 +1,5 @@
-import { eq } from "drizzle-orm";
import { FastifyRequest } from "fastify";
import * as psl from "psl";
-import { db } from "./db/postgres/postgres.js";
-import { sites } from "./db/postgres/schema.js";
const desktopOS = new Set([
"AIX",
@@ -139,38 +136,6 @@ export const extractSiteId = (path: string) => {
return null;
};
-// Cache for string ID to numeric ID lookups to avoid repeated DB queries
-const siteIdCache = new Map<string, number>();
-
-// Resolve a site identifier (string ID or numeric ID) to its numeric siteId
-// Returns the numeric siteId or null if not found
-export const resolveNumericSiteId = async (siteIdentifier: string): Promise<number | null> => {
- // Check cache first
- if (siteIdCache.has(siteIdentifier)) {
- return siteIdCache.get(siteIdentifier)!;
- }
-
- // Look up the string ID in the database
- try {
- const site = await db.select({ siteId: sites.siteId }).from(sites).where(eq(sites.id, siteIdentifier)).limit(1);
-
- if (site.length > 0) {
- const numericId = site[0].siteId;
- // Cache the result
- siteIdCache.set(siteIdentifier, numericId);
- return numericId;
- }
- } catch (error) {
- console.error("Error resolving site ID:", error);
- }
-
- if (/^\d+$/.test(siteIdentifier)) {
- return parseInt(siteIdentifier, 10);
- }
-
- return null;
-};
-
// Replace site ID in URL path with numeric ID
export const replacePathSiteId = (path: string, numericId: number): string => {
const [pathPart, queryPart] = path.split("?");