πΈ Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Refactor custom query validation and enhance table reference collection
Bill Yange325d6645478
diff --git a/server/src/api/analytics/utils/customQueryValidation.test.ts b/server/src/api/analytics/utils/customQueryValidation.test.tsnew file mode 100644index 000000000..b2248cd6f--- /dev/null+++ b/server/src/api/analytics/utils/customQueryValidation.test.ts@@ -0,0 +1,69 @@+import { describe, expect, it } from "vitest";+import { validateScopedQuery } from "./customQueryValidation.js";++const SCOPED_ONLY_ERROR = "Queries can only read from scoped_events";++describe("validateScopedQuery β table reference scoping", () => {+ it("allows reading from scoped_events", () => {+ expect(validateScopedQuery("SELECT count(*) FROM scoped_events")).toBeNull();+ });++ it("allows CTEs and self-joins on scoped_events", () => {+ expect(+ validateScopedQuery(+ "WITH t AS (SELECT user_id, count() c FROM scoped_events GROUP BY user_id) SELECT * FROM t"+ )+ ).toBeNull();+ expect(+ validateScopedQuery(+ "SELECT a.user_id FROM scoped_events a JOIN scoped_events b ON a.user_id = b.user_id"+ )+ ).toBeNull();+ });++ it("does not flag commas in SELECT lists, GROUP BY, ORDER BY, or function args", () => {+ expect(+ validateScopedQuery(+ "SELECT count(*), uniq(user_id) FROM scoped_events GROUP BY pathname ORDER BY pathname, count() LIMIT 10"+ )+ ).toBeNull();+ });++ it("blocks comma-join to a materialized view target table (RYB-015)", () => {+ expect(+ validateScopedQuery(+ "SELECT sessions_mv_target.site_id FROM scoped_events, sessions_mv_target WHERE sessions_mv_target.site_id > 0 LIMIT 100"+ )+ ).toBe(SCOPED_ONLY_ERROR);+ expect(+ validateScopedQuery(+ "SELECT pathname_hourly_mv_target.pathname FROM scoped_events, pathname_hourly_mv_target LIMIT 100"+ )+ ).toBe(SCOPED_ONLY_ERROR);+ });++ it("blocks comma-join without surrounding whitespace", () => {+ expect(+ validateScopedQuery("SELECT * FROM scoped_events,sessions_mv_target LIMIT 1")+ ).toBe(SCOPED_ONLY_ERROR);+ });++ it("blocks an unauthorized table inside a subquery comma-join", () => {+ expect(+ validateScopedQuery(+ "SELECT * FROM (SELECT * FROM scoped_events, sessions_mv_target) x LIMIT 1"+ )+ ).toBe(SCOPED_ONLY_ERROR);+ });++ it("still blocks plain FROM/JOIN to unauthorized tables", () => {+ expect(validateScopedQuery("SELECT * FROM events")).toBe(SCOPED_ONLY_ERROR);+ expect(+ validateScopedQuery("SELECT * FROM scoped_events JOIN events ON 1=1")+ ).toBe(SCOPED_ONLY_ERROR);+ });++ it("requires the query to reference scoped_events at all", () => {+ expect(validateScopedQuery("SELECT 1")).toBe("Query must read from scoped_events");+ });+});diff --git a/server/src/api/analytics/utils/customQueryValidation.ts b/server/src/api/analytics/utils/customQueryValidation.tsindex 6f873a8e9..b708f2cf6 100644--- a/server/src/api/analytics/utils/customQueryValidation.ts+++ b/server/src/api/analytics/utils/customQueryValidation.ts@@ -77,18 +77,6 @@ const blockedFunctions = ["viewIfPermitted",] as const;-const blockedTableNames = [- "bot_events",- "events",- "hourly_events_by_site_mv",- "hourly_events_by_site_mv_target",- "information_schema",- "monitor_events",- "session_replay_events",- "session_replay_metadata",- "system",-] as const;-function stripSqlLiteralsAndComments(query: string) {let result = "";let index = 0;@@ -186,6 +174,112 @@ function getCteNames(query: string) {return cteNames;}+// Identifiers that end a FROM clause's comma-separated table list. Once one of these+// appears at the top paren level, later commas belong to another clause (GROUP BY,+// ORDER BY, a UNIONed SELECT, β¦) rather than the table list.+const fromClauseTerminators = new Set([+ "where",+ "prewhere",+ "group",+ "having",+ "order",+ "limit",+ "settings",+ "union",+ "intersect",+ "except",+ "window",+ "qualify",+ "format",+ "into",+]);++function isIdentifierStart(char: string) {+ return /[A-Za-z_]/.test(char);+}++function isIdentifierChar(char: string) {+ return /[A-Za-z0-9_.]/.test(char);+}++// Collect every directly-named table reference in the query. Covers the table after+// each FROM/JOIN and every comma-separated entry in a FROM list (`FROM a, b`), which+// a FROM/JOIN-keyword-only scan would miss. Subquery references (`FROM ( SELECT β¦ )`)+// are skipped here β their inner FROM/JOIN clauses are reached by this same scan.+function collectTableReferences(query: string): string[] {+ const references: string[] = [];+ const length = query.length;++ const readIdentifier = (start: number): [string, number] => {+ let end = start;+ while (end < length && isIdentifierChar(query[end])) {+ end++;+ }+ return [query.slice(start, end), end];+ };++ const skipWhitespace = (index: number): number => {+ while (index < length && /\s/.test(query[index])) {+ index++;+ }+ return index;+ };++ // Record the table reference that follows a FROM / JOIN / comma when it is a plain+ // identifier; subqueries and anything else are left to the surrounding scan.+ const readReference = (index: number) => {+ const start = skipWhitespace(index);+ if (start < length && isIdentifierStart(query[start])) {+ references.push(readIdentifier(start)[0]);+ }+ };++ const keywordPattern = /\b(FROM|JOIN)\b/gi;+ let match: RegExpExecArray | null;+ while ((match = keywordPattern.exec(query)) !== null) {+ const afterKeyword = match.index + match[0].length;++ // A JOIN introduces exactly one table reference.+ if (match[1].toLowerCase() === "join") {+ readReference(afterKeyword);+ continue;+ }++ // A FROM introduces a comma-separated table list. Read the first reference, then+ // walk the clause tracking paren depth and pick up every top-level comma entry.+ readReference(afterKeyword);++ let depth = 0;+ let index = afterKeyword;+ while (index < length) {+ const char = query[index];+ if (char === "(") {+ depth++;+ index++;+ } else if (char === ")") {+ if (depth === 0) {+ break; // a closing paren that ends an enclosing subquery β clause is done+ }+ depth--;+ index++;+ } else if (char === "," && depth === 0) {+ readReference(index + 1);+ index++;+ } else if (depth === 0 && isIdentifierStart(char)) {+ const [word, end] = readIdentifier(index);+ if (fromClauseTerminators.has(word.toLowerCase())) {+ break;+ }+ index = end; // an alias or join keyword β skip past it+ } else {+ index++;+ }+ }+ }++ return references;+}+export function validateScopedQuery(query: string): string | null {const normalizedQuery = normalizeCustomQuery(query);const queryWithoutLiterals = stripSqlLiteralsAndComments(normalizedQuery);@@ -229,25 +323,12 @@ export function validateScopedQuery(query: string): string | null {return "scoped_events is reserved and cannot be redefined";}- for (const tableName of blockedTableNames) {- if (new RegExp(`,\\s*${tableName}\\b`, "i").test(compactQuery)) {- return "Queries can only read from scoped_events";- }- }-- // Match `FROM <identifier>` (requires whitespace, so `FROMx` won't match) and- // `FROM(` / `FROM (` (a subquery, captured as undefined β skipped). Every named- // target must be scoped_events or a declared CTE.- const tableReferencePattern = /\b(?:FROM|JOIN)(?:\s+([a-zA-Z_][a-zA-Z0-9_.]*)|\s*\()/gi;- let match: RegExpExecArray | null;- while ((match = tableReferencePattern.exec(compactQuery)) !== null) {- const tableName = match[1];- if (!tableName) {- // Paren branch: a subquery, validated by subsequent FROM/JOIN matches.- continue;- }-- const normalizedTableName = tableName.toLowerCase();+ // Every table reference must be scoped_events or a declared CTE. collectTableReferences+ // walks the full FROM list, so comma-separated targets (`FROM scoped_events, other`) are+ // validated too β a FROM/JOIN-keyword-only scan captured only the first table and let the+ // rest through. Subqueries are validated by their own inner FROM/JOIN clauses.+ for (const reference of collectTableReferences(compactQuery)) {+ const normalizedTableName = reference.toLowerCase();if (normalizedTableName !== "scoped_events" && !cteNames.has(normalizedTableName)) {return "Queries can only read from scoped_events";}diff --git a/server/src/services/usageService.ts b/server/src/services/usageService.tsindex 218ef9e14..b59a42bc0 100644--- a/server/src/services/usageService.ts+++ b/server/src/services/usageService.ts@@ -137,11 +137,6 @@ class UsageService {},stripeSubscriptions: Map<string, Stripe.Subscription>): Promise<[number, string | null]> {- // Special case for specific organizations- if (orgData.name === "rybbit" || orgData.name === "Zam") {- return [Infinity, this.getStartOfMonth()];- }-// Resolve this org's Stripe subscription from the bulk snapshot (no per-org Stripe call),// then layer in custom plan / override / AppSumo via the same priority rules as elsewhere.const stripeSub = stripeSubscriptionInfoFromSnapshot(stripeSubscriptions, orgData.stripeCustomerId);@@ -267,12 +262,7 @@ class UsageService {const isOverLimit = eventCount > eventLimit;let sendApproaching = false;- if (- !alreadyNotifiedApproaching &&- !isOverLimit &&- Number.isFinite(eventLimit) &&- daysRemaining >= 2- ) {+ if (!alreadyNotifiedApproaching && !isOverLimit && Number.isFinite(eventLimit) && daysRemaining >= 2) {const projected = daysElapsed >= 1 ? eventCount * (totalDaysInMonth / daysElapsed) : 0;const trigger90 = eventCount >= eventLimit * 0.9;const triggerProjection = daysElapsed >= 7 && projected >= eventLimit;@@ -328,9 +318,7 @@ class UsageService {}}} else {- this.logger.warn(- `No owners found for organization ${orgData.name}, skipping approaching-limit email`- );+ this.logger.warn(`No owners found for organization ${orgData.name}, skipping approaching-limit email`);}}