OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix: close security gaps across ingest, auth, imports and integrations (#632)
Carl-Gerhard Lindesvärd298ba6a7f0da
diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.ymlindex 9fcb35982..c532aa322 100644--- a/.github/workflows/docker-build.yml+++ b/.github/workflows/docker-build.yml@@ -159,8 +159,14 @@ jobs:CLICKHOUSE_URL: http://localhost:23123/openpanelSELF_HOSTED: "true"- # No env block, deliberately. Every workspace's test preload pins DATABASE_URL / CLICKHOUSE_URL / REDIS_URL itself and- # creates and migrates the isolated `openpanel_test` databases (test/bun-preload.ts, packages/core/test/preload.ts).+ # Core's preload only pins the URLs so infrastructure-free files run offline, yet many of its database tests+ # assume `openpanel_test` exists, whichever shard they land in.+ - name: Create test databases+ if: matrix.directory == 'packages/core'+ run: bun -e "await (await import('./test/bootstrap-databases.ts')).bootstrapTestDatabases()"++ # No env block, deliberately. Every workspace's test preload pins DATABASE_URL / CLICKHOUSE_URL / REDIS_URL itself;+ # test/bun-preload.ts also creates and migrates the isolated `openpanel_test` databases for the other workspaces.# Env set here would be silently overwritten.- name: Run testsworking-directory: ${{ matrix.directory }}diff --git a/apps/api/e2e/ingest-load.ts b/apps/api/e2e/ingest-load.tsindex 76332e453..708e49c14 100644--- a/apps/api/e2e/ingest-load.ts+++ b/apps/api/e2e/ingest-load.ts@@ -21,13 +21,20 @@* --project acme-shop project id, for the ClickHouse counts* --client / --secret write client; defaults to the seeded project's* --label run prefix for the output files in --out- * --out /tmp/openpanel-ingest-load+ * --out <dir> default: a fresh private directory under $TMPDIR* Env: API_URL (or E2E_API_URL), REDIS_URL, CLICKHOUSE_URL, and the Kafka* topic and group from the api's own config loader.*/-import { appendFileSync, existsSync, mkdirSync, readFileSync } from 'node:fs';-import { resolve } from 'node:path';+import {+ appendFileSync,+ existsSync,+ mkdirSync,+ mkdtempSync,+ readFileSync,+} from 'node:fs';+import { tmpdir } from 'node:os';+import { join, resolve } from 'node:path';import { parseArgs } from 'node:util';import { createKafkaAdmin, sampleConsumerGroupLag } from '@openpanel/core';import { getRedisCache } from '@openpanel/redis';@@ -59,7 +66,7 @@ const { values } = parseArgs({client: { type: 'string' },secret: { type: 'string' },label: { type: 'string', default: 'run' },- out: { type: 'string', default: '/tmp/openpanel-ingest-load' },+ out: { type: 'string' },timeout: { type: 'string', default: String(DEFAULT_TIMEOUT_S) },},});@@ -80,8 +87,13 @@ const timeoutS = integerOption('timeout', values.timeout, 1);const total = devices * perDevice;const projectId = values.project;const runId = `${values.label}_${Date.now().toString(36)}`;-const outPrefix = resolve(values.out, runId);-mkdirSync(values.out, { recursive: true });+// A fixed name under /tmp could be pre-created by another local user, who would+// then decide where the samples are appended.+const outDir =+ values.out ?? mkdtempSync(join(tmpdir(), 'openpanel-ingest-load-'));+mkdirSync(outDir, { recursive: true });+const outPrefix = resolve(outDir, runId);+console.log(`Writing ${outPrefix}.samples and ${outPrefix}.json`);const apiUrl = process.env.E2E_API_URL ?? process.env.API_URL;if (!apiUrl) {diff --git a/apps/public/src/app/(content)/compare/[slug]/page.tsx b/apps/public/src/app/(content)/compare/[slug]/page.tsxindex 72137c1c6..db7ddddd4 100644--- a/apps/public/src/app/(content)/compare/[slug]/page.tsx+++ b/apps/public/src/app/(content)/compare/[slug]/page.tsx@@ -91,7 +91,7 @@ export default async function ComparePage({<div><ScriptdangerouslySetInnerHTML={{- __html: JSON.stringify(buildJsonLd(data, url(`/compare/${slug}`))),+ __html: JSON.stringify(buildJsonLd(data, url(data.url))),}}id="compare-schema"strategy="beforeInteractive"diff --git a/apps/public/src/lib/compare.ts b/apps/public/src/lib/compare.tsindex 90f274534..d84a2c2ba 100644--- a/apps/public/src/lib/compare.ts+++ b/apps/public/src/lib/compare.ts@@ -1,5 +1,6 @@import { readdirSync, readFileSync } from 'node:fs';import { join } from 'node:path';+import { isContentSlug } from './utils';export interface CompareSeo {title: string;@@ -228,6 +229,9 @@ const contentDir = join(process.cwd(), 'content', 'compare');export async function getCompareData(slug: string): Promise<CompareData | null> {+ if (!isContentSlug(slug)) {+ return null;+ }try {const filePath = join(contentDir, `${slug}.json`);const fileContents = readFileSync(filePath, 'utf8');diff --git a/apps/public/src/lib/features.ts b/apps/public/src/lib/features.tsindex 51c2756c1..71b9fb500 100644--- a/apps/public/src/lib/features.ts+++ b/apps/public/src/lib/features.ts@@ -1,5 +1,6 @@import { readdirSync, readFileSync } from 'node:fs';import { join } from 'node:path';+import { isContentSlug } from './utils';export interface FeatureSeo {title: string;@@ -105,6 +106,9 @@ const contentDir = join(process.cwd(), 'content', 'features');export async function getFeatureData(slug: string): Promise<FeatureData | null> {+ if (!isContentSlug(slug)) {+ return null;+ }try {const filePath = join(contentDir, `${slug}.json`);const fileContents = readFileSync(filePath, 'utf8');diff --git a/apps/public/src/lib/for.ts b/apps/public/src/lib/for.tsindex 31dfd2e7b..d47f95ec1 100644--- a/apps/public/src/lib/for.ts+++ b/apps/public/src/lib/for.ts@@ -1,5 +1,6 @@import { readdirSync, readFileSync } from 'node:fs';import { join } from 'node:path';+import { isContentSlug } from './utils';export interface ForSeo {title: string;@@ -87,6 +88,9 @@ export interface ForData {const contentDir = join(process.cwd(), 'content', 'for');export async function getForData(slug: string): Promise<ForData | null> {+ if (!isContentSlug(slug)) {+ return null;+ }try {const filePath = join(contentDir, `${slug}.json`);const fileContents = readFileSync(filePath, 'utf8');diff --git a/apps/public/src/lib/utils.ts b/apps/public/src/lib/utils.tsindex e5ef23ced..95be6d99e 100644--- a/apps/public/src/lib/utils.ts+++ b/apps/public/src/lib/utils.ts@@ -14,3 +14,9 @@ export const formatEventsCount = (value: number) => {}return value.toString();};++// Content slugs reach the filesystem and the page's JSON-LD straight from the+// URL, where Next has already decoded `%2F`; only plain file names get through.+const CONTENT_SLUG = /^[a-z0-9-]+$/;++export const isContentSlug = (slug: string) => CONTENT_SLUG.test(slug);diff --git a/apps/start/e2e/chat.spec.ts b/apps/start/e2e/chat.spec.tsindex c868bae54..9ff6c5a35 100644--- a/apps/start/e2e/chat.spec.ts+++ b/apps/start/e2e/chat.spec.ts@@ -3,7 +3,7 @@ import {MODEL_STORAGE_KEY,} from '@openpanel/core/modules/assistant/assistant.constants';import { expect as baseExpect, test } from './fixtures';-import { shot, trpcQuery } from './org-helpers';+import { shot, trpcMutation, trpcQuery } from './org-helpers';/** The model list the API offers the dashboard, and whether a self-hoster may pick one. */async function chatModels(page: Parameters<typeof trpcQuery>[0]) {@@ -43,28 +43,54 @@ test.describe('AI chat', () => {}).toPass({ timeout: 30_000 });});- test('the sidebar composer carries the question into the chat', async ({+ test('onboarding sends its diagnose question into the chat', async ({page,}) => {- await page.goto('/acme/acme-web', { waitUntil: 'networkidle' });- const composer = page.getByPlaceholder('Ask AI anything…');- // Before hydration the form submits natively and hydration can clear a- // typed value, so type and submit until the question shows up in the chat- // (or the chat reports it is not configured).- const question = page.getByText('What changed this week?');+ // The troubleshooting help only shows while a project has no events.+ const created = await trpcMutation<{ id: string }>(page, 'project.create', {+ organizationId: 'acme',+ project: 'Chat diagnose e2e',+ domain: 'https://diagnose.example.com',+ cors: [],+ website: true,+ app: false,+ backend: false,+ });+ expect(created.status).toBe(200);+ const projectId = created.data?.id ?? '';+ try {+ await page.goto(`/onboarding/${projectId}/connect`, {+ waitUntil: 'networkidle',+ });+ await page.getByText("I'm not receiving events").click();+ await page+ .getByRole('link', { name: 'Diagnose with the assistant' })+ .click();+ await expect(page).toHaveURL(new RegExp(`/acme/${projectId}/chat`));+ expect(new URL(page.url()).searchParams.has('q')).toBe(false);+ const question = page.getByText(/No events have arrived/).first();+ const notConfigured = page.getByText(NOT_CONFIGURED);+ await expect(question.or(notConfigured)).toBeVisible();+ if (!(await notConfigured.isVisible())) {+ // Sent, not just prefilled: the URL settles on the new conversation.+ await expect(page).toHaveURL(/\/chat\?.*c=/);+ }+ } finally {+ await trpcMutation(page, 'project.delete', { projectId });+ }+ });++ test('a ?q= link only prefills the composer', async ({ page }) => {+ await page.goto(`${CHAT}?q=${encodeURIComponent('Export every profile')}`);+ const composer = page.getByRole('textbox', { name: 'Message' });const notConfigured = page.getByText(NOT_CONFIGURED);- await expect(async () => {- await composer.fill('What changed this week?');- await expect(composer).toHaveValue('What changed this week?');- await composer.press('Enter');- await expect(question.or(notConfigured)).toBeVisible({ timeout: 2000 });- }).toPass({ timeout: 30_000 });+ await expect(composer.or(notConfigured)).toBeVisible();if (await notConfigured.isVisible()) {- test.skip();return;}- // The URL settles on the new conversation.- await expect(page).toHaveURL(/\/chat\?.*c=/);+ await expect(composer).toHaveValue('Export every profile');+ await page.waitForTimeout(1000);+ expect(new URL(page.url()).searchParams.has('c')).toBe(false);});test('a question gets an answer and the conversation is listed', async ({diff --git a/apps/start/src/components/chat/chat-composer.tsx b/apps/start/src/components/chat/chat-composer.tsxindex 0d9af8e72..0b1959124 100644--- a/apps/start/src/components/chat/chat-composer.tsx+++ b/apps/start/src/components/chat/chat-composer.tsx@@ -15,6 +15,7 @@ export function ChatComposer({modelId,onModelChange,autoFocus,+ initialText = '',}: {onSend: (text: string) => void;onStop: () => void;@@ -23,10 +24,19 @@ export function ChatComposer({modelId: string | null;onModelChange: (id: string) => void;autoFocus?: boolean;+ initialText?: string;}) {- const [text, setText] = useState('');+ const [text, setText] = useState(initialText);const textareaRef = useRef<HTMLTextAreaElement | null>(null);+ // A new ?q= on an open chat replaces the draft; the q being cleared after a+ // send must not.+ useEffect(() => {+ if (initialText) {+ setText(initialText);+ }+ }, [initialText]);+useEffect(() => {if (autoFocus) {textareaRef.current?.focus();diff --git a/apps/start/src/components/chat/chat-launch.ts b/apps/start/src/components/chat/chat-launch.tsindex c8861ba3d..f64d4b438 100644--- a/apps/start/src/components/chat/chat-launch.ts+++ b/apps/start/src/components/chat/chat-launch.ts@@ -3,10 +3,28 @@ import { useCallback } from 'react';import { usePageContextRef } from '@/contexts/page-context';import { useAppParams } from '@/hooks/use-app-params';-/**- * Opens the chat page. A question travels in `q` and is sent on arrival; the- * page the user came from travels in `hint`.- */+// A question the user asked inside the app is sent on arrival. It is handed+// over in memory, not in the URL, so a crafted link can only prefill one.+// Bound to its project: a launch whose chat never opened must not be sent in+// another project's chat later.+let launchedQuestion: { projectId: string; question: string } | null = null;++/** Call right before navigating to the chat page. */+export function setLaunchedQuestion(+ projectId: string,+ question?: string+): void {+ const text = question?.trim();+ launchedQuestion = text ? { projectId, question: text } : null;+}++export function takeLaunchedQuestion(projectId: string): string | null {+ const pending = launchedQuestion;+ launchedQuestion = null;+ return pending?.projectId === projectId ? pending.question : null;+}++/** Opens the chat page; the page the user came from travels in `hint`. */export function useChatLaunch() {const navigate = useNavigate();const { organizationId, projectId } = useAppParams();@@ -14,14 +32,11 @@ export function useChatLaunch() {return useCallback((question?: string) => {- const text = question?.trim();+ setLaunchedQuestion(projectId, question);navigate({to: '/$organizationId/$projectId/chat',params: { organizationId, projectId },- search: {- q: text || undefined,- hint: hintRef.current ?? undefined,- },+ search: { hint: hintRef.current ?? undefined },});},[navigate, organizationId, projectId, hintRef]diff --git a/apps/start/src/components/chat/chat-view.tsx b/apps/start/src/components/chat/chat-view.tsxindex 848327471..4521a143c 100644--- a/apps/start/src/components/chat/chat-view.tsx+++ b/apps/start/src/components/chat/chat-view.tsx@@ -7,9 +7,11 @@ import {import { useCallback, useEffect, useRef, useState } from 'react';import { ChatComposer } from './chat-composer';import { ChatEmpty } from './chat-empty';+import { takeLaunchedQuestion } from './chat-launch';import { ChatMessages } from './chat-messages';import type { AssistantUIMessage } from './types';import { useChatSession } from './use-chat-session';+import { useAppParams } from '@/hooks/use-app-params';function readStoredModel(): string | null {if (typeof window === 'undefined') {@@ -37,7 +39,10 @@ export function ChatView({models: readonly ChatModelEntry[];defaultModelId: string | null;hint?: ChatLaunchHint;- /** Sent once on mount; the launcher puts it in the URL. */+ /**+ * Prefills the composer. It comes from the URL, so anyone can write it into a+ * link: the user reads it and presses Send before it runs with their access.+ */initialQuestion?: string;onFirstMessage: () => void;onTitleChange: (title: string | null) => void;@@ -81,14 +86,18 @@ export function ChatView({[messages.length, onFirstMessage, sendMessage]);- // The launcher's question is sent exactly once, however the page re-renders.- const sentInitial = useRef(false);+ const { projectId } = useAppParams();+ const tookLaunchedQuestion = useRef(false);useEffect(() => {- if (initialQuestion && !sentInitial.current && messages.length === 0) {- sentInitial.current = true;- send(initialQuestion);+ if (tookLaunchedQuestion.current || messages.length > 0) {+ return;}- }, [initialQuestion, messages.length, send]);+ tookLaunchedQuestion.current = true;+ const question = takeLaunchedQuestion(projectId);+ if (question) {+ send(question);+ }+ }, [messages.length, projectId, send]);return (<>@@ -112,6 +121,7 @@ export function ChatView({<ChatComposerautoFocusbusy={busy}+ initialText={initialQuestion}modelId={modelId}models={models}onModelChange={selectModel}diff --git a/apps/start/src/components/chat/tool-labels.ts b/apps/start/src/components/chat/tool-labels.tsindex 4110dd8aa..d9c981877 100644--- a/apps/start/src/components/chat/tool-labels.ts+++ b/apps/start/src/components/chat/tool-labels.ts@@ -104,6 +104,8 @@ export function approvalPrompt(toolName: string, input: unknown): string {return 'Update the project settings?';case 'change_plan':return 'Change the workspace plan?';+ case 'diagnose_tracking':+ return `Load ${String(fields.url ?? '')} to check its tracking?`;default:return `Run ${toolName.replace(/_/g, ' ')}?`;}diff --git a/apps/start/src/components/integrations/forms/discord-integration.tsx b/apps/start/src/components/integrations/forms/discord-integration.tsxindex a740462b6..a5b2bcfc8 100644--- a/apps/start/src/components/integrations/forms/discord-integration.tsx+++ b/apps/start/src/components/integrations/forms/discord-integration.tsx@@ -61,7 +61,7 @@ export function DiscordIntegrationForm({const handleTest = async () => {const url = form.getValues('config.url');if (!url) {- return toast.error('Webhook URL is required');+ return toast.error('Enter the webhook URL to test it');}const res = await testMutation.mutateAsync({projectId,@@ -87,6 +87,11 @@ export function DiscordIntegrationForm({/><InputWithLabellabel="Discord Webhook URL"+ // The URL is a credential and never sent back to the browser, so an+ // edit starts blank and blank means "keep the stored URL".+ placeholder={+ defaultValues?.id ? 'Leave blank to keep the current URL' : undefined+ }{...form.register('config.url')}error={path(['config', 'url', 'message'], form.formState.errors)}/>diff --git a/apps/start/src/components/onboarding/install-troubleshooting.tsx b/apps/start/src/components/onboarding/install-troubleshooting.tsxindex 692e2b8c1..94f8d5c32 100644--- a/apps/start/src/components/onboarding/install-troubleshooting.tsx+++ b/apps/start/src/components/onboarding/install-troubleshooting.tsx@@ -4,6 +4,7 @@ import { Link } from '@tanstack/react-router';import { SparklesIcon } from 'lucide-react';import { toast } from 'sonner';import { curlSnippet, type SnippetContext } from './install-snippets';+import { setLaunchedQuestion } from '@/components/chat/chat-launch';import CopyInput from '@/components/forms/copy-input';import { WithLabel } from '@/components/forms/input-with-label';import TagInput from '@/components/forms/tag-input';@@ -116,20 +117,18 @@ export function InstallTroubleshooting({</div><LinkclassName={cn(buttonVariants({ size: 'default' }))}- onClick={() =>+ onClick={() => {+ setLaunchedQuestion(project.id, diagnoseQuestion(project));trackFunnel('onboarding_assistant_opened', {projectId: project.id,hasDomain: Boolean(project.domain),- })- }+ });+ }}params={{organizationId: project.organizationId,projectId: project.id,}}- search={{- q: diagnoseQuestion(project),- hint: { page: 'onboarding' },- }}+ search={{ hint: { page: 'onboarding' } }}to="/$organizationId/$projectId/chat"><SparklesIcon className="mr-2 size-4" />diff --git a/apps/start/src/components/report-table/csv.ts b/apps/start/src/components/report-table/csv.tsindex a035550da..aea59354f 100644--- a/apps/start/src/components/report-table/csv.ts+++ b/apps/start/src/components/report-table/csv.ts@@ -1,8 +1,5 @@import type { ReportColumn, ReportRowIdentity } from './columns';--function escapeCell(cell: string) {- return /[",\n]/.test(cell) ? `"${cell.replace(/"/g, '""')}"` : cell;-}+import { escapeCsvValue } from '@/utils/csv-download';/** Raw values, not the formatted ones, so the file can be summed in a spreadsheet. */export function buildCsv<Row>(@@ -18,10 +15,10 @@ export function buildCsv<Row>(];const lines = rows.map((row) => [identity.searchText(row),- ...dataColumns.map((column) => String(column.value(row) ?? '')),+ ...dataColumns.map((column) => column.value(row)),]);return [header, ...lines]- .map((line) => line.map(escapeCell).join(','))+ .map((line) => line.map(escapeCsvValue).join(',')).join('\n');}diff --git a/apps/start/src/components/sessions/replay/index.tsx b/apps/start/src/components/sessions/replay/index.tsxindex 7d02bdd9b..fb26c8067 100644--- a/apps/start/src/components/sessions/replay/index.tsx+++ b/apps/start/src/components/sessions/replay/index.tsx@@ -63,24 +63,30 @@ function ReplayChunkLoader({const { addEvent, refreshDuration } = useReplayContext();useEffect(() => {- function recursive(fromIndex: number) {+ let cancelled = false;++ function loadFrom(index: number) {queryClient.fetchQuery(trpc.session.replayChunksFrom.queryOptions({sessionId,projectId,- fromIndex,+ fromIndex: index,})).then((res) => {- res.data.forEach((row) => {- row?.events?.forEach((event) => {+ if (cancelled) {+ return;+ }+ for (const row of res.data) {+ for (const event of row?.events ?? []) {addEvent(event);- });- });+ }+ }refreshDuration();- if (res.hasMore) {- recursive(fromIndex + res.data.length);+ const advanced = res.nextIndex > index;+ if (res.hasMore && advanced) {+ loadFrom(res.nextIndex);}}).catch(() => {@@ -88,7 +94,10 @@ function ReplayChunkLoader({});}- recursive(fromIndex);+ loadFrom(fromIndex);+ return () => {+ cancelled = true;+ };}, []);return null;@@ -222,7 +231,7 @@ function ReplayContent({</div>{hasReplay && hasMore && (<ReplayChunkLoader- fromIndex={firstBatch?.data?.length ?? 0}+ fromIndex={firstBatch?.nextIndex ?? 0}projectId={projectId}sessionId={sessionId}/>diff --git a/apps/start/src/components/sessions/replay/replay-player.tsx b/apps/start/src/components/sessions/replay/replay-player.tsxindex 0df5187b1..056da91aa 100644--- a/apps/start/src/components/sessions/replay/replay-player.tsx+++ b/apps/start/src/components/sessions/replay/replay-player.tsx@@ -92,7 +92,6 @@ export function ReplayPlayer({autoPlay: false,showController: false,speedOption: [0.5, 1, 2, 4, 8],- UNSAFE_replayCanvas: true,skipInactive: false,},}) as ReplayPlayerInstance;diff --git a/apps/start/src/hooks/use-live-counter.ts b/apps/start/src/hooks/use-live-counter.tsindex d8e0d7eda..857d79c25 100644--- a/apps/start/src/hooks/use-live-counter.ts+++ b/apps/start/src/hooks/use-live-counter.ts@@ -35,7 +35,9 @@ export function useLiveCounter({}, [query.data]);useWS<number>(- `/live/visitors/${projectId}`,+ shareId+ ? `/live/visitors/${projectId}?shareId=${shareId}`+ : `/live/visitors/${projectId}`,(value) => {if (!Number.isNaN(value)) {counter.set(value);diff --git a/apps/start/src/modals/add-notification-rule.tsx b/apps/start/src/modals/add-notification-rule.tsxindex 3d2a60547..f3c716057 100644--- a/apps/start/src/modals/add-notification-rule.tsx+++ b/apps/start/src/modals/add-notification-rule.tsx@@ -33,7 +33,7 @@ import { Textarea } from '@/components/ui/textarea';import { useAppParams } from '@/hooks/use-app-params';import { useEventNames } from '@/hooks/use-event-names';import { useEventProperties } from '@/hooks/use-event-properties';-import { useTRPC } from '@/integrations/trpc/react';+import { handleError, useTRPC } from '@/integrations/trpc/react';import type { RouterOutputs } from '@/trpc/client';interface Props {@@ -88,6 +88,7 @@ export default function AddNotificationRule({ rule }: Props) {);popModal();},+ onError: handleError,}));const integrationsQuery = useQuery(diff --git a/apps/start/src/modals/share-dashboard-modal.tsx b/apps/start/src/modals/share-dashboard-modal.tsxindex 4acb4edc1..f538f5f65 100644--- a/apps/start/src/modals/share-dashboard-modal.tsx+++ b/apps/start/src/modals/share-dashboard-modal.tsx@@ -51,7 +51,12 @@ export default function ShareDashboardModal({? `${window.location.origin}/share/dashboard/${existingShare.id}`: '';- const { register, handleSubmit, watch } = useForm<IForm>({+ const {+ register,+ handleSubmit,+ watch,+ formState: { dirtyFields },+ } = useForm<IForm>({resolver: zodResolver(validator),// `values` rather than `defaultValues`: the share settings arrive after// the first render, and the field must then show the stored password.@@ -173,7 +178,10 @@ export default function ShareDashboardModal({onSubmit={handleSubmit((values) => {mutation.mutate({...values,- password: getSharePasswordToSubmit(values.password),+ password: getSharePasswordToSubmit(+ values.password,+ !!dirtyFields.password+ ),});})}>diff --git a/apps/start/src/modals/share-overview-modal.tsx b/apps/start/src/modals/share-overview-modal.tsxindex 4a4557574..4bd66da9f 100644--- a/apps/start/src/modals/share-overview-modal.tsx+++ b/apps/start/src/modals/share-overview-modal.tsx@@ -46,7 +46,12 @@ export default function ShareOverviewModal() {? `${window.location.origin}/share/overview/${existingShare.id}`: '';- const { register, handleSubmit, watch } = useForm<IForm>({+ const {+ register,+ handleSubmit,+ watch,+ formState: { dirtyFields },+ } = useForm<IForm>({resolver: zodResolver(validator),// `values` rather than `defaultValues`: the share settings arrive after// the first render, and the field must then show the stored password.@@ -164,7 +169,10 @@ export default function ShareOverviewModal() {onSubmit={handleSubmit((values) => {mutation.mutate({...values,- password: getSharePasswordToSubmit(values.password),+ password: getSharePasswordToSubmit(+ values.password,+ !!dirtyFields.password+ ),});})}>diff --git a/apps/start/src/modals/share-report-modal.tsx b/apps/start/src/modals/share-report-modal.tsxindex fc3a8cf2a..58f07aa3b 100644--- a/apps/start/src/modals/share-report-modal.tsx+++ b/apps/start/src/modals/share-report-modal.tsx@@ -47,7 +47,12 @@ export default function ShareReportModal({ reportId }: { reportId: string }) {? `${window.location.origin}/share/report/${existingShare.id}`: '';- const { register, handleSubmit, watch } = useForm<IForm>({+ const {+ register,+ handleSubmit,+ watch,+ formState: { dirtyFields },+ } = useForm<IForm>({resolver: zodResolver(validator),// `values` rather than `defaultValues`: the share settings arrive after// the first render, and the field must then show the stored password.@@ -165,7 +170,10 @@ export default function ShareReportModal({ reportId }: { reportId: string }) {onSubmit={handleSubmit((values) => {mutation.mutate({...values,- password: getSharePasswordToSubmit(values.password),+ password: getSharePasswordToSubmit(+ values.password,+ !!dirtyFields.password+ ),});})}>diff --git a/apps/start/src/routeTree.gen.ts b/apps/start/src/routeTree.gen.tsindex 9fe298ed6..19e03d1fa 100644--- a/apps/start/src/routeTree.gen.ts+++ b/apps/start/src/routeTree.gen.ts@@ -33,6 +33,7 @@ import { Route as ShareReportShareIdRouteImport } from './routes/share.report.$simport { Route as ShareOverviewShareIdRouteImport } from './routes/share.overview.$shareId'import { Route as ShareDashboardShareIdRouteImport } from './routes/share.dashboard.$shareId'import { Route as StepsOnboardingProjectRouteImport } from './routes/_steps.onboarding.project'+import { Route as PublicInviteInviteIdRouteImport } from './routes/_public.invite.$inviteId'import { Route as AppOrganizationIdSettingsRouteImport } from './routes/_app.$organizationId.settings'import { Route as AppOrganizationIdBillingRouteImport } from './routes/_app.$organizationId.billing'import { Route as AppOrganizationIdProjectIdRouteImport } from './routes/_app.$organizationId.$projectId'@@ -251,6 +252,11 @@ const StepsOnboardingProjectRoute = StepsOnboardingProjectRouteImport.update({path: '/onboarding/project',getParentRoute: () => StepsRoute,} as any)+const PublicInviteInviteIdRoute = PublicInviteInviteIdRouteImport.update({+ id: '/invite/$inviteId',+ path: '/invite/$inviteId',+ getParentRoute: () => PublicRoute,+} as any)const AppOrganizationIdSettingsRoute =AppOrganizationIdSettingsRouteImport.update({id: '/settings',@@ -715,6 +721,7 @@ export interface FileRoutesByFullPath {'/$organizationId/$projectId': typeof AppOrganizationIdProjectIdRouteWithChildren'/$organizationId/billing': typeof AppOrganizationIdBillingRoute'/$organizationId/settings': typeof AppOrganizationIdSettingsRoute+ '/invite/$inviteId': typeof PublicInviteInviteIdRoute'/onboarding/project': typeof StepsOnboardingProjectRoute'/share/dashboard/$shareId': typeof ShareDashboardShareIdRoute'/share/overview/$shareId': typeof ShareOverviewShareIdRoute@@ -801,6 +808,7 @@ export interface FileRoutesByTo {'/widget/realtime': typeof WidgetRealtimeRoute'/$organizationId/billing': typeof AppOrganizationIdBillingRoute'/$organizationId/settings': typeof AppOrganizationIdSettingsRoute+ '/invite/$inviteId': typeof PublicInviteInviteIdRoute'/onboarding/project': typeof StepsOnboardingProjectRoute'/share/dashboard/$shareId': typeof ShareDashboardShareIdRoute'/share/overview/$shareId': typeof ShareOverviewShareIdRoute@@ -884,6 +892,7 @@ export interface FileRoutesById {'/_app/$organizationId/$projectId': typeof AppOrganizationIdProjectIdRouteWithChildren'/_app/$organizationId/billing': typeof AppOrganizationIdBillingRoute'/_app/$organizationId/settings': typeof AppOrganizationIdSettingsRoute+ '/_public/invite/$inviteId': typeof PublicInviteInviteIdRoute'/_steps/onboarding/project': typeof StepsOnboardingProjectRoute'/share/dashboard/$shareId': typeof ShareDashboardShareIdRoute'/share/overview/$shareId': typeof ShareOverviewShareIdRoute@@ -984,6 +993,7 @@ export interface FileRouteTypes {| '/$organizationId/$projectId'| '/$organizationId/billing'| '/$organizationId/settings'+ | '/invite/$inviteId'| '/onboarding/project'| '/share/dashboard/$shareId'| '/share/overview/$shareId'@@ -1070,6 +1080,7 @@ export interface FileRouteTypes {| '/widget/realtime'| '/$organizationId/billing'| '/$organizationId/settings'+ | '/invite/$inviteId'| '/onboarding/project'| '/share/dashboard/$shareId'| '/share/overview/$shareId'@@ -1152,6 +1163,7 @@ export interface FileRouteTypes {| '/_app/$organizationId/$projectId'| '/_app/$organizationId/billing'| '/_app/$organizationId/settings'+ | '/_public/invite/$inviteId'| '/_steps/onboarding/project'| '/share/dashboard/$shareId'| '/share/overview/$shareId'@@ -1421,6 +1433,13 @@ declare module '@tanstack/react-router' {preLoaderRoute: typeof StepsOnboardingProjectRouteImportparentRoute: typeof StepsRoute}+ '/_public/invite/$inviteId': {+ id: '/_public/invite/$inviteId'+ path: '/invite/$inviteId'+ fullPath: '/invite/$inviteId'+ preLoaderRoute: typeof PublicInviteInviteIdRouteImport+ parentRoute: typeof PublicRoute+ }'/_app/$organizationId/settings': {id: '/_app/$organizationId/settings'path: '/settings'@@ -2466,10 +2485,12 @@ const LoginRouteWithChildren = LoginRoute._addFileChildren(LoginRouteChildren)interface PublicRouteChildren {PublicOnboardingRoute: typeof PublicOnboardingRoute+ PublicInviteInviteIdRoute: typeof PublicInviteInviteIdRoute}const PublicRouteChildren: PublicRouteChildren = {PublicOnboardingRoute: PublicOnboardingRoute,+ PublicInviteInviteIdRoute: PublicInviteInviteIdRoute,}const PublicRouteWithChildren =diff --git a/apps/start/src/routes/_app.$organizationId.$projectId.chat.tsx b/apps/start/src/routes/_app.$organizationId.$projectId.chat.tsxindex f7db1138b..87e9255e9 100644--- a/apps/start/src/routes/_app.$organizationId.$projectId.chat.tsx+++ b/apps/start/src/routes/_app.$organizationId.$projectId.chat.tsx@@ -14,7 +14,7 @@ import { createProjectTitle } from '@/utils/title';const searchSchema = z.object({/** Conversation id. Absent means a new chat. */c: z.string().optional(),- /** A question to send on arrival. */+ /** A question to prefill the composer with; the user sends it. */q: z.string().optional(),hint: zChatLaunchHint.optional(),});diff --git a/apps/start/src/routes/_app.$organizationId.$projectId.tsx b/apps/start/src/routes/_app.$organizationId.$projectId.tsxindex d73f9308a..bf42020a6 100644--- a/apps/start/src/routes/_app.$organizationId.$projectId.tsx+++ b/apps/start/src/routes/_app.$organizationId.$projectId.tsx@@ -2,7 +2,7 @@ import { FREE_PRODUCT_IDS } from '@openpanel/payments';import { subscriptionBlocksDashboard } from '@openpanel/payments/subscription-state';import { getSubscriptionStateMeta } from '@openpanel/payments/subscription-state-meta';import { useSuspenseQuery } from '@tanstack/react-query';-import { createFileRoute, Outlet } from '@tanstack/react-router';+import { createFileRoute, Outlet, redirect } from '@tanstack/react-router';import BillingPrompt from '@/components/organization/billing-prompt';import { useProjectDocumentTitle } from '@/hooks/use-project-document-title';import { useTRPC } from '@/integrations/trpc/react';@@ -20,18 +20,30 @@ export const Route = createFileRoute('/_app/$organizationId/$projectId')({};},loader: async ({ context, params }) => {+ const projectQuery =+ context.trpc.project.getProjectWithClients.queryOptions({+ projectId: params.projectId,+ });await Promise.all([context.queryClient.prefetchQuery(context.trpc.organization.get.queryOptions({organizationId: params.organizationId,})),- context.queryClient.prefetchQuery(- context.trpc.project.getProjectWithClients.queryOptions({- projectId: params.projectId,- })- ),+ context.queryClient.prefetchQuery(projectQuery),]);+ // The billing gate below reads the URL's organization, so it must be the+ // one that owns the project.+ const project = context.queryClient.getQueryData(projectQuery.queryKey);+ if (project && project.organizationId !== params.organizationId) {+ throw redirect({+ to: '/$organizationId/$projectId',+ params: {+ organizationId: project.organizationId,+ projectId: params.projectId,+ },+ });+ }},});diff --git a/apps/start/src/routes/_public.invite.$inviteId.tsx b/apps/start/src/routes/_public.invite.$inviteId.tsxnew file mode 100644index 000000000..bd9bf97ff--- /dev/null+++ b/apps/start/src/routes/_public.invite.$inviteId.tsx@@ -0,0 +1,132 @@+import { useMutation, useQuery, useSuspenseQuery } from '@tanstack/react-query';+import { createFileRoute, redirect, useNavigate } from '@tanstack/react-router';+import { AlertCircleIcon } from 'lucide-react';+import { AuthHeading } from '@/components/auth/auth-shell';+import FullPageLoadingState from '@/components/full-page-loading-state';+import { Button } from '@/components/ui/button';+import { useLogout } from '@/hooks/use-logout';+import { handleError, useTRPC } from '@/integrations/trpc/react';+import { isSignedIn } from '@/utils/route-session';+import { createEntityTitle, PAGE_TITLES } from '@/utils/title';++export const Route = createFileRoute('/_public/invite/$inviteId')({+ head: () => ({+ meta: [+ { title: createEntityTitle('Accept invitation', PAGE_TITLES.ONBOARDING) },+ { name: 'robots', content: 'noindex, follow' },+ ],+ }),+ beforeLoad: async ({ context, params }) => {+ if (!(await isSignedIn(context))) {+ throw redirect({+ to: '/onboarding',+ search: { inviteId: params.inviteId },+ });+ }+ },+ loader: async ({ context, params }) => {+ await context.queryClient.prefetchQuery(+ context.trpc.organization.getInvite.queryOptions({+ inviteId: params.inviteId,+ })+ );+ },+ component: Component,+ pendingComponent: FullPageLoadingState,+});++function Component() {+ const { inviteId } = Route.useParams();+ const trpc = useTRPC();+ const navigate = useNavigate();+ const logout = useLogout();+ const { data: session } = useSuspenseQuery(trpc.auth.session.queryOptions());+ const { data: invite, isError } = useQuery(+ trpc.organization.getInvite.queryOptions({ inviteId })+ );+ const accept = useMutation(+ trpc.organization.acceptInvite.mutationOptions({+ onSuccess({ organizationId }) {+ navigate({ to: '/$organizationId', params: { organizationId } });+ },+ onError: handleError,+ })+ );++ if (isError) {+ return (+ <InviteProblem+ description="It may have been revoked or already accepted."+ title="Invitation not found"+ />+ );+ }+ if (!invite) {+ return null;+ }++ const organizationName = invite.organization?.name ?? 'the workspace';+ if (invite.isExpired) {+ return (+ <InviteProblem+ description="Ask the workspace owner for a new one."+ title={`The invitation to ${organizationName} has expired`}+ />+ );+ }++ const signedInEmail = session.user?.email ?? '';+ const isInvitedAccount =+ signedInEmail.toLowerCase() === invite.email.toLowerCase();++ return (+ <div className="col w-full gap-8">+ <AuthHeading+ description={+ isInvitedAccount+ ? `You were invited as ${invite.email}.`+ : `This invitation was sent to ${invite.email}, but you are signed in as ${signedInEmail}. Sign in with that address to accept it.`+ }+ title={`Join ${organizationName}`}+ />+ {isInvitedAccount ? (+ <Button+ className="w-full"+ loading={accept.isPending}+ onClick={() => accept.mutate({ inviteId })}+ size="xl"+ >+ Accept invitation+ </Button>+ ) : (+ <Button+ className="w-full"+ loading={logout.isPending}+ onClick={() => logout.mutate()}+ size="xl"+ variant="outline"+ >+ Sign out+ </Button>+ )}+ </div>+ );+}++function InviteProblem({+ title,+ description,+}: {+ title: string;+ description: string;+}) {+ return (+ <div className="row items-start gap-3 rounded-xl bg-destructive/10 p-4">+ <AlertCircleIcon className="mt-0.5 shrink-0 text-destructive" size={18} />+ <div className="col gap-0.5">+ <div className="font-medium">{title}</div>+ <p className="text-muted-foreground text-sm">{description}</p>+ </div>+ </div>+ );+}diff --git a/apps/start/src/routes/_public.onboarding.tsx b/apps/start/src/routes/_public.onboarding.tsxindex 3e2c4e19d..226f726ea 100644--- a/apps/start/src/routes/_public.onboarding.tsx+++ b/apps/start/src/routes/_public.onboarding.tsx@@ -24,10 +24,18 @@ export const Route = createFileRoute('/_public/onboarding')({{ name: 'robots', content: 'noindex, follow' },],}),- beforeLoad: async ({ context }) => {- if (await isSignedIn(context)) {- throw redirect({ to: '/' });+ beforeLoad: async ({ context, search }) => {+ if (!(await isSignedIn(context))) {+ return;}+ // A signed-in user opening an invite link accepts it explicitly there.+ if (search.inviteId) {+ throw redirect({+ to: '/invite/$inviteId',+ params: { inviteId: search.inviteId },+ });+ }+ throw redirect({ to: '/' });},component: Component,validateSearch,diff --git a/apps/start/src/routes/widget/counter.tsx b/apps/start/src/routes/widget/counter.tsxindex 65a69cd08..2a1d7bde9 100644--- a/apps/start/src/routes/widget/counter.tsx+++ b/apps/start/src/routes/widget/counter.tsx@@ -67,7 +67,7 @@ function CounterWidget({ shareId, data }: RealtimeWidgetProps) {// WebSocket subscription for real-time updatesuseWS<number>(- `/live/visitors/${data.projectId}`,+ `/live/visitors/${data.projectId}?shareId=${shareId}`,() => {if (!document.hidden) {queryClient.refetchQueries(diff --git a/apps/start/src/routes/widget/realtime.tsx b/apps/start/src/routes/widget/realtime.tsxindex da57537a9..f54ff5de5 100644--- a/apps/start/src/routes/widget/realtime.tsx+++ b/apps/start/src/routes/widget/realtime.tsx@@ -95,7 +95,7 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {// WebSocket subscription for real-time updatesuseWS<number>(- `/live/visitors/${data.projectId}`,+ `/live/visitors/${data.projectId}?shareId=${shareId}`,() => {if (!document.hidden) {queryClient.refetchQueries(diff --git a/apps/start/src/types/rrweb-player.d.ts b/apps/start/src/types/rrweb-player.d.tsindex f561662ee..1d404666b 100644--- a/apps/start/src/types/rrweb-player.d.ts+++ b/apps/start/src/types/rrweb-player.d.ts@@ -6,7 +6,6 @@ declare module 'rrweb-player' {autoPlay?: boolean;showController?: boolean;speedOption?: number[];- UNSAFE_replayCanvas?: boolean;skipInactive?: boolean;}diff --git a/apps/start/src/utils/csv-download.test.ts b/apps/start/src/utils/csv-download.test.tsnew file mode 100644index 000000000..ed92a655a--- /dev/null+++ b/apps/start/src/utils/csv-download.test.ts@@ -0,0 +1,49 @@+import { describe, expect, it } from 'vitest';+import { cohortMembersToCSV, escapeCsvValue } from './csv-download';++describe('escapeCsvValue', () => {+ it('keeps numbers numeric, negative ones included', () => {+ expect(escapeCsvValue(-12.5)).toBe('-12.5');+ });++ it.each([+ '-5',+ '-3.2',+ '+7',+ '-.5',+ '-1e3',+ ])('leaves the numeric string %j unprefixed', (value) => {+ expect(escapeCsvValue(value)).toBe(value);+ });++ it('leaves ordinary text alone', () => {+ expect(escapeCsvValue('/pricing')).toBe('/pricing');+ });++ it.each([+ '=HYPERLINK("https://evil.test")',+ '+1+1',+ '-1+1',+ '-5+cmd|"/c calc"!A0',+ ' -5',+ '@SUM(A1)',+ '\t=1',+ ' =1+1',+ ])('makes %j literal text', (value) => {+ expect(escapeCsvValue(value).replace(/^"/, '')).toMatch(/^'/);+ });++ it('quotes a neutralised cell that holds a delimiter', () => {+ expect(escapeCsvValue('=1,2')).toBe(`"'=1,2"`);+ });++ it('quotes and doubles embedded quotes', () => {+ expect(escapeCsvValue('say "hi"')).toBe('"say ""hi"""');+ });+});++describe('cohortMembersToCSV', () => {+ it('neutralises a formula-shaped profile id', () => {+ expect(cohortMembersToCSV(['=cmd'])).toBe("profile_id\n'=cmd");+ });+});diff --git a/apps/start/src/utils/csv-download.ts b/apps/start/src/utils/csv-download.tsindex e097560f1..abf01c2e9 100644--- a/apps/start/src/utils/csv-download.ts+++ b/apps/start/src/utils/csv-download.ts@@ -1,12 +1,27 @@-function escapeCsvValue(value: string | number | null | undefined): string {+// A spreadsheet runs a cell that starts with one of these as a formula, even+// after leading whitespace, so exported text from tracked data could compute or+// fetch. A leading quote makes it literal text.+const SPREADSHEET_FORMULA_PREFIX = /^\s*[=+\-@\t\r]/;+// A signed number held as a string ("-5") is not a formula, and quoting it+// would stop the spreadsheet from summing it.+const PLAIN_NUMBER = /^[+-]?(\d+\.?\d*|\.\d+)(e[+-]?\d+)?$/i;+const CSV_QUOTING_REQUIRED = /[",\n\r]/;++export function escapeCsvValue(+ value: string | number | null | undefined+): string {if (value === null || value === undefined) {return '';}- const str = String(value);- if (str.includes(',') || str.includes('"') || str.includes('\n')) {- return `"${str.replace(/"/g, '""')}"`;+ if (typeof value === 'number') {+ return String(value);}- return str;+ const isFormula =+ SPREADSHEET_FORMULA_PREFIX.test(value) && !PLAIN_NUMBER.test(value);+ const text = isFormula ? `'${value}` : value;+ return CSV_QUOTING_REQUIRED.test(text)+ ? `"${text.replace(/"/g, '""')}"`+ : text;}function buildCSV(rows: (string | number | null | undefined)[][]): string {diff --git a/apps/start/src/utils/op-filter.test.ts b/apps/start/src/utils/op-filter.test.tsnew file mode 100644index 000000000..9d234a984--- /dev/null+++ b/apps/start/src/utils/op-filter.test.ts@@ -0,0 +1,65 @@+// @vitest-environment jsdom++import type { TrackHandlerPayload } from '@openpanel/web';+import { afterEach, describe, expect, it } from 'vitest';+import { shouldSendAnalyticsEvent } from './op-filter';++function screenView(properties: Record<string, unknown>): TrackHandlerPayload {+ return { type: 'track', payload: { name: 'screen_view', properties } };+}++describe('shouldSendAnalyticsEvent', () => {+ afterEach(() => {+ window.history.replaceState(null, '', '/');+ });++ it('sends ordinary page views', () => {+ expect(+ shouldSendAnalyticsEvent(+ screenView({ __path: `${window.location.origin}/login` })+ )+ ).toBe(true);+ });++ it('drops a page view of the reset-password page', () => {+ expect(+ shouldSendAnalyticsEvent(+ screenView({+ __path: `${window.location.origin}/reset-password?token=secret`,+ })+ )+ ).toBe(false);+ });++ it('drops events whose referrer is the unsubscribe page', () => {+ expect(+ shouldSendAnalyticsEvent(+ screenView({+ __path: `${window.location.origin}/login`,+ __referrer: `${window.location.origin}/unsubscribe?token=secret`,+ })+ )+ ).toBe(false);+ });++ it('drops outgoing-link events that point at the reset-password page', () => {+ expect(+ shouldSendAnalyticsEvent(+ screenView({+ __path: `${window.location.origin}/login`,+ href: `${window.location.origin}/reset-password?token=secret`,+ })+ )+ ).toBe(false);+ });++ it('drops every event while the browser is on a credential page', () => {+ window.history.replaceState(null, '', '/reset-password?token=secret');+ expect(+ shouldSendAnalyticsEvent({+ type: 'track',+ payload: { name: 'link_out', properties: {} },+ })+ ).toBe(false);+ });+});diff --git a/apps/start/src/utils/op-filter.ts b/apps/start/src/utils/op-filter.tsnew file mode 100644index 000000000..b6a0471b3--- /dev/null+++ b/apps/start/src/utils/op-filter.ts@@ -0,0 +1,32 @@+import type { TrackHandlerPayload } from '@openpanel/web';++// These pages carry a credential (reset token, unsubscribe token) in the URL.+const PATHS_WITH_URL_CREDENTIALS = new Set(['/reset-password', '/unsubscribe']);++// Outgoing-link events carry their destination in `href`.+const URL_PROPERTIES = ['__path', '__referrer', 'href'] as const;++function pointsAtCredentialPage(url: unknown): boolean {+ if (typeof url !== 'string' || url === '') {+ return false;+ }+ try {+ const { pathname } = new URL(url, window.location.origin);+ return PATHS_WITH_URL_CREDENTIALS.has(pathname);+ } catch {+ return false;+ }+}++/** Drops any event that would send a credential-bearing URL to analytics. */+export function shouldSendAnalyticsEvent(event: TrackHandlerPayload): boolean {+ const onCredentialPage = pointsAtCredentialPage(window.location.href);+ if (onCredentialPage) {+ return false;+ }+ const properties: Record<string, unknown> =+ 'properties' in event.payload && event.payload.properties+ ? event.payload.properties+ : {};+ return !URL_PROPERTIES.some((key) => pointsAtCredentialPage(properties[key]));+}diff --git a/apps/start/src/utils/op.ts b/apps/start/src/utils/op.tsindex 745f712e7..5181a1b7e 100644--- a/apps/start/src/utils/op.ts+++ b/apps/start/src/utils/op.ts@@ -1,4 +1,5 @@import { OpenPanel } from '@openpanel/web';+import { shouldSendAnalyticsEvent } from '@/utils/op-filter';const clientId = import.meta.env.VITE_OP_CLIENT_ID;@@ -9,6 +10,7 @@ export const op = new OpenPanel({trackScreenViews: true,trackOutgoingLinks: true,trackAttributes: true,+ filter: shouldSendAnalyticsEvent,// sessionReplay: {// enabled: true,// }diff --git a/apps/start/src/utils/share-password.test.ts b/apps/start/src/utils/share-password.test.tsindex 213b55b9d..64b1ee984 100644--- a/apps/start/src/utils/share-password.test.ts+++ b/apps/start/src/utils/share-password.test.ts@@ -7,17 +7,21 @@ import {describe('getSharePasswordToSubmit', () => {it('omits the password when the placeholder is untouched', () => {- expect(getSharePasswordToSubmit(STORED_PASSWORD_PLACEHOLDER)).toBe(+ expect(getSharePasswordToSubmit(STORED_PASSWORD_PLACEHOLDER, false)).toBe(undefined);});+ it('omits the password when an empty field was never edited', () => {+ expect(getSharePasswordToSubmit('', false)).toBe(undefined);+ });+it('removes the password when the field was cleared', () => {- expect(getSharePasswordToSubmit('')).toBe(null);+ expect(getSharePasswordToSubmit('', true)).toBe(null);});it('sends a typed password', () => {- expect(getSharePasswordToSubmit('hunter2')).toBe('hunter2');+ expect(getSharePasswordToSubmit('hunter2', true)).toBe('hunter2');});});diff --git a/apps/start/src/utils/share-password.ts b/apps/start/src/utils/share-password.tsindex 674aeff24..ad0c0bdc4 100644--- a/apps/start/src/utils/share-password.ts+++ b/apps/start/src/utils/share-password.ts@@ -9,12 +9,14 @@ export function getSharePasswordFieldValue(/*** The share mutations keep the stored password when the field is omitted and- * remove it on null, so the untouched placeholder must not become null.+ * remove it on null. Only a deliberate edit may remove it: an untouched field+ * is empty whenever the settings have not loaded, and is not a removal.*/export function getSharePasswordToSubmit(- fieldValue: string | null | undefined+ fieldValue: string | null | undefined,+ isEdited: boolean): string | null | undefined {- if (fieldValue === STORED_PASSWORD_PLACEHOLDER) {+ if (!isEdited || fieldValue === STORED_PASSWORD_PLACEHOLDER) {return undefined;}return fieldValue || null;diff --git a/apps/start/tsconfig.json b/apps/start/tsconfig.jsonindex 315163304..65b1b266b 100644--- a/apps/start/tsconfig.json+++ b/apps/start/tsconfig.json@@ -5,7 +5,7 @@"target": "ES2022","jsx": "react-jsx","module": "ESNext",- "lib": ["ES2022", "DOM", "DOM.Iterable"],+ "lib": ["ES2022", "ES2024.String", "DOM", "DOM.Iterable"],"types": ["vite/client"],/* Bundler mode */diff --git a/bun.lock b/bun.lockindex 575512108..518de9a61 100644--- a/bun.lock+++ b/bun.lock@@ -352,6 +352,7 @@"prisma-error-enum": "0.1.3","prom-client": "15.1.3","ramda": "0.29.1",+ "re2js": "2.9.0","short-unique-id": "5.0.3","superjson": "2.2.2","unzipper": "0.12.5",@@ -5100,6 +5101,8 @@"rc9": ["[email protected]", "", { "dependencies": { "defu": "^6.1.4", "destr": "^2.0.3" } }, "sha512-btXCnMmRIBINM2LDZoEmOogIZU7Qe7zn4BpomSKZ/ykbLObuBdvG+mFq11DL6fjH1DRwHhrlgtYWG96bJiC7Cg=="],+ "re2js": ["[email protected]", "", {}, "sha512-rZpqkd4ETP3GrQ5mDLzADr14OleeKdb/40lG84WE8absRfsVLJpWt2ifjtYYeoPqeouIkcxhTlurxIXfR5kR5Q=="],+"react": ["[email protected]", "", {}, "sha512-Ku/hhYbVjOQnXDZFv2+RibmLFGwFdeeKHFcOTlrt7xplBnya5OGn/hIRDsqDiSUcfORsDC7MPxwork8jBwsIWA=="],"react-animate-height": ["[email protected]", "", { "peerDependencies": { "react": ">=16.8.0", "react-dom": ">=16.8.0" } }, "sha512-R6DSvr7ud07oeCixScyvXWEMJY/Mt2+GyOWC1KMaRc69gOBw+SsCg4TJmrp4rKUM1hyd6p+YKw90brjPH93Y2A=="],diff --git a/packages/core/package.json b/packages/core/package.jsonindex 57eef1533..01205ac00 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -61,6 +61,7 @@"prisma-error-enum": "0.1.3","prom-client": "15.1.3","ramda": "0.29.1",+ "re2js": "2.9.0","short-unique-id": "5.0.3","superjson": "2.2.2","unzipper": "0.12.5",diff --git a/packages/core/src/buffers/session-buffer.ts b/packages/core/src/buffers/session-buffer.tsindex eff8030e6..e87f0b67e 100644--- a/packages/core/src/buffers/session-buffer.ts+++ b/packages/core/src/buffers/session-buffer.ts@@ -25,6 +25,9 @@ export function resolveSessionTimeoutMs(config: CoreConfig): number {return config.session.timeoutMs ?? DEFAULT_SESSION_TIMEOUT_MS;}+/** A session row is rewritten on every event, so its group union is capped. */+const MAX_SESSION_GROUPS = 100;+const sessionKey = (projectId: string, deviceId: string) =>`session:${projectId}:${deviceId}`;const profileIndexKey = (projectId: string, profileId: string) =>@@ -317,7 +320,7 @@ export class SessionBuffer extends BaseBuffer {if (payload.groups?.length) {current.groups = [...new Set([...(current.groups ?? []), ...payload.groups]),- ];+ ].slice(0, MAX_SESSION_GROUPS);}return { current, chRows: [current, oldRow] };@@ -369,7 +372,7 @@ export class SessionBuffer extends BaseBuffer {utm_campaign: pickUtm(payload, 'utm_campaign'),utm_content: pickUtm(payload, 'utm_content'),utm_term: pickUtm(payload, 'utm_term'),- groups: payload.groups ?? [],+ groups: (payload.groups ?? []).slice(0, MAX_SESSION_GROUPS),sign: 1,version: 1,};diff --git a/packages/core/src/http/cors.test.ts b/packages/core/src/http/cors.test.tsnew file mode 100644index 000000000..271b41526--- /dev/null+++ b/packages/core/src/http/cors.test.ts@@ -0,0 +1,36 @@+import { expect, test } from 'bun:test';+import { Elysia } from 'elysia';+import { corsDelegator } from './cors';++const DASHBOARD_ORIGIN = 'https://dashboard.example.com';+const OTHER_ORIGIN = 'https://evil.example.com';++const app = new Elysia()+ .use(corsDelegator({ dashboardOrigins: [DASHBOARD_ORIGIN] }))+ .get('/admin/memory/heap-snapshot', () => 'snapshot')+ .get('/bullboard', () => 'board')+ .get('/trpc/x', () => 'trpc')+ .get('/track', () => 'track');++async function allowedOrigin(path: string, origin: string) {+ const response = await app.handle(+ new Request(`http://localhost${path}`, { headers: { origin } })+ );+ return response.headers.get('access-control-allow-origin');+}++test('operator routes are readable from no other origin, not even the dashboard', async () => {+ for (const path of ['/admin/memory/heap-snapshot', '/bullboard']) {+ for (const origin of [DASHBOARD_ORIGIN, OTHER_ORIGIN]) {+ expect(await allowedOrigin(path, origin)).toBeNull();+ }+ }+});++test('private routes admit only the dashboard, open routes any origin', async () => {+ expect(await allowedOrigin('/trpc/x', DASHBOARD_ORIGIN)).toBe(+ DASHBOARD_ORIGIN+ );+ expect(await allowedOrigin('/trpc/x', OTHER_ORIGIN)).toBeNull();+ expect(await allowedOrigin('/track', OTHER_ORIGIN)).toBe(OTHER_ORIGIN);+});diff --git a/packages/core/src/http/cors.ts b/packages/core/src/http/cors.tsindex 40cd280d3..67cbd21ea 100644--- a/packages/core/src/http/cors.ts+++ b/packages/core/src/http/cors.ts@@ -1,5 +1,9 @@-// `CORS_PRIVATE_PATHS` is the allowlist: a request under one of those prefixes-// is only allowed from a configured dashboard origin; everything else (`/track`,+// `CORS_OPERATOR_PATHS` (Basic-authenticated operator tooling) is readable+// from no other origin: a browser replays cached Basic credentials on a+// credentialed cross-origin request, so echoing the origin would hand a heap+// snapshot to any page. `CORS_PRIVATE_PATHS` is the allowlist: a request+// under one of those prefixes is only allowed from a configured dashboard+// origin; everything else (`/track`,// `/export`, `/insights`, `/import`, `/manage`, `/profile`, `/event`, `/tools`)// is open to any origin. `/gsc` and `/mcp` are absent from the list, and so open.//@@ -9,6 +13,10 @@// server-side SDK) still gets `*`.import { cors } from '@elysiajs/cors';+import { BULL_BOARD_BASE_PATH } from './bull-board';+import { MEMORY_ROUTES_BASE_PATH } from './memory.routes';++const DEBUG_ROUTES_BASE_PATH = '/debug';/** The allowlist of private-route prefixes. */export const CORS_PRIVATE_PATHS = [@@ -21,6 +29,13 @@ export const CORS_PRIVATE_PATHS = ['/ai',] as const;+/** Operator-only prefixes: memory diagnostics, the queue board, dev debug routes. */+export const CORS_OPERATOR_PATHS = [+ MEMORY_ROUTES_BASE_PATH,+ BULL_BOARD_BASE_PATH,+ DEBUG_ROUTES_BASE_PATH,+] as const;+/** Preflight cache duration for the open paths (7 days). */const CORS_MAX_AGE_SECONDS = 86_400 * 7;@@ -35,6 +50,9 @@ export function corsDelegator({ dashboardOrigins }: CorsOptions) {return cors({origin: (request) => {const path = new URL(request.url).pathname;+ if (CORS_OPERATOR_PATHS.some((prefix) => path.startsWith(prefix))) {+ return false;+ }const isPrivatePath = CORS_PRIVATE_PATHS.some((prefix) =>path.startsWith(prefix));diff --git a/packages/core/src/http/memory.routes.test.ts b/packages/core/src/http/memory.routes.test.tsindex 81afad7e5..42d5f4a3e 100644--- a/packages/core/src/http/memory.routes.test.ts+++ b/packages/core/src/http/memory.routes.test.ts@@ -59,3 +59,17 @@ test('a local gc returns before/after for this process and logs it', async () =>expect(body.freed.heapUsed).toBe(body.before.heapUsed - body.after.heapUsed);expect(info).toHaveBeenCalledTimes(1);});++test('a request another site started is refused even with valid credentials', async () => {+ for (const [method, url] of ROUTES) {+ for (const site of ['cross-site', 'same-site']) {+ const response = await app.handle(+ new Request(url, {+ method,+ headers: { ...basic('ops', 'secret-pass'), 'sec-fetch-site': site },+ })+ );+ expect(response.status).toBe(403);+ }+ }+});diff --git a/packages/core/src/http/memory.routes.ts b/packages/core/src/http/memory.routes.tsindex d097718cb..44c2e125f 100644--- a/packages/core/src/http/memory.routes.ts+++ b/packages/core/src/http/memory.routes.ts@@ -20,6 +20,10 @@ import {export const MEMORY_ROUTES_BASE_PATH = '/admin/memory';const UNAUTHORIZED = 401;+const FORBIDDEN = 403;+// What a browser sends for a request another site's page started. curl sends+// no `sec-fetch-site`, and opening the URL directly sends `none`.+const CROSS_ORIGIN_FETCH_SITES = new Set(['cross-site', 'same-site']);const MEMORY_REALM = 'OpenPanel ops';const HIDDEN = { hide: true } as const;const MAX_TOP_OBJECT_TYPES = 200;@@ -53,6 +57,12 @@ export function memoryRoutes(deps: AppDeps, adminAuth: BasicAuthCredentials) {stopListening?.();}).onBeforeHandle(({ request, set, status }) => {+ // A browser attaches cached Basic credentials to another page's request,+ // so without this any site could trigger a gc (CSRF).+ const fetchSite = request.headers.get('sec-fetch-site');+ if (fetchSite && CROSS_ORIGIN_FETCH_SITES.has(fetchSite)) {+ return status(FORBIDDEN);+ }if (matchesBasicAuth(request.headers.get('authorization'), adminAuth)) {return;}diff --git a/packages/core/src/http/rate-limit.test.ts b/packages/core/src/http/rate-limit.test.tsindex 5c3089e28..9207975a8 100644--- a/packages/core/src/http/rate-limit.test.ts+++ b/packages/core/src/http/rate-limit.test.ts@@ -1,8 +1,22 @@// The prefix table IS the contract: it must match the published rate limits,// and the paths the docs call unlimited must stay absent from it.-import { describe, expect, test } from 'bun:test';-import { HTTP_RATE_LIMITS, limitFor } from './rate-limit';+import { afterAll, describe, expect, test } from 'bun:test';+import { randomUUID } from 'node:crypto';+import { getRedisCache } from '@openpanel/redis';+import type { Logger } from '../logger';+import {+ bucketsFor,+ fingerprintFor,+ HTTP_RATE_LIMITS,+ isRateLimited,+ limitFor,+} from './rate-limit';++const DEFAULT_IP_HEADERS = {+ attributionOrder: undefined,+ trustedOrder: undefined,+};// The published rate-limits page's numbers, for the first three entries;// `/mcp` and `/misc/favicon` are pinned here only.@@ -54,3 +68,181 @@ describe('HTTP rate limits', () => {expect(limitFor('/misc/stats')).toBeUndefined();});});++describe('rate-limit buckets', () => {+ const CLIENT_ID = '11111111-1111-4111-8111-111111111111';+ const ROTATED_ID = '22222222-2222-4222-8222-222222222222';+ const mcpToken = Buffer.from(`${CLIENT_ID}:wrong-secret`).toString('base64');++ test('a public route ignores the client-id header', () => {+ const request = new Request('https://api.test/misc/favicon', {+ headers: { 'openpanel-client-id': ROTATED_ID },+ });+ expect(+ fingerprintFor('ip', { request, ipHeaders: DEFAULT_IP_HEADERS })+ ).toBe('ip:unknown');+ });++ test('MCP buckets on the token, not on a rotated header', () => {+ const request = new Request(`https://api.test/mcp?token=${mcpToken}`, {+ headers: { 'openpanel-client-id': ROTATED_ID },+ });+ expect(+ fingerprintFor('mcp-token', { request, ipHeaders: DEFAULT_IP_HEADERS })+ ).toBe(`client:${CLIENT_ID}`);++ const bearer = new Request('https://api.test/mcp', {+ headers: { authorization: `Bearer ${mcpToken}` },+ });+ expect(+ fingerprintFor('mcp-token', {+ request: bearer,+ ipHeaders: DEFAULT_IP_HEADERS,+ })+ ).toBe(`client:${CLIENT_ID}`);+ });++ test('authenticated surfaces bucket on the client they authenticate', () => {+ const request = new Request('https://api.test/export/events', {+ headers: { 'openpanel-client-id': CLIENT_ID },+ });+ expect(+ fingerprintFor('client', { request, ipHeaders: DEFAULT_IP_HEADERS })+ ).toBe(`client:${CLIENT_ID}`);+ });++ test('the address budget ignores a caller-set ip header', () => {+ const request = new Request('https://api.test/export/events', {+ headers: { 'x-client-ip': '8.8.8.8', 'x-real-ip': '1.1.1.1' },+ });+ expect(+ fingerprintFor('ip', { request, ipHeaders: DEFAULT_IP_HEADERS })+ ).toBe('ip:1.1.1.1');+ });++ test('the address budget follows the deployment trusted header order', () => {+ const request = new Request('https://api.test/export/events', {+ headers: { 'cf-connecting-ip': '1.1.1.1', 'x-real-ip': '9.9.9.9' },+ });+ const ipHeaders = {+ attributionOrder: undefined,+ trustedOrder: ['x-real-ip'],+ };+ expect(fingerprintFor('ip', { request, ipHeaders })).toBe('ip:9.9.9.9');+ });++ test('client surfaces also count every request against the address', () => {+ for (const path of [+ '/manage/clients',+ '/export/events',+ '/insights/x',+ '/mcp',+ ]) {+ const limit = limitFor(path);+ if (!limit) {+ throw new Error(`no limit for ${path}`);+ }+ const buckets = (clientId: string) =>+ bucketsFor(limit, {+ request: new Request(+ `https://api.test${path}?token=${Buffer.from(`${clientId}:s`).toString('base64')}`,+ {+ headers: {+ 'openpanel-client-id': clientId,+ 'cf-connecting-ip': '1.1.1.1',+ },+ }+ ),+ ipHeaders: DEFAULT_IP_HEADERS,+ }).map((bucket) => bucket.key);++ const [firstClient, firstAddress] = buckets(CLIENT_ID);+ const [rotatedClient, rotatedAddress] = buckets(ROTATED_ID);+ expect(rotatedClient).not.toBe(firstClient);+ expect(rotatedAddress).toBe(firstAddress);+ expect(firstAddress).toBe(`rl:http:${limit.scope}:budget:ip:1.1.1.1`);+ expect(limit.ipBudget?.max ?? 0).toBeGreaterThan(limit.max);+ }+ });++ test('the address budget falls back to the socket address', () => {+ const request = new Request('https://api.test/manage/clients', {+ headers: { 'openpanel-client-id': CLIENT_ID },+ });+ const limit = limitFor('/manage/clients');+ if (!limit) {+ throw new Error('no limit for /manage');+ }+ const keys = bucketsFor(limit, {+ request,+ ipHeaders: DEFAULT_IP_HEADERS,+ remoteAddress: '10.0.0.5',+ }).map((bucket) => bucket.key);+ expect(keys).toEqual([+ `rl:http:manage:client:${CLIENT_ID}`,+ 'rl:http:manage:budget:ip:10.0.0.5',+ ]);+ });++ test('a public route has no second bucket', () => {+ const limit = limitFor('/misc/favicon');+ if (!limit) {+ throw new Error('no limit for /misc/favicon');+ }+ const request = new Request('https://api.test/misc/favicon');+ expect(+ bucketsFor(limit, { request, ipHeaders: DEFAULT_IP_HEADERS })+ ).toHaveLength(1);+ });+});++describe('rate-limit address budget', () => {+ const scope = `test-${randomUUID()}`;+ const silentLogger: Logger = {+ fatal: () => undefined,+ error: () => undefined,+ warn: () => undefined,+ info: () => undefined,+ debug: () => undefined,+ trace: () => undefined,+ child: () => silentLogger,+ };++ afterAll(async () => {+ const redis = getRedisCache();+ const keys = await redis.keys(`rl:http:${scope}:*`);+ if (keys.length > 0) {+ await redis.del(...keys);+ }+ });++ test('rotating the client id does not escape the address budget', async () => {+ const limit = {+ scope,+ keyBy: 'client',+ max: 100,+ windowMs: 10_000,+ ipBudget: { max: 3, windowMs: 10_000 },+ } as const;+ const attempt = () =>+ isRateLimited(+ limit,+ {+ request: new Request('https://api.test/manage/clients', {+ headers: {+ 'openpanel-client-id': randomUUID(),+ 'cf-connecting-ip': '1.1.1.1',+ },+ }),+ ipHeaders: DEFAULT_IP_HEADERS,+ },+ silentLogger+ );++ const results: boolean[] = [];+ for (let index = 0; index < 5; index++) {+ results.push(await attempt());+ }+ expect(results).toEqual([false, false, false, true, true]);+ });+});diff --git a/packages/core/src/http/rate-limit.ts b/packages/core/src/http/rate-limit.tsindex 8930d60bf..2ab1dc5d8 100644--- a/packages/core/src/http/rate-limit.ts+++ b/packages/core/src/http/rate-limit.ts@@ -3,6 +3,7 @@import { getRedisCache } from '@openpanel/redis';import { Elysia } from 'elysia';+import type { IpHeaderConfig } from '../config';import type { AppDeps } from '../context';import type { Logger } from '../logger';import { getTrustedIpFromHeaders } from '../shared/get-client-ip';@@ -19,29 +20,124 @@ const TOO_MANY_REQUESTS_BODY = {message: 'You have exceeded the rate limit for this endpoint.',} as const;+const MCP_TOKEN_SEPARATOR = ':';+/**- * The API client if there is one, otherwise the TRUSTED ip.- * `x-client-ip` and `x-forwarded-for` are caller-set, so keying on the- * attribution address would hand out a fresh bucket per request.+ * Per-address budgets on top of the per-client limits. A client id is+ * unverified when the limiter runs, so rotating it buys fresh client buckets;+ * the address budget caps how often one caller reaches the secret check. Set+ * well above one client's limit because many clients share an egress+ * (offices, CI, hosted AI clients for MCP).*/-const TRUSTED_ONLY = { attributionOrder: undefined, trustedOrder: undefined };+const MANAGE_IP_BUDGET = { max: 200, windowMs: 10_000 } as const;+const EXPORT_IP_BUDGET = { max: 1000, windowMs: 10_000 } as const;+const INSIGHTS_IP_BUDGET = { max: 1000, windowMs: 10_000 } as const;+const MCP_IP_BUDGET = { max: 3000, windowMs: 60_000 } as const;-function fingerprintFor(headers: Headers, remoteAddress?: string): string {- const clientId = headers.get('openpanel-client-id');- if (clientId) {- return `client:${clientId}`;+/**+ * Who a bucket belongs to. The limiter runs before authentication, so a+ * caller-supplied id only names the bucket where it also names the credential+ * that request is checked against:+ * - `client`: the `openpanel-client-id` header the route authenticates.+ * - `mcp-token`: the client id inside the MCP token, which MCP authenticates;+ * its callers share egress addresses (hosted AI clients), so not only the ip.+ * - `ip`: an unauthenticated route, where any header is free to rotate.+ */+export type RateLimitKey = 'client' | 'mcp-token' | 'ip';++function mcpTokenClientId(request: Request): string | undefined {+ const token =+ new URL(request.url).searchParams.get('token') ??+ request.headers.get('authorization')?.replace(/^Bearer\s+/i, '');+ if (!token) {+ return undefined;+ }+ const decoded = Buffer.from(token, 'base64').toString('utf-8');+ const separator = decoded.indexOf(MCP_TOKEN_SEPARATOR);+ return separator > 0 ? decoded.slice(0, separator) : undefined;+}++function callerIdFor(+ keyBy: RateLimitKey,+ request: Request+): string | undefined {+ if (keyBy === 'client') {+ return request.headers.get('openpanel-client-id') ?? undefined;}- const { ip } = getTrustedIpFromHeaders(TRUSTED_ONLY, headers, remoteAddress);+ if (keyBy === 'mcp-token') {+ return mcpTokenClientId(request);+ }+ return undefined;+}++export interface RateLimitCaller {+ request: Request;+ ipHeaders: IpHeaderConfig;+ remoteAddress?: string;+}++/** Never a caller-set header: an attribution address would rotate freely. */+function ipFingerprint({+ request,+ ipHeaders,+ remoteAddress,+}: RateLimitCaller): string {+ const { ip } = getTrustedIpFromHeaders(+ ipHeaders,+ request.headers,+ remoteAddress+ );// Fail closed: an edge that stops forwarding IPs should throttle, not open.return `ip:${ip || 'unknown'}`;}-export interface HttpRateLimit {+export function fingerprintFor(+ keyBy: RateLimitKey,+ caller: RateLimitCaller+): string {+ const clientId = callerIdFor(keyBy, caller.request);+ if (clientId) {+ return `client:${clientId}`;+ }+ return ipFingerprint(caller);+}++interface RateLimitWindow {/** Requests allowed per window. */max: number;windowMs: number;+}++export interface HttpRateLimit extends RateLimitWindow {/** Groups every route under one bucket. */scope: string;+ keyBy: RateLimitKey;+ /** A second bucket per trusted address, counted on every request. */+ ipBudget?: RateLimitWindow;+}++interface Bucket extends RateLimitWindow {+ key: string;+}++export function bucketsFor(+ { max, windowMs, scope, keyBy, ipBudget }: HttpRateLimit,+ caller: RateLimitCaller+): Bucket[] {+ const buckets: Bucket[] = [+ {+ key: `rl:http:${scope}:${fingerprintFor(keyBy, caller)}`,+ max,+ windowMs,+ },+ ];+ if (ipBudget) {+ buckets.push({+ key: `rl:http:${scope}:budget:${ipFingerprint(caller)}`,+ ...ipBudget,+ });+ }+ return buckets;}/**@@ -50,32 +146,45 @@ export interface HttpRateLimit {* credential.*/export async function isRateLimited(- { max, windowMs, scope }: HttpRateLimit,- headers: Headers,- remoteAddress: string | undefined,+ limit: HttpRateLimit,+ caller: RateLimitCaller,logger: Logger): Promise<boolean> {- const key = `rl:http:${scope}:${fingerprintFor(headers, remoteAddress)}`;+ const buckets = bucketsFor(limit, caller);try {const redis = getRedisCache();- const results = await redis.pipeline().incr(key).pttl(key).exec();- const hits = Number(results?.[0]?.[1] ?? 1);- const ttlMs = Number(results?.[1]?.[1] ?? KEY_MISSING);-- // The window opens on the first hit. Re-arm a counter that lost its- // expiry, or it would count up forever and lock the caller out for good.- if (hits === 1 || ttlMs === KEY_WITHOUT_EXPIRY || ttlMs === KEY_MISSING) {- await redis.pexpire(key, windowMs);+ const pipeline = redis.pipeline();+ for (const { key } of buckets) {+ pipeline.incr(key).pttl(key);}+ const results = await pipeline.exec();++ let isLimited = false;+ for (const [index, { key, max, windowMs }] of buckets.entries()) {+ const hits = Number(results?.[index * 2]?.[1] ?? 1);+ const ttlMs = Number(results?.[index * 2 + 1]?.[1] ?? KEY_MISSING);++ // The window opens on the first hit. Re-arm a counter that lost its+ // expiry, or it would count up forever and lock the caller out for good.+ if (hits === 1 || ttlMs === KEY_WITHOUT_EXPIRY || ttlMs === KEY_MISSING) {+ await redis.pexpire(key, windowMs);+ }- if (hits > max) {- logger.warn({ scope, key, hits, max, windowMs }, 'rate limit exceeded');- return true;+ if (hits > max) {+ logger.warn(+ { scope: limit.scope, key, hits, max, windowMs },+ 'rate limit exceeded'+ );+ isLimited = true;+ }}- return false;+ return isLimited;} catch (error) {- logger.error({ err: error, scope }, 'rate limit store unavailable');+ logger.error(+ { err: error, scope: limit.scope },+ 'rate limit store unavailable'+ );return false;}}@@ -90,22 +199,63 @@ export async function isRateLimited(export const HTTP_RATE_LIMITS: readonly (HttpRateLimit & {prefix: string;})[] = [- { prefix: '/manage', scope: 'manage', max: 20, windowMs: 10_000 },- { prefix: '/export', scope: 'export', max: 100, windowMs: 10_000 },- { prefix: '/insights', scope: 'insights', max: 100, windowMs: 10_000 },- { prefix: '/mcp', scope: 'mcp', max: 60, windowMs: 60_000 },- { prefix: '/misc/favicon', scope: 'favicon', max: 100, windowMs: 10_000 },+ {+ prefix: '/manage',+ scope: 'manage',+ keyBy: 'client',+ ipBudget: MANAGE_IP_BUDGET,+ max: 20,+ windowMs: 10_000,+ },+ {+ prefix: '/export',+ scope: 'export',+ keyBy: 'client',+ ipBudget: EXPORT_IP_BUDGET,+ max: 100,+ windowMs: 10_000,+ },+ {+ prefix: '/insights',+ scope: 'insights',+ keyBy: 'client',+ ipBudget: INSIGHTS_IP_BUDGET,+ max: 100,+ windowMs: 10_000,+ },+ {+ prefix: '/mcp',+ scope: 'mcp',+ keyBy: 'mcp-token',+ ipBudget: MCP_IP_BUDGET,+ max: 60,+ windowMs: 60_000,+ },+ {+ prefix: '/misc/favicon',+ scope: 'favicon',+ keyBy: 'ip',+ max: 100,+ windowMs: 10_000,+ },];export function limitFor(pathname: string): HttpRateLimit | undefined {return HTTP_RATE_LIMITS.find((limit) => pathname.startsWith(limit.prefix));}+/** The part of Bun's server Elysia hands a hook: the peer address. */+interface SocketAddressSource {+ requestIP(request: Request): { address: string } | null;+}+/** Mounted once per route surface so a new `/manage` route cannot arrive without a limit. */export function httpRateLimit(deps: AppDeps) {return new Elysia({ name: 'core/http/rate-limit' }).use(requestContext(deps))- .onBeforeHandle({ as: 'global' }, async ({ ctx, request, path, set }) => {+ .onBeforeHandle({ as: 'global' }, async (context) => {+ const { ctx, request, path, set } = context;+ const server: SocketAddressSource | null = context.server;const limit = limitFor(path);if (!limit) {return;@@ -115,12 +265,13 @@ export function httpRateLimit(deps: AppDeps) {if (!ctx) {return;}- // No socket address at this layer; without a trusted header, callers- // share the fail-closed `unknown` bucket.const limited = await isRateLimited(limit,- request.headers,- undefined,+ {+ request,+ ipHeaders: deps.config.ipHeaders,+ remoteAddress: server?.requestIP(request)?.address,+ },ctx.logger);if (limited) {diff --git a/packages/core/src/modules/assistant/assistant.constants.test.ts b/packages/core/src/modules/assistant/assistant.constants.test.tsnew file mode 100644index 000000000..acd9527c7--- /dev/null+++ b/packages/core/src/modules/assistant/assistant.constants.test.ts@@ -0,0 +1,34 @@+import { expect, test } from 'bun:test';+import { zChatRequest } from './assistant.constants';++function chatRequest(parts: Record<string, unknown>[]) {+ return {+ id: 'conversation-1',+ projectId: 'project-1',+ organizationId: 'organization-1',+ message: { id: 'message-1', role: 'user', parts },+ };+}++test('a text message is a valid chat request', () => {+ expect(+ zChatRequest.safeParse(chatRequest([{ type: 'text', text: 'Top pages?' }]))+ .success+ ).toBe(true);+});++// The AI SDK would fetch the URL from the API's own network.+test('a file part is refused', () => {+ expect(+ zChatRequest.safeParse(+ chatRequest([+ { type: 'text', text: 'Summarize this' },+ {+ type: 'file',+ mediaType: 'text/plain',+ url: 'http://169.254.169.254/latest/meta-data',+ },+ ])+ ).success+ ).toBe(false);+});diff --git a/packages/core/src/modules/assistant/assistant.constants.ts b/packages/core/src/modules/assistant/assistant.constants.tsindex ea664798b..026dcbdb8 100644--- a/packages/core/src/modules/assistant/assistant.constants.ts+++ b/packages/core/src/modules/assistant/assistant.constants.ts@@ -161,11 +161,19 @@ export const CHAT_TRIGGERS = ['submit-message', 'regenerate-message'] as const;const MAX_MESSAGE_ID_LENGTH = 64;+// The AI SDK downloads a file part's URL from the API's network before the+// model sees it, and the chat has no attachments, so a file part is refused.+const zChatRequestPart = z+ .record(z.string(), z.unknown())+ .refine((part) => part.type !== 'file', {+ message: 'File attachments are not supported',+ });+/** The one message the client sends per turn: the new user message, or the assistant message it responded to (tool approvals, client tool results). */export const zChatRequestMessage = z.object({id: z.string().min(1).max(MAX_MESSAGE_ID_LENGTH),role: z.enum(CHAT_MESSAGE_ROLES),- parts: z.array(z.record(z.string(), z.unknown())),+ parts: z.array(zChatRequestPart),metadata: z.record(z.string(), z.unknown()).optional(),});diff --git a/packages/core/src/modules/assistant/src/chat.ts b/packages/core/src/modules/assistant/src/chat.tsindex daa25b59c..1b80dc2e7 100644--- a/packages/core/src/modules/assistant/src/chat.ts+++ b/packages/core/src/modules/assistant/src/chat.ts@@ -28,7 +28,7 @@ import {resolveChatModel,} from './providers';import { generateConversationTitle } from './title';-import { CHAT_TOOL_APPROVAL, createChatTools } from './tools';+import { createChatToolApproval, createChatTools } from './tools';/** Dashboard-building turns chain many tool calls; the ceiling stops runaway loops. */const MAX_STEPS = 30;@@ -186,7 +186,7 @@ export async function runChat(}),messages: modelMessages,tools,- toolApproval: CHAT_TOOL_APPROVAL,+ toolApproval: createChatToolApproval(toolContext),stopWhen: isStepCount(MAX_STEPS),reasoning: entry.reasoning,providerOptions: reasoningProviderOptions(entry),diff --git a/packages/core/src/modules/assistant/src/persistence.postgres.test.ts b/packages/core/src/modules/assistant/src/persistence.postgres.test.tsindex ee76cdae7..cfdb7da6b 100644--- a/packages/core/src/modules/assistant/src/persistence.postgres.test.ts+++ b/packages/core/src/modules/assistant/src/persistence.postgres.test.ts@@ -8,6 +8,8 @@ const ORGANIZATION_ID = 'chat-persistence-org';const PROJECT_ID = 'chat-persistence-project';const USER_ID = 'chat-persistence-user';const CONVERSATION_ID = 'chat-persistence-conversation';+const OTHER_USER_ID = 'chat-persistence-other-user';+const CLAIMED_CONVERSATION_ID = 'chat-persistence-claimed-conversation';let deps: ServiceDeps;@@ -28,13 +30,25 @@ beforeAll(async () => {create: { id: USER_ID, email: '[email protected]' },update: {},});+ await deps.db.user.upsert({+ where: { id: OTHER_USER_ID },+ create: {+ id: OTHER_USER_ID,+ email: '[email protected]',+ },+ update: {},+ });});afterAll(async () => {- await deps.db.conversation.deleteMany({ where: { id: CONVERSATION_ID } });+ await deps.db.conversation.deleteMany({+ where: { id: { in: [CONVERSATION_ID, CLAIMED_CONVERSATION_ID] } },+ });await deps.db.project.deleteMany({ where: { id: PROJECT_ID } });await deps.db.organization.deleteMany({ where: { id: ORGANIZATION_ID } });- await deps.db.user.deleteMany({ where: { id: USER_ID } });+ await deps.db.user.deleteMany({+ where: { id: { in: [USER_ID, OTHER_USER_ID] } },+ });});// Anthropic binds a thinking block to the bytes of the conversation before@@ -75,3 +89,33 @@ test('message parts survive storage byte for byte, including key order', async (expect(JSON.stringify(loaded?.parts)).toBe(JSON.stringify(message.parts));});++// The route checks ownership before the model runs; a conversation created+// with the same id while it runs must not receive this turn.+test('messages are not saved into a conversation someone else owns', async () => {+ await deps.db.conversation.create({+ data: {+ id: CLAIMED_CONVERSATION_ID,+ projectId: PROJECT_ID,+ organizationId: ORGANIZATION_ID,+ userId: OTHER_USER_ID,+ },+ });++ await expect(+ saveMessages(deps, {+ conversationId: CLAIMED_CONVERSATION_ID,+ projectId: PROJECT_ID,+ organizationId: ORGANIZATION_ID,+ userId: USER_ID,+ messages: [+ {+ id: 'chat-persistence-claimed-message',+ role: 'user',+ parts: [{ type: 'text', text: 'private question' }],+ },+ ],+ })+ ).rejects.toThrow('belongs to another user');+ expect(await loadMessages(deps, CLAIMED_CONVERSATION_ID)).toEqual([]);+});diff --git a/packages/core/src/modules/assistant/src/persistence.ts b/packages/core/src/modules/assistant/src/persistence.tsindex d3ecfd250..cf92742fc 100644--- a/packages/core/src/modules/assistant/src/persistence.ts+++ b/packages/core/src/modules/assistant/src/persistence.ts@@ -64,7 +64,9 @@ export async function saveMessages(throw new Error(`Message ${foreign.id} belongs to another conversation`);}const title = stripNullCharacters(input.title ?? undefined);- await tx.conversation.upsert({+ // The id comes from the client and is only checked before the model runs;+ // someone else may have created it since, and must not get these messages.+ const conversation = await tx.conversation.upsert({where: { id: input.conversationId },create: {id: input.conversationId,@@ -77,7 +79,16 @@ export async function saveMessages(updatedAt: new Date(),...(title ? { title } : {}),},+ select: { userId: true, projectId: true },});+ const ownedByCaller =+ conversation.userId === input.userId &&+ conversation.projectId === input.projectId;+ if (!ownedByCaller) {+ throw new Error(+ `Conversation ${input.conversationId} belongs to another user`+ );+ }// Rows saved together must load in this order; the default timestamp can tie.const baseTime = Date.now();for (const [index, message] of input.messages.entries()) {diff --git a/packages/core/src/modules/assistant/src/tools/index.ts b/packages/core/src/modules/assistant/src/tools/index.tsindex 8b9a88b74..26c98e1f6 100644--- a/packages/core/src/modules/assistant/src/tools/index.ts+++ b/packages/core/src/modules/assistant/src/tools/index.ts@@ -10,7 +10,7 @@ import { entityTools } from './entities';import { referenceTools } from './references';import { searchTools } from './search';import { seoTools } from './seo';-import { trackingTools } from './tracking';+import { trackingToolApproval, trackingTools } from './tracking';/** One tool set per turn; every tool closes over the checked context. */export function createChatTools(ctx: ChatToolContext) {@@ -33,7 +33,7 @@ export function createChatTools(ctx: ChatToolContext) {export type ChatTools = ReturnType<typeof createChatTools>;/** Writes wait for the user to confirm in the chat. */-export const CHAT_TOOL_APPROVAL = {+const CHAT_WRITE_APPROVAL = {create_dashboard: 'user-approval',save_chart: 'user-approval',create_reference: 'user-approval',@@ -41,3 +41,7 @@ export const CHAT_TOOL_APPROVAL = {update_project: 'user-approval',change_plan: 'user-approval',} as const satisfies Partial<Record<keyof ChatTools, 'user-approval'>>;++export function createChatToolApproval(ctx: ChatToolContext) {+ return { ...CHAT_WRITE_APPROVAL, ...trackingToolApproval(ctx) };+}diff --git a/packages/core/src/modules/assistant/src/tools/tracking.test.ts b/packages/core/src/modules/assistant/src/tools/tracking.test.tsindex d7798a892..636ec0d89 100644--- a/packages/core/src/modules/assistant/src/tools/tracking.test.ts+++ b/packages/core/src/modules/assistant/src/tools/tracking.test.ts@@ -1,5 +1,5 @@import { describe, expect, test } from 'bun:test';-import { assessTracking, type TrackingFacts } from './tracking';+import { assessTracking, isProjectSite, type TrackingFacts } from './tracking';const CLIENT = 'a6d8af68-dfdb-4794-baac-809f503fb7bf';@@ -66,6 +66,25 @@ describe('assessTracking', () => {).toContain('another project');});+ test('a client id the user cannot place reads the same whether or not it exists', () => {+ const findings = assessTracking(+ facts({+ snippet: {+ scriptTag: true,+ initCall: true,+ sdkPackage: false,+ clientIds: ['11111111-1111-4111-8111-111111111111'],+ },+ })+ );+ const message = findings.find(+ (finding) => finding.check === 'clientId'+ )?.message;+ expect(levels(findings, 'clientId')).toEqual(['error']);+ expect(message).not.toContain('another project');+ expect(message).not.toContain('does not exist');+ });+test('an origin outside the allowed list is an error, a wildcard is fine', () => {expect(levels(assessTracking(facts({ cors: ['https://other.test'] })), 'cors')@@ -111,3 +130,21 @@ describe('assessTracking', () => {).toEqual(['error']);});});++describe('isProjectSite', () => {+ test("the project's own origin loads without asking, whatever the path", () => {+ expect(isProjectSite('https://acme.test', 'acme.test/pricing?x=1')).toBe(+ true+ );+ });++ test('any other address needs the user to confirm it', () => {+ expect(+ isProjectSite('https://acme.test', 'https://evil.test/?d=secret')+ ).toBe(false);+ expect(+ isProjectSite('https://acme.test', 'https://acme.test.evil.test')+ ).toBe(false);+ expect(isProjectSite(null, 'https://acme.test')).toBe(false);+ });+});diff --git a/packages/core/src/modules/assistant/src/tools/tracking.ts b/packages/core/src/modules/assistant/src/tools/tracking.tsindex dd6743a8a..df16de5e6 100644--- a/packages/core/src/modules/assistant/src/tools/tracking.ts+++ b/packages/core/src/modules/assistant/src/tools/tracking.ts@@ -69,6 +69,35 @@ function normalizeUrl(input: string): string {return /^https?:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}`;}+/**+ * What the model reads (event properties, page HTML) can steer it, and a URL's+ * path and query can carry what it has read to whoever runs that server. Only+ * the project's own site is loaded without the user confirming the address.+ */+export function isProjectSite(projectDomain: string | null, url: string) {+ if (!projectDomain) {+ return false;+ }+ const siteOrigin = originOf(normalizeUrl(projectDomain));+ return siteOrigin !== null && siteOrigin === originOf(normalizeUrl(url));+}++function diagnoseTrackingApproval(ctx: ChatToolContext) {+ return async ({ url }: { url?: string }) => {+ const target = url?.trim();+ if (!target) {+ return 'not-applicable' as const;+ }+ const project = await ctx.deps.db.project.findUniqueOrThrow({+ where: { id: ctx.projectId },+ select: { domain: true },+ });+ return isProjectSite(project.domain, target)+ ? ('not-applicable' as const)+ : ('user-approval' as const);+ };+}+/** Turns the gathered facts into the checklist the user sees and the model explains. */export function assessTracking(facts: TrackingFacts): Finding[] {const findings: Finding[] = [];@@ -158,7 +187,7 @@ export function assessTracking(facts: TrackingFacts): Finding[] {check: 'clientId',message: foreign? `Client id ${id} belongs to another project${foreign.projectId ? ` (${foreign.projectId})` : ''}, so its events land there, not here.`- : `Client id ${id} does not exist. Events sent with it are rejected.`,+ : `Client id ${id} is not a client of this project or of any project you can open, so its events do not land here.`,});}}@@ -329,9 +358,10 @@ async function lastEventAt(ctx: ChatToolContext): Promise<string | null> {}/**- * Client ids found on the page that are not this project's. The owning- * project is named only when the user may open it: a `selected`-scope- * member must not learn sibling projects from a client id on a web page.+ * Client ids found on the page that belong to another project the user may+ * open. One in a project they cannot open is left out, so it reads the same as+ * an unknown id: a `selected`-scope member must not learn sibling projects+ * from a client id on a web page.*/async function findForeignClients(ctx: ChatToolContext,@@ -353,10 +383,7 @@ async function findForeignClients(const canOpen = (projectId: string | null) =>grants?.projectScope === 'all' ||(grants?.projects.some((grant) => grant.projectId === projectId) ?? false);- return clients.map((client) => ({- id: client.id,- projectId: canOpen(client.projectId) ? client.projectId : null,- }));+ return clients.filter((client) => canOpen(client.projectId));}export async function gatherTrackingFacts(@@ -412,11 +439,15 @@ export async function gatherTrackingFacts(};}+export function trackingToolApproval(ctx: ChatToolContext) {+ return { diagnose_tracking: diagnoseTrackingApproval(ctx) };+}+export function trackingTools(ctx: ChatToolContext) {return {diagnose_tracking: tool({description:- "Check why events from a website are not showing up. Loads the page, looks for the OpenPanel snippet and client id, and checks them against this project's clients, allowed origins (CORS), event filters and the page views that actually arrived. Pass the URL the user mentions; without one it uses the project's configured domain, so confirm that domain with the user first when it is unclear.",+ "Check why events from a website are not showing up. Loads the page, looks for the OpenPanel snippet and client id, and checks them against this project's clients, allowed origins (CORS), event filters and the page views that actually arrived. Pass the URL the user mentions; without one it uses the project's configured domain, so confirm that domain with the user first when it is unclear. A URL on another site waits for the user to confirm it.",inputSchema: z.object({url: z.string()diff --git a/packages/core/src/modules/auth/src/registration.test.ts b/packages/core/src/modules/auth/src/registration.test.tsindex e69682ee4..93ab6778f 100644--- a/packages/core/src/modules/auth/src/registration.test.ts+++ b/packages/core/src/modules/auth/src/registration.test.ts@@ -3,9 +3,10 @@ import { testCoreConfig } from '../../../../test/config-fixture';import type { ServiceDeps } from '../../../services';import { getIsRegistrationAllowed } from './registration';+const HOUR_MS = 60 * 60 * 1000;const mockUserCount = mock(async () => 5);const mockInviteFindUnique = mock(- async (): Promise<{ id: string } | null> => null+ async (): Promise<{ id: string; expiresAt: Date } | null> => null);/**@@ -56,7 +57,10 @@ describe('getIsRegistrationAllowed', () => {});it('allows a new user holding a valid invite when registration is disabled', async () => {- mockInviteFindUnique.mockResolvedValue({ id: 'invite-1' });+ mockInviteFindUnique.mockResolvedValue({+ id: 'invite-1',+ expiresAt: new Date(Date.now() + HOUR_MS),+ });expect(await getIsRegistrationAllowed(@@ -66,6 +70,20 @@ describe('getIsRegistrationAllowed', () => {).toBe(true);});+ it('blocks an expired invite', async () => {+ mockInviteFindUnique.mockResolvedValue({+ id: 'invite-1',+ expiresAt: new Date(Date.now() - HOUR_MS),+ });++ expect(+ await getIsRegistrationAllowed(+ depsWith({ allowRegistration: false, allowInvitation: true }),+ 'invite-1'+ )+ ).toBe(false);+ });+it('blocks an unknown invite id', async () => {mockInviteFindUnique.mockResolvedValue(null);@@ -78,7 +96,10 @@ describe('getIsRegistrationAllowed', () => {});it('blocks a valid invite when invitations are disabled', async () => {- mockInviteFindUnique.mockResolvedValue({ id: 'invite-1' });+ mockInviteFindUnique.mockResolvedValue({+ id: 'invite-1',+ expiresAt: new Date(Date.now() + HOUR_MS),+ });expect(await getIsRegistrationAllowed(diff --git a/packages/core/src/modules/auth/src/registration.ts b/packages/core/src/modules/auth/src/registration.tsindex b495acdb9..016617adb 100644--- a/packages/core/src/modules/auth/src/registration.ts+++ b/packages/core/src/modules/auth/src/registration.ts@@ -25,7 +25,7 @@ export async function getIsRegistrationAllowed(return true;}- // 2. If there is an invite, check if it is valid+ // 2. If there is an invite, it must exist and be unexpiredif (inviteId) {if (allowInvitation === false) {return false;@@ -37,7 +37,7 @@ export async function getIsRegistrationAllowed(},});- return !!invite;+ return !!invite && invite.expiresAt >= new Date();}return allowRegistration;diff --git a/packages/core/src/modules/chart/chart.service.test.ts b/packages/core/src/modules/chart/chart.service.test.tsindex b0c79dcf1..14e8f42ca 100644--- a/packages/core/src/modules/chart/chart.service.test.ts+++ b/packages/core/src/modules/chart/chart.service.test.ts@@ -441,3 +441,81 @@ describe('getChartBucketProfiles', () => {).rejects.toThrow(/Refusing to inline identifier/);});});++describe('getRetentionChart from a shared report', () => {+ const nameFilter = (operator: 'is' | 'isNot', value: string) => ({+ id: 'n',+ name: 'name',+ operator,+ value: [value],+ });+ const pathFilter = {+ id: 'p',+ name: 'path',+ operator: 'is' as const,+ value: ['/signup'],+ };+ const savedReport = (+ chartType: IReportInput['chartType'],+ operator: 'is' | 'isNot',+ leadingFilters: (typeof pathFilter)[] = []+ ) =>+ ({+ ...report({+ chartType,+ series: [+ pageView({+ id: 'A',+ filters: [...leadingFilters, nameFilter(operator, 'signup')],+ }),+ pageView({ id: 'B', filters: [nameFilter('is', 'purchase')] }),+ ],+ }),+ id: 'report_1',+ }) as unknown as Parameters<typeof service.getRetentionChart>[0];+ const input = {+ projectId: TEST_PROJECT_ID,+ firstEvent: ['page_view'],+ secondEvent: ['page_view'],+ criteria: 'on_or_after' as const,+ interval: 'day' as const,+ range: '30d' as const,+ };++ it('refuses a report that is not a retention report', async () => {+ await expect(+ service.getRetentionChart(savedReport('linear', 'is'), input)+ ).rejects.toThrow('Report is not a retention report');+ });++ it('refuses an excluded event as the cohort selector', async () => {+ await expect(+ service.getRetentionChart(savedReport('retention', 'isNot'), input)+ ).rejects.toThrow('Report must have at least 2 event series');+ });++ it('refuses a series whose selected events are all excluded', async () => {+ const excludedSelection = {+ ...savedReport('retention', 'is'),+ series: [+ pageView({+ id: 'A',+ filters: [nameFilter('isNot', 'signup'), nameFilter('is', 'signup')],+ }),+ pageView({ id: 'B', filters: [nameFilter('is', 'purchase')] }),+ ],+ } as unknown as Parameters<typeof service.getRetentionChart>[0];+ await expect(+ service.getRetentionChart(excludedSelection, input)+ ).rejects.toThrow('Report must have at least 2 event series');+ });++ it('finds the event selector behind a property filter', async () => {+ await expect(+ service.getRetentionChart(+ savedReport('retention', 'is', [pathFilter]),+ input+ )+ ).resolves.toBeDefined();+ });+});diff --git a/packages/core/src/modules/chart/chart.service.ts b/packages/core/src/modules/chart/chart.service.tsindex 18981a8c3..5fbc1bca1 100644--- a/packages/core/src/modules/chart/chart.service.ts+++ b/packages/core/src/modules/chart/chart.service.ts@@ -545,20 +545,47 @@ export interface RetentionChartInput {filters?: IChartEventFilter[];}+/**+ * The events a retention series selects: its `is` filter on `name`, wherever+ * it sits among the filters. Reading any other filter as the selector would+ * turn an exclusion into the events the cohort is built from.+ */+function retentionEventNames(+ serie: ReturnType<typeof onlyEventSeries>[number] | undefined+): string[] {+ const selector = serie?.filters?.find(+ (filter) => filter.name === 'name' && filter.operator === 'is'+ );+ if (!selector) {+ return [];+ }+ // A series' filters combine with AND, so its `isNot` names are not events+ // of that series either.+ const excluded = new Set(+ (serie?.filters ?? [])+ .filter((filter) => filter.name === 'name' && filter.operator === 'isNot')+ .flatMap((filter) => filter.value.map(String))+ );+ return selector.value.map(String).filter((name) => !excluded.has(name));+}+/*** A shared retention report takes its events from the saved series: each- * series' `filters[0]` is the event-name selector, everything else is an- * audience filter.+ * series' name filters select its events, everything else is an audience+ * filter.*/function retentionInputFromReport(report: NonNullable<IServiceReport>,input: RetentionChartInput): RetentionChartInput {+ if (report.chartType !== 'retention') {+ throw new Error('Report is not a retention report');+ }const retentionOptions =report.options?.type === 'retention' ? report.options : undefined;const eventSeries = onlyEventSeries(report.series);- const firstEvent = (eventSeries[0]?.filters?.[0]?.value ?? []).map(String);- const secondEvent = (eventSeries[1]?.filters?.[0]?.value ?? []).map(String);+ const firstEvent = retentionEventNames(eventSeries[0]);+ const secondEvent = retentionEventNames(eventSeries[1]);if (firstEvent.length === 0 || secondEvent.length === 0) {throw new Error('Report must have at least 2 event series');}diff --git a/packages/core/src/modules/chart/conversion.service.ts b/packages/core/src/modules/chart/conversion.service.tsindex 1f1a8c960..6e3f1adbb 100644--- a/packages/core/src/modules/chart/conversion.service.ts+++ b/packages/core/src/modules/chart/conversion.service.ts@@ -108,7 +108,7 @@ export async function getConversion();const cohortIds = collectBreakdownCohortIds(knownBreakdowns);- const cohortMetadata = await fetchCohortsMetadata(deps, cohortIds);+ const cohortMetadata = await fetchCohortsMetadata(deps, projectId, cohortIds);const cohortJoins = cohortIds.map((id) =>buildInlineCohortJoin(id, projectId, 'events'));diff --git a/packages/core/src/modules/chart/funnel.service.ts b/packages/core/src/modules/chart/funnel.service.tsindex d44e88d6d..c049bc64a 100644--- a/packages/core/src/modules/chart/funnel.service.ts+++ b/packages/core/src/modules/chart/funnel.service.ts@@ -120,6 +120,7 @@ export async function buildFunnelBase();const cohortMetadata = await fetchCohortsMetadata(deps,+ projectId,collectBreakdownCohortIds(breakdowns));diff --git a/packages/core/src/modules/chart/retention.service.ts b/packages/core/src/modules/chart/retention.service.tsindex 5b6dcf097..ae059a9da 100644--- a/packages/core/src/modules/chart/retention.service.ts+++ b/packages/core/src/modules/chart/retention.service.ts@@ -4,6 +4,7 @@import { DateTime, round } from '@openpanel/shared';import { range } from 'ramda';import type { ServiceDeps, Services } from '../../services';+import { TimeRangeError } from '../../shared/time-buckets';import type { IChartEventFilter } from '../report/report.constants';import {type IRetentionCriteria,@@ -209,6 +210,10 @@ const LUXON_UNIT: Record<IRetentionInterval, 'days' | 'weeks' | 'months'> = {month: 'months',};+// Each period is two columns of `groupUniqArrayIf` over every cohort member,+// so the matrix width is the statement's cost. A year by the day is 366.+const MAX_RETENTION_PERIODS = 1000;+const CLICKHOUSE_DATE_TIME_LENGTH = 'yyyy-MM-dd HH:mm:ss'.length;const ISO_DATE_LENGTH = 'yyyy-MM-dd'.length;@@ -250,6 +255,11 @@ export async function getRetentionCohort(} = input;const diffInterval = diffIntervalCount(startDate, endDate, interval);+ if (diffInterval > MAX_RETENTION_PERIODS) {+ throw new TimeRangeError(+ `Retention covers at most ${MAX_RETENTION_PERIODS} ${interval} periods and the selected range spans ${diffInterval}. Pick a coarser interval or a shorter range.`+ );+ }const cohortData = await runQuery<{cohort_interval: string;diff --git a/packages/core/src/modules/chart/sankey.service.ts b/packages/core/src/modules/chart/sankey.service.tsindex ea960447f..7f3d86e82 100644--- a/packages/core/src/modules/chart/sankey.service.ts+++ b/packages/core/src/modules/chart/sankey.service.ts@@ -231,14 +231,20 @@ function getNodeId(event: string, step: number) {return `${event}::step${step}`;}-function groupTransitionsByStep(transitions: Transition[]) {- const byStep = new Map<number, Transition[]>();+/** Keyed by step, then source, so each active node reads only its own edges. */+function groupTransitionsByStepAndSource(transitions: Transition[]) {+ const byStep = new Map<number, Map<string, Transition[]>>();for (const transition of transitions) {- const bucket = byStep.get(transition.step);- if (bucket) {- bucket.push(transition);+ let bySource = byStep.get(transition.step);+ if (!bySource) {+ bySource = new Map();+ byStep.set(transition.step, bySource);+ }+ const fromSource = bySource.get(transition.source);+ if (fromSource) {+ fromSource.push(transition);} else {- byStep.set(transition.step, [transition]);+ bySource.set(transition.source, [transition]);}}return byStep;@@ -253,7 +259,7 @@ function expandFlow(): { nodes: Map<string, NodeData>; links: SankeyLink[] } {const nodes = new Map<string, NodeData>();const links: SankeyLink[] = [];- const transitionsByStep = groupTransitionsByStep(transitions);+ const transitionsByStep = groupTransitionsByStepAndSource(transitions);const activeNodes = new Map<string, string>();topEntries.forEach((entry, index) => {@@ -268,12 +274,11 @@ function expandFlow(});for (let step = FIRST_STEP; step < steps; step++) {- const stepTransitions = transitionsByStep.get(step) || [];+ const stepTransitions = transitionsByStep.get(step);const nextActiveNodes = new Map<string, string>();for (const [sourceEvent, sourceNodeId] of activeNodes) {- const fromSource = stepTransitions- .filter((transition) => transition.source === sourceEvent)+ const fromSource = (stepTransitions?.get(sourceEvent) ?? []).sort((a, b) => b.value - a.value).slice(0, TOP_DESTINATIONS_PER_NODE);diff --git a/packages/core/src/modules/chart/src/engine/compute.ts b/packages/core/src/modules/chart/src/engine/compute.tsindex e3dd5a68a..0a1827ef7 100644--- a/packages/core/src/modules/chart/src/engine/compute.ts+++ b/packages/core/src/modules/chart/src/engine/compute.ts@@ -67,40 +67,43 @@ export function compute((a, b) => new Date(a).getTime() - new Date(b).getTime());+ // Each dependency's points are indexed by date once: a lookup per+ // (date, dependency) would make long minute-level ranges quadratic.+ const dependencies = definitions+ .slice(0, formulaIndex)+ .flatMap((_depDef, depIndex) => {+ const readableId = alphabetIds[depIndex];+ if (!readableId) {+ return [];+ }+ // Could be a formula from a previous breakdown group: match by+ // definitionIndex AND breakdown signature.+ const source =+ seriesByIndex.get(depIndex) ??+ results.find(+ (s) =>+ s.definitionIndex === depIndex &&+ 'type' in s.definition &&+ s.definition.type === 'formula' &&+ s.name.slice(1).join(':::') === breakdownSignature+ );+ const countByDate = new Map<string, number>();+ for (const point of source?.data ?? []) {+ if (!countByDate.has(point.date)) {+ countByDate.set(point.date, point.count);+ }+ }+ const totalCount =+ source?.data.find((d) => d.total_count != null)?.total_count ?? 0;+ return [{ readableId, countByDate, totalCount }];+ });+// total_count is constant across all dates for a breakdown group, so// it's computed once here rather than per date.const totalCountScope: Record<string, number> = {};- definitions.slice(0, formulaIndex).forEach((_depDef, depIndex) => {- const readableId = alphabetIds[depIndex];- if (!readableId) {- return;- }-- const depSeries = seriesByIndex.get(depIndex);- if (depSeries) {- const totalCount = depSeries.data.find(- (d) => d.total_count != null- )?.total_count;- totalCountScope[readableId] = totalCount ?? 0;- } else {- // Could be a formula from a previous breakdown group - find it in results- const formulaSerie = results.find(- (s) =>- s.definitionIndex === depIndex &&- 'type' in s.definition &&- s.definition.type === 'formula' &&- s.name.slice(1).join(':::') === breakdownSignature- );- if (formulaSerie) {- const totalCount = formulaSerie.data.find(- (d) => d.total_count != null- )?.total_count;- totalCountScope[readableId] = totalCount ?? 0;- } else {- totalCountScope[readableId] = 0;- }- }- });+ for (const dependency of dependencies) {+ totalCountScope[dependency.readableId] = dependency.totalCount;+ }const totalCountResult = evaluateFormula(formula.formula,@@ -111,35 +114,9 @@ export function compute(const formulaData = sortedDates.map((date) => {const scope: Record<string, number> = {};-- definitions.slice(0, formulaIndex).forEach((_depDef, depIndex) => {- const readableId = alphabetIds[depIndex];- if (!readableId) {- return;- }-- const depSeries = seriesByIndex.get(depIndex);- if (depSeries) {- const dataPoint = depSeries.data.find((d) => d.date === date);- scope[readableId] = dataPoint?.count ?? 0;- } else {- // Could be a formula from a previous breakdown group - find it in results- // Match by definitionIndex AND breakdown signature- const formulaSerie = results.find(- (s) =>- s.definitionIndex === depIndex &&- 'type' in s.definition &&- s.definition.type === 'formula' &&- s.name.slice(1).join(':::') === breakdownSignature- );- if (formulaSerie) {- const dataPoint = formulaSerie.data.find((d) => d.date === date);- scope[readableId] = dataPoint?.count ?? 0;- } else {- scope[readableId] = 0;- }- }- });+ for (const dependency of dependencies) {+ scope[dependency.readableId] = dependency.countByDate.get(date) ?? 0;+ }const result = evaluateFormula(formula.formula, scope);diff --git a/packages/core/src/modules/chart/src/sql.ts b/packages/core/src/modules/chart/src/sql.tsindex 58761f4d4..9bda625a4 100644--- a/packages/core/src/modules/chart/src/sql.ts+++ b/packages/core/src/modules/chart/src/sql.ts@@ -13,6 +13,7 @@// clauses by key keeps a stable clause order.import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';+import { assertTimeBucketBudget } from '../../../shared/time-buckets';import type {IChartBreakdown,IChartEvent,@@ -512,8 +513,12 @@ export function chartSeriesQuery(input: ChartSeriesQueryInput): SqlFragment {const bucket = intervalBucket(interval, timezone);// ClickHouse rejects WITH FILL when TO < FROM, so only emit a fill clause// for a valid range. The SELECT bucket truncation is always safe.+ const hasDateRange = !!startDate && !!endDate;+ if (hasDateRange) {+ assertTimeBucketBudget({ startDate, endDate, interval });+ }const hasValidFillRange =- !!startDate && !!endDate && new Date(endDate) >= new Date(startDate);+ hasDateRange && new Date(endDate) >= new Date(startDate);const fill = hasValidFillRange? sql`WITH FILL ${bucket.fill(startDate, endDate)}`: sql.empty;diff --git a/packages/core/src/modules/chart/src/statement.test.ts b/packages/core/src/modules/chart/src/statement.test.tsnew file mode 100644index 000000000..64e94095b--- /dev/null+++ b/packages/core/src/modules/chart/src/statement.test.ts@@ -0,0 +1,17 @@+import { expect, mock, test } from 'bun:test';+import type { ServiceDeps } from '../../../services';+import { fetchCohortsMetadata } from './statement';++test('cohort names are looked up inside the chart project only', async () => {+ const findMany = mock(async () => [{ id: 'c1', name: 'Local cohort' }]);+ const deps = { db: { cohort: { findMany } } } as unknown as ServiceDeps;++ const metadata = await fetchCohortsMetadata(deps, 'proj_1', ['c1', 'c2']);++ expect(findMany).toHaveBeenCalledWith({+ where: { projectId: 'proj_1', id: { in: ['c1', 'c2'] } },+ select: { id: true, name: true },+ });+ expect(metadata.get('c1')?.name).toBe('Local cohort');+ expect(metadata.has('c2')).toBe(false);+});diff --git a/packages/core/src/modules/chart/src/statement.ts b/packages/core/src/modules/chart/src/statement.tsindex 656b41e42..93658a938 100644--- a/packages/core/src/modules/chart/src/statement.ts+++ b/packages/core/src/modules/chart/src/statement.ts@@ -23,15 +23,17 @@ export type AggregateChartSqlInput = Omit<'interval' | 'chartType'> & { timezone: string };+/** Ids from another project resolve to nothing, so their names never reach a label. */export async function fetchCohortsMetadata(deps: ServiceDeps,+ projectId: string,cohortIds: string[]): Promise<Map<string, CohortMetadata>> {if (cohortIds.length === 0) {return new Map();}const cohorts = await deps.db.cohort.findMany({- where: { id: { in: cohortIds } },+ where: { projectId, id: { in: cohortIds } },select: { id: true, name: true },});return new Map(cohorts.map((c) => [c.id, { id: c.id, name: c.name }]));@@ -59,6 +61,7 @@ export async function resolveChartBreakdowns(const breakdowns = withoutEmptyAllCohortsBreakdown(known, allCohorts);const cohortMetadata = await fetchCohortsMetadata(deps,+ projectId,collectBreakdownCohortIds(breakdowns));return { breakdowns, allCohorts, cohortMetadata };diff --git a/packages/core/src/modules/client/client.rpc.ts b/packages/core/src/modules/client/client.rpc.tsindex 74880b412..f56f00206 100644--- a/packages/core/src/modules/client/client.rpc.ts+++ b/packages/core/src/modules/client/client.rpc.ts@@ -21,17 +21,25 @@ export const clientRouter = createTRPCRouter({.input(z.object({ id: z.string(), name: z.string() })).mutation(async ({ input, ctx }) => {const userId = ctx.session.userId;- const access = await ctx.services.auth.getClientAccess({- userId,- clientId: input.id,- });- if (!access) {+ const client = await ctx.services.client.getClientById(input.id);+ if (!client?.organizationId) {throw new TRPCForbiddenError('You do not have access to this client');}- const client = await ctx.services.client.getClientById(input.id);- if (!client) {- throw new TRPCForbiddenError('Client not found');+ // A project's client is renamed by whoever may write that project; an+ // organization-wide one only by the admins who create and delete them.+ if (client.projectId) {+ await ctx.services.auth.requireProjectAccess({+ userId,+ projectId: client.projectId,+ level: 'write',+ });+ } else {+ await ctx.services.auth.requireOrganizationAdmin({+ userId,+ organizationId: client.organizationId,+ message: 'Only organization admins can rename API clients',+ });}return ctx.services.client.updateClientForOrganization(diff --git a/packages/core/src/modules/cohort/cohort.service.ts b/packages/core/src/modules/cohort/cohort.service.tsindex c8eff6d4c..064d7e7a5 100644--- a/packages/core/src/modules/cohort/cohort.service.ts+++ b/packages/core/src/modules/cohort/cohort.service.ts@@ -360,9 +360,11 @@ function profileColumnAccess(normalizedName: string): SqlFragment {return sql.id(normalizedName, COHORT_PROFILE_COLUMNS);}+const LATEST_ROW_KEY = sql`latest_row_key`;+function buildProfileCohortHavingClause(definition: PropertyBasedCohortDefinition-): SqlFragment | null {+): { latestRowKey: SqlFragment; having: SqlFragment } | null {const { properties, operator } = definition.criteria;// Every argMax below must order candidate rows IDENTICALLY, or equal-version@@ -370,14 +372,15 @@ function buildProfileCohortHavingClause(// AND cohort against a combination no stored row contains. One shared key (the// version column, tie-broken by a hash of every referenced column) makes all// aggregates pick the same winning row. The hash keeps comparison state at a- // fixed 8 bytes.+ // fixed 8 bytes. The key is written once, in WITH, and referenced by alias:+ // inlined into every argMax it grew the statement quadratically.const referencedColumns = Array.from(new Set(properties.map((f) => normalizeProfileColumn(f.name)))).map(profileColumnAccess);const latestRowKey = sql`tuple(last_seen_at, cityHash64(${sql.join(referencedColumns)}))`;const filterWhere = getProfileFiltersWhereClause(properties, {- latestPerProfileKey: latestRowKey,+ latestPerProfileKey: LATEST_ROW_KEY,});const filterClauses = Object.values(filterWhere);@@ -385,7 +388,10 @@ function buildProfileCohortHavingClause(return null;}- return sql.join(filterClauses, operator === 'and' ? ' AND ' : ' OR ');+ return {+ latestRowKey,+ having: sql.join(filterClauses, operator === 'and' ? ' AND ' : ' OR '),+ };}export function buildPropertyBasedCohortQuery(@@ -393,9 +399,9 @@ export function buildPropertyBasedCohortQuery(definition: PropertyBasedCohortDefinition,limit?: number): SqlFragment {- const havingClause = buildProfileCohortHavingClause(definition);+ const clause = buildProfileCohortHavingClause(definition);- if (!havingClause) {+ if (!clause) {return sql`SELECT id as profile_id FROM ${sql.id(TABLE.profiles)} WHERE 1=0`;}@@ -404,11 +410,12 @@ export function buildPropertyBasedCohortQuery(// is what runs out of memory. The aggregate shape spills normally under// profileCohortQuerySettings, and filters on aggregates move to HAVING.return sql`+ WITH ${clause.latestRowKey} AS ${LATEST_ROW_KEY}SELECT id as profile_idFROM ${sql.id(TABLE.profiles)}WHERE project_id = ${sql.string(projectId)}GROUP BY id- HAVING (${havingClause})+ HAVING (${clause.having})${limit ? sql`LIMIT ${sql.uint64(limit)}` : sql.empty}`;}diff --git a/packages/core/src/modules/cohort/src/cohort-sql.test.ts b/packages/core/src/modules/cohort/src/cohort-sql.test.tsindex 5330df1e3..ecc19ba6b 100644--- a/packages/core/src/modules/cohort/src/cohort-sql.test.ts+++ b/packages/core/src/modules/cohort/src/cohort-sql.test.ts@@ -52,7 +52,10 @@ test('resolves the newest row per profile without FINAL', () => {expect(sql).not.toContain('FINAL');expect(sql).toContain('GROUP BY id');expect(sql).toContain(- 'argMax(profiles.properties[{p2:String}], tuple(last_seen_at, cityHash64(profiles.properties[{p3:String}])))'+ 'WITH tuple(last_seen_at, cityHash64(profiles.properties[{p1:String}])) AS latest_row_key'+ );+ expect(sql).toContain(+ 'argMax(profiles.properties[{p3:String}], latest_row_key)');});@@ -74,13 +77,16 @@ test('orders every aggregate by ONE shared row key', () => {],});- // One shared key per aggregate: same columns, same order, both times.+ // The key is written once and every aggregate orders by its alias, so the+ // statement grows linearly with the filter count.expect(sql).toContain(- 'argMax(profiles.properties[{p2:String}], tuple(last_seen_at, cityHash64(profiles.properties[{p3:String}], profiles.email)))'+ 'WITH tuple(last_seen_at, cityHash64(profiles.properties[{p1:String}], profiles.email)) AS latest_row_key');+ expect(sql.match(/cityHash64/g)).toHaveLength(1);expect(sql).toContain(- 'argMax(profiles.email, tuple(last_seen_at, cityHash64(profiles.properties[{p5:String}], profiles.email)))'+ 'argMax(profiles.properties[{p3:String}], latest_row_key)');+ expect(sql).toContain('argMax(profiles.email, latest_row_key)');});test('filters aggregates in HAVING, not WHERE', () => {@@ -105,7 +111,7 @@ test('wraps plain columns too, so mixed cohorts stay on one scan', () => {],});- expect(sql).toContain('argMax(profiles.email, tuple(last_seen_at,');+ expect(sql).toContain('argMax(profiles.email, latest_row_key)');expect(sql).toContain(' OR ');expect(sql).not.toContain('FINAL');});@@ -124,7 +130,7 @@ test('wraps numeric comparisons inside the cast', () => {});expect(sql).toContain(- 'toFloat64OrNull(argMax(profiles.properties[{p2:String}], tuple(last_seen_at,'+ 'toFloat64OrNull(argMax(profiles.properties[{p3:String}], latest_row_key))');});@@ -143,7 +149,7 @@ test('binds user-controlled property keys instead of escaping them', () => {// The key never reaches the SQL text, escaped or not — it is a bound param.expect(query).not.toContain("pl'an");- expect(query).toContain('profiles.properties[{p2:String}]');+ expect(query).toContain('profiles.properties[{p3:String}]');expect(Object.values(query_params)).toContain("pl'an");});diff --git a/packages/core/src/modules/gsc/gsc.routes.ts b/packages/core/src/modules/gsc/gsc.routes.tsindex 6b7e5e29d..127dcc3d0 100644--- a/packages/core/src/modules/gsc/gsc.routes.ts+++ b/packages/core/src/modules/gsc/gsc.routes.ts@@ -1,6 +1,6 @@-// Per-route cookie signing is not wired yet, so `ctx.cookies.get` reads the-// raw, unsigned cookie value; this route only re-checks the state match and is-// not a signature-verified equivalent.+// The OAuth cookies are signed only when COOKIE_SECRET is set, so the callback+// does not trust them for authorization: the signed-in user must be able to+// write the project they name.import { redirect } from 'elysia';import { z } from 'zod';@@ -25,6 +25,14 @@ export const gscRoutes = defineRoutes((app) =>app.get('/gsc/callback',async ({ query, ctx }) => {+ const userId = (await ctx.session())?.userId;+ if (!userId) {+ return loginErrorRedirect(+ ctx.config.dashboardUrl,+ 'Sign in to finish connecting Google Search Console'+ );+ }+const storedState = ctx.cookies.get('gsc_oauth_state');const codeVerifier = ctx.cookies.get('gsc_code_verifier');const projectId = ctx.cookies.get('gsc_project_id');@@ -43,6 +51,7 @@ export const gscRoutes = defineRoutes((app) =>storedState,codeVerifier,projectId,+ userId,});return redirect(`${ctx.config.dashboardUrl}/${organizationId}/${projectId}/settings/gsc`diff --git a/packages/core/src/modules/gsc/gsc.service.test.ts b/packages/core/src/modules/gsc/gsc.service.test.tsindex f5702b19c..8010a9759 100644--- a/packages/core/src/modules/gsc/gsc.service.test.ts+++ b/packages/core/src/modules/gsc/gsc.service.test.ts@@ -151,9 +151,17 @@ const validateAuthorizationCode = mock(async () => ({// other code path that imports this same module for an export this file's// chain doesn't happen to need.const actualAuthService = await import('../auth/auth.service');+const requireProjectAccess = mock(+ async ({ userId }: { userId: string; projectId: string; level: string }) => {+ if (userId !== 'u_writer') {+ throw new Error('forbidden');+ }+ }+);mock.module('../auth/auth.service', () => ({...actualAuthService,googleGscClient: () => ({ validateAuthorizationCode }),+ getAccessChecks: async () => ({ requireProjectAccess }),}));const getSettingsForProject = mock(@@ -326,11 +334,34 @@ test('completeGscOAuthCallback rejects a state mismatch before calling Google',storedState: 'b',codeVerifier: 'verifier',projectId: 'p1',+ userId: 'u_writer',})).rejects.toThrow('GSC OAuth state mismatch');expect(validateAuthorizationCode).not.toHaveBeenCalled();});+test('completeGscOAuthCallback refuses a user who cannot write the cookie-named project', async () => {+ validateAuthorizationCode.mockClear();+ gscConnectionStore.clear();+ await expect(+ subject.completeGscOAuthCallback(deps, {+ code: 'code',+ state: 'a',+ storedState: 'a',+ codeVerifier: 'verifier',+ projectId: 'p1',+ userId: 'u_reader',+ })+ ).rejects.toThrow('forbidden');+ expect(requireProjectAccess).toHaveBeenLastCalledWith({+ userId: 'u_reader',+ projectId: 'p1',+ level: 'write',+ });+ expect(validateAuthorizationCode).not.toHaveBeenCalled();+ expect(gscConnectionStore.has('p1')).toBe(false);+});+test('completeGscOAuthCallback rejects an unknown project', async () => {await expect(subject.completeGscOAuthCallback(deps, {@@ -339,6 +370,7 @@ test('completeGscOAuthCallback rejects an unknown project', async () => {storedState: 'a',codeVerifier: 'verifier',projectId: 'proj_missing',+ userId: 'u_writer',})).rejects.toThrow('Project not found for GSC connection');});@@ -351,6 +383,7 @@ test('completeGscOAuthCallback upserts the connection and returns the organizatistoredState: 'a',codeVerifier: 'verifier',projectId: 'p1',+ userId: 'u_writer',});expect(result).toEqual({ organizationId: 'org_1' });diff --git a/packages/core/src/modules/gsc/gsc.service.ts b/packages/core/src/modules/gsc/gsc.service.tsindex edcf393a9..ca1eb18b9 100644--- a/packages/core/src/modules/gsc/gsc.service.ts+++ b/packages/core/src/modules/gsc/gsc.service.ts@@ -13,7 +13,7 @@ import type { Logger } from '../../logger';import { TRPCNotFoundError } from '../../rpc/errors';import type { ServiceDeps, Services } from '../../services';import { TABLE_NAMES } from '../../shared/ch-tables';-import { googleGscClient } from '../auth/auth.service';+import { getAccessChecks, googleGscClient } from '../auth/auth.service';const BACKFILL_MONTHS = 6;const CHUNK_DAYS = 14;@@ -1384,6 +1384,7 @@ export interface GscOAuthCallbackInput {storedState: string;codeVerifier: string;projectId: string;+ userId: string;}export interface GscOAuthCallbackResult {@@ -1398,6 +1399,15 @@ export async function completeGscOAuthCallback(throw new Error('GSC OAuth state mismatch');}+ // The cookies are only signed when COOKIE_SECRET is set, so the project they+ // name is not proof that this user may connect it.+ const { requireProjectAccess } = await getAccessChecks();+ await requireProjectAccess({+ userId: input.userId,+ projectId: input.projectId,+ level: 'write',+ });+const tokens = await googleGscClient(deps.config).validateAuthorizationCode(input.code,input.codeVerifierdiff --git a/packages/core/src/modules/import/import.constants.test.ts b/packages/core/src/modules/import/import.constants.test.tsnew file mode 100644index 000000000..a5d0a5322--- /dev/null+++ b/packages/core/src/modules/import/import.constants.test.ts@@ -0,0 +1,54 @@+import { expect, test } from 'bun:test';+import {+ MAX_IMPORT_RANGE_DAYS,+ zAmplitudeImportConfig,+ zMixpanelImportConfig,+} from './import.constants';++const mixpanel = {+ provider: 'mixpanel',+ type: 'api',+ serviceAccount: 'account',+ serviceSecret: 'secret',+ projectId: '123',+} as const;++const amplitude = {+ provider: 'amplitude',+ type: 'api',+ apiKey: 'key',+ secretKey: 'secret',+} as const;++test('an API import accepts a single day and an ordinary range', () => {+ for (const range of [+ { from: '2025-01-01', to: '2025-01-01' },+ { from: '2024-01-01', to: '2025-01-01' },+ ]) {+ expect(+ zMixpanelImportConfig.safeParse({ ...mixpanel, ...range }).success+ ).toBe(true);+ expect(+ zAmplitudeImportConfig.safeParse({ ...amplitude, ...range }).success+ ).toBe(true);+ }+});++test('an API import rejects a non-date, a reversed range and an oversized span', () => {+ const longestTo = new Date(Date.UTC(2000, 0, MAX_IMPORT_RANGE_DAYS + 1))+ .toISOString()+ .slice(0, 10);+ for (const range of [+ { from: 'yesterday', to: '2025-01-01' },+ { from: '2025-01-02', to: '2025-01-01' },+ { from: '0001-01-01', to: '9999-12-31' },+ { from: '2000-01-01', to: longestTo },+ ]) {+ expect(+ zMixpanelImportConfig.safeParse({ ...mixpanel, ...range }).success+ ).toBe(false);+ expect(+ zAmplitudeImportConfig.safeParse({ ...amplitude, ...range }).success+ ).toBe(false);+ }+});diff --git a/packages/core/src/modules/import/import.constants.ts b/packages/core/src/modules/import/import.constants.tsindex 454530353..4de8c993e 100644--- a/packages/core/src/modules/import/import.constants.ts+++ b/packages/core/src/modules/import/import.constants.ts@@ -1,4 +1,5 @@import { z } from 'zod';+import { MS_PER_DAY } from '../../shared/time.constants';export type ImportProviderId = 'umami' | 'mixpanel' | 'amplitude';export type ImportProviderType = 'file' | 'api';@@ -83,35 +84,57 @@ export type IUmamiImportConfig = z.infer<typeof zUmamiImportConfig>;export const zPlausibleImportConfig = createFileImportConfig('plausible');export type IPlausibleImportConfig = z.infer<typeof zPlausibleImportConfig>;+/** Twenty years of daily export chunks; a wider range is a typo, not a history. */+export const MAX_IMPORT_RANGE_DAYS = 20 * 366;++/** Days covered by a `YYYY-MM-DD` range, inclusive; `NaN` when either end is not a date. */+export function getImportRangeDays(from: string, to: string): number {+ return (Date.parse(to) - Date.parse(from)) / MS_PER_DAY + 1;+}++const isImportRangeSupported = ({ from, to }: { from: string; to: string }) => {+ const days = getImportRangeDays(from, to);+ return days >= 1 && days <= MAX_IMPORT_RANGE_DAYS;+};++const IMPORT_RANGE_ISSUE = {+ path: ['to'],+ message: `The end date must be on or after the start date and at most ${MAX_IMPORT_RANGE_DAYS} days later`,+};+export const zAmplitudeDataResidency = z.enum(['us', 'eu']);export type IAmplitudeDataResidency = z.infer<typeof zAmplitudeDataResidency>;-export const zAmplitudeImportConfig = z.object({- provider: z.literal('amplitude'),- type: z.literal('api'),- apiKey: z.string().min(1),- secretKey: z.string().min(1),- from: z.string().min(1),- to: z.string().min(1),- mapScreenViewProperty: z.string().optional(),- dataResidency: zAmplitudeDataResidency.optional(),-});+export const zAmplitudeImportConfig = z+ .object({+ provider: z.literal('amplitude'),+ type: z.literal('api'),+ apiKey: z.string().min(1),+ secretKey: z.string().min(1),+ from: z.iso.date(),+ to: z.iso.date(),+ mapScreenViewProperty: z.string().optional(),+ dataResidency: zAmplitudeDataResidency.optional(),+ })+ .refine(isImportRangeSupported, IMPORT_RANGE_ISSUE);export type IAmplitudeImportConfig = z.infer<typeof zAmplitudeImportConfig>;export const zMixpanelDataResidency = z.enum(['us', 'eu', 'in']);export type IMixpanelDataResidency = z.infer<typeof zMixpanelDataResidency>;-export const zMixpanelImportConfig = z.object({- provider: z.literal('mixpanel'),- type: z.literal('api'),- serviceAccount: z.string().min(1),- serviceSecret: z.string().min(1),- projectId: z.string().min(1),- from: z.string().min(1),- to: z.string().min(1),- mapScreenViewProperty: z.string().optional(),- dataResidency: zMixpanelDataResidency.optional(),-});+export const zMixpanelImportConfig = z+ .object({+ provider: z.literal('mixpanel'),+ type: z.literal('api'),+ serviceAccount: z.string().min(1),+ serviceSecret: z.string().min(1),+ projectId: z.string().min(1),+ from: z.iso.date(),+ to: z.iso.date(),+ mapScreenViewProperty: z.string().optional(),+ dataResidency: zMixpanelDataResidency.optional(),+ })+ .refine(isImportRangeSupported, IMPORT_RANGE_ISSUE);export type IMixpanelImportConfig = z.infer<typeof zMixpanelImportConfig>;export type IImportConfig =diff --git a/packages/core/src/modules/import/import.rpc.test.ts b/packages/core/src/modules/import/import.rpc.test.tsindex ccf16e050..014ed1446 100644--- a/packages/core/src/modules/import/import.rpc.test.ts+++ b/packages/core/src/modules/import/import.rpc.test.ts@@ -1,8 +1,7 @@-// Only the "is anyone logged in" boundary is exercised here, with no database.--import { expect, test } from 'bun:test';+import { expect, mock, test } from 'bun:test';import { stubHttpCtx } from '../../../test/rpc-fixtures';import { makeTrpcContext } from '../../rpc/base';+import { TRPCForbiddenError } from '../../rpc/errors';import type { CookieOptions } from '../../shared/cookie';import { importRouter } from './import.rpc';@@ -61,3 +60,110 @@ test('retry rejects an unauthenticated caller before re-enqueuing', async () =>code: 'UNAUTHORIZED',});});++function memberCaller(+ overrides: Record<string, unknown>,+ writableProjectIds: string[] = ['proj_1']+) {+ const requireProjectAccess = mock(({ projectId }: { projectId: string }) =>+ writableProjectIds.includes(projectId)+ ? Promise.resolve({ level: 'write' })+ : Promise.reject(+ new TRPCForbiddenError('You do not have access to this project')+ )+ );+ const { ctx } = stubHttpCtx({+ services: { auth: { requireProjectAccess } },+ ...overrides,+ } as unknown as Parameters<typeof stubHttpCtx>[0]);+ return {+ requireProjectAccess,+ caller: async () =>+ importRouter.createCaller(+ await makeTrpcContext(ctx, new Headers(), {+ cookieOptions: COOKIE_OPTIONS,+ })+ ),+ };+}++test('list never returns provider credentials or a presigned query string', async () => {+ const findMany = mock(() =>+ Promise.resolve([+ {+ id: 'imp_1',+ projectId: 'proj_1',+ config: {+ provider: 'mixpanel',+ type: 'api',+ serviceAccount: 'svc-account',+ serviceSecret: 'svc-secret',+ projectId: '123',+ from: '2025-01-01',+ to: '2025-01-31',+ },+ },+ {+ id: 'imp_2',+ projectId: 'proj_1',+ config: {+ provider: 'amplitude',+ type: 'api',+ apiKey: 'amp-key',+ secretKey: 'amp-secret',+ from: '2025-01-01',+ to: '2025-01-31',+ },+ },+ {+ id: 'imp_3',+ projectId: 'proj_1',+ config: {+ provider: 'umami',+ type: 'file',+ fileUrl:+ 'https://bucket.s3.amazonaws.com/export.csv?X-Amz-Signature=abc',+ projectMapper: [],+ },+ },+ ])+ );+ const t = memberCaller({ db: { import: { findMany } } });+ const caller = await t.caller();++ const imports = await caller.list({ projectId: 'proj_1' });++ expect(findMany).toHaveBeenCalledTimes(1);+ const serialized = JSON.stringify(imports);+ for (const secret of ['svc-account', 'svc-secret', 'amp-key', 'amp-secret']) {+ expect(serialized).not.toContain(secret);+ }+ expect(imports[2]?.config).toMatchObject({+ fileUrl: 'https://bucket.s3.amazonaws.com/export.csv',+ });+});++test('create refuses an Umami mapping into a project the caller cannot write', async () => {+ const create = mock(() => Promise.resolve({ id: 'imp_1' }));+ const t = memberCaller({ db: { import: { create } } });+ const caller = await t.caller();++ await expect(+ caller.create({+ projectId: 'proj_1',+ provider: 'umami',+ config: {+ provider: 'umami',+ type: 'file',+ fileUrl: 'https://example.com/export.csv',+ projectMapper: [{ from: 'website_1', to: 'proj_other' }],+ },+ })+ ).rejects.toMatchObject({ code: 'FORBIDDEN' });+ expect(t.requireProjectAccess).toHaveBeenCalledWith({+ userId: 'user_1',+ projectId: 'proj_other',+ level: 'write',+ });+ expect(create).not.toHaveBeenCalled();+});diff --git a/packages/core/src/modules/import/import.rpc.ts b/packages/core/src/modules/import/import.rpc.tsindex cec630297..4703268d9 100644--- a/packages/core/src/modules/import/import.rpc.ts+++ b/packages/core/src/modules/import/import.rpc.ts@@ -5,6 +5,10 @@ import { z } from 'zod';import { createTRPCRouter, protectedProcedure } from '../../rpc/base';import { TRPCBadRequestError, TRPCNotFoundError } from '../../rpc/errors';import { zCreateImport } from './import.constants';+import {+ requireMappedProjectsWriteAccess,+ toPublicImport,+} from './import.service';export const importRouter = createTRPCRouter({list: protectedProcedure@@ -17,11 +21,11 @@ export const importRouter = createTRPCRouter({level: 'read',});- const db = ctx.db;- return db.import.findMany({+ const imports = await ctx.db.import.findMany({where: { projectId: input.projectId },orderBy: { createdAt: 'desc' },});+ return imports.map(toPublicImport);}),create: protectedProcedure@@ -33,6 +37,11 @@ export const importRouter = createTRPCRouter({projectId: input.projectId,level: 'write',});+ await requireMappedProjectsWriteAccess(+ ctx.services.auth,+ userId,+ input.config+ );const db = ctx.db;const organization = await db.organization.findFirst({@@ -66,7 +75,7 @@ export const importRouter = createTRPCRouter({data: { jobId },});- return { ...importRecord, jobId };+ return toPublicImport({ ...importRecord, jobId });}),delete: protectedProcedure@@ -88,7 +97,9 @@ export const importRouter = createTRPCRouter({await ctx.queues.import.import.remove(importRecord.jobId);}- return db.import.delete({ where: { id: input.id } });+ return toPublicImport(+ await db.import.delete({ where: { id: input.id } })+ );}),retry: protectedProcedure@@ -106,15 +117,23 @@ export const importRouter = createTRPCRouter({level: 'write',});+ await requireMappedProjectsWriteAccess(+ ctx.services.auth,+ userId,+ importRecord.config+ );+if (importRecord.status !== 'failed') {throw new Error('Only failed imports can be retried');}const jobId = await ctx.services.import.enqueue(importRecord.id);- return db.import.update({- where: { id: importRecord.id },- data: { jobId, status: 'pending', errorMessage: null },- });+ return toPublicImport(+ await db.import.update({+ where: { id: importRecord.id },+ data: { jobId, status: 'pending', errorMessage: null },+ })+ );}),});diff --git a/packages/core/src/modules/import/import.service.ts b/packages/core/src/modules/import/import.service.tsindex 3ea66711a..06c13c08e 100644--- a/packages/core/src/modules/import/import.service.ts+++ b/packages/core/src/modules/import/import.service.ts@@ -609,33 +609,30 @@ export async function backfillSessionsToProduction(}}-export async function getImportDateBounds(+/**+ * The days that actually hold staged events, ascending. Walking every calendar+ * day between the first and last event instead lets a file with a few far-apart+ * timestamps schedule thousands of empty moves. Session events count: a+ * session_end can be the only event of the day after its session.+ */+export async function getImportStagedDates(deps: ServiceDeps,importId: string,- fromCreatedAt?: string-): Promise<{ min: string | null; max: string | null }> {- const ch = deps.ch;- const res = await ch.query({+ fromDate?: string+): Promise<string[]> {+ const res = await deps.ch.query({query: `- SELECT min(created_at) AS min, max(created_at) AS max+ SELECT DISTINCT toString(toDate(created_at)) AS dateFROM ${TABLE_NAMES.events_imports}WHERE import_id = {importId:String}- AND name NOT IN ('session_start', 'session_end')- ${fromCreatedAt ? 'AND created_at >= {fromCreatedAt:String}' : ''}+ ${fromDate ? 'AND toDate(created_at) >= {fromDate:Date}' : ''}+ ORDER BY date ASC`,- query_params: { importId, fromCreatedAt },+ query_params: { importId, fromDate },format: 'JSONEachRow',});- const rows = (await res.json()) as Array<{- min: string | null;- max: string | null;- }>;- return rows.length > 0- ? {- min: fromCreatedAt ?? rows[0]?.min ?? null,- max: rows[0]?.max ?? null,- }- : { min: null, max: null };+ const rows = (await res.json()) as Array<{ date: string }>;+ return rows.map(({ date }) => date);}/**@@ -850,7 +847,12 @@ export async function runImportJob(include: { project: true },});- const jobLogger = logger.child({ importId, config: record.config });+ // Bindings bypass the redacting log formatter, and the config carries+ // provider credentials.+ const jobLogger = logger.child({+ importId,+ provider: record.config.provider,+ });jobLogger.info('Starting import job');const providerInstance = await createImportProvider(@@ -1069,25 +1071,19 @@ export async function runImportJob(return next.toISOString().split('T')[0]!;})();- const bounds = await getImportDateBounds(deps, importId, moveFromDate);- if (bounds.min && bounds.max) {- const startDate = bounds.min.split(' ')[0]!;- const endDate = bounds.max.split(' ')[0]!;- const cursor = new Date(`${startDate}T12:00:00Z`);- const end = new Date(`${endDate}T12:00:00Z`);-- while (cursor <= end) {- const dateStr = cursor.toISOString().split('T')[0]!;-- await moveImportsToProduction(deps, importId, dateStr);- await updateImportStatus(deps, jobLogger, progress, importId, {- step: 'moving',- batch: dateStr,- });+ const stagedDates = await getImportStagedDates(+ deps,+ importId,+ moveFromDate+ );+ for (const dateStr of stagedDates) {+ await moveImportsToProduction(deps, importId, dateStr);+ await updateImportStatus(deps, jobLogger, progress, importId, {+ step: 'moving',+ batch: dateStr,+ });- await yieldToEventLoop();- cursor.setUTCDate(cursor.getUTCDate() + 1);- }+ await yieldToEventLoop();}jobLogger.info('Move to production complete');@@ -1163,6 +1159,61 @@ export async function insertRawEventsBatch(return { writtenRows };}+/** Provider credentials are write-only: the worker needs them, a project reader does not. */+function toPublicImportConfig(config: IImportConfig) {+ switch (config.provider) {+ case 'amplitude': {+ const { apiKey: _apiKey, secretKey: _secretKey, ...rest } = config;+ return rest;+ }+ case 'mixpanel': {+ const {+ serviceAccount: _serviceAccount,+ serviceSecret: _serviceSecret,+ ...rest+ } = config;+ return rest;+ }+ default:+ return { ...config, fileUrl: stripUrlCredentials(config.fileUrl) };+ }+}++export function toPublicImport<T extends { config: IImportConfig }>(+ importRecord: T+) {+ return { ...importRecord, config: toPublicImportConfig(importRecord.config) };+}++/** A presigned export URL carries its signature in the query string. */+function stripUrlCredentials(fileUrl: string): string {+ try {+ const { origin, pathname } = new URL(fileUrl);+ return `${origin}${pathname}`;+ } catch {+ return '';+ }+}++/** An Umami import writes into every mapped project, not just the one it was started from. */+export async function requireMappedProjectsWriteAccess(+ auth: Services['auth'],+ userId: string,+ config: IImportConfig+): Promise<void> {+ if (!('projectMapper' in config)) {+ return;+ }+ const mappedProjectIds = new Set(config.projectMapper.map(({ to }) => to));+ for (const projectId of mappedProjectIds) {+ await auth.requireProjectAccess({+ userId,+ projectId,+ level: 'write',+ });+ }+}+/** `ctx.services.import` — a thin binding of the job body above to a Ctx's queues. */export function createImportService(deps: ServiceDeps,diff --git a/packages/core/src/modules/import/src/base-provider.ts b/packages/core/src/modules/import/src/base-provider.tsindex 06ab346d1..795a3b583 100644--- a/packages/core/src/modules/import/src/base-provider.ts+++ b/packages/core/src/modules/import/src/base-provider.ts@@ -1,4 +1,5 @@import type { IClickhouseEvent } from '../../event/event.service';+import { getImportRangeDays, MAX_IMPORT_RANGE_DAYS } from '../import.constants';import type { BaseRawEvent } from './types';export abstract class BaseImportProvider<@@ -45,6 +46,13 @@ export abstract class BaseImportProvider<chunkSizeDays?: number;}): [string, string][] {+ // Imports stored before the range was validated still reach this on retry.+ if (getImportRangeDays(from, to) > MAX_IMPORT_RANGE_DAYS) {+ throw new Error(+ `Import range exceeds ${MAX_IMPORT_RANGE_DAYS} days (${from} to ${to})`+ );+ }+const chunks: [string, string][] = [];const startDate = new Date(from);diff --git a/packages/core/src/modules/import/src/providers/umami.ts b/packages/core/src/modules/import/src/providers/umami.tsindex a69e4ba21..9c6462d42 100644--- a/packages/core/src/modules/import/src/providers/umami.ts+++ b/packages/core/src/modules/import/src/providers/umami.ts@@ -24,6 +24,8 @@ import { BaseImportProvider } from '../base-provider';/** Bounds for a remote Umami export. */const MAX_IMPORT_BYTES = 2_000_000_000; // 2 GBconst MAX_IMPORT_ROWS = 50_000_000;+/** Characters per CSV row; caps what the parser buffers while waiting for a row to end. */+const MAX_IMPORT_RECORD_SIZE = 1_000_000;/** Bounds the connection and redirect walk only, not the download itself. */const IMPORT_CONNECT_TIMEOUT_MS = 30_000;@@ -204,6 +206,7 @@ export class UmamiProvider extends BaseImportProvider<UmamiRawEvent> {bom: true, // handle UTF-8 BOMrelax_column_count: true,skip_empty_lines: true,+ max_record_size: MAX_IMPORT_RECORD_SIZE,});// Wire the pipeline for proper backpressure & error propagationdiff --git a/packages/core/src/modules/ingest/ingest.constants.test.ts b/packages/core/src/modules/ingest/ingest.constants.test.tsnew file mode 100644index 000000000..1e2dee21f--- /dev/null+++ b/packages/core/src/modules/ingest/ingest.constants.test.ts@@ -0,0 +1,54 @@+import { describe, expect, test } from 'bun:test';+import {+ MAX_PROFILE_AVATAR_LENGTH,+ MAX_PROFILE_EMAIL_LENGTH,+ MAX_PROFILE_NAME_LENGTH,+ zIdentifyPayload,+} from './ingest.constants';++describe('zIdentifyPayload', () => {+ test('accepts a well-formed identify', () => {+ expect(+ zIdentifyPayload.safeParse({+ profileId: 'user-1',+ firstName: 'Ada',+ lastName: 'Lovelace',+ email: '[email protected]',+ avatar: 'https://example.com/ada.png',+ properties: { plan: 'pro', nested: { seats: 3 } },+ }).success+ ).toBe(true);+ });++ test('refuses profile fields past their length caps', () => {+ for (const oversized of [+ { firstName: 'x'.repeat(MAX_PROFILE_NAME_LENGTH + 1) },+ { lastName: 'x'.repeat(MAX_PROFILE_NAME_LENGTH + 1) },+ { email: `${'x'.repeat(MAX_PROFILE_EMAIL_LENGTH)}@example.com` },+ {+ avatar: `https://example.com/${'x'.repeat(MAX_PROFILE_AVATAR_LENGTH)}`,+ },+ ]) {+ const result = zIdentifyPayload.safeParse({+ profileId: 'user-1',+ ...oversized,+ });+ expect([oversized, result.success]).toEqual([oversized, false]);+ }+ });++ test('refuses non-string profile fields and non-record properties', () => {+ for (const malformed of [+ { firstName: { nested: true } },+ { email: ['[email protected]'] },+ { properties: ['a', 'b'] },+ { properties: 'plan=pro' },+ ]) {+ const result = zIdentifyPayload.safeParse({+ profileId: 'user-1',+ ...malformed,+ });+ expect([malformed, result.success]).toEqual([malformed, false]);+ }+ });+});diff --git a/packages/core/src/modules/ingest/ingest.constants.ts b/packages/core/src/modules/ingest/ingest.constants.tsindex e27c48731..fed5b3f43 100644--- a/packages/core/src/modules/ingest/ingest.constants.ts+++ b/packages/core/src/modules/ingest/ingest.constants.ts@@ -205,15 +205,22 @@ export const zProfileId = z.union([z.number(),]) satisfies z.ZodType<IProfileId>;+/** Group membership is copied onto every session write, so both are bounded. */+export const MAX_GROUPS_PER_EVENT = 100;+export const MAX_GROUP_ID_LENGTH = 256;++const zGroupId = z.string().min(1).max(MAX_GROUP_ID_LENGTH);+const zGroupIds = z.array(zGroupId).max(MAX_GROUPS_PER_EVENT);+export const zGroupPayload = z.object({- id: z.string().min(1),+ id: zGroupId,type: z.string().min(1),name: z.string().min(1),properties: z.record(z.string(), z.unknown()).optional(),}) satisfies z.ZodType<IGroupPayload>;export const zAssignGroupPayload = z.object({- groupIds: z.array(z.string().min(1)),+ groupIds: zGroupIds,profileId: zProfileId.optional(),}) satisfies z.ZodType<IAssignGroupPayload>;@@ -222,7 +229,7 @@ export const zTrackPayload = zname: z.string().min(1),properties: z.record(z.string(), z.unknown()).optional(),profileId: zProfileId.optional(),- groups: z.array(z.string().min(1)).nullish(),+ groups: zGroupIds.nullish(),}).refine((data) => !(RESERVED_EVENT_NAMES as readonly string[]).includes(data.name),@@ -256,13 +263,36 @@ export const zTrackPayload = z}) satisfies z.ZodType<ITrackPayload>;+/** Profile columns are plain strings that every later write merges into. */+export const MAX_PROFILE_NAME_LENGTH = 256;+export const MAX_PROFILE_EMAIL_LENGTH = 320;+export const MAX_PROFILE_AVATAR_LENGTH = 2048;++const zProfileName = z.string().max(MAX_PROFILE_NAME_LENGTH);+const zProfileProperties = z.record(z.string(), z.unknown());+export const zIdentifyPayload = z.object({profileId: zProfileId,- firstName: z.string().optional(),- lastName: z.string().optional(),- email: z.string().email().optional(),- avatar: z.string().url().optional(),- properties: z.record(z.string(), z.unknown()).optional(),+ firstName: zProfileName.optional(),+ lastName: zProfileName.optional(),+ email: z.string().email().max(MAX_PROFILE_EMAIL_LENGTH).optional(),+ avatar: z.string().url().max(MAX_PROFILE_AVATAR_LENGTH).optional(),+ properties: zProfileProperties.optional(),+}) satisfies z.ZodType<IIdentifyPayload>;++/**+ * An identity that arrives without a body schema: `/track`'s+ * `properties.__identify` and `/profile`. A field of the wrong type or size is+ * dropped rather than failing the request, as for the other `__` properties;+ * only a missing or malformed `profileId` rejects the identity as a whole.+ */+export const zLooseIdentifyPayload = z.object({+ profileId: zProfileId,+ firstName: zProfileName.optional().catch(undefined),+ lastName: zProfileName.optional().catch(undefined),+ email: z.string().max(MAX_PROFILE_EMAIL_LENGTH).optional().catch(undefined),+ avatar: z.string().max(MAX_PROFILE_AVATAR_LENGTH).optional().catch(undefined),+ properties: zProfileProperties.optional().catch(undefined),}) satisfies z.ZodType<IIdentifyPayload>;export const zIncrementPayload = z.object({diff --git a/packages/core/src/modules/ingest/ingest.service.test.ts b/packages/core/src/modules/ingest/ingest.service.test.tsindex d03b9b50a..df1b517aa 100644--- a/packages/core/src/modules/ingest/ingest.service.test.ts+++ b/packages/core/src/modules/ingest/ingest.service.test.ts@@ -309,3 +309,66 @@ describe('ingestTrack — events transport', () => {expect(identifyProfile).toHaveBeenCalledTimes(1);});});++// `__identify` has no body schema, and what it carries lands in the shared+// profile buffer and in the event's profile id.+describe('ingestTrack — __identify', () => {+ beforeEach(() => {+ identifyProfile.mockClear();+ });++ const trackWithIdentify = async (identify: unknown, profileId?: string) => {+ const produce = mock(async () => undefined);+ await ingestTrack(+ request({+ type: 'track',+ payload: {+ name: 'page_view',+ profileId,+ properties: { __deviceId: DEVICE_ID, __identify: identify },+ },+ } as ITrackHandlerPayload),+ transport(produce)+ );+ const [queueData] = produce.mock.calls[0] as unknown as [+ { event: { profileId?: unknown } },+ ];+ return queueData.event.profileId;+ };++ it('drops fields of the wrong type or size and keeps the rest', async () => {+ await trackWithIdentify({+ profileId: 'user-1',+ firstName: 'Ada',+ lastName: { nested: true },+ email: ['[email protected]'],+ avatar: 'x'.repeat(5000),+ properties: ['not', 'a', 'record'],+ groups: ['smuggled'],+ });++ expect(identifyProfile).toHaveBeenCalledTimes(1);+ const [, , identity] = identifyProfile.mock.calls[0] as unknown as [+ unknown,+ string,+ Record<string, unknown>,+ ];+ expect(identity).toEqual({+ profileId: 'user-1',+ firstName: 'Ada',+ lastName: undefined,+ email: undefined,+ avatar: undefined,+ properties: undefined,+ });+ });++ it('ignores an identity without a usable profile id', async () => {+ for (const identify of [{ profileId: { id: 1 } }, 'user-1', null, {}]) {+ identifyProfile.mockClear();+ const eventProfileId = await trackWithIdentify(identify, 'fallback');+ expect([identify, eventProfileId]).toEqual([identify, 'fallback']);+ expect(identifyProfile).not.toHaveBeenCalled();+ }+ });+});diff --git a/packages/core/src/modules/ingest/ingest.service.ts b/packages/core/src/modules/ingest/ingest.service.tsindex ef894b980..830dd450f 100644--- a/packages/core/src/modules/ingest/ingest.service.ts+++ b/packages/core/src/modules/ingest/ingest.service.ts@@ -27,16 +27,17 @@ import {} from '../profile/profile.service';import { getSalts } from '../salt/salt.service';import { convertClickhouseDateToJs } from '../session/src/dates';-import type {- DeprecatedPostEventPayload,- IAssignGroupPayload,- IDecrementPayload,- IGroupPayload,- IIdentifyPayload,- IIncrementPayload,- IReplayPayload,- ITrackHandlerPayload,- ITrackPayload,+import {+ type DeprecatedPostEventPayload,+ type IAssignGroupPayload,+ type IDecrementPayload,+ type IGroupPayload,+ type IIdentifyPayload,+ type IIncrementPayload,+ type IReplayPayload,+ type ITrackHandlerPayload,+ type ITrackPayload,+ zLooseIdentifyPayload,} from './ingest.constants';import { isBot } from './src/bots/detect';import { applyBotSuspicion, stripBotProperties } from './src/bots/suspicion';@@ -119,12 +120,12 @@ export function getStringHeaders(function getIdentity(body: ITrackHandlerPayload): IIdentifyPayload | undefined {if (body.type === 'track') {- const identity = body.payload.properties?.__identify as- | IIdentifyPayload- | undefined;-- if (identity) {- return identity;+ const submitted = body.payload.properties?.__identify;+ if (submitted !== undefined) {+ const identity = zLooseIdentifyPayload.safeParse(submitted);+ if (identity.success) {+ return identity.data;+ }}return body.payload.profileId@@ -613,6 +614,12 @@ export type LegacyEventOutcome =| { status: 'ok' }| { status: 'missing-project-id' };+function legacyProfileId(raw: unknown): string | undefined {+ return typeof raw === 'string' || typeof raw === 'number'+ ? String(raw)+ : undefined;+}+export async function ingestLegacyEvent(request: {projectId: string | null | undefined;@@ -686,8 +693,14 @@ export async function ingestLegacyEvent(id: uuid(),projectId,headers,+ // Picked, not spread: this route has no body schema, and a stray field+ // (`groups` as an object, the body's `clientSecret`) would otherwise+ // ride into Kafka and onto the shared buffers' insert batches.event: {- ...request.body,+ name,+ profileId: legacyProfileId(request.body?.profileId),+ properties: request.body?.properties,+ groups: [],timestamp,isTimestampFromThePast,},diff --git a/packages/core/src/modules/ingest/legacy-event.test.ts b/packages/core/src/modules/ingest/legacy-event.test.tsindex 82cda43a0..972774d5a 100644--- a/packages/core/src/modules/ingest/legacy-event.test.ts+++ b/packages/core/src/modules/ingest/legacy-event.test.ts@@ -146,6 +146,25 @@ describe('ingestLegacyEvent', () => {expect(partitionKey).toBe(payload.deviceId);});+ it('forwards only the legacy fields, never a stray groups or secret', async () => {+ await ingestLegacyEvent(+ {+ ...makeRequest(),+ body: {+ ...legacyBody({ profileId: 'user-1' }),+ groups: { not: 'an array' },+ clientSecret: 'sec_from_body',+ } as unknown as DeprecatedPostEventPayload,+ },+ { buffers, produceIncomingEvent, deps: transportDeps }+ );++ const { event } = produced[0]!.payload;+ expect(event.groups).toEqual([]);+ expect(event.profileId).toBe('user-1');+ expect(JSON.stringify(event)).not.toContain('sec_from_body');+ });+it('mints the event id at the producer, like /track', async () => {await ingestLegacyEvent(makeRequest(), {buffers,diff --git a/packages/core/src/modules/ingest/src/client-auth.test.ts b/packages/core/src/modules/ingest/src/client-auth.test.tsindex 66beae360..4dd36af3f 100644--- a/packages/core/src/modules/ingest/src/client-auth.test.ts+++ b/packages/core/src/modules/ingest/src/client-auth.test.ts@@ -243,6 +243,48 @@ describe('validateIngestRequest', () => {});});+describe('validateIngestRequest — client type and profile filters', () => {+ const blockedProfile = () =>+ getClientByIdCached.mockResolvedValueOnce({+ ...client,+ project: {+ ...client.project,+ filters: [{ type: 'profile_id', profileId: '123' }],+ },+ } as unknown as typeof client);++ for (const [label, trackBody] of [+ ['a numeric profileId', { type: 'track', payload: { profileId: 123 } }],+ [+ 'the id inside __identify',+ {+ type: 'track',+ payload: {+ profileId: 'other',+ properties: { __identify: { profileId: '123' } },+ },+ },+ ],+ ['a root profileId', { profileId: '123' }],+ ] as const) {+ it(`blocks ${label} listed in the project filters`, async () => {+ blockedProfile();++ const result = await validateIngestRequest({+ deps,+ headers: { 'openpanel-client-id': CLIENT_ID },+ clientIp: '1.2.3.4',+ body: trackBody,+ });++ expect(result).toMatchObject({+ ok: false,+ message: 'Ingestion: Profile id is blocked by project filter',+ });+ });+ }+});+describe('validateIngestRequest — secret verification (main #481)', () => {it('does not mark a request authenticated when the secret does not match', async () => {guarded();diff --git a/packages/core/src/modules/ingest/src/client-auth.ts b/packages/core/src/modules/ingest/src/client-auth.tsindex a18617607..46e5d426c 100644--- a/packages/core/src/modules/ingest/src/client-auth.ts+++ b/packages/core/src/modules/ingest/src/client-auth.ts@@ -1,6 +1,5 @@// None of the behaviour below is decoration: the `ignoreCorsAndSecret`-// short-circuit, the unanchored wildcard origin regex, CORS-before-secret-// ordering, the ip and profile_id project filters, the `__revenue` gate, the+// short-circuit, CORS-before-secret ordering, the ip and profile_id project filters, the `__revenue` gate, the// body-field credential fallback, the `mixan-*` header fallback, and the// 5-minute verify cache whose key holds base64(<plaintext secret>) are all// deliberate.@@ -110,6 +109,26 @@ async function verifyClientSecret(return isVerified;}+/**+ * Every profile id the request can end up stored under, as the string the+ * pipeline stores: `/track`'s `__identify` wins over its `profileId`, and+ * `/event` and `/profile` carry it at the root. Numbers are stringified there+ * too, so `123` must be refused by a `"123"` filter.+ */+function submittedProfileIds(body: unknown): string[] {+ const candidates = [+ path(['payload', 'properties', '__identify', 'profileId'], body),+ path(['payload', 'profileId'], body),+ path(['profileId'], body),+ ];+ return candidates+ .filter(+ (candidate): candidate is string | number =>+ typeof candidate === 'string' || typeof candidate === 'number'+ )+ .map(String);+}+export async function validateIngestRequest({deps,headers,@@ -190,11 +209,12 @@ export async function validateIngestRequest({const profileFilter = client.project.filters.filter((filter): filter is IProjectFilterProfileId => filter.type === 'profile_id');- const profileId =- path<string | undefined>(['payload', 'profileId'], body) || // Track handler- path<string | undefined>(['profileId'], body); // Event handler-- if (profileFilter.some((filter) => filter.profileId === profileId)) {+ const profileIds = submittedProfileIds(body);+ if (+ profileFilter.some((filter) =>+ profileIds.includes(String(filter.profileId))+ )+ ) {return refuse('Ingestion: Profile id is blocked by project filter',secretVerifieddiff --git a/packages/core/src/modules/integration/integration.constants.ts b/packages/core/src/modules/integration/integration.constants.tsindex e1d29f521..4192f2471 100644--- a/packages/core/src/modules/integration/integration.constants.ts+++ b/packages/core/src/modules/integration/integration.constants.ts@@ -135,7 +135,9 @@ export type IWebhookConfig = z.infer<typeof zWebhookConfig>;export const zDiscordConfig = z.object({type: z.literal('discord'),- url: z.string().url(),+ // The webhook URL is a bearer credential, so it is write-only too: blank on+ // update keeps the stored one.+ url: z.string().url().or(z.literal('')),});export type IDiscordConfig = z.infer<typeof zDiscordConfig>;diff --git a/packages/core/src/modules/integration/integration.routes.ts b/packages/core/src/modules/integration/integration.routes.tsindex 4f37ce2eb..2b9af2378 100644--- a/packages/core/src/modules/integration/integration.routes.ts+++ b/packages/core/src/modules/integration/integration.routes.ts@@ -21,10 +21,18 @@ export const integrationRoutes = defineRoutes((app) =>app.get('/webhook/slack',async ({ query, ctx }) => {+ const userId = (await ctx.session())?.userId;+ if (!userId) {+ return loginErrorRedirect(+ ctx.config.dashboardUrl,+ 'Sign in to finish connecting Slack'+ );+ }+try {const { organizationId, projectId } = await completeSlackOAuthCallback(ctx,- query+ { ...query, userId });// Installs started before integrations were project-scoped may lackdiff --git a/packages/core/src/modules/integration/integration.service.test.ts b/packages/core/src/modules/integration/integration.service.test.tsindex 2807b132b..9f44e8a13 100644--- a/packages/core/src/modules/integration/integration.service.test.ts+++ b/packages/core/src/modules/integration/integration.service.test.ts@@ -6,7 +6,15 @@// Access is a real `getAccessChecks()` over faked lookups; storage is an// in-memory fake `deps.db`.-import { afterAll, beforeAll, beforeEach, expect, mock, test } from 'bun:test';+import {+ afterAll,+ beforeAll,+ beforeEach,+ expect,+ mock,+ spyOn,+ test,+} from 'bun:test';import { testCoreConfig } from '../../../test/config-fixture';interface FakeIntegration {@@ -33,6 +41,20 @@ mock.module('../../shared/access-lookups', () => ({getProjectById: mock(async () => ({ organizationId: 'org_1' })),}));+let slackStateMetadata: Record<string, string> = {};+const slackCodeExchange = mock(async () => new Response('{}'));+const realSlack = { ...(await import('./src/slack')) };+mock.module('./src/slack', () => ({+ ...realSlack,+ slackInstaller: () => ({+ stateStore: {+ verifyStateParam: async () => ({+ metadata: JSON.stringify(slackStateMetadata),+ }),+ },+ }),+}));+const integration = {findUniqueOrThrow: mock(async ({ where: { id } }: { where: { id: string } }) => {@@ -91,6 +113,7 @@ beforeAll(async () => {});afterAll(() => {+ mock.module('./src/slack', () => realSlack);mock.module('../../shared/access-lookups', () => realAccessLookups);resetAccessChecksForTests();});@@ -221,3 +244,60 @@ test('delete returns only the id, not the deleted row', async () => {expect(result).toEqual({ id: 'int_1' });expect(integrationStore.has('int_1')).toBe(false);});++test('createOrUpdate does not send stored header values to a new URL', async () => {+ storeWebhook();++ await expect(+ subject.upsertIntegration(deps, 'u1', {+ id: 'int_1',+ name: 'Hook',+ projectId: 'proj_1',+ config: {+ type: 'webhook',+ url: 'https://attacker.example/hook',+ headers: { Authorization: '', 'X-Trace': '' },+ mode: 'message',+ },+ })+ ).rejects.toThrow('Missing credentials: headers.Authorization');+ expect(integrationStore.get('int_1')?.config).toMatchObject({+ url: 'https://example.com/hook',+ });+});++test('the Slack callback refuses an install started by another user', async () => {+ storeSlack();+ slackStateMetadata = {+ organizationId: 'org_1',+ integrationId: 'int_slack',+ projectId: 'proj_1',+ userId: 'u_initiator',+ };+ const fetchSpy = spyOn(globalThis, 'fetch').mockImplementation(+ slackCodeExchange as unknown as typeof fetch+ );++ try {+ await expect(+ subject.completeSlackOAuthCallback(deps, {+ code: 'code',+ state: 'state',+ userId: 'u_victim',+ })+ ).rejects.toThrow('started by another user');++ projectAccessLevel = 'read';+ await expect(+ subject.completeSlackOAuthCallback(deps, {+ code: 'code',+ state: 'state',+ userId: 'u_initiator',+ })+ ).rejects.toThrow();++ expect(fetchSpy).not.toHaveBeenCalled();+ } finally {+ fetchSpy.mockRestore();+ }+});diff --git a/packages/core/src/modules/integration/integration.service.ts b/packages/core/src/modules/integration/integration.service.tsindex 4f4dd9cba..1f084432a 100644--- a/packages/core/src/modules/integration/integration.service.ts+++ b/packages/core/src/modules/integration/integration.service.ts@@ -294,6 +294,7 @@ export async function createOrUpdateSlackIntegration(integrationId: res.id,organizationId,projectId,+ userId,});if (!installUrl) {throw new TRPCBadRequestError(@@ -378,6 +379,9 @@ const slackOAuthMetadataSchema = z.object({// Optional for back-compat with install URLs generated before integrations// became project-scoped; the post-install redirect falls back to the org page.projectId: z.string().optional(),+ // The signed state proves the install URL is ours, not that the browser+ // completing it belongs to whoever generated it.+ userId: z.string(),});export class SlackOAuthCallbackError extends Error {}@@ -392,6 +396,7 @@ export async function completeSlackOAuthCallback(params: {code: string;state: string;+ userId: string;}): Promise<CompleteSlackOAuthCallbackResult> {const verifiedState = await slackInstaller(@@ -405,6 +410,22 @@ export async function completeSlackOAuthCallback(throw new SlackOAuthCallbackError('Invalid metadata');}+ const { organizationId, integrationId, projectId, userId } =+ parsedMetadata.data;++ if (userId !== params.userId) {+ throw new SlackOAuthCallbackError(+ 'This Slack installation was started by another user'+ );+ }++ const db = deps.db;+ const integration = await db.integration.findUniqueOrThrow({+ where: { id: integrationId },+ select: { projectId: true, organizationId: true },+ });+ await assertIntegrationAccess(params.userId, integration, 'write');+const slack = deps.config.slack;const slackOauthAccessUrl = ['https://slack.com/api/oauth.v2.access',@@ -429,9 +450,6 @@ export async function completeSlackOAuthCallback('👋 Hello. You have successfully connected OpenPanel.dev to your Slack workspace.',});- const { organizationId, integrationId, projectId } = parsedMetadata.data;-- const db = deps.db;await db.integration.update({where: {id: integrationId,diff --git a/packages/core/src/modules/integration/src/registry.test.ts b/packages/core/src/modules/integration/src/registry.test.tsindex e015d4a70..c8afc4152 100644--- a/packages/core/src/modules/integration/src/registry.test.ts+++ b/packages/core/src/modules/integration/src/registry.test.ts@@ -143,7 +143,6 @@ describe('config secret handling', () => {const submitted = carryOverConfigSecrets({...stored,- url: 'https://acme.test/hook-v2',// Authorization came back blank (redacted, untouched); X-Env retyped;// a third header added; nothing removed.headers: { Authorization: '', 'X-Env': 'staging', 'X-New': 'v' },@@ -156,7 +155,43 @@ describe('config secret handling', () => {'X-Env': 'staging','X-New': 'v',});- expect(submitted.url).toBe('https://acme.test/hook-v2');+ });++ it('drops stored header values when the webhook URL changes', () => {+ const stored = {+ type: 'webhook',+ url: 'https://acme.test/hook',+ mode: 'message',+ headers: { Authorization: 'Bearer sk-live-123' },+ };+ const submitted = carryOverConfigSecrets(+ {+ ...stored,+ url: 'https://attacker.test/hook',+ headers: { Authorization: '' },+ },+ stored+ );++ expect(submitted.headers).toEqual({ Authorization: '' });+ expect(findMissingSecretFields(submitted)).toEqual([+ 'headers.Authorization',+ ]);+ });++ it('redacts the Discord webhook URL and carries it over when left blank', () => {+ const stored = {+ type: 'discord',+ url: 'https://discord.com/api/webhooks/1/secret',+ };++ expect(redactConfigSecrets(stored).url).toBe('');+ expect(+ carryOverConfigSecrets({ type: 'discord', url: '' }, stored).url+ ).toBe(stored.url);+ expect(findMissingSecretFields({ type: 'discord', url: '' })).toEqual([+ 'url',+ ]);});it('lets a removed header stay removed', () => {diff --git a/packages/core/src/modules/integration/src/registry.ts b/packages/core/src/modules/integration/src/registry.tsindex d93633567..fb3410310 100644--- a/packages/core/src/modules/integration/src/registry.ts+++ b/packages/core/src/modules/integration/src/registry.ts@@ -101,9 +101,8 @@ type SecretPath<C> =export interface IConfigSecret<C> {path: SecretPath<C>;/**- * Also encrypt at rest, and treat the value as a required credential:- * rejected if it arrives already encrypted (a replay), and rejected if it is- * still blank after carry-over.+ * Also encrypt at rest, and reject the value if it arrives already encrypted+ * (a replay).** Only for values whose readers decrypt at use. The object-store adapters* do; the notification senders read `config.url` / `config.headers` /@@ -117,6 +116,12 @@ export interface IConfigSecret<C> {* are set, and carry-over is per key.*/record?: boolean;+ /**+ * The config value that receives this secret. The stored secret is carried+ * over only while that value is unchanged: otherwise an edit could point a+ * credential the editor never saw at a destination they control.+ */+ boundTo?: SecretPath<C>;}/**@@ -163,6 +168,9 @@ const slackServer: IServerIntegration<'slack'> = {const discordServer: IServerIntegration<'discord'> = {type: 'discord',+ // The webhook URL is the credential: anyone holding it can post to the+ // channel. Redact-only, the sender reads it raw.+ secretFields: [{ path: 'url' }],testConnection: async (config) => {const res = await sendTestDiscordNotification(config.url,@@ -191,8 +199,9 @@ const webhookServer: IServerIntegration<'webhook'> = {type: 'webhook',// Header VALUES routinely carry an Authorization bearer. Keys stay visible so// the form still shows which headers are configured; blank values carry over- // per key on update. Redact-only — postWebhook sends them raw.- secretFields: [{ path: 'headers', record: true }],+ // per key on update, but only to the same URL. Redact-only — postWebhook+ // sends them raw.+ secretFields: [{ path: 'headers', record: true, boundTo: 'url' }],validateConfig: (config) => {if (config.mode === 'javascript' && config.javascriptTemplate) {const result = validateJavaScriptTemplate(config.javascriptTemplate);@@ -203,12 +212,12 @@ const webhookServer: IServerIntegration<'webhook'> = {return { valid: true };},notification: {- deliver: ({ config, notification, payload }) => {+ deliver: async ({ config, notification, payload }) => {let body: unknown;if (config.mode === 'javascript') {// We only transform event payloads for now (not funnel)if (config.javascriptTemplate && payload.type === 'event') {- body = executeJavaScriptTemplate(+ body = await executeJavaScriptTemplate(config.javascriptTemplate,payload.event as Record<string, unknown>);@@ -424,6 +433,12 @@ export function carryOverConfigSecrets<C>(next: C, stored: unknown): C {if (value !== '') {return undefined;}+ const isDestinationChanged =+ secret.boundTo !== undefined &&+ readPath(next, secret.boundTo) !== readPath(stored, secret.boundTo);+ if (isDestinationChanged) {+ return undefined;+ }const previous = readPath(stored, secret.path);if (recordKey === undefined) {return typeof previous === 'string' ? previous : undefined;@@ -456,7 +471,9 @@ export function findEncryptedSecretField(config: unknown): string | undefined {}/**- * Paths of the required credentials that are present but blank.+ * Paths of the credentials that are present but blank. Blank is what a read+ * returns, so a blank credential is indistinguishable from "keep the stored+ * one" and is never a value worth saving.** An ABSENT field is not missing — it is not applicable to this config variant* (an `iam_role` S3 config carries no `secretAccessKey` at all), and the zod@@ -466,12 +483,21 @@ export function findMissingSecretFields(config: unknown): string[] {if (typeof config !== 'object' || config === null) {return [];}- return secretsFor(config)- .filter(- (secret) =>- secret.encrypted &&- readPath(config, secret.path) === '' &&- !secret.record- )- .map((secret) => secret.path);+ const missing: string[] = [];+ for (const secret of secretsFor(config)) {+ const value = readPath(config, secret.path);+ if (value === '') {+ missing.push(secret.path);+ continue;+ }+ if (!secret.record || typeof value !== 'object' || value === null) {+ continue;+ }+ for (const [key, recordValue] of Object.entries(value as LooseConfig)) {+ if (recordValue === '') {+ missing.push(`${secret.path}.${key}`);+ }+ }+ }+ return missing;}diff --git a/packages/core/src/modules/integration/src/slack.ts b/packages/core/src/modules/integration/src/slack.tsindex 266ed46c4..a07a1cc00 100644--- a/packages/core/src/modules/integration/src/slack.ts+++ b/packages/core/src/modules/integration/src/slack.ts@@ -52,11 +52,13 @@ export const getSlackInstallUrl = ({integrationId,organizationId,projectId,+ userId,}: {config: CoreConfig;integrationId: string;organizationId: string;projectId: string;+ userId: string;}): Promise<string> | null => {if (!config.slack.clientId) {return null;@@ -64,7 +66,12 @@ export const getSlackInstallUrl = ({return slackInstaller(config).generateInstallUrl({scopes: INSTALL_SCOPES,redirectUri: config.slack.oauthRedirectUrl,- metadata: JSON.stringify({ integrationId, organizationId, projectId }),+ metadata: JSON.stringify({+ integrationId,+ organizationId,+ projectId,+ userId,+ }),});};diff --git a/packages/core/src/modules/mcp/src/tools/analytics/overview.ts b/packages/core/src/modules/mcp/src/tools/analytics/overview.tsindex 677514d5f..21c1063a9 100644--- a/packages/core/src/modules/mcp/src/tools/analytics/overview.ts+++ b/packages/core/src/modules/mcp/src/tools/analytics/overview.ts@@ -1,5 +1,6 @@import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';import { z } from 'zod';+import { countTimeBuckets } from '../../../../../shared/time-buckets';import { getAnalyticsOverviewCore } from '../../../../overview/overview.service';import {MAX_SERIES_POINTS,@@ -17,13 +18,6 @@ type Interval = 'hour' | 'day' | 'week' | 'month';/** Coarsest-to-finest, so we can step down when a request is too fine-grained. */const INTERVAL_ORDER: Interval[] = ['hour', 'day', 'week', 'month'];-const POINTS_PER_DAY: Record<Interval, number> = {- hour: 24,- day: 1,- week: 1 / 7,- month: 1 / 30,-};-const SERIES_COLUMNS = ['date','unique_visitors',@@ -48,12 +42,8 @@ function fitInterval(startDate: string,endDate: string): { interval: Interval; note?: string } {- const days =- Math.max(- 0,- (new Date(endDate).getTime() - new Date(startDate).getTime()) / 86_400_000- ) + 1;- const points = (i: Interval) => Math.ceil(days * POINTS_PER_DAY[i]);+ const points = (interval: Interval) =>+ countTimeBuckets({ startDate, endDate, interval });if (points(requested) <= MAX_SERIES_POINTS) {return { interval: requested };@@ -69,7 +59,7 @@ function fitInterval(}}- const coarsest = INTERVAL_ORDER[INTERVAL_ORDER.length - 1] as Interval;+ const coarsest = INTERVAL_ORDER.at(-1) as Interval;return {interval: coarsest,note: `Requested interval "${requested}" is far too fine for this range; coarsened to "${coarsest}". Narrow the date range for finer resolution.`,diff --git a/packages/core/src/modules/mcp/src/tools/dashboard-management.test.ts b/packages/core/src/modules/mcp/src/tools/dashboard-management.test.tsindex 042344ba3..c3a466e42 100644--- a/packages/core/src/modules/mcp/src/tools/dashboard-management.test.ts+++ b/packages/core/src/modules/mcp/src/tools/dashboard-management.test.ts@@ -581,6 +581,20 @@ describe('dashboard management behavior', () => {layout: { x: -1, y: 0, w: 4, h: 3 },})).rejects.toThrow();+ await expect(+ server.invoke('update_report_layout', {+ projectId: 'project-1',+ reportId: '11111111-1111-4111-8111-111111111111',+ layout: { x: 0, y: 1_000_000, w: 4, h: 3 },+ })+ ).rejects.toThrow();+ await expect(+ server.invoke('update_report_layout', {+ projectId: 'project-1',+ reportId: '11111111-1111-4111-8111-111111111111',+ layout: { x: 0, y: 0, w: 4, h: 1_000_000 },+ })+ ).rejects.toThrow();expect(mockDb.reportLayout.upsert).not.toHaveBeenCalled();await server.invoke('update_report_layout', {diff --git a/packages/core/src/modules/mcp/src/tools/dashboard-management.ts b/packages/core/src/modules/mcp/src/tools/dashboard-management.tsindex c91291b04..525f8c3bb 100644--- a/packages/core/src/modules/mcp/src/tools/dashboard-management.ts+++ b/packages/core/src/modules/mcp/src/tools/dashboard-management.ts@@ -6,6 +6,7 @@ import { getId } from '../../../../slug-id';import { getProjectById } from '../../../project/project.service';import {GRID_COLUMNS,+ GRID_MAX_ROWS,GRID_ROW_HEIGHT,zReport,} from '../../../report/report.constants';@@ -57,17 +58,19 @@ const reportSchema = zReportconst GRID_DESCRIPTION = `Grid units: ${GRID_COLUMNS} columns across the dashboard, rows ${GRID_ROW_HEIGHT}px tall. A half-width chart is w ${GRID_COLUMNS / 2}, h 8.`;const zColumn = z.number().int().nonnegative().max(GRID_COLUMNS);const zColumnSpan = z.number().int().positive().max(GRID_COLUMNS);+const zRow = z.number().int().nonnegative().max(GRID_MAX_ROWS);+const zRowSpan = z.number().int().positive().max(GRID_MAX_ROWS);const layoutSchema = z.object({x: zColumn,- y: z.number().int().nonnegative(),+ y: zRow,w: zColumnSpan,- h: z.number().int().positive(),+ h: zRowSpan,minW: zColumnSpan.optional(),- minH: z.number().int().positive().optional(),+ minH: zRowSpan.optional(),maxW: zColumnSpan.optional(),- maxH: z.number().int().positive().optional(),+ maxH: zRowSpan.optional(),}).superRefine((layout, ctx) => {if (diff --git a/packages/core/src/modules/misc/src/data-health.ts b/packages/core/src/modules/misc/src/data-health.tsindex c3a9af854..304da9d2c 100644--- a/packages/core/src/modules/misc/src/data-health.ts+++ b/packages/core/src/modules/misc/src/data-health.ts@@ -2,14 +2,19 @@// the job is testable without `mock.module`. `misc.jobs.ts` binds them off// the job's ctx.+import type {+ AccessLevel,+ ProjectScope,+} from '@openpanel/db/src/prisma-client';import type { CoreConfig } from '../../../config';import type { Logger } from '../../../logger';+import { resolveProjectAccess } from '../../../shared/access-lookups';+import { MS_PER_DAY } from '../../../shared/time.constants';-const DAY_MS = 24 * 60 * 60 * 1000;// A brand-new project gets 48h to send its first event before we reach out.-const NO_DATA_AFTER_MS = 2 * DAY_MS;+const NO_DATA_AFTER_MS = 2 * MS_PER_DAY;// An active project whose newest event is older than this counts as stalled.-const STALLED_AFTER_MS = 7 * DAY_MS;+const STALLED_AFTER_MS = 7 * MS_PER_DAY;const DEFAULT_DASHBOARD_URL = 'https://dashboard.openpanel.dev';interface OrgAlert {@@ -34,9 +39,17 @@ export interface DataHealthProject {}export interface DataHealthMember {+ role: string;+ projectScope: ProjectScope;+ projects: { projectId: string; level: AccessLevel | null }[];user: { email: string | null; firstName: string | null } | null;}+interface Recipient {+ firstName: string | undefined;+ canOpen(projectId: string): boolean;+}+/** The Prisma surface this job touches — nothing wider. */export interface DataHealthDb {member: { findMany(args: unknown): Promise<DataHealthMember[]> };@@ -59,21 +72,41 @@ export interface DataHealthDeps {): Promise<unknown>;}-async function recipientsForOrg(organizationId: string, deps: DataHealthDeps) {+/** Members by email; each alert lists only the projects that member can open. */+async function recipientsForOrg(+ organizationId: string,+ deps: DataHealthDeps+): Promise<Map<string, Recipient>> {const members = await deps.db.member.findMany({where: {organizationId,user: { deletedAt: null },},- include: { user: { select: { email: true, firstName: true } } },+ select: {+ role: true,+ projectScope: true,+ projects: { select: { projectId: true, level: true } },+ user: { select: { email: true, firstName: true } },+ },});- const seen = new Map<string, string | undefined>();+ const recipients = new Map<string, Recipient>();for (const member of members) {- if (member.user?.email && !seen.has(member.user.email)) {- seen.set(member.user.email, member.user.firstName ?? undefined);+ const email = member.user?.email;+ if (!email || recipients.has(email)) {+ continue;}+ recipients.set(email, {+ firstName: member.user?.firstName ?? undefined,+ canOpen: (projectId) =>+ resolveProjectAccess({+ role: member.role,+ projectScope: member.projectScope,+ grant:+ member.projects.find((row) => row.projectId === projectId) ?? null,+ }) !== null,+ });}- return seen;+ return recipients;}export interface DataHealthResult extends Record<string, number> {@@ -182,15 +215,25 @@ export async function runDataHealthCron(const dashboardUrl = `${deps.config.dashboardUrl || DEFAULT_DASHBOARD_URL}/${alert.organizationId}`;if (alert.noData.length > 0) {- for (const [to, firstName] of recipients) {+ // Only projects someone was told about: one nobody here can open+ // stays unmarked, so a member granted access later still hears of it.+ const alertedIds = new Set<string>();+ for (const [to, recipient] of recipients) {+ const visible = alert.noData.filter((p) => recipient.canOpen(p.id));+ if (visible.length === 0) {+ continue;+ }+ for (const p of visible) {+ alertedIds.add(p.id);+ }// Per-recipient guard: one bad address must not block the marker// update below — that would re-email everyone tomorrow.try {await deps.sendEmail('tracking-no-data', {to,data: {- firstName,- projectNames: alert.noData.map((p) => p.name),+ firstName: recipient.firstName,+ projectNames: visible.map((p) => p.name),dashboardUrl,},});@@ -203,22 +246,30 @@ export async function runDataHealthCron(}}await deps.db.project.updateMany({- where: { id: { in: alert.noData.map((p) => p.id) } },+ where: { id: { in: [...alertedIds] } },data: { noDataNotifiedAt: new Date() },});}if (alert.stalled.length > 0) {- const newestLastEvent = alert.stalled- .map((p) => p.lastEventAt)- .sort((a, b) => b.getTime() - a.getTime())[0];- for (const [to, firstName] of recipients) {+ const alertedIds = new Set<string>();+ for (const [to, recipient] of recipients) {+ const visible = alert.stalled.filter((p) => recipient.canOpen(p.id));+ if (visible.length === 0) {+ continue;+ }+ for (const p of visible) {+ alertedIds.add(p.id);+ }+ const newestLastEvent = visible+ .map((p) => p.lastEventAt)+ .sort((a, b) => b.getTime() - a.getTime())[0];try {await deps.sendEmail('tracking-data-stopped', {to,data: {- firstName,- projectNames: alert.stalled.map((p) => p.name),+ firstName: recipient.firstName,+ projectNames: visible.map((p) => p.name),lastEventDate: newestLastEvent?.toLocaleDateString('en-US', {month: 'long',day: 'numeric',@@ -235,7 +286,7 @@ export async function runDataHealthCron(}}await deps.db.project.updateMany({- where: { id: { in: alert.stalled.map((p) => p.id) } },+ where: { id: { in: [...alertedIds] } },data: { dataStoppedNotifiedAt: new Date() },});}diff --git a/packages/core/src/modules/notification/notification.service.test.ts b/packages/core/src/modules/notification/notification.service.test.tsindex 2925724ed..55f7e57b7 100644--- a/packages/core/src/modules/notification/notification.service.test.ts+++ b/packages/core/src/modules/notification/notification.service.test.ts@@ -228,6 +228,41 @@ test('matchEventFilters: "is" matches the exact property value', () => {).toBe(false);});+test('matchEventFilters: regex matches through RE2, slashes stripped', () => {+ const payload = { path: '/product/42' } as never;+ const regex = (value: string) =>+ [{ name: 'path', operator: 'regex', value: [value] }] as never;++ expect(+ subject.matchEventFilters(payload, regex('/^\\/product\\/\\d+$/'))+ ).toBe(true);+ expect(subject.matchEventFilters(payload, regex('^/cart'))).toBe(false);+});++test('matchEventFilters: a stored regex RE2 cannot compile never matches and is logged once', () => {+ const payload = { path: '/product/42' } as never;+ const lookahead = [+ { name: 'path', operator: 'regex', value: ['(?=/product)/p'] },+ ] as never;+ const warn = mock(() => undefined);+ const logger = { warn } as never;++ expect(subject.matchEventFilters(payload, lookahead, logger)).toBe(false);+ expect(subject.matchEventFilters(payload, lookahead, logger)).toBe(false);+ expect(warn).toHaveBeenCalledTimes(1);+});++test('matchEventFilters: a backtracking-prone regex stays fast', () => {+ const payload = { name: `${'a'.repeat(50_000)}!` } as never;+ const startedAt = performance.now();+ expect(+ subject.matchEventFilters(payload, [+ { name: 'name', operator: 'regex', value: ['(a+)+$'] },+ ] as never)+ ).toBe(false);+ expect(performance.now() - startedAt).toBeLessThan(250);+});+test('matchEventFilters: has_profile compares profileId against deviceId', () => {const identified = { profileId: 'p1', deviceId: 'd1' } as never;const anonymous = { profileId: 'd1', deviceId: 'd1' } as never;@@ -385,6 +420,36 @@ test('createOrUpdateNotificationRule rejects an export-only integration', async).rejects.toMatchObject({ code: 'BAD_REQUEST' });});+test('createOrUpdateNotificationRule rejects a regex filter RE2 cannot compile', async () => {+ notificationRule.create.mockClear();+ await expect(+ subject.createOrUpdateNotificationRule(deps, {+ name: 'Rule',+ config: {+ type: 'events',+ events: [+ {+ name: 'screen_view',+ segment: 'event',+ filters: [+ {+ name: 'path',+ operator: 'regex',+ value: ['(a)\\1'],+ },+ ],+ },+ ],+ } as never,+ integrations: ['app'],+ sendToApp: true,+ sendToEmail: false,+ projectId: 'proj_1',+ })+ ).rejects.toMatchObject({ code: 'BAD_REQUEST' });+ expect(notificationRule.create).not.toHaveBeenCalled();+});+test('createOrUpdateNotificationRule creates a new rule when id is absent', async () => {const created = await subject.createOrUpdateNotificationRule(deps, {name: 'Rule',diff --git a/packages/core/src/modules/notification/notification.service.ts b/packages/core/src/modules/notification/notification.service.tsindex 62a9fcc3c..1cfe8affd 100644--- a/packages/core/src/modules/notification/notification.service.ts+++ b/packages/core/src/modules/notification/notification.service.ts@@ -2,10 +2,10 @@// out of the eager barrel chain (jobs.registry.ts, services.ts) that pulls this module in.import type { Integration, Prisma } from '@openpanel/db/src/prisma-client';-import { stripLeadingAndTrailingSlashes } from '@openpanel/shared';import { pathOr } from 'ramda';import { cacheablePerDb, type DbScope } from '../../cacheable-per-deps';import { sendEmail } from '../../clients/email';+import type { Logger } from '../../logger';import { TRPCBadRequestError, TRPCForbiddenError } from '../../rpc/errors';import type { ServiceDeps, Services } from '../../services';import { listProjectMemberEmails } from '../../shared/access-lookups';@@ -26,6 +26,11 @@ import type {IChartEventFilter,} from '../report/report.constants';import type { ICreateNotificationRule } from './notification.constants';+import {+ testFilterPattern,+ toFilterPattern,+ validateFilterPattern,+} from './src/filter-regex';export const APP_NOTIFICATION_INTEGRATION_ID = 'app';export const EMAIL_NOTIFICATION_INTEGRATION_ID = 'email';@@ -87,7 +92,8 @@ export const getNotificationRulesByProjectId = cacheablePerDb(export function matchEventFilters(payload: IServiceCreateEventPayload,- filters: IChartEventFilter[]+ filters: IChartEventFilter[],+ logger?: Logger): boolean {return filters.every((filter) => {const { name, value, operator } = filter;@@ -122,17 +128,10 @@ export function matchEventFilters(return value.some((val) => propertyValue.startsWith(String(val)));case 'endsWith':return value.some((val) => propertyValue.endsWith(String(val)));- case 'regex': {- return value- .map((val) => stripLeadingAndTrailingSlashes(String(val)))- .some((val) => {- try {- return new RegExp(val).test(propertyValue);- } catch {- return false;- }- });- }+ case 'regex':+ return value.some((val) =>+ testFilterPattern(toFilterPattern(val), propertyValue, logger)+ );case 'isNull':return propertyValue === '';case 'isNotNull':@@ -153,14 +152,15 @@ export function matchEventFilters(export function matchEvent(payload: IServiceCreateEventPayload,- chartEvent: IChartEvent+ chartEvent: IChartEvent,+ logger?: Logger): boolean {if (payload.name !== chartEvent.name && chartEvent.name !== '*') {return false;}if (chartEvent.filters.length > 0) {- return matchEventFilters(payload, chartEvent.filters);+ return matchEventFilters(payload, chartEvent.filters, logger);}return true;@@ -372,15 +372,34 @@ export async function getNotificationRuleByIdOrThrow(return await deps.db.notificationRule.findUniqueOrThrow({ where: { id } });}+function assertRegexFiltersAreSafe(input: ICreateNotificationRule) {+ for (const event of input.config.events) {+ for (const filter of event.filters) {+ if (filter.operator !== 'regex') {+ continue;+ }+ for (const value of filter.value) {+ const error = validateFilterPattern(toFilterPattern(value));+ if (error) {+ throw new TRPCBadRequestError(error);+ }+ }+ }+ }+}+/*** Authorization (`requireProjectAccess`) is the caller's job. Validates that every* connected integration belongs to `input.projectId` or is a legacy org-wide one- * in the same org, then writes the rule.+ * in the same org and that regex filters compile under RE2 and run fast+ * enough, then writes the rule.*/export async function createOrUpdateNotificationRule(deps: ServiceDeps,input: ICreateNotificationRule) {+ assertRegexFiltersAreSafe(input);+const db = deps.db;const project = await db.project.findUniqueOrThrow({where: { id: input.projectId },diff --git a/packages/core/src/modules/notification/src/filter-regex.test.ts b/packages/core/src/modules/notification/src/filter-regex.test.tsnew file mode 100644index 000000000..f24b8d3f1--- /dev/null+++ b/packages/core/src/modules/notification/src/filter-regex.test.ts@@ -0,0 +1,47 @@+import { expect, spyOn, test } from 'bun:test';+import {+ REGEX_VALIDATION_BUDGET_MS,+ testFilterPattern,+ toFilterPattern,+ validateFilterPattern,+} from './filter-regex';++test('accepts ordinary patterns', () => {+ expect(validateFilterPattern('^/product/\\d+$')).toBeUndefined();+ expect(validateFilterPattern('(a+)+$')).toBeUndefined();+ expect(validateFilterPattern(toFilterPattern('/checkout|cart/'))).toBe(+ undefined+ );+});++test('rejects syntax RE2 does not support', () => {+ expect(validateFilterPattern('(?=foo)bar')).toContain(+ 'Invalid regular expression'+ );+ expect(validateFilterPattern('(a)\\1')).toContain(+ 'Invalid regular expression'+ );+ expect(validateFilterPattern('((a{100}){100}){100}')).toContain(+ 'Invalid regular expression'+ );+});++test('rejects a pattern that runs over the time budget on the probes', () => {+ let now = 0;+ const clock = spyOn(performance, 'now').mockImplementation(() => {+ now += REGEX_VALIDATION_BUDGET_MS;+ return now;+ });++ try {+ expect(validateFilterPattern('abc')).toContain('too slow');+ expect(clock).toHaveBeenCalled();+ } finally {+ clock.mockRestore();+ }+});++test('matches anywhere in the input, like RegExp.test', () => {+ expect(testFilterPattern('product', '/shop/product/1')).toBe(true);+ expect(testFilterPattern('^product', '/shop/product/1')).toBe(false);+});diff --git a/packages/core/src/modules/notification/src/filter-regex.ts b/packages/core/src/modules/notification/src/filter-regex.tsnew file mode 100644index 000000000..c025b16a3--- /dev/null+++ b/packages/core/src/modules/notification/src/filter-regex.ts@@ -0,0 +1,100 @@+// Filter patterns are written by project members and run against tracking+// payloads on the event consumer, so they go through RE2 (linear time, no+// backtracking) rather than the native engine. RE2 is also what ClickHouse+// uses for the same `regex` operator in reports.++import { stripLeadingAndTrailingSlashes } from '@openpanel/shared';+import { LRUCache } from 'lru-cache';+import { RE2JS } from 're2js';+import type { Logger } from '../../../logger';++const COMPILED_PATTERN_CACHE_MAX_ENTRIES = 1000;++/** Total time a pattern may take to compile and run over every probe. */+export const REGEX_VALIDATION_BUDGET_MS = 50;++const PROBE_LENGTH = 20_000;++// Long runs of one character class and near-misses: what makes a slow+// pattern visible, even in a linear-time engine.+const PROBE_INPUTS = [+ 'a'.repeat(PROBE_LENGTH),+ `${'a'.repeat(PROBE_LENGTH)}!`,+ 'ab'.repeat(PROBE_LENGTH / 2),+ `${'0'.repeat(PROBE_LENGTH)}x`,+ `${' '.repeat(PROBE_LENGTH)}\n`,+ `${'/path'.repeat(PROBE_LENGTH / 5)}?`,+];++type CompiledPattern = { regex: RE2JS } | { error: string };++const compiledPatterns = new LRUCache<string, CompiledPattern>({+ max: COMPILED_PATTERN_CACHE_MAX_ENTRIES,+});++/** The dashboard lets people write `/pattern/`; the slashes are not part of it. */+export function toFilterPattern(value: unknown): string {+ return stripLeadingAndTrailingSlashes(String(value));+}++function compilePattern(pattern: string): CompiledPattern {+ try {+ return { regex: RE2JS.compile(pattern) };+ } catch (error) {+ return { error: error instanceof Error ? error.message : String(error) };+ }+}++/**+ * Compiled once per process per pattern. A pattern RE2 cannot compile (a rule+ * saved before RE2, using lookaround or backreferences) never matches, and is+ * logged the first time it is seen.+ */+export function testFilterPattern(+ pattern: string,+ input: string,+ logger?: Logger+): boolean {+ let compiled = compiledPatterns.get(pattern);+ if (!compiled) {+ compiled = compilePattern(pattern);+ compiledPatterns.set(pattern, compiled);+ if ('error' in compiled) {+ logger?.warn(+ { pattern, error: compiled.error },+ 'Filter regex is not valid RE2, treating it as non-matching'+ );+ }+ }+ return 'regex' in compiled && compiled.regex.test(input);+}++// Timed twice and judged on the faster run: the first one also pays for JIT+// warm-up, and either can land on a GC pause or a busy event loop. A pattern+// that is slow by construction is slow both times.+const VALIDATION_TIMING_RUNS = 2;++function probeDurationMs(regex: { test: (input: string) => boolean }): number {+ const startedAt = performance.now();+ for (const probe of PROBE_INPUTS) {+ regex.test(probe);+ if (performance.now() - startedAt > REGEX_VALIDATION_BUDGET_MS) {+ break;+ }+ }+ return performance.now() - startedAt;+}++/** An error message for a pattern that cannot be saved, or undefined. */+export function validateFilterPattern(pattern: string): string | undefined {+ const compiled = compilePattern(pattern);+ if ('error' in compiled) {+ return `Invalid regular expression "${pattern}": ${compiled.error}`;+ }+ for (let run = 0; run < VALIDATION_TIMING_RUNS; run++) {+ if (probeDurationMs(compiled.regex) <= REGEX_VALIDATION_BUDGET_MS) {+ return undefined;+ }+ }+ return `Regular expression "${pattern}" is too slow to evaluate. Simplify it, for example by removing nested or large repetitions.`;+}diff --git a/packages/core/src/modules/notification/src/notification-dispatch.ts b/packages/core/src/modules/notification/src/notification-dispatch.tsindex 67b72d45e..b5db3a498 100644Binary files a/packages/core/src/modules/notification/src/notification-dispatch.ts and b/packages/core/src/modules/notification/src/notification-dispatch.ts differdiff --git a/packages/core/src/modules/onboarding/onboarding.service.test.ts b/packages/core/src/modules/onboarding/onboarding.service.test.tsindex 672c58738..07fb96a2d 100644--- a/packages/core/src/modules/onboarding/onboarding.service.test.ts+++ b/packages/core/src/modules/onboarding/onboarding.service.test.ts@@ -67,12 +67,12 @@ function org(overrides: Record<string, unknown> = {}) {onboarding: null,subscriptionStatus: 'trialing',subscriptionEndsAt: new Date('2026-06-16T12:00:00Z'),- createdBy: {- id: 'user-1',- email: '[email protected]',- firstName: 'Alex',- deletedAt: null,- },+ createdByUserId: 'user-1',+ members: [+ {+ user: { id: 'user-1', email: '[email protected]', firstName: 'Alex' },+ },+ ],projects: [{ id: 'proj-1' }],...overrides,};diff --git a/packages/core/src/modules/onboarding/onboarding.service.ts b/packages/core/src/modules/onboarding/onboarding.service.tsindex 6f94795f9..f6a9bdf4c 100644--- a/packages/core/src/modules/onboarding/onboarding.service.ts+++ b/packages/core/src/modules/onboarding/onboarding.service.ts@@ -15,6 +15,12 @@ import {getOrganizationById,getOrganizationEventsCount,} from '../organization/organization.service';+import {+ getOrganizationContact,+ ORGANIZATION_ADMINS_INCLUDE,+ type OrganizationContact,+ type OrganizationWithAdmins,+} from '../organization/src/contact';import { getUserById } from '../user/user.service';import type { IOnboardingProject } from './onboarding.constants';@@ -195,24 +201,18 @@ interface OnboardingUsage {hasData: boolean;}-interface OnboardingCronOrg {+interface OnboardingCronOrg extends OrganizationWithAdmins {id: string;createdAt: Date;onboarding: string | null;subscriptionStatus: string | null;subscriptionEndsAt: Date | null;- createdBy: {- id: string;- email: string;- firstName: string | null;- deletedAt: Date | null;- } | null;projects: { id: string }[];}interface OnboardingContext {org: OnboardingCronOrg;- user: NonNullable<OnboardingCronOrg['createdBy']>;+ user: OrganizationContact;dashboardUrl: string;// Lazy + memoized: only emails past the day gate pay for the ClickHouse count.getUsage: () => Promise<OnboardingUsage>;@@ -356,40 +356,33 @@ export async function runOnboardingCron(where: {OR: [{ onboarding: null }, { onboarding: { notIn: ['completed'] } }],deleteAt: null,- createdBy: { deletedAt: null },},include: {- createdBy: {- select: { id: true, email: true, firstName: true, deletedAt: true },- },+ ...ORGANIZATION_ADMINS_INCLUDE,projects: { select: { id: true } },},});logger.info(`Found ${orgs.length} organizations in onboarding`);- const contactable = orgs.filter(- (org) => org.createdBy && !org.createdBy.deletedAt- );- const withoutCreator = orgs.length - contactable.length;-- const subjects: SequenceSubject<OnboardingContext>[] = contactable.map(- (org) => {- const user = org.createdBy as NonNullable<OnboardingCronOrg['createdBy']>;- return {- id: org.id,- email: user.email,- anchor: org.createdAt,- pointer: org.onboarding,- ctx: {- org,- user,- dashboardUrl: deps.config.dashboardUrl,- getUsage: createUsageGetter(deps, org),- },- };+ const subjects: SequenceSubject<OnboardingContext>[] = orgs.flatMap((org) => {+ const user = getOrganizationContact(org);+ if (!user) {+ return [];}- );+ return {+ id: org.id,+ email: user.email,+ anchor: org.createdAt,+ pointer: org.onboarding,+ ctx: {+ org,+ user,+ dashboardUrl: deps.config.dashboardUrl,+ getUsage: createUsageGetter(deps, org),+ },+ };+ });const result = await runSequence({name: 'onboarding',@@ -414,7 +407,8 @@ export async function runOnboardingCron(totalOrgs: orgs.length,emailsSent: result.emailsSent,orgsCompleted: result.completed,- orgsSkipped: result.deferred + result.stepsSkipped + withoutCreator,+ orgsSkipped:+ result.deferred + result.stepsSkipped + orgs.length - subjects.length,};logger.info({ ...summary }, 'Completed onboarding email job');diff --git a/packages/core/src/modules/organization/organization.rpc.ts b/packages/core/src/modules/organization/organization.rpc.tsindex 436c55c20..164f2b2db 100644--- a/packages/core/src/modules/organization/organization.rpc.ts+++ b/packages/core/src/modules/organization/organization.rpc.ts@@ -170,6 +170,17 @@ export const organizationRouter = createTRPCRouter({return ctx.services.organization.invitations(input.organizationId);}),+ // Membership-exempt: the caller is not a member yet. The service binds the+ // invite to the session's email.+ acceptInvite: protectedProcedureWithoutAccess+ .input(z.object({ inviteId: z.string() }))+ .mutation(({ input, ctx }) =>+ ctx.services.organization.acceptInvite({+ userId: ctx.session.userId,+ inviteId: input.inviteId,+ })+ ),+getInvite: publicProcedure.input(z.object({ inviteId: z.string().optional() })).query(async ({ input, ctx }) => {diff --git a/packages/core/src/modules/organization/organization.service.test.ts b/packages/core/src/modules/organization/organization.service.test.tsindex 5a5047c13..b5a96313a 100644--- a/packages/core/src/modules/organization/organization.service.test.ts+++ b/packages/core/src/modules/organization/organization.service.test.ts@@ -55,6 +55,7 @@ interface FakeInvite {interface FakeUser {id: string;email: string;+ emailVerified?: boolean;}const EPOCH = new Date('2026-09-03T00:00:00.000Z');@@ -377,6 +378,9 @@ const invite = {inviteStore.delete(id);return found;}),+ deleteMany: mock(async ({ where: { id } }: { where: { id: string } }) => ({+ count: inviteStore.delete(id) ? 1 : 0,+ })),};const user = {@@ -385,6 +389,15 @@ const user = {[...userStore.values()].find((u) => u.email === where.email.equals) ??null),+ findUniqueOrThrow: mock(+ async ({ where: { id } }: { where: { id: string } }) => {+ const found = userStore.get(id);+ if (!found) {+ throw new Error(`user ${id} not found`);+ }+ return found;+ }+ ),};const memberProject = {@@ -886,8 +899,12 @@ test('inviteUserToOrganization refuses a duplicate pending invite', async () =>).rejects.toThrow(/already invited/);});-test('inviteUserToOrganization connects an existing user immediately, with no email sent', async () => {- userStore.set('user_2', { id: 'user_2', email: '[email protected]' });+test('inviteUserToOrganization connects an existing verified user immediately, with no email sent', async () => {+ userStore.set('user_2', {+ id: 'user_2',+ email: '[email protected]',+ emailVerified: true,+ });const result = await subject.inviteUserToOrganization(deps, {organizationId: 'org_1',@@ -920,6 +937,95 @@ test('inviteUserToOrganization emails a new user and leaves the invite pending',});});+test('inviteUserToOrganization emails an unverified existing user instead of connecting them', async () => {+ userStore.set('user_2', { id: 'user_2', email: '[email protected]' });++ const result = await subject.inviteUserToOrganization(deps, {+ organizationId: 'org_1',+ email: '[email protected]',+ role: 'member',+ projectScope: 'all',+ access: [],+ invitedById: 'user_admin',+ });++ expect(result.type).toBe('is_invited');+ expect(memberStore.size).toBe(0);+ expect(sentEmails).toMatchObject([+ { templateKey: 'invite', to: '[email protected]' },+ ]);+});++function seedPendingInvite(overrides: Partial<FakeInvite> = {}) {+ inviteStore.set('invite_1', {+ id: 'invite_1',+ email: '[email protected]',+ organizationId: 'org_1',+ role: 'viewer',+ createdById: 'user_admin',+ projectScope: 'selected',+ projectAccess: [{ projectId: 'proj_1', level: 'write' }],+ expiresAt: new Date(Date.now() + 60_000),+ createdAt: EPOCH,+ ...overrides,+ });+}++test('acceptInvite joins with the invite role and grants, matching the email case-insensitively', async () => {+ userStore.set('user_3', { id: 'user_3', email: '[email protected]' });+ seedPendingInvite();++ const result = await subject.acceptInvite(deps, {+ userId: 'user_3',+ inviteId: 'invite_1',+ });++ expect(result).toEqual({ organizationId: 'org_1' });+ const member = [...memberStore.values()].find((m) => m.userId === 'user_3');+ expect(member).toMatchObject({ role: 'viewer', projectScope: 'selected' });+ expect(memberProjectStore).toMatchObject([{ projectId: 'proj_1' }]);+ expect(inviteStore.has('invite_1')).toBe(false);+});++test('acceptInvite twice at once creates one membership and both succeed', async () => {+ userStore.set('user_3', { id: 'user_3', email: '[email protected]' });+ seedPendingInvite();++ const results = await Promise.all([+ subject.acceptInvite(deps, { userId: 'user_3', inviteId: 'invite_1' }),+ subject.acceptInvite(deps, { userId: 'user_3', inviteId: 'invite_1' }),+ ]);++ expect(results).toEqual([+ { organizationId: 'org_1' },+ { organizationId: 'org_1' },+ ]);+ expect(+ [...memberStore.values()].filter((m) => m.userId === 'user_3')+ ).toHaveLength(1);+});++test('acceptInvite refuses a session whose email is not the invited one', async () => {+ userStore.set('user_3', { id: 'user_3', email: '[email protected]' });+ seedPendingInvite();++ await expect(+ subject.acceptInvite(deps, { userId: 'user_3', inviteId: 'invite_1' })+ ).rejects.toMatchObject({ code: 'FORBIDDEN' });+ expect(memberStore.size).toBe(0);+ expect(inviteStore.has('invite_1')).toBe(true);+});++test('acceptInvite refuses an expired invite', async () => {+ userStore.set('user_3', { id: 'user_3', email: '[email protected]' });+ seedPendingInvite({ expiresAt: new Date(Date.now() - 60_000) });++ await expect(+ subject.acceptInvite(deps, { userId: 'user_3', inviteId: 'invite_1' })+ ).rejects.toMatchObject({ code: 'BAD_REQUEST' });+ expect(memberStore.size).toBe(0);+});+test('revokeInvite deletes the invite row', async () => {inviteStore.set('invite_1', {id: 'invite_1',diff --git a/packages/core/src/modules/organization/organization.service.ts b/packages/core/src/modules/organization/organization.service.tsindex d33b7c1c1..0dd2a3d6e 100644--- a/packages/core/src/modules/organization/organization.service.ts+++ b/packages/core/src/modules/organization/organization.service.ts@@ -16,7 +16,11 @@ import {} from '../../cacheable-per-deps';import { chQuery } from '../../ch-query';import { sendEmail } from '../../clients/email';-import { TRPCBadRequestError, TRPCNotFoundError } from '../../rpc/errors';+import {+ TRPCBadRequestError,+ TRPCForbiddenError,+ TRPCNotFoundError,+} from '../../rpc/errors';import type { ServiceDeps, Services } from '../../services';type MembershipDeps = Pick<ServiceDeps, 'db' | 'config'>;@@ -318,7 +322,8 @@ export async function connectUserToOrganization(organizationId: invite.organizationId,});- await db.invite.delete({+ // deleteMany: a concurrent acceptance may already have consumed it.+ await db.invite.deleteMany({where: {id: inviteId,},@@ -327,6 +332,39 @@ export async function connectUserToOrganization(return member;}+/**+ * Join an organization from an invite while signed in. The session's email must+ * be the invited address, so an account that merely registered that address+ * without proving it still cannot take the invite without the link.+ */+export async function acceptInvite(+ deps: ServiceDeps,+ { userId, inviteId }: { userId: string; inviteId: string }+): Promise<{ organizationId: string }> {+ const [user, invite] = await Promise.all([+ deps.db.user.findUniqueOrThrow({ where: { id: userId } }),+ deps.db.invite.findUnique({ where: { id: inviteId } }),+ ]);++ if (!invite) {+ throw new TRPCNotFoundError('Invite not found');+ }+ if (invite.email.toLowerCase() !== user.email.toLowerCase()) {+ throw new TRPCForbiddenError(+ 'This invitation was sent to a different email address'+ );+ }+ if (deps.config.auth.allowInvitation === false) {+ throw new TRPCForbiddenError('Invitations are not allowed');+ }+ if (invite.expiresAt < new Date()) {+ throw new TRPCBadRequestError('This invitation has expired');+ }++ await connectUserToOrganization(deps, { user, inviteId });+ return { organizationId: invite.organizationId };+}+export async function getOrganizationBillingEventsCount(deps: ServiceDeps,organization: IServiceOrganization & { projects: { id: string }[] }@@ -917,7 +955,9 @@ export async function inviteUserToOrganization(},});- if (userExists) {+ // Only a verified address may join without the link: anyone can register+ // an unverified account under someone else's email.+ if (userExists?.emailVerified) {const member = await connectUserToOrganization(deps, {user: userExists,inviteId: invite.id,@@ -1101,6 +1141,10 @@ export function createOrganizationService(getInvites(deps, organizationId),getInvite: (inviteId: string): ReturnType<typeof getInviteById> =>getInviteById(deps, inviteId),+ acceptInvite: (input: {+ userId: string;+ inviteId: string;+ }): Promise<{ organizationId: string }> => acceptInvite(deps, input),runDeleteCron: (): Promise<DeleteCronResult> => runDeleteCron(deps),};}diff --git a/packages/core/src/modules/organization/src/contact.ts b/packages/core/src/modules/organization/src/contact.tsnew file mode 100644index 000000000..79d555389--- /dev/null+++ b/packages/core/src/modules/organization/src/contact.ts@@ -0,0 +1,39 @@+import { STORED_ADMIN_ROLES } from '../organization.constants';++export interface OrganizationContact {+ id: string;+ email: string;+ firstName: string | null;+}++/** The Prisma `include` `getOrganizationContact` reads: current admins, longest-standing first. */+export const ORGANIZATION_ADMINS_INCLUDE = {+ members: {+ where: { role: { in: STORED_ADMIN_ROLES }, user: { deletedAt: null } },+ orderBy: { createdAt: 'asc' },+ select: { user: { select: { id: true, email: true, firstName: true } } },+ },+} as const;++export interface OrganizationWithAdmins {+ createdByUserId: string | null;+ members: { user: OrganizationContact | null }[];+}++/**+ * Who an organization's lifecycle mail goes to: the creator while they are+ * still an admin, else the longest-standing admin. `createdBy` outlives the+ * membership, so mailing it would keep sending usage to someone removed.+ */+export function getOrganizationContact(+ organization: OrganizationWithAdmins+): OrganizationContact | null {+ const admins = organization.members.flatMap((member) =>+ member.user ? [member.user] : []+ );+ return (+ admins.find((admin) => admin.id === organization.createdByUserId) ??+ admins[0] ??+ null+ );+}diff --git a/packages/core/src/modules/organization/src/wind-down.test.ts b/packages/core/src/modules/organization/src/wind-down.test.tsindex 12f762c82..489f99a62 100644--- a/packages/core/src/modules/organization/src/wind-down.test.ts+++ b/packages/core/src/modules/organization/src/wind-down.test.ts@@ -69,12 +69,12 @@ function makeOrg(windDownStartedAt: null,windDownStep: null,deleteAt: null,- createdBy: {- id: 'user-1',- email: '[email protected]',- firstName: 'Alex',- deletedAt: null,- },+ createdByUserId: 'user-1',+ members: [+ {+ user: { id: 'user-1', email: '[email protected]', firstName: 'Alex' },+ },+ ],projects: [{ id: 'project-1', name: 'acme-web' }],...overrides,} as WindDownOrganization;@@ -247,16 +247,8 @@ describe('runWindDownCron', () => {it('skips organizations with no one to warn', async () => {organizationFindMany.mockResolvedValue([- makeOrg({ createdBy: null }),- makeOrg({- id: 'org-2',- createdBy: {- id: 'user-2',- email: '[email protected]',- firstName: null,- deletedAt: new Date(),- },- }),+ makeOrg({ createdByUserId: null, members: [] }),+ makeOrg({ id: 'org-2', members: [] }),]);const result = await runWindDownCron(deps());@@ -265,6 +257,30 @@ describe('runWindDownCron', () => {expect(sendEmail).not.toHaveBeenCalled();});+ it('warns a current admin, not a creator who was removed', async () => {+ organizationFindMany.mockResolvedValue([+ makeOrg({+ createdByUserId: 'user-removed',+ members: [+ {+ user: {+ id: 'user-2',+ email: '[email protected]',+ firstName: 'Sam',+ },+ },+ ],+ }),+ ]);++ await runWindDownCron(deps());++ expect(sendEmail).toHaveBeenCalled();+ for (const [, options] of sendEmail.mock.calls) {+ expect(options).toMatchObject({ to: '[email protected]' });+ }+ });+describe('still-tracking organizations', () => {// The population this whole sequence is aimed at: trial lapsed months ago,// SDKs never stopped, never paid a cent.diff --git a/packages/core/src/modules/organization/src/wind-down.ts b/packages/core/src/modules/organization/src/wind-down.tsindex b9030f3de..7a87cf9cb 100644--- a/packages/core/src/modules/organization/src/wind-down.ts+++ b/packages/core/src/modules/organization/src/wind-down.ts@@ -9,6 +9,12 @@ import {type SequenceSubject,step,} from '../../email/src/sequence';+import {+ getOrganizationContact,+ ORGANIZATION_ADMINS_INCLUDE,+ type OrganizationContact,+ type OrganizationWithAdmins,+} from './contact';import type { HighlightProject } from './win-back-highlight';/**@@ -43,18 +49,12 @@ function getMaxPerRun(config: CoreConfig): number {}/** The organization row shape this job reads. */-export interface WindDownOrganization {+export interface WindDownOrganization extends OrganizationWithAdmins {id: string;subscriptionState: string;subscriptionEndsAt: Date | null;windDownStartedAt: Date | null;windDownStep: string | null;- createdBy: {- id: string;- email: string;- firstName: string | null;- deletedAt: Date | null;- } | null;projects: { id: string; name: string }[];}@@ -94,7 +94,7 @@ export interface WindDownDeps {interface WindDownContext {org: WindDownOrganization;- user: NonNullable<WindDownOrganization['createdBy']>;+ user: OrganizationContact;dashboardUrl: string;/** Day 0 of this org's schedule. */startedAt: Date;@@ -309,9 +309,7 @@ export async function runWindDownCron(],},include: {- createdBy: {- select: { id: true, email: true, firstName: true, deletedAt: true },- },+ ...ORGANIZATION_ADMINS_INCLUDE,projects: { select: { id: true, name: true } },},});@@ -336,11 +334,16 @@ export async function runWindDownCron(});}- // Without a creator we have nobody to warn, so the sequence can't run. Such+ // Without an admin we have nobody to warn, so the sequence can't run. Such// orgs are left alone; the delete cron already sweeps ones with no admin.- const contactable = expired.filter(- (org) => org.createdBy && !org.createdBy.deletedAt- );+ const contacts = new Map<string, OrganizationContact>();+ for (const org of expired) {+ const contact = getOrganizationContact(org);+ if (contact) {+ contacts.set(org.id, contact);+ }+ }+ const contactable = expired.filter((org) => contacts.has(org.id));// One cheap aggregate for the whole instance, same source the data-health// job uses. It answers "are the SDKs still live", which decides both the@@ -394,10 +397,11 @@ export async function runWindDownCron((org) => org.windDownStartedAt !== null || enteringIds.has(org.id));- const subjects: SequenceSubject<WindDownContext>[] = active.map((org) => {- const user = org.createdBy as NonNullable<- WindDownOrganization['createdBy']- >;+ const subjects: SequenceSubject<WindDownContext>[] = active.flatMap((org) => {+ const user = contacts.get(org.id);+ if (!user) {+ return [];+ }const startedAt = org.windDownStartedAt ?? now;const seen = activity.get(org.id);diff --git a/packages/core/src/modules/overview/overview.rpc.ts b/packages/core/src/modules/overview/overview.rpc.tsindex cc6f10be8..ba7075968 100644--- a/packages/core/src/modules/overview/overview.rpc.ts+++ b/packages/core/src/modules/overview/overview.rpc.ts@@ -78,6 +78,29 @@ const overviewProcedure = publicProcedure.use(});+interface DateRange {+ startDate: string;+ endDate: string;+}++/** Ends a range at the organization's subscription cutoff, if it has one. */+async function clampToSubscriptionEnd(+ ctx: ServiceDeps,+ projectId: string,+ range: DateRange+): Promise<DateRange> {+ const endDate = await getOrganizationSubscriptionChartEndDate(+ ctx,+ projectId,+ range.endDate+ );+ if (!endDate) {+ return range;+ }+ const startsAfterCutoff = new Date(range.startDate) > new Date(endDate);+ return { startDate: startsAfterCutoff ? endDate : range.startDate, endDate };+}+function getCurrentAndPrevious<T extends {startDate?: string | null;@@ -90,36 +113,22 @@ function getCurrentAndPrevious<const previous = getChartPrevStartEndDate(current);return async <R>(- fn: (input: T & { startDate: string; endDate: string }) => Promise<R>+ fn: (input: T & DateRange) => Promise<R>): Promise<{current: R;previous: R | null;}> => {- const endDate = await getOrganizationSubscriptionChartEndDate(- ctx,- input.projectId,- current.endDate- );- if (endDate) {- current.endDate = endDate;- // Only expired trial scenarios- if (new Date(current.startDate) > new Date(current.endDate)) {- current.startDate = current.endDate;- }- }- const res = await Promise.all([- fn({- ...input,- startDate: current.startDate,- endDate: current.endDate,- }),+ // Both ranges are clamped: after an expired trial the previous period can+ // still end past the cutoff.+ const [currentRange, previousRange] = await Promise.all([+ clampToSubscriptionEnd(ctx, input.projectId, current),fetchPrevious- ? fn({- ...input,- startDate: previous.startDate,- endDate: previous.endDate,- })- : Promise.resolve(null),+ ? clampToSubscriptionEnd(ctx, input.projectId, previous)+ : null,+ ]);+ const res = await Promise.all([+ fn({ ...input, ...currentRange }),+ previousRange ? fn({ ...input, ...previousRange }) : null,]);return {diff --git a/packages/core/src/modules/overview/overview.service.ts b/packages/core/src/modules/overview/overview.service.tsindex 00c413ca6..b302e81bf 100644--- a/packages/core/src/modules/overview/overview.service.ts+++ b/packages/core/src/modules/overview/overview.service.ts@@ -908,12 +908,23 @@ export async function getUserJourney(const getNodeId = (path: string, step: number) => `${path}::step${step}`;- const transitionsByStep = new Map<number, typeof transitions>();+ // Keyed by step, then source, so each active node reads only its own edges.+ const transitionsByStepAndSource = new Map<+ number,+ Map<string, typeof transitions>+ >();for (const t of transitions) {- if (!transitionsByStep.has(t.step)) {- transitionsByStep.set(t.step, []);+ let bySource = transitionsByStepAndSource.get(t.step);+ if (!bySource) {+ bySource = new Map();+ transitionsByStepAndSource.set(t.step, bySource);+ }+ const fromSource = bySource.get(t.source);+ if (fromSource) {+ fromSource.push(t);+ } else {+ bySource.set(t.source, [t]);}- transitionsByStep.get(t.step)!.push(t);}const activeNodes = new Map<string, string>(); // path -> nodeId@@ -929,12 +940,11 @@ export async function getUserJourney(});for (let step = 1; step < steps; step++) {- const stepTransitions = transitionsByStep.get(step) || [];+ const stepTransitions = transitionsByStepAndSource.get(step);const nextActiveNodes = new Map<string, string>();for (const [sourcePath, sourceNodeId] of activeNodes) {- const fromSource = stepTransitions- .filter((t) => t.source === sourcePath)+ const fromSource = (stepTransitions?.get(sourcePath) ?? []).sort((a, b) => b.value - a.value).slice(0, TOP_DESTINATIONS_PER_NODE);diff --git a/packages/core/src/modules/overview/src/overview.sql.ts b/packages/core/src/modules/overview/src/overview.sql.tsindex 87c5a01c7..fcd7c501d 100644--- a/packages/core/src/modules/overview/src/overview.sql.ts+++ b/packages/core/src/modules/overview/src/overview.sql.ts@@ -10,6 +10,7 @@ import {toRangeBoundaryDate,toRangeBoundaryLiteral,} from '../../../shared/ch-dates';+import { assertTimeBucketBudget } from '../../../shared/time-buckets';import { compiledText } from '../../chart/src/compiled';import { getSelectPropertyKey } from '../../chart/src/field-resolution';import type { IInterval } from '../../report/report.constants';@@ -97,6 +98,7 @@ export function fillClause(startDate: string,endDate: string): SqlFragment {+ assertTimeBucketBudget({ startDate, endDate, interval });return sql`WITH FILL FROM ${fillFrom(interval, startDate)} TO ${fillBoundaryParam(interval, endDate, 'end')} STEP ${compiledText(toIntervalStep(interval))}`;}diff --git a/packages/core/src/modules/overview/src/pages.sql.ts b/packages/core/src/modules/overview/src/pages.sql.tsindex f28a08374..aa4698b85 100644--- a/packages/core/src/modules/overview/src/pages.sql.ts+++ b/packages/core/src/modules/overview/src/pages.sql.ts@@ -145,12 +145,15 @@ export interface PageTimeseriesQueryInput {export function pageTimeseriesQuery(input: PageTimeseriesQueryInput): SqlFragment {- const originFilter = input.filterOrigin- ? sql`AND e.origin = ${sql.string(input.filterOrigin)}`- : sql.empty;- const pathFilter = input.filterPath- ? sql`AND e.path = ${sql.string(input.filterPath)}`- : sql.empty;+ // An empty selector still selects: '' is a page's origin or path, not "any".+ const originFilter =+ input.filterOrigin === undefined+ ? sql.empty+ : sql`AND e.origin = ${sql.string(input.filterOrigin)}`;+ const pathFilter =+ input.filterPath === undefined+ ? sql.empty+ : sql`AND e.path = ${sql.string(input.filterPath)}`;// `WITH FILL` attaches to the ORDER BY expression it follows, so the// ranking columns go after it, not before. ClickHouse fills AFTER@@ -206,16 +209,19 @@ export function pageConversionsQuery(AND ${dateRangeWhere('created_at', input.startDate, input.endDate)}`;+ // ASOF pairs each view with only its next conversion: whether that one is+ // inside the window decides attribution, and a plain join would enumerate+ // every view x conversion pair of a profile first.const viewsBeforeConversions = sql`SELECT DISTINCT e.profile_id, e.path, e.originFROM events AS e- INNER JOIN conversion_events AS c ON e.profile_id = c.profile_id+ ASOF INNER JOIN conversion_events AS c+ ON e.profile_id = c.profile_id AND e.created_at < c.conv_timeWHERE e.project_id = ${sql.string(input.projectId)}AND e.name = 'screen_view'AND e.path != ''AND ${dateRangeWhere('e.created_at', input.startDate, input.endDate)}- AND e.created_at < c.conv_time- AND e.created_at >= c.conv_time - INTERVAL ${sql.uint64(input.windowHours)} HOUR+ AND c.conv_time <= e.created_at + INTERVAL ${sql.uint64(input.windowHours)} HOUR`;const totalVisitors = sql`diff --git a/packages/core/src/modules/profile/profile.routes.test.ts b/packages/core/src/modules/profile/profile.routes.test.tsindex 2ecdf2779..2558c2b40 100644--- a/packages/core/src/modules/profile/profile.routes.test.ts+++ b/packages/core/src/modules/profile/profile.routes.test.ts@@ -111,6 +111,36 @@ test('POST /profile identifies with geo + user agent and answers 202 + profileIdexpect(getGeoLocation).toHaveBeenCalledTimes(1);});+test('POST /profile drops malformed optional fields before the service', async () => {+ const response = await post('/profile', {+ profileId: 'prof-1',+ firstName: 'Ann',+ lastName: ['x'],+ email: { address: '[email protected]' },+ properties: 'plan=pro',+ groups: ['smuggled'],+ });++ expect(response.status).toBe(202);+ expect(identifyProfile).toHaveBeenCalledTimes(1);+ const [, , received] = identifyProfile.mock.calls[0]!;+ expect(received).toEqual({+ profileId: 'prof-1',+ firstName: 'Ann',+ lastName: undefined,+ email: undefined,+ properties: undefined,+ });+});++test('POST /profile refuses a body without a usable profileId', async () => {+ for (const body of [{}, { profileId: { id: 1 } }, { profileId: '' }]) {+ const response = await post('/profile', body);+ expect([body, response.status]).toEqual([body, 400]);+ }+ expect(identifyProfile).not.toHaveBeenCalled();+});+test('POST /profile/increment passes value through; /decrement negates it', async () => {const body = { profileId: 'prof-1', property: 'visits', value: 3 };diff --git a/packages/core/src/modules/profile/profile.routes.ts b/packages/core/src/modules/profile/profile.routes.tsindex 457b76d86..92e8dbf6f 100644--- a/packages/core/src/modules/profile/profile.routes.ts+++ b/packages/core/src/modules/profile/profile.routes.ts@@ -2,20 +2,19 @@// filtering and subscription-limit checks for this router live in the// ingest module, not this one.//-// /profile has no request-body schema, so `payload` below is typed but not-// shape-validated at the boundary — unlike /profile/increment and-// /profile/decrement, which validate via `zAdjustProperty`.+// /profile has no Elysia body schema: it parses loosely (zLooseIdentifyPayload)+// so a bad optional field is dropped rather than refusing the legacy SDKs'+// identify.import { parseUserAgent } from '@openpanel/shared/server';import { z } from 'zod';import { getGeoLocation } from '../../clients/geo';import { defineRoutes } from '../../http/define';-import { zProfileId } from '../ingest/ingest.constants';+import { zLooseIdentifyPayload, zProfileId } from '../ingest/ingest.constants';import { validateIngestRequest } from '../ingest/src/client-auth';import {type AdjustProfilePropertyResult,adjustProfileProperty,- type IdentifyProfileInput,identifyProfile,} from './profile.service';@@ -61,7 +60,11 @@ export const profileRoutes = defineRoutes((app) =>if (body === null || body === undefined) {return status(400, 'Missing body');}- const payload = body as IdentifyProfileInput;+ const parsed = zLooseIdentifyPayload.safeParse(body);+ if (!parsed.success) {+ return status(400, 'Invalid profileId');+ }+ const payload = parsed.data;const userAgent = parseUserAgent(ctx.headers.get('user-agent'),payload.propertiesdiff --git a/packages/core/src/modules/profile/profile.service.ts b/packages/core/src/modules/profile/profile.service.tsindex 4773cb692..0f7b44eda 100644--- a/packages/core/src/modules/profile/profile.service.ts+++ b/packages/core/src/modules/profile/profile.service.ts@@ -687,8 +687,13 @@ export async function identifyProfile(payload: IdentifyProfileInput,request: ProfileRequestContext): Promise<void> {+ // Picked, not spread: a spread would let any extra field (`groups`) reach+ // the shared profile buffer.await upsertProfile(deps, {- ...payload,+ firstName: payload.firstName,+ lastName: payload.lastName,+ email: payload.email,+ avatar: payload.avatar,id: payload.profileId,isExternal: true,projectId,diff --git a/packages/core/src/modules/project/project.service.ts b/packages/core/src/modules/project/project.service.tsindex 35e97c096..c3f1f1d36 100644--- a/packages/core/src/modules/project/project.service.ts+++ b/packages/core/src/modules/project/project.service.ts@@ -47,9 +47,13 @@ export async function getProjectById(deps: DbScope, id: string) {return res;}-/** L1 LRU (60s) + L2 Redis, one instance per Postgres client. Empty cache name, so the Redis key is `cachable::<id>`. */+/**+ * L1 LRU (60s) + L2 Redis, one instance per Postgres client. Named so its keys+ * never meet `getClientByIdCached`'s `cachable::<id>`: a project id is a slug+ * of a user-chosen name and can spell out a client id.+ */export const getProjectByIdCached = cacheablePerDb(- '',+ 'getProjectById',getProjectById,DAY_IN_SECONDS);diff --git a/packages/core/src/modules/realtime/realtime.routes.test.ts b/packages/core/src/modules/realtime/realtime.routes.test.tsindex 81fc6054d..f70a9f518 100644--- a/packages/core/src/modules/realtime/realtime.routes.test.ts+++ b/packages/core/src/modules/realtime/realtime.routes.test.ts@@ -33,7 +33,13 @@ const subscribeToProjectEventBatches = mock(});-const subscribeToProjectNotifications = mock(async () => () => undefined);+let notificationCallback: ((notification: unknown) => void) | undefined;+const subscribeToProjectNotifications = mock(+ async (_projectId: string, onNotification: (n: unknown) => void) => {+ notificationCallback = onNotification;+ return () => undefined;+ }+);const subscribeToOrganizationSubscriptionUpdates = mock(async (_organizationId: string, _onUpdate: (message: unknown) => void) =>() =>@@ -83,11 +89,22 @@ mock.module('../../http/session', () => ({resolveSession,}));+// The share rows `canShareViewLiveVisitors` reads, keyed by share id.+let shareWidgets: Record<string, { projectId: string; public: boolean }> = {};+const findShareWidget = mock(+ async ({ where }: { where: { id: string } }) => shareWidgets[where.id] ?? null+);+const findShareOverview = mock(async () => null);+let app: ReturnType<typeof realtimeRoutes>;let port: number;beforeEach(async () => {const { deps } = stubAppDeps();+ deps.db = {+ shareWidget: { findUnique: findShareWidget },+ shareOverview: { findUnique: findShareOverview },+ } as unknown as typeof deps.db;app = realtimeRoutes(deps);await new Promise<void>((resolve) => {app.listen(0, () => resolve());@@ -100,6 +117,8 @@ beforeEach(async () => {activeVisitorCount = 0;visitorActivityCallback = undefined;eventBatchCallback = undefined;+ notificationCallback = undefined;+ shareWidgets = {};for (const m of [subscribeToVisitorActivity,subscribeToProjectEventBatches,@@ -110,6 +129,7 @@ beforeEach(async () => {getProjectAccess,getOrganizationAccess,getActiveVisitorCount,+ findShareWidget,]) {m.mockClear();}@@ -179,16 +199,21 @@ function connect(path: string): Promise<WsHarness> {});}-test('wsVisitors: no session required (ADR-011 invariant 10) — upgrades and streams the active count', async () => {- const { ws, nextMessage } = await connect('/live/visitors/proj_1');- activeVisitorCount = 5;-- // The subscription is registered async (behind `await subscribeTo...`);- // give the open handler's microtask a turn before firing the callback.- while (!visitorActivityCallback) {+async function waitFor(condition: () => boolean) {+ while (!condition()) {await new Promise((resolve) => setTimeout(resolve, 1));}- visitorActivityCallback();+}++test('wsVisitors: a public widget share of the project streams the active count without a session', async () => {+ shareWidgets = { widget_1: { projectId: 'proj_1', public: true } };+ const { ws, nextMessage } = await connect(+ '/live/visitors/proj_1?shareId=widget_1'+ );+ activeVisitorCount = 5;++ await waitFor(() => !!visitorActivityCallback);+ visitorActivityCallback?.();await expect(nextMessage()).resolves.toBe('5');expect(subscribeToVisitorActivity).toHaveBeenCalledWith(@@ -199,6 +224,38 @@ test('wsVisitors: no session required (ADR-011 invariant 10) — upgrades and stws.close();});+test('wsVisitors: a project member streams the active count', async () => {+ session = { userId: 'user_1' };+ projectAccess = { level: 'read' };+ const { ws } = await connect('/live/visitors/proj_1');++ await waitFor(() => !!visitorActivityCallback);+ expect(getProjectAccess).toHaveBeenCalledWith({+ userId: 'user_1',+ projectId: 'proj_1',+ });++ ws.close();+});++test("wsVisitors: a bare project id, a private widget or another project's widget is refused", async () => {+ shareWidgets = {+ private_widget: { projectId: 'proj_1', public: false },+ other_widget: { projectId: 'proj_2', public: true },+ };++ for (const path of [+ '/live/visitors/proj_1',+ '/live/visitors/proj_1?shareId=private_widget',+ '/live/visitors/proj_1?shareId=other_widget',+ ]) {+ const { nextMessage, nextClose } = await connect(path);+ await expect(nextMessage()).resolves.toBe('No access');+ await nextClose();+ }+ expect(subscribeToVisitorActivity).not.toHaveBeenCalled();+});+test('wsProjectEvents: no session -> "No active session" then close (reject-after-upgrade)', async () => {const { nextMessage, nextClose } = await connect('/live/events/proj_1');@@ -326,3 +383,24 @@ test('wsProjectNotifications: no session -> "No active session" then close', asyawait nextClose();expect(subscribeToProjectNotifications).not.toHaveBeenCalled();});++test('wsProjectNotifications: a member removed after connecting gets no further notifications', async () => {+ session = { userId: 'user_1' };+ projectAccess = { level: 'read' };++ const { nextMessage, nextClose } = await connect(+ '/live/notifications/proj_1'+ );+ await waitFor(() => !!notificationCallback);++ notificationCallback?.({ projectId: 'proj_1', title: 'first' });+ expect(getSuperJson<{ title: string }>(await nextMessage())?.title).toBe(+ 'first'+ );++ projectAccess = null;+ notificationCallback?.({ projectId: 'proj_1', title: 'second' });++ await expect(nextMessage()).resolves.toBe('No access');+ await nextClose();+});diff --git a/packages/core/src/modules/realtime/realtime.routes.ts b/packages/core/src/modules/realtime/realtime.routes.tsindex 278291cfd..cf0ba1001 100644--- a/packages/core/src/modules/realtime/realtime.routes.ts+++ b/packages/core/src/modules/realtime/realtime.routes.ts@@ -1,5 +1,6 @@-// The `/live` websocket surface. `/live/visitors/:projectId` stays-// unauthenticated by intent; the other three require a session plus+// The `/live` websocket surface. `/live/visitors/:projectId` admits a+// `?shareId=` of a public widget or overview share of that project as an+// alternative to a session; the other three require a session plus// project/org access. All four use REJECT-AFTER-UPGRADE: the socket accepts// the upgrade, then the open handler sends a text frame and closes it. None// of these routes uses the `session` macro, because that macro would 401 the@@ -8,6 +9,7 @@import { setSuperJson } from '@openpanel/shared';import { z } from 'zod';+import type { HttpCtx } from '../../context';import { defineRoutes } from '../../http/define';import {getActiveVisitorCount,@@ -20,7 +22,12 @@ import {const NO_ACTIVE_SESSION_MESSAGE = 'No active session';const NO_ACCESS_MESSAGE = 'No access';+// A visitor socket outlives the share or membership that admitted it, and+// fires on every event batch, so access is re-proved at most this often.+const VISITOR_ACCESS_RECHECK_MS = 60_000;+const projectParams = z.object({ projectId: z.string() });+const visitorQuery = z.object({ shareId: z.string().optional() });const organizationParams = z.object({ organizationId: z.string() });interface ClosableSocket {@@ -53,20 +60,69 @@ function rejectNoAccess(ws: ClosableSocket) {ws.close();}+// The delivery callbacks are fire-and-forget, so a failed lookup must+// resolve to "no access" rather than reject unhandled.+function hasMemberProjectAccess(+ ctx: HttpCtx,+ userId: string,+ projectId: string+): Promise<boolean> {+ return ctx.services.auth+ .getProjectAccess({ userId, projectId })+ .then((access) => !!access)+ .catch(() => false);+}++async function canWatchVisitors(+ ctx: HttpCtx,+ projectId: string,+ shareId: string | undefined+): Promise<boolean> {+ const sharedWithViewer =+ !!shareId &&+ (await ctx.services.share+ .canShareViewLiveVisitors({ projectId, shareId }, ctx.cookies)+ .catch(() => false));+ if (sharedWithViewer) {+ return true;+ }+ const userId = (await ctx.session().catch(() => null))?.userId;+ return !!userId && (await hasMemberProjectAccess(ctx, userId, projectId));+}+export const realtimeRoutes = defineRoutes((app) =>app.ws('/live/visitors/:projectId', {params: projectParams,+ query: visitorQuery,async open(ws) {const { projectId } = ws.data.params;+ const { shareId } = ws.data.query;const { ctx } = ws.data;+ if (!(await canWatchVisitors(ctx, projectId, shareId))) {+ rejectNoAccess(ws);+ return;+ }++ let accessCheckedAt = Date.now();+ const sendCount = () =>+ getActiveVisitorCount(ctx, projectId).then(+ (count) => ws.send(String(count)),+ () => ws.send('0')+ );rememberUnsubscribe(ws,- await subscribeToVisitorActivity(projectId, () => {- getActiveVisitorCount(ctx, projectId).then(- (count) => ws.send(String(count)),- () => ws.send('0')- );+ await subscribeToVisitorActivity(projectId, async () => {+ const recheckDue =+ Date.now() - accessCheckedAt > VISITOR_ACCESS_RECHECK_MS;+ if (recheckDue) {+ accessCheckedAt = Date.now();+ if (!(await canWatchVisitors(ctx, projectId, shareId))) {+ rejectNoAccess(ws);+ return;+ }+ }+ await sendCount();}));},@@ -123,11 +179,21 @@ export const realtimeRoutes = defineRoutes((app) =>return;}+ // Re-proved per delivery: notifications are rare, and a member removed+ // from the project must stop receiving them on a socket opened before.rememberUnsubscribe(ws,- await subscribeToProjectNotifications(projectId, (notification) => {- ws.send(setSuperJson(notification));- })+ await subscribeToProjectNotifications(+ projectId,+ async (notification) => {+ const ctx = ws.data.ctx;+ if (!(await hasMemberProjectAccess(ctx, userId, projectId))) {+ rejectNoAccess(ws);+ return;+ }+ ws.send(setSuperJson(notification));+ }+ ));},close(ws) {diff --git a/packages/core/src/modules/session/session.jobs.ts b/packages/core/src/modules/session/session.jobs.tsindex 7fc24db5e..99488e057 100644--- a/packages/core/src/modules/session/session.jobs.ts+++ b/packages/core/src/modules/session/session.jobs.ts@@ -86,7 +86,12 @@ export const sessionQueueJobs = {session: defineJob({payload: sessionEndPayload,handler: async ({ payload: data, ctx }) => {- const logger = ctx.logger.child({ payload: data.event });+ // Child bindings bypass log redaction, so only ids: the event carries+ // the visitor's IP and properties.+ const logger = ctx.logger.child({+ projectId: data.event.projectId,+ sessionId: data.snapshot.id,+ });await createSessionEnd(data, await sessionEndDeps(ctx, logger));diff --git a/packages/core/src/modules/session/session.service.ts b/packages/core/src/modules/session/session.service.tsindex 9a8c60e1d..12d27747d 100644--- a/packages/core/src/modules/session/session.service.ts+++ b/packages/core/src/modules/session/session.service.ts@@ -361,11 +361,11 @@ export async function getSessionReplayChunksFrom(}));- // `chunkIndex` is the row's position, not `row.chunk_index`, and `hasMore` counts rows- // the JSON filter may drop, so an unparseable chunk rewinds the client.+ const page = rows.slice(0, REPLAY_CHUNKS_PAGE_SIZE);+ // `nextIndex` counts every row read, so an unparseable chunk is skipped rather+ // than refetched.return {- data: rows- .slice(0, REPLAY_CHUNKS_PAGE_SIZE)+ data: page.map((row, index) => {const events = getSafeJson<{ type: number; data: unknown; timestamp: number }[]@@ -376,6 +376,7 @@ export async function getSessionReplayChunksFrom(return { chunkIndex: index + fromIndex, events };}).filter(Boolean),+ nextIndex: fromIndex + page.length,hasMore: rows.length > REPLAY_CHUNKS_PAGE_SIZE,};}diff --git a/packages/core/src/modules/share/share.service.ts b/packages/core/src/modules/share/share.service.tsindex 7d09424eb..e379117c1 100644--- a/packages/core/src/modules/share/share.service.ts+++ b/packages/core/src/modules/share/share.service.ts@@ -315,6 +315,38 @@ export async function hasAnonymousShareAccessToProject();}+/**+ * Whether `shareId` lets an anonymous viewer watch a project's live visitor+ * count: a public widget of that project, or a public overview share of it+ * that is not locked behind a password the viewer has not entered.+ */+export async function canShareViewLiveVisitors(+ deps: ServiceDeps,+ { projectId, shareId }: { projectId: string; shareId: string },+ cookies: CookieReader+): Promise<boolean> {+ const db = deps.db;+ const [widget, overview] = await Promise.all([+ db.shareWidget.findUnique({+ where: { id: shareId },+ select: { projectId: true, public: true },+ }),+ db.shareOverview.findUnique({+ where: { id: shareId },+ select: { id: true, projectId: true, public: true, password: true },+ }),+ ]);++ if (widget?.public && widget.projectId === projectId) {+ return true;+ }+ return (+ !!overview?.public &&+ overview.projectId === projectId &&+ isUnlocked(deps, cookies, 'overview', overview)+ );+}+/** Shape returned to a viewer who has not unlocked a password-protected share. */function lockedShare(id: string,@@ -695,6 +727,10 @@ export function createShareService(kinds?: ShareType[]): Promise<boolean> =>hasAnonymousShareAccessToProject(deps, projectId, cookies, kinds),+ canShareViewLiveVisitors: (+ input: { projectId: string; shareId: string },+ cookies: CookieReader+ ): Promise<boolean> => canShareViewLiveVisitors(deps, input, cookies),getShareOverview: (shareId: string,cookies: CookieReaderdiff --git a/packages/core/src/modules/subscription/subscription.rpc.ts b/packages/core/src/modules/subscription/subscription.rpc.tsindex 02b9d7d90..cd9b8d6dd 100644--- a/packages/core/src/modules/subscription/subscription.rpc.ts+++ b/packages/core/src/modules/subscription/subscription.rpc.ts@@ -39,7 +39,10 @@ export const subscriptionRouter = createTRPCRouter({usage: protectedProcedure.input(z.object({ organizationId: z.string() })).query(({ input, ctx }) =>- ctx.services.subscription.getUsage(input.organizationId)+ ctx.services.subscription.getUsage(+ ctx.session.userId,+ input.organizationId+ )),cancelSubscription: protectedProcedurediff --git a/packages/core/src/modules/subscription/subscription.service.ts b/packages/core/src/modules/subscription/subscription.service.tsindex dd867d0ac..e9ff925df 100644--- a/packages/core/src/modules/subscription/subscription.service.ts+++ b/packages/core/src/modules/subscription/subscription.service.ts@@ -6,6 +6,8 @@ import {createPortal,getProduct,getProducts,+ type IPolarProduct,+ isCustomProduct,pauseSubscription as pausePolarSubscription,reactivateSubscription,resumeSubscription as resumePolarSubscription,@@ -37,6 +39,10 @@ import {const POLAR_PRODUCTS_CACHE_KEY = 'polar:products';const POLAR_PRODUCTS_CACHE_TTL_SECONDS = 60 * 60 * 24;const TRAILING_USAGE_WINDOW_DAYS = 30;+const PRODUCT_OVER_LIMIT_MESSAGE =+ 'This product is not applicable since you have exceeded the limits for this subscription.';+const PRODUCT_UNAVAILABLE_MESSAGE =+ 'This product is not available for this organization';export type IOrganizationUsage = IOrganizationBillingUsage & {projects: { id: string; name: string }[];@@ -52,6 +58,40 @@ async function requireOrgAdmin(await requireOrganizationAdmin({ userId, organizationId });}+/**+ * Why an organization may not move to a product, or null when it may. The+ * catalog shows it and checkout enforces it, so the two cannot drift.+ */+function getProductIneligibility(+ product: Pick<IPolarProduct, 'metadata' | 'isArchived'>,+ organization: { id: string; subscriptionPeriodEventsCount: number }+): string | null {+ const isOtherOrganizationsProduct =+ isCustomProduct(product) &&+ product.metadata.organizationId !== organization.id;+ if (product.isArchived || isOtherOrganizationsProduct) {+ return PRODUCT_UNAVAILABLE_MESSAGE;+ }+ const eventsLimit = product.metadata.eventsLimit;+ const isOverLimit =+ typeof eventsLimit === 'number' &&+ organization.subscriptionPeriodEventsCount >= eventsLimit;+ return isOverLimit ? PRODUCT_OVER_LIMIT_MESSAGE : null;+}++async function requireEligibleProduct(+ productId: string,+ organization: { id: string; subscriptionPeriodEventsCount: number }+): Promise<void> {+ const ineligibility = getProductIneligibility(+ await getProduct(productId),+ organization+ );+ if (ineligibility) {+ throw new TRPCBadRequestError(ineligibility);+ }+}+export async function getCurrentSubscriptionProduct(deps: ServiceDeps,organizationId: string@@ -112,10 +152,12 @@ export async function checkout(return null;}+ await requireEligibleProduct(input.productId, organization);await changeSubscription(organization.subscriptionId, input.productId);return null;}+ await requireEligibleProduct(input.productId, organization);const checkoutSession = await createCheckout({productId: input.productId,organizationId: input.organizationId,@@ -129,7 +171,7 @@ export async function checkout(export async function listProducts(deps: ServiceDeps, organizationId: string) {const organization = await deps.db.organization.findUniqueOrThrow({where: { id: organizationId },- select: { subscriptionPeriodEventsCount: true },+ select: { id: true, subscriptionPeriodEventsCount: true },});return (@@ -138,23 +180,19 @@ export async function listProducts(deps: ServiceDeps, organizationId: string) {POLAR_PRODUCTS_CACHE_TTL_SECONDS,() => getProducts())- ).map((product) => {- const eventsLimit = product.metadata.eventsLimit;- return {- ...product,- disabled:- typeof eventsLimit === 'number' &&- organization.subscriptionPeriodEventsCount >= eventsLimit- ? 'This product is not applicable since you have exceeded the limits for this subscription.'- : null,- };- });+ ).map((product) => ({+ ...product,+ disabled: getProductIneligibility(product, organization),+ }));}+/** Admin-only: it names and counts every project, including ones a `selected`-scope member cannot open. */export async function getUsage(deps: ServiceDeps,+ userId: string,organizationId: string): Promise<IOrganizationUsage> {+ await requireOrgAdmin(userId, organizationId);const organization = await deps.db.organization.findUniqueOrThrow({where: { id: organizationId },include: {@@ -797,8 +835,10 @@ export function createSubscriptionService(): ReturnType<typeof checkout> => checkout(deps, userId, input, ipAddress),listProducts: (organizationId: string): ReturnType<typeof listProducts> =>listProducts(deps, organizationId),- getUsage: (organizationId: string): ReturnType<typeof getUsage> =>- getUsage(deps, organizationId),+ getUsage: (+ userId: string,+ organizationId: string+ ): ReturnType<typeof getUsage> => getUsage(deps, userId, organizationId),cancelSubscription: (userId: string,input: ICancelSubscriptiondiff --git a/packages/core/src/modules/widget/widget.rpc.ts b/packages/core/src/modules/widget/widget.rpc.tsindex 6f9e63e42..8796f060e 100644--- a/packages/core/src/modules/widget/widget.rpc.ts+++ b/packages/core/src/modules/widget/widget.rpc.ts@@ -33,19 +33,26 @@ const REALTIME_TOP_LIST_LIMIT = 10;const EVENTS_TABLE = sql.id(TABLE_NAMES.events);const REALTIME_WINDOW = sql`created_at >= (now() - INTERVAL 30 MINUTE)`;-async function findWidgetByType(+// The type lives in the `options` JSON, so nothing stops two concurrent+// enables from each creating a row; reads take the oldest, writes hit all.+function widgetsOfType(projectId: string, type: z.infer<typeof zWidgetType>) {+ return { projectId, options: { path: ['type'], equals: type } };+}++function findWidgetByType(db: Ctx['db'],projectId: string,- type: string+ type: z.infer<typeof zWidgetType>) {- const widgets = await db.shareWidget.findMany({- where: { projectId },+ return db.shareWidget.findFirst({+ where: widgetsOfType(projectId, type),+ orderBy: { createdAt: 'asc' },});- return widgets.find((w) => w.options.type === type);}-// Shared by toggle/updateOptions: same find-then-update-or-create shape,-// differing only in which field is written and the create-time default.+// Shared by toggle/updateOptions: same update-or-create shape, differing only+// in which field is written and the create-time default. The update covers+// every row of the type, so disabling a widget revokes duplicate links too.async function upsertWidgetByType(db: Ctx['db'],params: {@@ -56,13 +63,16 @@ async function upsertWidgetByType(create: { public: boolean; options: z.infer<typeof zWidgetOptions> };}) {- const existing = await findWidgetByType(db, params.projectId, params.type);-+ const { count } = await db.shareWidget.updateMany({+ where: widgetsOfType(params.projectId, params.type),+ data: params.update,+ });+ const existing =+ count > 0+ ? await findWidgetByType(db, params.projectId, params.type)+ : null;if (existing) {- return db.shareWidget.update({- where: { id: existing.id },- data: params.update,- });+ return existing;}return db.shareWidget.create({diff --git a/packages/core/src/pino-logger.test.ts b/packages/core/src/pino-logger.test.tsindex 4d880e58f..aa15f613d 100644--- a/packages/core/src/pino-logger.test.ts+++ b/packages/core/src/pino-logger.test.ts@@ -22,6 +22,12 @@ describe('redactSensitive', () => {});});+ it('redacts a service account key', () => {+ expect(+ redactSensitive({ serviceAccountKey: '{"private_key":"x"}' })+ ).toEqual({ serviceAccountKey: '[REDACTED]' });+ });+it('still redacts sensitive keys by name', () => {expect(redactSensitive({ authorization: 'Bearer abc', page: 2 })).toEqual({authorization: '[REDACTED]',diff --git a/packages/core/src/rpc/base.test.ts b/packages/core/src/rpc/base.test.tsindex a547849f2..48d84c5ab 100644--- a/packages/core/src/rpc/base.test.ts+++ b/packages/core/src/rpc/base.test.ts@@ -4,6 +4,7 @@ import { z } from 'zod';import { testCoreConfig } from '../../test/config-fixture';import { stubHttpCtx, TEST_SESSION } from '../../test/rpc-fixtures';import type { CookieOptions } from '../shared/cookie';+import { BadRequestError } from '../shared/errors';import * as baseModule from './base';import {createRateLimitMiddleware,@@ -336,6 +337,7 @@ test('the rate limit middleware hands the limiter ctx fields, not a request', asexpect(calls[0]).toEqual({headers: ctx.headers,ip: '203.0.113.7',+ ipHeaders: ctx.config.ipHeaders,logger: ctx.logger,path: 'signIn',max: 5,@@ -372,3 +374,20 @@ test('a limiter rejection stops the procedure and surfaces as 429', async () =>expect(response.status).toBe(429);expect(logger.lines[0]?.message).toBe('trpc rate limited');});++test('a BadRequestError from below the transport answers BAD_REQUEST with its message', async () => {+ const { ctx } = stubHttpCtx();+ const router = createTRPCRouter({+ series: procedure.query(() => {+ throw new BadRequestError('Pick a shorter range.');+ }),+ });+ const caller = router.createCaller(+ await makeTrpcContext(ctx, new Headers(), { cookieOptions: COOKIE_OPTIONS })+ );++ await expect(caller.series()).rejects.toMatchObject({+ code: 'BAD_REQUEST',+ message: 'Pick a shorter range.',+ });+});diff --git a/packages/core/src/rpc/base.ts b/packages/core/src/rpc/base.tsindex 41eb95984..9cc8265f4 100644--- a/packages/core/src/rpc/base.ts+++ b/packages/core/src/rpc/base.ts@@ -2,11 +2,13 @@ import { initTRPC, TRPCError } from '@trpc/server';import { has } from 'ramda';import superjson from 'superjson';import { ZodError, z } from 'zod';+import type { IpHeaderConfig } from '../config';import type { HttpCtx, Session } from '../context';import type { Logger } from '../logger';import { runWithAlsSession } from '../shared/als-session';import { type CookieOptions, serializeCookie } from '../shared/cookie';import { classifyDriverError } from '../shared/driver-errors';+import { BadRequestError } from '../shared/errors';import { EMPTY_SESSION } from '../shared/session';import { cancelledCallError, raceCancellation } from './deadline';import { TRPCForbiddenError } from './errors';@@ -89,20 +91,24 @@ const stopWhenCancelled = t.middleware(async ({ ctx, next }) => {});/**- * Turns a driver failure caused by the CALLER into the status it deserves.+ * Turns a failure caused by the CALLER into the status it deserves. Code below+ * the transport says so with a `BadRequestError`; drivers only label theirs.** A malformed id raises Prisma P2023 before any `findUnique` null check can* run, and the driver's message (for ClickHouse, the generated SQL) would* otherwise reach the client as a 500. Anything `classifyDriverError` does not* recognise stays a 500, because a query bug is not the caller's mistake.*/-const mapDriverErrors = t.middleware(async ({ next }) => {+const mapCallerErrors = t.middleware(async ({ next }) => {const result = await next();if (result.ok) {return result;}const cause = result.error.cause;+ if (cause instanceof BadRequestError) {+ throw new TRPCError({ code: 'BAD_REQUEST', message: cause.message, cause });+ }const failure = classifyDriverError(cause);if (!failure) {return result;@@ -114,7 +120,7 @@ const mapDriverErrors = t.middleware(async ({ next }) => {/** The bare procedure. It authenticates NOTHING; prefer the builders below. */export const procedure = t.procedure.use(stopWhenCancelled)- .use(mapDriverErrors);+ .use(mapCallerErrors);// Authentication runs before input parsing: tRPC runs `.use` middleware ahead// of the `.input` parser, so an anonymous caller gets UNAUTHORIZED whatever it@@ -189,15 +195,24 @@ const enforceAccess = t.middleware(});+// Only the input's field names are logged: mutation inputs carry credentials+// (integration configs, second-factor codes) under names and inside values+// that no key-based redaction can recognise.+function getInputKeys(rawInput: unknown): string[] | undefined {+ if (rawInput === null || typeof rawInput !== 'object') {+ return undefined;+ }+ return Object.keys(rawInput);+}+const loggerMiddleware = t.middleware(- async ({ ctx, next, getRawInput, path, input, type }) => {+ async ({ ctx, next, getRawInput, path, type }) => {const rawInput = await getRawInput();if (type === 'mutation') {ctx.logger.info({path,- rawInput,- input,+ inputKeys: getInputKeys(rawInput),userId: ctx.session?.userId,organizationId: has('organizationId', rawInput)? rawInput.organizationId@@ -428,6 +443,8 @@ export type EnforceRateLimit = (headers: Headers;ip: string;remoteAddress: string | undefined;+ /** The deployment's `TRUSTED_IP_HEADER_ORDER`, which picks the bucket. */+ ipHeaders: IpHeaderConfig;logger: Logger;/** tRPC procedure path - blocks are scoped to it. */path: string;@@ -442,6 +459,7 @@ export const createRateLimitMiddleware =headers: ctx.headers,ip: ctx.ip,remoteAddress: ctx.remoteAddress,+ ipHeaders: ctx.config.ipHeaders,logger: ctx.logger,path,...options,diff --git a/packages/core/src/rpc/rate-limit.ts b/packages/core/src/rpc/rate-limit.tsindex c108d4174..321569c28 100644--- a/packages/core/src/rpc/rate-limit.ts+++ b/packages/core/src/rpc/rate-limit.ts@@ -40,9 +40,6 @@ const fallbackCounters = new LRUCache<string, number>({ttl: 5 * MINUTE,});-/** The trusted-header defaults: the attribution address is client-forgeable. */-const TRUSTED_ONLY = { attributionOrder: undefined, trustedOrder: undefined };-const keyFor = (kind: string, path: string, fingerprint: string) =>`rl:${kind}:${path}:${fingerprint}`;@@ -105,13 +102,14 @@ async function escalate(export const enforceRateLimit: EnforceRateLimit = async ({headers,remoteAddress,+ ipHeaders,logger,path,max,windowMs,}) => {const { ip: trustedIp, header: ipHeader } = getTrustedIpFromHeaders(- TRUSTED_ONLY,+ ipHeaders,headers,remoteAddress);diff --git a/packages/core/src/shared/access-lookups.ts b/packages/core/src/shared/access-lookups.tsindex 3f81cf19c..ca71c9e26 100644--- a/packages/core/src/shared/access-lookups.ts+++ b/packages/core/src/shared/access-lookups.ts@@ -27,6 +27,12 @@ export async function getProjectById(return db.project.findUnique({ where: { id: projectId } });}+/**+ * Removing a member or narrowing their grants must take effect on every api+ * process at once, so access answers live in Redis only.+ */+const REVOCABLE_CACHE_OPTIONS = { localCache: false } as const;+export interface IProjectAccess {level: AccessLevel;}@@ -177,7 +183,8 @@ export const getProjectAccess = cacheable(return null;}},- 60 * 5+ 60 * 5,+ REVOCABLE_CACHE_OPTIONS);export type IOrganizationAccess = Member;@@ -200,7 +207,8 @@ export const getOrganizationAccess = cacheable(},});},- 60 * 5+ 60 * 5,+ REVOCABLE_CACHE_OPTIONS);export async function getClientAccess({diff --git a/packages/core/src/shared/errors.test.ts b/packages/core/src/shared/errors.test.tsindex bd3e2e8ec..d2fc8bd64 100644--- a/packages/core/src/shared/errors.test.ts+++ b/packages/core/src/shared/errors.test.ts@@ -1,5 +1,5 @@import { describe, expect, test } from 'bun:test';-import { HttpError, LogError, normalizeError } from './errors';+import { BadRequestError, HttpError, LogError, normalizeError } from './errors';describe('LogError', () => {test('carries a payload and keeps its name and prototype chain', () => {@@ -33,6 +33,15 @@ describe('HttpError', () => {});describe('normalizeError', () => {+ test('answers a BadRequestError with 400 and its message', () => {+ expect(+ normalizeError(new BadRequestError('Pick a shorter range.'))+ ).toMatchObject({+ status: 400,+ message: 'Pick a shorter range.',+ });+ });+test('reads statusCode off an Error', () => {const error = Object.assign(new Error('nope'), { statusCode: 404 });expect(normalizeError(error)).toEqual({diff --git a/packages/core/src/shared/errors.ts b/packages/core/src/shared/errors.tsindex cb5e5c7f2..52ab7d602 100644--- a/packages/core/src/shared/errors.ts+++ b/packages/core/src/shared/errors.ts@@ -4,6 +4,7 @@import { classifyDriverError } from './driver-errors';const DEFAULT_HTTP_STATUS = 500;+const BAD_REQUEST_STATUS = 400;export class LogError extends Error {public readonly payload?: Record<string, unknown>;@@ -45,6 +46,16 @@ export class HttpError extends Error {}}+/** The caller asked for something unusable: 400 over REST, BAD_REQUEST over tRPC (rpc/base.ts). */+export class BadRequestError extends Error {+ readonly status = BAD_REQUEST_STATUS;++ constructor(message: string) {+ super(message);+ this.name = 'BadRequestError';+ }+}+export interface NormalizedError {status: number;code: string | undefined;diff --git a/packages/core/src/shared/js-runtime/concurrency-limit.test.ts b/packages/core/src/shared/js-runtime/concurrency-limit.test.tsnew file mode 100644index 000000000..231aab3fe--- /dev/null+++ b/packages/core/src/shared/js-runtime/concurrency-limit.test.ts@@ -0,0 +1,55 @@+import { describe, expect, it } from 'bun:test';+import { createConcurrencyLimit } from './concurrency-limit';++const MAX_CONCURRENT = 2;+const TASK_COUNT = 7;++function deferred() {+ let resolve: () => void = () => undefined;+ const promise = new Promise<void>((done) => {+ resolve = done;+ });+ return { promise, resolve };+}++describe('createConcurrencyLimit', () => {+ it('runs at most the limit at once and queues the rest in order', async () => {+ const limit = createConcurrencyLimit(MAX_CONCURRENT);+ const gates = Array.from({ length: TASK_COUNT }, deferred);+ const started: number[] = [];+ let running = 0;+ let peak = 0;++ const runs = gates.map((gate, index) =>+ limit(async () => {+ started.push(index);+ running++;+ peak = Math.max(peak, running);+ await gate.promise;+ running--;+ return index;+ })+ );++ await Bun.sleep(0);+ expect(started).toEqual([0, 1]);++ for (const gate of gates) {+ gate.resolve();+ await Bun.sleep(0);+ }++ expect(await Promise.all(runs)).toEqual([0, 1, 2, 3, 4, 5, 6]);+ expect(started).toEqual([0, 1, 2, 3, 4, 5, 6]);+ expect(peak).toBe(MAX_CONCURRENT);+ });++ it('frees the slot when a task throws', async () => {+ const limit = createConcurrencyLimit(1);+ const failing = limit(() => Promise.reject(new Error('boom')));+ const next = limit(async () => 'ran');++ await expect(failing).rejects.toThrow('boom');+ expect(await next).toBe('ran');+ });+});diff --git a/packages/core/src/shared/js-runtime/concurrency-limit.ts b/packages/core/src/shared/js-runtime/concurrency-limit.tsnew file mode 100644index 000000000..5c8d50a07--- /dev/null+++ b/packages/core/src/shared/js-runtime/concurrency-limit.ts@@ -0,0 +1,32 @@+/** Runs at most `maxConcurrent` tasks at once; the rest wait in FIFO order. */+export function createConcurrencyLimit(maxConcurrent: number) {+ let running = 0;+ const waiting: Array<() => void> = [];++ const acquire = async (): Promise<void> => {+ if (running < maxConcurrent) {+ running++;+ return;+ }+ // The releasing task hands its slot over, so `running` stays unchanged.+ await new Promise<void>((resolve) => waiting.push(resolve));+ };++ const release = (): void => {+ const next = waiting.shift();+ if (next) {+ next();+ return;+ }+ running--;+ };++ return async function limit<T>(task: () => Promise<T>): Promise<T> {+ await acquire();+ try {+ return await task();+ } finally {+ release();+ }+ };+}diff --git a/packages/core/src/shared/js-runtime/execute.test.ts b/packages/core/src/shared/js-runtime/execute.test.tsnew file mode 100644index 000000000..e0497ee30--- /dev/null+++ b/packages/core/src/shared/js-runtime/execute.test.ts@@ -0,0 +1,45 @@+import { describe, expect, it } from 'bun:test';+import { execute, runInSandbox } from './execute';++const IS_LINUX = process.platform === 'linux';++describe('template sandbox', () => {+ it('fails a template that allocates past the memory limit', async () => {+ const run = execute("(payload) => 'x'.repeat(2 ** 30)", {});+ await expect(run).rejects.toThrow('Error executing JavaScript template');+ // The OS cap is Linux-only; elsewhere the output cap rejects it after the+ // child allocates.+ if (IS_LINUX) {+ await expect(run).rejects.toThrow(/memory/i);+ }+ });++ it.if(IS_LINUX)(+ 'fails a template that doubles a string until it runs out of memory',+ async () => {+ const doubling = `(payload) => [${Array.from({ length: 32 }, () => 0).join(',')}].reduce((a) => a + a, 'x'.repeat(1024))`;+ await expect(execute(doubling, {})).rejects.toThrow(/memory/i);+ }+ );++ it('kills a child that outlives its deadline', async () => {+ const started = performance.now();+ await expect(+ runInSandbox(+ // Exponential backtracking on a short string: slow, but no memory.+ "(payload) => 'a'.repeat(40).replace(/(a+)+b/, '')",+ {},+ { timeoutMs: 10_000, deadlineMs: 100, maxOutputBytes: 1000 }+ )+ ).rejects.toThrow('timed out');+ expect(performance.now() - started).toBeLessThan(2000);+ });++ it('runs an honest template under the limits', async () => {+ expect(+ await execute('(payload) => ({ n: payload.n + 1 })', { n: 1 })+ ).toEqual({+ n: 2,+ });+ });+});diff --git a/packages/core/src/shared/js-runtime/execute.ts b/packages/core/src/shared/js-runtime/execute.tsindex 4e198b88f..6aeebacec 100644--- a/packages/core/src/shared/js-runtime/execute.ts+++ b/packages/core/src/shared/js-runtime/execute.ts@@ -1,54 +1,169 @@-import { runInNewContext } from 'node:vm';+import { spawn } from 'node:child_process';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { createConcurrencyLimit } from './concurrency-limit';+import { SANDBOX_EXIT_CODE } from './sandbox-protocol';import { validate } from './validate';/**- * Wall-clock budget for one template run. The validator refuses loops,- * recursion and calls to template-defined functions, so an honest template- * finishes in a few milliseconds; this is the backstop for a pathological- * regex or a huge .repeat().+ * Wall-clock budget for one template run inside the sandbox. The validator+ * refuses loops, recursion and calls to template-defined functions, so an+ * honest template finishes in a few milliseconds.*/const EXECUTION_TIMEOUT_MS = 250;+/**+ * When the parent SIGKILLs the child: the template budget plus process start+ * (~10 ms idle, more on a busy worker). Catches what the in-child timeout+ * cannot interrupt, such as one huge native `.repeat()` or `.replace()`.+ */+const PROCESS_DEADLINE_MS = 2000;+/** Cap on the serialized output so a template cannot balloon a webhook body. */const MAX_OUTPUT_BYTES = 1_000_000;-export function execute(+/**+ * Address-space cap (RLIMIT_AS) for the child, Linux only. JSC sizes its heap+ * reservations from the limit and throws "Out of memory" past it. Bun 1.4.4+ * aborts at startup under some smaller caps (768 MiB among them), so change+ * this only together with the sandbox tests, which run under it on Linux.+ */+const MEMORY_LIMIT_KIB = 1024 * 1024;++/**+ * Children alive at once per process, each up to MEMORY_LIMIT_KIB, whatever+ * the notification worker's concurrency. Further runs wait for a slot.+ */+const MAX_CONCURRENT_SANDBOXES = 3;++const limitSandboxes = createConcurrencyLimit(MAX_CONCURRENT_SANDBOXES);++// `apps/start` typechecks this source without Bun's types, which declare `dir`.+const { dir: MODULE_DIR } = import.meta as ImportMeta & { dir: string };+const SANDBOX_CHILD_PATH = join(MODULE_DIR, 'sandbox-child.ts');++/**+ * `ulimit` applies to the shell and survives `exec`, so the cap is in place+ * before Bun starts. A raised oom_score_adj makes the kernel's OOM killer pick+ * the sandbox over the worker if the container runs out regardless.+ */+const LINUX_LIMIT_WRAPPER = `ulimit -v ${MEMORY_LIMIT_KIB} || exit 1; echo 1000 > /proc/self/oom_score_adj 2>/dev/null; exec "$0" "$@"`;++export interface SandboxLimits {+ timeoutMs: number;+ deadlineMs: number;+ maxOutputBytes: number;+}++const DEFAULT_LIMITS: SandboxLimits = {+ timeoutMs: EXECUTION_TIMEOUT_MS,+ deadlineMs: PROCESS_DEADLINE_MS,+ maxOutputBytes: MAX_OUTPUT_BYTES,+};++function sandboxCommand(): [string, string[]] {+ const bunArgs = ['--no-env-file', SANDBOX_CHILD_PATH];+ if (process.platform !== 'linux') {+ return [process.execPath, bunArgs];+ }+ return ['/bin/sh', ['-c', LINUX_LIMIT_WRAPPER, process.execPath, ...bunArgs]];+}++/**+ * Runs an already-validated template in a fresh child process. The child gets+ * an empty environment and a neutral cwd (no `.env`, no bunfig preload), so+ * nothing but the template and the payload crosses into it.+ */+export function runInSandbox(+ code: string,+ payload: Record<string, unknown>,+ limits: SandboxLimits = DEFAULT_LIMITS+): Promise<string> {+ return limitSandboxes(() => spawnSandbox(code, payload, limits));+}++function spawnSandbox(+ code: string,+ payload: Record<string, unknown>,+ limits: SandboxLimits+): Promise<string> {+ const [command, args] = sandboxCommand();+ const child = spawn(command, args, {+ env: {},+ cwd: tmpdir(),+ stdio: ['pipe', 'pipe', 'ignore'],+ });++ return new Promise((resolve, reject) => {+ const chunks: Buffer[] = [];+ let outputBytes = 0;+ let failure: string | undefined;++ const kill = (reason: string) => {+ failure ??= reason;+ child.kill('SIGKILL');+ };+ const deadline = setTimeout(+ () => kill('Template timed out'),+ limits.deadlineMs+ );++ child.stdout.on('data', (chunk: Buffer) => {+ outputBytes += chunk.length;+ if (outputBytes > limits.maxOutputBytes) {+ kill('Template output is too large');+ return;+ }+ chunks.push(chunk);+ });+ // The child may die before it has read its input; the exit code reports that.+ child.stdin.on('error', () => undefined);+ child.on('error', (error) => kill(error.message));++ child.on('close', (exitCode, signal) => {+ clearTimeout(deadline);+ const output = Buffer.concat(chunks).toString('utf8');+ if (failure) {+ reject(new Error(failure));+ } else if (exitCode === SANDBOX_EXIT_CODE.ok) {+ resolve(output);+ } else if (exitCode === SANDBOX_EXIT_CODE.templateError) {+ reject(new Error(output));+ } else {+ reject(+ new Error(+ `Template sandbox crashed (${signal ?? `exit ${exitCode}`}), likely by exceeding its memory limit`+ )+ );+ }+ });++ child.stdin.end(+ JSON.stringify({+ code,+ payload,+ timeoutMs: limits.timeoutMs,+ maxOutputBytes: limits.maxOutputBytes,+ })+ );+ });+}++export async function execute(code: string,payload: Record<string, unknown>-): unknown {+): Promise<unknown> {// Re-check at run time: the stored string is what runs, not what passed- // validation at save time.+ // validation at save time. The child process is the second layer: a+ // validator miss or a runaway allocation can only take the child down.const validation = validate(code);if (!validation.valid) {throw new Error(`Invalid JavaScript template: ${validation.error}`);}try {- // A fresh V8 context with its own globals and eval/new Function disabled.- // Payload and result cross as JSON, so the template never holds a host-realm- // reference. Not a security boundary on its own (the validator is), but a- // validator miss lands in an empty realm instead of the worker process.- const script = `'use strict'; JSON.stringify((${code})(JSON.parse(payloadJson)));`;- const resultJson: unknown = runInNewContext(- script,- { payloadJson: JSON.stringify(payload) },- {- timeout: EXECUTION_TIMEOUT_MS,- contextCodeGeneration: { strings: false, wasm: false },- microtaskMode: 'afterEvaluate',- }- );-- if (resultJson === undefined) {- return undefined;- }- if (typeof resultJson !== 'string') {- throw new Error('Template did not return a JSON-serializable value');- }- if (Buffer.byteLength(resultJson, 'utf8') > MAX_OUTPUT_BYTES) {- throw new Error('Template output is too large');- }- return JSON.parse(resultJson);+ const resultJson = await runInSandbox(code, payload);+ return resultJson === '' ? undefined : JSON.parse(resultJson);} catch (error) {throw new Error(`Error executing JavaScript template: ${diff --git a/packages/core/src/shared/js-runtime/sandbox-child.ts b/packages/core/src/shared/js-runtime/sandbox-child.tsnew file mode 100644index 000000000..ad1277e07--- /dev/null+++ b/packages/core/src/shared/js-runtime/sandbox-child.ts@@ -0,0 +1,89 @@+// Entry point of the child process `execute` spawns per template run. It reads+// `{ code, payload, timeoutMs, maxOutputBytes }` as JSON from stdin and answers+// through its exit code: the parent treats any other exit (a signal, an abort+// on allocation failure) as the template having crashed the sandbox.+import { runInNewContext } from 'node:vm';+import { SANDBOX_EXIT_CODE } from './sandbox-protocol';++interface SandboxInput {+ code: string;+ payload: unknown;+ timeoutMs: number;+ maxOutputBytes: number;+}++// A type guard rather than zod: importing zod would add to every child's start.+function isSandboxInput(value: unknown): value is SandboxInput {+ return (+ typeof value === 'object' &&+ value !== null &&+ 'code' in value &&+ typeof value.code === 'string' &&+ 'payload' in value &&+ 'timeoutMs' in value &&+ typeof value.timeoutMs === 'number' &&+ 'maxOutputBytes' in value &&+ typeof value.maxOutputBytes === 'number'+ );+}++function parseSandboxInput(json: string): SandboxInput {+ const input: unknown = JSON.parse(json);+ if (!isSandboxInput(input)) {+ throw new Error('Malformed sandbox input');+ }+ return input;+}++async function readStdin(): Promise<string> {+ const chunks: Buffer[] = [];+ for await (const chunk of process.stdin) {+ chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)));+ }+ return Buffer.concat(chunks).toString('utf8');+}++function finish(exitCode: number, output: string): void {+ process.stdout.write(output, () => process.exit(exitCode));+}++function run(input: SandboxInput): void {+ // Payload and result cross as JSON, so the template never holds a reference+ // to this realm, and eval/new Function stay disabled inside it.+ const script = `'use strict'; JSON.stringify((${input.code})(JSON.parse(payloadJson)));`;+ const resultJson: unknown = runInNewContext(+ script,+ { payloadJson: JSON.stringify(input.payload) },+ {+ timeout: input.timeoutMs,+ contextCodeGeneration: { strings: false, wasm: false },+ microtaskMode: 'afterEvaluate',+ }+ );++ if (resultJson === undefined) {+ finish(SANDBOX_EXIT_CODE.ok, '');+ return;+ }+ if (typeof resultJson !== 'string') {+ finish(+ SANDBOX_EXIT_CODE.templateError,+ 'Template did not return a JSON-serializable value'+ );+ return;+ }+ if (Buffer.byteLength(resultJson, 'utf8') > input.maxOutputBytes) {+ finish(SANDBOX_EXIT_CODE.templateError, 'Template output is too large');+ return;+ }+ finish(SANDBOX_EXIT_CODE.ok, resultJson);+}++try {+ run(parseSandboxInput(await readStdin()));+} catch (error) {+ finish(+ SANDBOX_EXIT_CODE.templateError,+ error instanceof Error ? error.message : String(error)+ );+}diff --git a/packages/core/src/shared/js-runtime/sandbox-protocol.ts b/packages/core/src/shared/js-runtime/sandbox-protocol.tsnew file mode 100644index 000000000..bf8339b7b--- /dev/null+++ b/packages/core/src/shared/js-runtime/sandbox-protocol.ts@@ -0,0 +1,5 @@+/** Exit codes `sandbox-child.ts` answers with; stdout carries the result or the error message. */+export const SANDBOX_EXIT_CODE = {+ ok: 0,+ templateError: 3,+} as const;diff --git a/packages/core/src/shared/js-runtime/validate.test.ts b/packages/core/src/shared/js-runtime/validate.test.tsindex 866a4077f..9be47d7ad 100644--- a/packages/core/src/shared/js-runtime/validate.test.ts+++ b/packages/core/src/shared/js-runtime/validate.test.ts@@ -455,25 +455,25 @@ describe('execute', () => {};describe('Basic transformations', () => {- it('should execute simple arrow function', () => {+ it('should execute simple arrow function', async () => {const code = '(payload) => ({ event: payload.name })';- const result = execute(code, basePayload);+ const result = await execute(code, basePayload);expect(result).toEqual({ event: 'page_view' });});- it('should access nested properties', () => {+ it('should access nested properties', async () => {const code = '(payload) => ({ plan: payload.properties.plan })';- const result = execute(code, basePayload);+ const result = await execute(code, basePayload);expect(result).toEqual({ plan: 'premium' });});- it('should handle multiple properties', () => {+ it('should handle multiple properties', async () => {const code = `(payload) => ({event: payload.name,user: payload.profileId,location: payload.city})`;- const result = execute(code, basePayload);+ const result = await execute(code, basePayload);expect(result).toEqual({event: 'page_view',user: 'user-123',@@ -483,11 +483,11 @@ describe('execute', () => {});describe('Date operations', () => {- it('should format dates', () => {+ it('should format dates', async () => {const code = `(payload) => ({timestamp: new Date(payload.createdAt).toISOString()})`;- const result = execute(code, basePayload);+ const result = await execute(code, basePayload);expect(result).toHaveProperty('timestamp');expect((result as { timestamp: string }).timestamp).toBe('2024-01-15T10:30:00.000Z'@@ -496,61 +496,61 @@ describe('execute', () => {});describe('Array operations', () => {- it('should transform arrays', () => {+ it('should transform arrays', async () => {const code = `(payload) => ({tags: payload.tags?.map(t => t.toUpperCase()) || []})`;- const result = execute(code, basePayload);+ const result = await execute(code, basePayload);expect(result).toEqual({ tags: ['TAG1', 'TAG2'] });});- it('should filter arrays', () => {+ it('should filter arrays', async () => {const code = `(payload) => ({filtered: payload.tags?.filter(t => t.includes('tag1')) || []})`;- const result = execute(code, basePayload);+ const result = await execute(code, basePayload);expect(result).toEqual({ filtered: ['tag1'] });});});describe('String operations', () => {- it('should use template literals', () => {+ it('should use template literals', async () => {const code = `(payload) => ({location: \`\${payload.city}, \${payload.country}\`})`;- const result = execute(code, basePayload);+ const result = await execute(code, basePayload);expect(result).toEqual({ location: 'New York, US' });});- it('should use string methods', () => {+ it('should use string methods', async () => {const code = `(payload) => ({upperName: payload.name.toUpperCase()})`;- const result = execute(code, basePayload);+ const result = await execute(code, basePayload);expect(result).toEqual({ upperName: 'PAGE_VIEW' });});});describe('Math operations', () => {- it('should use Math functions', () => {+ it('should use Math functions', async () => {const code = `(payload) => ({roundedLng: Math.round(payload.longitude)})`;- const result = execute(code, basePayload);+ const result = await execute(code, basePayload);expect(result).toEqual({ roundedLng: -74 });});});describe('Conditional logic', () => {- it('should handle ternary operators', () => {+ it('should handle ternary operators', async () => {const code = `(payload) => ({plan: payload.properties?.plan || 'free'})`;- const result = execute(code, basePayload);+ const result = await execute(code, basePayload);expect(result).toEqual({ plan: 'premium' });});- it('should handle if conditions', () => {+ it('should handle if conditions', async () => {const code = `(payload) => {const result = { event: payload.name };if (payload.properties?.plan === 'premium') {@@ -558,13 +558,13 @@ describe('execute', () => {}return result;}`;- const result = execute(code, basePayload);+ const result = await execute(code, basePayload);expect(result).toEqual({ event: 'page_view', plan: 'premium' });});});describe('Complex transformations', () => {- it('should handle nested object construction', () => {+ it('should handle nested object construction', async () => {const code = `(payload) => ({event: payload.name,user: {@@ -577,7 +577,7 @@ describe('execute', () => {device: payload.device}})`;- const result = execute(code, basePayload);+ const result = await execute(code, basePayload);expect(result).toEqual({event: 'page_view',user: {@@ -594,20 +594,20 @@ describe('execute', () => {});describe('Error handling', () => {- it('should handle runtime errors gracefully', () => {+ it('should handle runtime errors gracefully', async () => {const code = '(payload) => payload.nonexistent.property';- expect(() => {- execute(code, basePayload);- }).toThrow('Error executing JavaScript template');+ await expect(execute(code, basePayload)).rejects.toThrow(+ 'Error executing JavaScript template'+ );});- it('should refuse to run a stored template that fails validation', () => {+ it('should refuse to run a stored template that fails validation', async () => {// A template that was persisted at some point but does not pass// validation must not reach new Function().const code = `(payload) => payload['constructor']['constructor']('return 1')()`;- expect(() => {- execute(code, basePayload);- }).toThrow('Invalid JavaScript template');+ await expect(execute(code, basePayload)).rejects.toThrow(+ 'Invalid JavaScript template'+ );});});@@ -621,16 +621,16 @@ describe('execute', () => {`(payload) => { return { event_name: payload.name, first_name: payload.properties?.first_name || '', last_name: payload.properties?.last_name || '', email: payload.properties?.email || '', phone: payload.properties?.phone || '', sms_consent: payload.properties?.sms_consent || '', form_name: payload.properties?.form_name || '', form_id: payload.properties?.form_id || '', page_url: payload.properties?.page_url || '' }; }`,];- it.each(productionTemplates)('validates and runs: %s', (code) => {+ it.each(productionTemplates)('validates and runs: %s', async (code) => {expect(validate(code)).toEqual({ valid: true });- expect(() => execute(code, basePayload)).not.toThrow();+ await execute(code, basePayload);});- it('produces the expected shape for the identify template', () => {- const result = execute(productionTemplates[0]!, basePayload) as Record<- string,- unknown- >;+ it('produces the expected shape for the identify template', async () => {+ const result = (await execute(+ productionTemplates[0]!,+ basePayload+ )) as Record<string, unknown>;expect(result.name).toBe('page_view');expect(result.timestamp).toBe('2024-01-15T10:30:00.000Z');expect(result.properties).toMatchObject({@@ -640,34 +640,34 @@ describe('execute', () => {});});- it('returns null when the template returns null', () => {- expect(execute(productionTemplates[1]!, basePayload)).toBeNull();+ it('returns null when the template returns null', async () => {+ expect(await execute(productionTemplates[1]!, basePayload)).toBeNull();});});describe('Isolation', () => {- it('does not hand the template a host object', () => {+ it('does not hand the template a host object', async () => {// The payload crosses into the context as JSON, so mutations never// reach the caller's object.const payload = { name: 'x', nested: { a: 1 } };- execute(+ await execute('(payload) => { payload.nested.a = 2; return payload; }',payload);expect(payload.nested.a).toBe(1);});- it('measures the output cap in UTF-8 bytes', () => {+ it('measures the output cap in UTF-8 bytes', async () => {// 600k three-byte characters is 600k UTF-16 units but 1.8MB on the wire.- expect(() => execute("(payload) => 'ࠀ'.repeat(600000)", {})).toThrow(- 'too large'- );+ await expect(+ execute("(payload) => 'ࠀ'.repeat(600000)", {})+ ).rejects.toThrow('too large');});- it('stops a template that runs too long', () => {- expect(() =>+ it('stops a template that runs too long', async () => {+ await expect(execute("(payload) => 'a'.repeat(100000000).replace(/(a+)+b/, '')", {})- ).toThrow('Error executing JavaScript template');+ ).rejects.toThrow('Error executing JavaScript template');});});});diff --git a/packages/core/src/shared/object.test.ts b/packages/core/src/shared/object.test.tsindex 2b0a52634..0afbc9f65 100644--- a/packages/core/src/shared/object.test.ts+++ b/packages/core/src/shared/object.test.ts@@ -1,7 +1,12 @@// Tests @openpanel/shared's object helpers from here because the root `test` script// does not run packages/shared.import { describe, expect, it } from 'bun:test';-import { toDots } from '@openpanel/shared';+import {+ TO_DOTS_MAX_KEYS,+ TO_DOTS_MAX_KEYS_PER_OBJECT,+ toDots,+ toObject,+} from '@openpanel/shared';describe('toDots', () => {it('should convert an object to a dot object', () => {@@ -47,4 +52,90 @@ describe('toDots', () => {regularString: 'normal string',});});++ it('decodes a JSON-serialised Node Buffer instead of flattening its bytes', () => {+ const body = JSON.stringify({ signedPayload: 'abc' });+ const data = Array.from(new TextEncoder().encode(body));+ expect(toDots({ requestBody: { type: 'Buffer', data } })).toEqual({+ 'requestBody.signedPayload': 'abc',+ });+ });++ it('keeps every root property up to the total cap', () => {+ const wide: Record<string, number> = {};+ for (let i = 0; i < TO_DOTS_MAX_KEYS_PER_OBJECT * 2; i++) {+ wide[`k${i}`] = i;+ }+ expect(Object.keys(toDots(wide))).toHaveLength(+ TO_DOTS_MAX_KEYS_PER_OBJECT * 2+ );+ });++ it('caps a wide root object at the total cap', () => {+ const wide: Record<string, number> = {};+ for (let i = 0; i < TO_DOTS_MAX_KEYS * 2; i++) {+ wide[`k${i}`] = i;+ }+ const result = toDots(wide);+ expect(Object.keys(result)).toHaveLength(TO_DOTS_MAX_KEYS);+ expect(result.k0).toBe('0');+ expect(result[`k${TO_DOTS_MAX_KEYS}`]).toBeUndefined();+ });++ it('applies the per-object limit to nested objects and arrays', () => {+ const longArray = Array.from(+ { length: TO_DOTS_MAX_KEYS_PER_OBJECT + 10 },+ (_, i) => i+ );+ const child: Record<string, number> = {};+ for (let i = 0; i < TO_DOTS_MAX_KEYS_PER_OBJECT + 10; i++) {+ child[`c${i}`] = i;+ }+ const result = toDots({ longArray, child, short: [1, 2] });+ const keys = Object.keys(result);+ expect(keys.filter((k) => k.startsWith('longArray.'))).toHaveLength(+ TO_DOTS_MAX_KEYS_PER_OBJECT+ );+ expect(keys.filter((k) => k.startsWith('child.'))).toHaveLength(+ TO_DOTS_MAX_KEYS_PER_OBJECT+ );+ expect(result[`longArray.${TO_DOTS_MAX_KEYS_PER_OBJECT}`]).toBeUndefined();+ expect(result['short.1']).toBe('2');+ });++ it('caps the total number of keys for deeply nested input', () => {+ const level = (child: unknown) =>+ Object.fromEntries(+ Array.from({ length: TO_DOTS_MAX_KEYS_PER_OBJECT }, (_, i) => [+ `k${i}`,+ child,+ ])+ );+ expect(Object.keys(toDots(level(level(level(1)))))).toHaveLength(+ TO_DOTS_MAX_KEYS+ );+ });++ it('strips lone surrogates and keeps valid pairs', () => {+ expect(toDots({ s: 'a\uD800b👋' })).toEqual({ s: 'ab👋' });+ });+});++describe('toObject', () => {+ it('rebuilds nested objects from dotted keys', () => {+ expect(toObject({ a: '1', 'b.c': '2', 'b.d.e': '3' })).toEqual({+ a: '1',+ b: { c: '2', d: { e: '3' } },+ });+ });++ it('lets a later key replace a scalar on its path', () => {+ expect(toObject({ a: '1', 'a.b': '2' })).toEqual({ a: { b: '2' } });+ });++ it('stores a __proto__ segment as data, not as the prototype', () => {+ const result = toObject({ '__proto__.polluted': 'yes' });+ expect(({} as Record<string, unknown>).polluted).toBeUndefined();+ expect(Object.getPrototypeOf(result)).toBe(Object.prototype);+ });});diff --git a/packages/core/src/shared/origin-allowed.test.ts b/packages/core/src/shared/origin-allowed.test.tsnew file mode 100644index 000000000..f2ca0e83b--- /dev/null+++ b/packages/core/src/shared/origin-allowed.test.ts@@ -0,0 +1,54 @@+import { describe, expect, it } from 'bun:test';+import { isOriginAllowed } from './origin-allowed';++describe('isOriginAllowed', () => {+ it('matches an exact origin, ignoring protocol, www and trailing slash', () => {+ expect(isOriginAllowed(['https://www.foo.com/'], 'https://foo.com')).toBe(+ true+ );+ expect(isOriginAllowed(['foo.com'], 'https://bar.com')).toBe(false);+ });++ it('matches subdomains of a wildcard entry on any port', () => {+ expect(isOriginAllowed(['*.foo.com'], 'https://app.foo.com')).toBe(true);+ expect(isOriginAllowed(['*.foo.com'], 'https://a.b.foo.com')).toBe(true);+ expect(isOriginAllowed(['*.foo.com'], 'http://app.foo.com:3000')).toBe(+ true+ );+ });++ it('requires the whole host to match a wildcard entry', () => {+ for (const origin of [+ 'https://app.foo.com.evil.com',+ 'https://evil.com/?app.foo.com',+ 'https://evil-foo.com',+ 'https://foo.com',+ ]) {+ expect([origin, isOriginAllowed(['*.foo.com'], origin)]).toEqual([+ origin,+ false,+ ]);+ }+ });++ it('honours a port in a wildcard entry', () => {+ expect(isOriginAllowed(['localhost:*'], 'http://localhost:5173')).toBe(+ true+ );+ expect(isOriginAllowed(['*.foo.com:8080'], 'https://a.foo.com:3000')).toBe(+ false+ );+ });++ it('treats regex syntax in an entry as literal text', () => {+ expect(isOriginAllowed(['*.fo+.com'], 'https://a.fooo.com')).toBe(false);+ expect(isOriginAllowed(['(a+)+*'], `https://${'a'.repeat(5000)}!`)).toBe(+ false+ );+ });++ it('allows any origin for *, but not a missing one', () => {+ expect(isOriginAllowed(['*'], 'https://anything.dev')).toBe(true);+ expect(isOriginAllowed(['*'], undefined)).toBe(false);+ });+});diff --git a/packages/core/src/shared/origin-allowed.ts b/packages/core/src/shared/origin-allowed.tsindex c0b1b2941..263628658 100644--- a/packages/core/src/shared/origin-allowed.ts+++ b/packages/core/src/shared/origin-allowed.ts@@ -3,8 +3,8 @@const DOMAIN_PROTOCOL = /https?:\/\//;const DOMAIN_TRAILING_SLASH = /\/$/;-const DOMAIN_DOT = /\./g;-const DOMAIN_WILDCARD = /\*/g;+const DOMAIN_PORT = /:\d+$/;+const WILDCARD = '*';export const cleanDomain = (domain: string) =>domain@@ -12,20 +12,56 @@ export const cleanDomain = (domain: string) =>.replace(DOMAIN_PROTOCOL, '').replace(DOMAIN_TRAILING_SLASH, '');+/**+ * `*` matches any run of characters and the whole host must match. Plain string+ * scans rather than a regex built from the stored pattern, so a pattern cannot+ * carry regex syntax or backtrack.+ */+function matchesWildcard(pattern: string, host: string): boolean {+ const parts = pattern.split(WILDCARD);+ const head = parts[0] ?? '';+ const tail = parts.at(-1) ?? '';+ if (!host.startsWith(head)) {+ return false;+ }+ let position = head.length;+ for (const part of parts.slice(1, -1)) {+ const index = host.indexOf(part, position);+ if (index === -1) {+ return false;+ }+ position = index + part.length;+ }+ return host.length - tail.length >= position && host.endsWith(tail);+}++/** `host[:port]` of a parseable origin, so a path or query cannot reach the wildcard match. */+function originHost(origin: string | undefined): string | undefined {+ const host = origin ? URL.parse(origin)?.host : undefined;+ return host ? cleanDomain(host) : undefined;+}+/** Whether `origin` matches the project's allowed origins (`*.foo.com` wildcards and `*` included). */export function isOriginAllowed(cors: string[],origin: string | undefined): boolean {+ const cleanedOrigin = cleanDomain(origin || '');+ const host = originHost(origin);const domainAllowed = cors.find((domain) => {const cleanedDomain = cleanDomain(domain);- if (cleanedDomain.includes('*')) {- const regex = new RegExp(- `${cleanedDomain.replace(DOMAIN_DOT, '\\.').replace(DOMAIN_WILDCARD, '.+?')}`- );- return regex.test(origin || '');+ if (!cleanedDomain.includes(WILDCARD)) {+ return cleanedDomain === cleanedOrigin;+ }+ if (!host) {+ return false;}- return cleanedDomain === cleanDomain(origin || '');+ // A wildcard entry without a port allows any port, as `*.foo.com` always+ // has for a dev server on `app.foo.com:3000`.+ const comparedHost = cleanedDomain.includes(':')+ ? host+ : host.replace(DOMAIN_PORT, '');+ return matchesWildcard(cleanedDomain, comparedHost);});if (domainAllowed) {diff --git a/packages/core/src/shared/sanitize-url.ts b/packages/core/src/shared/sanitize-url.tsindex 01bcd98e5..ce9c13bad 100644--- a/packages/core/src/shared/sanitize-url.ts+++ b/packages/core/src/shared/sanitize-url.ts@@ -14,6 +14,7 @@ export const SENSITIVE_KEY_PATTERNS = ['apikey','accesskey','privatekey',+ 'serviceaccount','cookie','bearer','credential',diff --git a/packages/core/src/shared/time-buckets.test.ts b/packages/core/src/shared/time-buckets.test.tsnew file mode 100644index 000000000..e90a531d2--- /dev/null+++ b/packages/core/src/shared/time-buckets.test.ts@@ -0,0 +1,86 @@+import { describe, expect, it } from 'bun:test';+import {+ assertTimeBucketBudget,+ countTimeBuckets,+ TimeRangeError,+} from './time-buckets';++describe('countTimeBuckets', () => {+ it('counts a dashboard range the way WITH FILL fills it', () => {+ expect(+ countTimeBuckets({+ startDate: '2026-01-01 00:00:00',+ endDate: '2026-01-30 23:59:59',+ interval: 'day',+ })+ ).toBe(30);+ expect(+ countTimeBuckets({+ startDate: '2026-01-01',+ endDate: '2026-01-30',+ interval: 'hour',+ })+ ).toBe(720);+ });++ it('counts a reversed range by its span', () => {+ expect(+ countTimeBuckets({+ startDate: '2026-01-30',+ endDate: '2026-01-01',+ interval: 'day',+ })+ ).toBe(29);+ });++ it('rejects dates ClickHouse DateTime cannot hold', () => {+ expect(() =>+ countTimeBuckets({+ startDate: '1900-01-01',+ endDate: '2026-01-01',+ interval: 'day',+ })+ ).toThrow(TimeRangeError);+ expect(() =>+ countTimeBuckets({+ startDate: '2026-01-01',+ endDate: '2200-01-01',+ interval: 'day',+ })+ ).toThrow(TimeRangeError);+ expect(() =>+ countTimeBuckets({+ startDate: 'not a date',+ endDate: '2026-01-01',+ interval: 'day',+ })+ ).toThrow(TimeRangeError);+ });+});++describe('assertTimeBucketBudget', () => {+ it('serves every range the picker offers at its finest interval', () => {+ expect(() =>+ assertTimeBucketBudget({+ startDate: '2025-01-01 00:00:00',+ endDate: '2026-01-31 23:59:59',+ interval: 'hour',+ })+ ).not.toThrow();+ });++ it('refuses a minute series over years as a 400', () => {+ let error: unknown;+ try {+ assertTimeBucketBudget({+ startDate: '2020-01-01',+ endDate: '2026-01-01',+ interval: 'minute',+ });+ } catch (caught) {+ error = caught;+ }+ expect(error).toBeInstanceOf(TimeRangeError);+ expect((error as TimeRangeError).status).toBe(400);+ });+});diff --git a/packages/core/src/shared/time-buckets.ts b/packages/core/src/shared/time-buckets.tsnew file mode 100644index 000000000..72a7c19b7--- /dev/null+++ b/packages/core/src/shared/time-buckets.ts@@ -0,0 +1,80 @@+// `WITH FILL` turns a compact (startDate, endDate, interval) into one row per+// bucket, and the API buffers every row, so a time series' bucket count is+// checked before its statement is built.++import { convertClickhouseDateToJs, toRangeBoundaryLiteral } from './ch-dates';+import { BadRequestError } from './errors';+import { MS_PER_DAY, MS_PER_HOUR, MS_PER_MINUTE } from './time.constants';++export type TimeBucketInterval = 'minute' | 'hour' | 'day' | 'week' | 'month';++/** Hourly over two years or by the minute over two weeks: past any readable chart. */+export const MAX_TIME_BUCKETS = 20_000;++const SHORTEST_MONTH_DAYS = 28;++// A month counts as its shortest length so the estimate never undercounts.+const INTERVAL_MS: Record<TimeBucketInterval, number> = {+ minute: MS_PER_MINUTE,+ hour: MS_PER_HOUR,+ day: MS_PER_DAY,+ week: 7 * MS_PER_DAY,+ month: SHORTEST_MONTH_DAYS * MS_PER_DAY,+};++// ClickHouse `DateTime` ends in 2106 and starts at the epoch; past either end+// a boundary wraps, so the statement would fill a range other than the one+// counted here.+const EARLIEST_DATE_MS = 0;+const LATEST_DATE_MS = Date.UTC(2106, 0, 1);++export class TimeRangeError extends BadRequestError {+ constructor(message: string) {+ super(message);+ this.name = 'TimeRangeError';+ }+}++function boundaryTime(value: string, boundary: 'start' | 'end'): number {+ if (Number.isNaN(new Date(value).getTime())) {+ throw new TimeRangeError(`Invalid ${boundary} date: ${value}`);+ }+ const time = convertClickhouseDateToJs(+ toRangeBoundaryLiteral(value, boundary)+ ).getTime();+ const isSupported =+ !Number.isNaN(time) && time >= EARLIEST_DATE_MS && time < LATEST_DATE_MS;+ if (!isSupported) {+ throw new TimeRangeError(+ `The ${boundary} date ${value} is outside the supported range (1970 to 2105).`+ );+ }+ return time;+}++export interface TimeBucketRange {+ startDate: string;+ endDate: string;+ interval: TimeBucketInterval;+}++/** Buckets spanned by the range, either way round; rejects unusable dates. */+export function countTimeBuckets({+ startDate,+ endDate,+ interval,+}: TimeBucketRange): number {+ const span = Math.abs(+ boundaryTime(endDate, 'end') - boundaryTime(startDate, 'start')+ );+ return Math.floor(span / INTERVAL_MS[interval]) + 1;+}++export function assertTimeBucketBudget(range: TimeBucketRange): void {+ const bucketCount = countTimeBuckets(range);+ if (bucketCount > MAX_TIME_BUCKETS) {+ throw new TimeRangeError(+ `The selected range splits into ${bucketCount} ${range.interval} intervals, more than the ${MAX_TIME_BUCKETS} supported. Pick a coarser interval or a shorter range.`+ );+ }+}diff --git a/packages/core/src/shared/time.constants.ts b/packages/core/src/shared/time.constants.tsnew file mode 100644index 000000000..1692b16d9--- /dev/null+++ b/packages/core/src/shared/time.constants.ts@@ -0,0 +1,3 @@+export const MS_PER_MINUTE = 60_000;+export const MS_PER_HOUR = 60 * MS_PER_MINUTE;+export const MS_PER_DAY = 24 * MS_PER_HOUR;diff --git a/packages/core/src/slug-id.test.ts b/packages/core/src/slug-id.test.tsnew file mode 100644index 000000000..e8916046b--- /dev/null+++ b/packages/core/src/slug-id.test.ts@@ -0,0 +1,73 @@+import { describe, expect, it, mock } from 'bun:test';+import { Glob } from 'bun';+import { readFileSync } from 'node:fs';+import type { ServiceDeps } from './services';+import { getId, RESERVED_PROJECT_IDS } from './slug-id';++const MAX_ATTEMPTS = 10;+const FIXED_SESSION_KEY_SEGMENT = /session:([a-z_]+)[:'"`]/g;++function fixedSessionKeySegmentsInSource(): Set<string> {+ const segments = new Set<string>();+ const sourceDir = import.meta.dir;+ for (const file of new Glob('**/*.ts').scanSync(sourceDir)) {+ if (file.endsWith('.test.ts')) {+ continue;+ }+ const source = readFileSync(`${sourceDir}/${file}`, 'utf8');+ for (const [, segment] of source.matchAll(FIXED_SESSION_KEY_SEGMENT)) {+ if (segment) {+ segments.add(segment);+ }+ }+ }+ return segments;+}++function dbWithTakenIds(isTaken: (id: string) => boolean) {+ const findUnique = mock(({ where }: { where: { id: string } }) =>+ Promise.resolve(isTaken(where.id) ? { id: where.id } : null)+ );+ const db = {+ dashboard: { findUnique },+ project: { findUnique },+ } as unknown as ServiceDeps['db'];+ return { db, findUnique };+}++describe('getId', () => {+ it('returns the slug when it is free', async () => {+ const { db } = dbWithTakenIds(() => false);+ expect(await getId({ db }, 'dashboard', 'Overview')).toBe('overview');+ });++ it('suffixes a taken slug', async () => {+ const { db } = dbWithTakenIds((id) => id === 'overview');+ expect(await getId({ db }, 'dashboard', 'Overview')).toMatch(+ /^overview-\d{4}$/+ );+ });++ it('gives up after a bounded number of lookups', async () => {+ const { db, findUnique } = dbWithTakenIds(() => true);+ await expect(getId({ db }, 'dashboard', 'Overview')).rejects.toThrow(+ 'Could not generate a unique dashboard id'+ );+ expect(findUnique).toHaveBeenCalledTimes(MAX_ATTEMPTS);+ });++ it('never gives a project an id reserved by session buffer keys', async () => {+ const { db } = dbWithTakenIds(() => false);+ expect(await getId({ db }, 'project', 'Wallclock')).toMatch(+ /^wallclock-\d{4}$/+ );+ });++ it('reserves every fixed segment of a session Redis key', () => {+ const segments = fixedSessionKeySegmentsInSource();+ expect(segments.size).toBeGreaterThan(0);+ expect(+ [...segments].filter((segment) => !RESERVED_PROJECT_IDS.has(segment))+ ).toEqual([]);+ });+});diff --git a/packages/core/src/slug-id.ts b/packages/core/src/slug-id.tsindex 84bf3d828..834833752 100644--- a/packages/core/src/slug-id.ts+++ b/packages/core/src/slug-id.ts@@ -8,12 +8,36 @@import { slug } from '@openpanel/shared';import type { ServiceDeps } from './services';+// Dashboard ids are global and names like "Overview" repeat across every+// organization, so the 4-digit suffix space of a popular name can fill up.+const SHORT_SUFFIX_DIGITS = 4;+const LONG_SUFFIX_DIGITS = 8;+const SHORT_SUFFIX_ATTEMPTS = 5;+const MAX_ATTEMPTS = 10;+const NUMERIC_SUFFIX = /-\d{4}$/;+// Session buffer keys are `session:{projectId}:{deviceId}` next to fixed+// `session:<segment>:…` keys, so a project id equal to a segment could write+// over another project's session state. The session keys are spread over the+// buffer and the session module, so slug-id.test.ts scans the source for them.+export const RESERVED_PROJECT_IDS = new Set([+ 'end',+ 'hwm',+ 'profile',+ 'projects',+ 'reaper',+ 'wallclock',+]);++function randomSuffix(digits: number): string {+ const lowest = 10 ** (digits - 1);+ return String(lowest + Math.floor(Math.random() * 9 * lowest));+}+export async function getId(deps: Pick<ServiceDeps, 'db'>,tableName: 'project' | 'dashboard' | 'organization',name: string): Promise<string> {- const newId = slug(name);const db = deps.db;if (!db[tableName]) {throw new Error('Table does not exists');@@ -23,24 +47,24 @@ export async function getId(throw new Error('findUnique does not exists');}- // @ts-expect-error- const existingProject = await db[tableName].findUnique({- where: {- id: newId,- },- });-- function random(str: string) {- const numbers = Math.floor(1000 + Math.random() * 9000);- if (str.match(/-\d{4}$/g)) {- return str.replace(/-\d{4}$/g, `-${numbers}`);+ const base = slug(name);+ const unsuffixedBase = base.replace(NUMERIC_SUFFIX, '');+ const isReserved = tableName === 'project' && RESERVED_PROJECT_IDS.has(base);+ for (let attempt = isReserved ? 1 : 0; attempt < MAX_ATTEMPTS; attempt++) {+ const digits =+ attempt <= SHORT_SUFFIX_ATTEMPTS+ ? SHORT_SUFFIX_DIGITS+ : LONG_SUFFIX_DIGITS;+ const candidate =+ attempt === 0 ? base : `${unsuffixedBase}-${randomSuffix(digits)}`;+ // @ts-expect-error+ const existing = await db[tableName].findUnique({+ where: { id: candidate },+ });+ if (!existing) {+ return candidate;}- return `${str}-${numbers}`;- }-- if (existingProject) {- return getId(deps, tableName, random(name));}- return newId;+ throw new Error(`Could not generate a unique ${tableName} id`);}diff --git a/packages/db/prisma/migrations/20261028120000_invite_project_scope_backfill/migration.sql b/packages/db/prisma/migrations/20261028120000_invite_project_scope_backfill/migration.sqlnew file mode 100644index 000000000..57cb57995--- /dev/null+++ b/packages/db/prisma/migrations/20261028120000_invite_project_scope_backfill/migration.sql@@ -0,0 +1,9 @@+-- Invites created before projectScope existed got the column default `all`,+-- even when their projectAccess restricted them to selected projects. The api+-- accepts invites as soon as it serves, and code migrations only run after+-- that, so the conversion has to land here or an invite accepted in between+-- grants every project of the organization.+UPDATE "public"."invites"+SET "projectScope" = 'selected'+WHERE "projectScope" = 'all'+ AND jsonb_array_length("projectAccess") > 0;diff --git a/packages/db/scripts/ch-update-sessions-with-revenue.ts b/packages/db/scripts/ch-update-sessions-with-revenue.tsindex ad67bb275..55ae636ef 100644--- a/packages/db/scripts/ch-update-sessions-with-revenue.ts+++ b/packages/db/scripts/ch-update-sessions-with-revenue.ts@@ -35,12 +35,16 @@ async function main() {const nextHour = new Date(currentDate.getTime() + MS_PER_HOUR);console.log(`Processing hour: ${currentDate.toISOString()}`);+ // Imported session ids can repeat across projects, so the mutation repeats+ // the selection's project and hour rather than matching on id alone.+ const hourOfProject = sql`created_at >= ${sql.string(datetime(currentDate))}+ AND created_at < ${sql.string(datetime(nextHour))}+ AND project_id = ${sql.string(PROJECT_ID)}`;+const sessions = await chQuery<{ id: string }>(sql`SELECT idFROM ${sql.id(TABLE_NAMES.sessions)}- WHERE created_at >= ${sql.string(datetime(currentDate))}- AND created_at < ${sql.string(datetime(nextHour))}- AND project_id = ${sql.string(PROJECT_ID)}+ WHERE ${hourOfProject}LIMIT ${sql.uint64(SESSIONS_PER_HOUR)}`);if (sessions.length === 0) {@@ -70,7 +74,7 @@ async function main() {)}, revenue)`;const statement =- sql`ALTER TABLE ${sql.id(TABLE_NAMES.sessions)} UPDATE revenue = ${updateExpr} WHERE id IN ${sql.array('String', sessionIds)}`.toStatement();+ sql`ALTER TABLE ${sql.id(TABLE_NAMES.sessions)} UPDATE revenue = ${updateExpr} WHERE ${hourOfProject} AND id IN ${sql.array('String', sessionIds)}`.toStatement();console.log(`Executing update: ${statement.query}`);diff --git a/packages/db/scripts/find-duplicate-events.ts b/packages/db/scripts/find-duplicate-events.tsindex 7c24ff1e8..6bc8852a8 100644--- a/packages/db/scripts/find-duplicate-events.ts+++ b/packages/db/scripts/find-duplicate-events.ts@@ -21,7 +21,7 @@* --bucket=<day|hour> Reporting granularity (default: day)* --project=<projectId> Restrict to a single project (default: all)* --limit=<n> Max example duplicate groups to print (default: 20)- * --batch=<n> Ids per DELETE statement when deleting (default: 10000)+ * --batch=<n> Rows per DELETE statement when deleting (default: 2000)* --danger-yes-delete Actually delete the extra rows. Keeps the lowest `id`* per duplicate group, deletes the rest. Without this* flag the script only reports.@@ -79,9 +79,10 @@ function parseArgs(): Args {bucket,project: getArg('project') ?? null,limit: Number.parseInt(getArg('limit') ?? '20', 10),- // Ids are inlined into `id IN (...)`. ~39 bytes/uuid, so 5000 ≈ 195KB stays- // safely under ClickHouse's default max_query_size (256KB).- batch: Number.parseInt(getArg('batch') ?? '5000', 10),+ // Each row inlines (project_id, id, content hash), about 110 bytes with+ // uuid ids, so 2000 rows ≈ 220KB stays under ClickHouse's default+ // max_query_size (256KB).+ batch: Number.parseInt(getArg('batch') ?? '2000', 10),delete: hasFlag('danger-yes-delete'),};}@@ -113,8 +114,12 @@ const DEDUP_KEY = `cityHash64())`;+function sqlString(value: string): string {+ return `'${value.replace(/'/g, "''")}'`;+}+function projectClause(project: string | null): string {- return project ? ` AND project_id = '${project.replace(/'/g, "''")}'` : '';+ return project ? ` AND project_id = ${sqlString(project)}` : '';}async function query<T>(sql: string): Promise<T[]> {@@ -256,30 +261,43 @@ async function deleteDuplicates(args: Args): Promise<void> {`created_at >= '${startLiteral}' AND created_at < '${endLiteral}'` +projectClause(args.project);- // Extra rows = every row in a duplicate group except the lowest id.- const extras = await query<{ id: string }>(`- SELECT id FROM (- SELECT id, row_number() OVER (- PARTITION BY ${DEDUP_KEY} ORDER BY id ASC- ) AS rn+ // Extra rows = every row in a duplicate group except the lowest id. The+ // delete matches on (project_id, id, content), so a copy that shares the+ // survivor's id cannot be removed without removing the survivor too; it is+ // skipped. A distinct event that reuses the id has other content and stays.+ const extras = await query<{+ project_id: string;+ id: string;+ dedup_hash: string;+ }>(`+ SELECT project_id, id, dedup_hash FROM (+ SELECT project_id, id,+ ${DEDUP_KEY} AS dedup_hash,+ row_number() OVER w AS rn,+ min(id) OVER w AS survivor_idFROM ${TABLE_NAMES.events}WHERE ${where}- ) WHERE rn > 1+ WINDOW w AS (PARTITION BY ${DEDUP_KEY} ORDER BY id ASC)+ ) WHERE rn > 1 AND id != survivor_id`);if (extras.length === 0) {continue;}- const ids = extras.map((e) => e.id);console.log(- `${startLiteral} → ${ids.length.toLocaleString()} duplicate rows to delete`+ `${startLiteral} → ${extras.length.toLocaleString()} duplicate rows to delete`);- grandTotal += ids.length;+ grandTotal += extras.length;- for (let i = 0; i < ids.length; i += args.batch) {- const slice = ids.slice(i, i + args.batch);- const idList = slice.map((id) => `'${id}'`).join(',');+ for (let i = 0; i < extras.length; i += args.batch) {+ const slice = extras.slice(i, i + args.batch);+ const rowList = slice+ .map(+ (row) =>+ `(${sqlString(row.project_id)},${sqlString(row.id)},toUInt64(${sqlString(row.dedup_hash)}))`+ )+ .join(',');// Scope the mutation to this bucket's time range so it only touches the// relevant monthly partition (and the primary index, when --project is// set) instead of scanning all history — `id` alone is not in the sort@@ -288,7 +306,7 @@ async function deleteDuplicates(args: Args): Promise<void> {query:`ALTER TABLE ${target} DELETE WHERE ` +`created_at >= '${startLiteral}' AND created_at < '${endLiteral}'` +- `${projectClause(args.project)} AND id IN (${idList})`,+ `${projectClause(args.project)} AND (project_id, id, ${DEDUP_KEY}) IN (${rowList})`,});await new Promise((resolve) => setTimeout(resolve, 200));}diff --git a/packages/db/src/clickhouse/client.ts b/packages/db/src/clickhouse/client.tsindex 3856599c6..6202c92e5 100644--- a/packages/db/src/clickhouse/client.ts+++ b/packages/db/src/clickhouse/client.ts@@ -7,7 +7,11 @@ import { ClickHouseLogLevel, createClient } from '@clickhouse/client';import type { NodeClickHouseClientConfigOptions } from '@clickhouse/client/dist/config';import { getSafeJson } from '@openpanel/shared';import { createLogger } from '../logger';-import { RoundRobinPicker, withRoundRobinRetry } from './round-robin';+import {+ nodeLabel,+ RoundRobinPicker,+ withRoundRobinRetry,+} from './round-robin';import { type SqlFragment, toStatement } from './sql';export { createClient } from '@clickhouse/client';@@ -194,30 +198,10 @@ const clients: ClickHouseClient[] = clickhouseUrls.map((url) =>const picker = new RoundRobinPicker(clients, clickhouseUrls, unhealthyMarkMs);-function maskUrlCredentials(url: string): string {- if (!url) {- return '<unset>';- }- try {- const u = new URL(url);- if (u.username) {- u.username = u.username.slice(0, 2) + '*'.repeat(u.username.length - 2);- }- if (u.password) {- u.password = '***';- }- return u.toString();- } catch {- // Malformed URL — fall back to the original; better than crashing module- // load in test envs where CLICKHOUSE_URL may be empty/garbage.- return url;- }-}-logger.info({nodeCount: clients.length,- urls: clickhouseUrls.map(maskUrlCredentials),+ nodes: clickhouseUrls.map(nodeLabel),requestTimeoutMs,unhealthyMarkMs,options: { ...CLICKHOUSE_OPTIONS, log: undefined },diff --git a/packages/db/src/clickhouse/round-robin.test.ts b/packages/db/src/clickhouse/round-robin.test.tsindex 6a7d6f9d2..33bed4efb 100644--- a/packages/db/src/clickhouse/round-robin.test.ts+++ b/packages/db/src/clickhouse/round-robin.test.ts@@ -3,8 +3,23 @@ import {classifyError,type ErrorClass,isRetriableConnectionError,+ nodeLabel,} from './round-robin';+describe('nodeLabel', () => {+ it('keeps only host and port, dropping credentials', () => {+ expect(nodeLabel('http://a:secret@ch-1:8123/openpanel')).toBe('ch-1:8123');+ });++ it('never echoes a url it cannot parse', () => {+ expect(nodeLabel('not a url :secret@')).toBe('<unparseable>');+ });++ it('labels an unset url', () => {+ expect(nodeLabel('')).toBe('<unset>');+ });+});+/*** The contract: anything from the ClickHouse server (errors that traveled* back over a working connection) must NOT mark a node unhealthy. Only truediff --git a/packages/db/src/clickhouse/round-robin.ts b/packages/db/src/clickhouse/round-robin.tsindex 3a133cae4..2310843b9 100644--- a/packages/db/src/clickhouse/round-robin.ts+++ b/packages/db/src/clickhouse/round-robin.ts@@ -8,6 +8,24 @@ import type { ILogger } from '../logger';* unbounded tail latency.*/+const UNSET_NODE_LABEL = '<unset>';+const UNPARSEABLE_NODE_LABEL = '<unparseable>';++/**+ * `host:port` of a node URL, for logs. Connection URLs carry credentials, so+ * nothing else of the URL is kept, and an unparseable one is not logged at all.+ */+export function nodeLabel(url: string): string {+ if (!url) {+ return UNSET_NODE_LABEL;+ }+ try {+ return new URL(url).host;+ } catch {+ return UNPARSEABLE_NODE_LABEL;+ }+}+export interface ClientSlot {client: ClickHouseClient;url: string;@@ -302,7 +320,7 @@ export async function withRoundRobinRetry<T>(if (attempt > 0) {picker.markHealthy(slot);logger.info(- { url: slot.url, attempt: attempt + 1 },+ { node: nodeLabel(slot.url), attempt: attempt + 1 },'CH operation succeeded after retry');}@@ -337,7 +355,7 @@ export async function withRoundRobinRetry<T>(code: (error as { code?: string })?.code,errClass,markedUnhealthy: errClass === 'node-down',- url: slot.url,+ node: nodeLabel(slot.url),attempt: attempt + 1,maxAttempts,nextDelayMs: delay,diff --git a/packages/email/src/index.tsx b/packages/email/src/index.tsxindex 9061d75c7..9eee2313d 100644--- a/packages/email/src/index.tsx+++ b/packages/email/src/index.tsx@@ -23,14 +23,20 @@ export type EmailData<T extends TemplateKey> = z.infer<Templates[T]['schema']>;export type EmailTemplate = keyof Templates;function createSmtpTransport() {+ const secure = process.env.SMTP_SECURE === 'true';+ const auth =+ process.env.SMTP_USER && process.env.SMTP_PASS+ ? { user: process.env.SMTP_USER, pass: process.env.SMTP_PASS }+ : undefined;return createTransport({host: process.env.SMTP_HOST,port: process.env.SMTP_PORT ? Number(process.env.SMTP_PORT) : 587,- secure: process.env.SMTP_SECURE === 'true',- auth:- process.env.SMTP_USER && process.env.SMTP_PASS- ? { user: process.env.SMTP_USER, pass: process.env.SMTP_PASS }- : undefined,+ secure,+ auth,+ // Without implicit TLS, STARTTLS is opportunistic: an attacker who strips+ // it gets the credentials in plaintext. An unauthenticated relay (a local+ // sidecar) is left alone.+ requireTLS: !secure && auth !== undefined,connectionTimeout: 10_000,greetingTimeout: 10_000,socketTimeout: 30_000,@@ -65,7 +71,7 @@ export async function sendEmail<T extends TemplateKey>(if (unsubscribed) {console.log(- `Skipping email to ${to} - unsubscribed from ${template.category}`+ `Skipping email to ${redactEmail(to)} - unsubscribed from ${template.category}`);return null;}diff --git a/packages/payments/scripts/create-custom-pricing.ts b/packages/payments/scripts/create-custom-pricing.tsindex 3d9355c1d..c6665957c 100644--- a/packages/payments/scripts/create-custom-pricing.ts+++ b/packages/payments/scripts/create-custom-pricing.ts@@ -46,7 +46,8 @@ async function promptForInput() {message: 'Enter your Polar organization ID:',},{- type: 'string',+ type: 'password',+ mask: '*',name: 'polarApiKey',message: 'Enter your Polar API key:',validate: (input: string) => {@@ -187,8 +188,9 @@ async function main() {});console.log('\nReview the following settings:');+ const { polarApiKey: _polarApiKey, ...reviewable } = input;console.table({- ...input,+ ...reviewable,organization: organization?.name,email: user?.email,name:diff --git a/packages/payments/src/polar.ts b/packages/payments/src/polar.tsindex 57eb11986..09434ebd2 100644--- a/packages/payments/src/polar.ts+++ b/packages/payments/src/polar.ts@@ -22,11 +22,12 @@ export async function getProducts() {isArchived: false,sorting: ['price_amount'],});- return products.result.items.filter((product) => {- return (- product.metadata.custom !== 'true' && product.metadata.custom !== true- );- });+ return products.result.items.filter((product) => !isCustomProduct(product));+}++/** Made for one organization by `create-custom-pricing`; `metadata.organizationId` says which. */+export function isCustomProduct(product: Pick<IPolarProduct, 'metadata'>) {+ return product.metadata.custom === 'true' || product.metadata.custom === true;}export async function getProduct(id: string) {diff --git a/packages/payments/src/subscription-state.test.ts b/packages/payments/src/subscription-state.test.tsindex 2bfa6389e..97975a956 100644--- a/packages/payments/src/subscription-state.test.ts+++ b/packages/payments/src/subscription-state.test.ts@@ -103,11 +103,11 @@ describe('getSubscriptionState', () => {expected: 'canceling',},{- name: 'canceledAt wins even if period passed',+ name: 'canceledAt with the period passed is canceled (missed webhook)',status: 'active',canceledAt: past,endsAt: past,- expected: 'canceling',+ expected: 'canceled',},// Payment-problem statesdiff --git a/packages/payments/src/subscription-state.ts b/packages/payments/src/subscription-state.tsindex a40d99c2e..fa15d5ab4 100644--- a/packages/payments/src/subscription-state.ts+++ b/packages/payments/src/subscription-state.ts@@ -53,8 +53,10 @@ export function getSubscriptionState(// `canceledAt` is set; the subscription stays usable until it expires.// Cancellation wins over a scheduled pause — Polar clears the pause flag// on cancel, but a stale combination must not hide that it's ending.+ // Past the period end the status is stale (missed webhook): the paid+ // period is over, so it is `canceled`, not still `canceling`.if (subscriptionCanceledAt) {- return 'canceling';+ return endsInFuture ? 'canceling' : 'canceled';}// Pause-at-period-end works the same way: status stays `active` with the// flag set until the period ends, then Polar flips it to `paused`. If thediff --git a/packages/redis/cachable.test.ts b/packages/redis/cachable.test.tsindex edaf15157..85b5d8a5e 100644--- a/packages/redis/cachable.test.ts+++ b/packages/redis/cachable.test.ts@@ -512,6 +512,24 @@ describe('cachable', () => {expect(cached).toBeNull();});+ it('should not keep an L1 copy when localCache is off', async () => {+ let value = 'granted';+ const fn = async (arg1: string) => value;+ const cachedFn = cacheable('revocableFn', fn, 3600, {+ localCache: false,+ });+ const otherProcessFn = cacheable('revocableFn', fn, 3600, {+ localCache: false,+ });++ expect(await otherProcessFn('arg1')).toBe('granted');++ value = 'revoked';+ await cachedFn.clear('arg1');++ expect(await otherProcessFn('arg1')).toBe('revoked');+ });+it('should provide set method', async () => {const fn = async (arg1: string, arg2: string) => ({});const cachedFn = cacheable(fn, 3600);diff --git a/packages/redis/cachable.ts b/packages/redis/cachable.tsindex ac6254ac6..413ea3262 100644--- a/packages/redis/cachable.ts+++ b/packages/redis/cachable.ts@@ -103,6 +103,12 @@ function stringify(obj: unknown): string {export interface CacheableOptions {cacheEmptyArray?: boolean;+ /**+ * Keep an in-process L1 copy. Turn it off for answers that must be revocable:+ * `clear()` reaches Redis and this process only, so every other process would+ * keep serving its L1 copy for up to a minute.+ */+ localCache?: boolean;}function shouldCache(result: unknown, options: CacheableOptions = {}): boolean {@@ -212,19 +218,21 @@ export function cacheable<T extends (...args: any) => any>(const getKey = (...args: Parameters<T>) =>`${cachePrefix}:${stringify(args)}`.replaceAll(/\s/g, '');- const lruCache = new LRUCache<string, any>({- max: CACHEABLE_LRU_MAX,- ttl: CACHEABLE_LRU_TTL_MS,- });+ const lruCache =+ options.localCache === false+ ? null+ : new LRUCache<string, any>({+ max: CACHEABLE_LRU_MAX,+ ttl: CACHEABLE_LRU_TTL_MS,+ });- // L1 LRU (60s) + L2 Redis. clear() deletes Redis + local LRU; other nodes may serve stale from LRU for up to 60s.const cachedFn = async (...args: Parameters<T>): Promise<Awaited<ReturnType<T>>> => {const key = getKey(...args);// L1: in-memory LRU first (offloads Redis on hot keys)- const lruHit = lruCache.get(key);+ const lruHit = lruCache?.get(key);if (lruHit !== undefined && shouldCache(lruHit, options)) {return lruHit as Awaited<ReturnType<T>>;}@@ -234,7 +242,7 @@ export function cacheable<T extends (...args: any) => any>(if (cached) {const parsed = parseCache(cached);if (shouldCache(parsed, options)) {- lruCache.set(key, parsed);+ lruCache?.set(key, parsed);return parsed;}}@@ -242,7 +250,7 @@ export function cacheable<T extends (...args: any) => any>(const result = await fn(...(args as any));if (shouldCache(result, options)) {- lruCache.set(key, result);+ lruCache?.set(key, result);getRedisCache().setex(key, expireInSec, JSON.stringify(result)).catch(() => {@@ -256,7 +264,7 @@ export function cacheable<T extends (...args: any) => any>(cachedFn.getKey = getKey;cachedFn.clear = (...args: Parameters<T>) => {const key = getKey(...args);- lruCache.delete(key);+ lruCache?.delete(key);return getRedisCache().del(key);};cachedFn.set =@@ -264,7 +272,7 @@ export function cacheable<T extends (...args: any) => any>((payload: Awaited<ReturnType<T>>) => {const key = getKey(...args);if (shouldCache(payload, options)) {- lruCache.set(key, payload);+ lruCache?.set(key, payload);return getRedisCache().setex(key, expireInSec, JSON.stringify(payload)).catch(() => {diff --git a/packages/sdks/nextjs/index.tsx b/packages/sdks/nextjs/index.tsxindex e2023455b..4ab989162 100644--- a/packages/sdks/nextjs/index.tsx+++ b/packages/sdks/nextjs/index.tsx@@ -27,19 +27,20 @@ type OpenPanelComponentProps = Omit<OpenPanelOptions, 'filter'> & {strategy?: 'beforeInteractive' | 'afterInteractive' | 'lazyOnload' | 'worker';};-const stringify = (obj: unknown) => {- if (typeof obj === 'object' && obj !== null && obj !== undefined) {- const entries = Object.entries(obj).map(([key, value]) => {- if (key === 'filter') {- return `"${key}":${value}`;- }- return `"${key}":${JSON.stringify(value)}`;- });- return `{${entries.join(',')}}`;- }+// `<` is escaped so a value containing `</script>` cannot end the inline script.+function toScriptJson(value: unknown) {+ return JSON.stringify(value).replace(/</g, '\\u003c');+}- return JSON.stringify(obj);-};+// `filter` is the developer's own function source, so it is the one value+// inserted as code; everything else is data.+function initOptionsSource({+ filter,+ ...options+}: Record<string, unknown> & { filter?: string }) {+ const data = toScriptJson(options);+ return filter ? `Object.assign(${data},{filter:${filter}})` : data;+}export function OpenPanelComponent({profileId,@@ -49,28 +50,26 @@ export function OpenPanelComponent({strategy = 'afterInteractive',...options}: OpenPanelComponentProps) {- const methods: { name: OpenPanelMethodNames; value: unknown }[] = [+ const methods: { name: OpenPanelMethodNames; source: string }[] = [{name: 'init',- value: {+ source: initOptionsSource({...options,sdk: 'nextjs',sdkVersion: process.env.NEXTJS_VERSION!,- },+ }),},];if (profileId) {methods.push({name: 'identify',- value: {- profileId,- },+ source: toScriptJson({ profileId }),});}if (globalProperties) {methods.push({name: 'setGlobalProperties',- value: globalProperties,+ source: toScriptJson(globalProperties),});}@@ -89,7 +88,7 @@ export function OpenPanelComponent({__html: `${getInitSnippet()}${methods.map((method) => {- return `window.op('${method.name}', ${stringify(method.value)});`;+ return `window.op('${method.name}', ${method.source});`;}).join('\n')}`,}}@@ -106,7 +105,7 @@ export function IdentifyComponent(props: IdentifyComponentProps) {return (<ScriptdangerouslySetInnerHTML={{- __html: `window.op('identify', ${JSON.stringify(props)});`,+ __html: `window.op('identify', ${toScriptJson(props)});`,}}/>);@@ -116,7 +115,7 @@ export function SetGlobalPropertiesComponent(props: Record<string, unknown>) {return (<ScriptdangerouslySetInnerHTML={{- __html: `window.op('setGlobalProperties', ${JSON.stringify(props)});`,+ __html: `window.op('setGlobalProperties', ${toScriptJson(props)});`,}}/>);diff --git a/packages/sdks/web/src/replay/recorder.ts b/packages/sdks/web/src/replay/recorder.tsindex 9b914ed19..9917774d7 100644--- a/packages/sdks/web/src/replay/recorder.ts+++ b/packages/sdks/web/src/replay/recorder.ts@@ -22,6 +22,11 @@ export type ReplayChunkPayload = {payload: string;};+// rrweb's maskAllInputs skips hidden inputs, which carry CSRF tokens and+// similar. They render nothing, so blocking them costs the replay nothing.+const HIDDEN_INPUT_SELECTOR = 'input[type="hidden" i]';+const DEFAULT_BLOCK_SELECTOR = '[data-openpanel-replay-block]';+let stopRecording: (() => void) | null = null;export function startReplayRecorder(@@ -122,7 +127,7 @@ export function startReplayRecorder(return text.replace(/\S/g, '*');}: undefined,- blockSelector: config.blockSelector ?? '[data-openpanel-replay-block]',+ blockSelector: `${config.blockSelector ?? DEFAULT_BLOCK_SELECTOR}, ${HIDDEN_INPUT_SELECTOR}`,blockClass: config.blockClass,ignoreSelector: config.ignoreSelector,});diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.tsindex 4d1d1b999..8d51c0805 100644--- a/packages/shared/src/index.ts+++ b/packages/shared/src/index.ts@@ -9,7 +9,14 @@ export {export { generateId, generateSecureId, shortId } from './id';export { getSafeJson, getSuperJson, setSuperJson } from './json';export { average, ifNaN, max, min, round, sum } from './math';-export { deepMergeObjects, strip, toDots, toObject } from './object';+export {+ deepMergeObjects,+ strip,+ TO_DOTS_MAX_KEYS,+ TO_DOTS_MAX_KEYS_PER_OBJECT,+ toDots,+ toObject,+} from './object';export { slug } from './slug';export {stripLeadingAndTrailingSlashes,diff --git a/packages/shared/src/object.ts b/packages/shared/src/object.tsindex d89c5f6e3..da5175251 100644--- a/packages/shared/src/object.ts+++ b/packages/shared/src/object.ts@@ -1,78 +1,178 @@-// `toDots` is the ClickHouse property flattener — its invalid-surrogate strip-// is load-bearing on the insert path.-import { anyPass, assocPath, isEmpty, isNil, reject } from 'ramda';+import { anyPass, isEmpty, isNil, reject } from 'ramda';++// One event's properties become one ClickHouse Map, flattened per event on the+// ingest path, so the flattener must stay linear and bounded: an unbounded+// payload (a Node Buffer serialised byte by byte) once took ~40s per event and+// stalled the Kafka consumers. Entries past either cap are dropped. The+// per-object cap skips the root: an event's own top-level properties are what+// the user sent on purpose, and the total cap already bounds them.+export const TO_DOTS_MAX_KEYS_PER_OBJECT = 50;+export const TO_DOTS_MAX_KEYS = 1000;++// ClickHouse rejects the whole insert when a property holds a lone surrogate.+// Almost every string is well formed, so that check runs first.+const LONE_SURROGATE_REGEX =+ /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g;+function removeInvalidSurrogates(value: string): string {+ if (value.isWellFormed()) {+ return value;+ }+ return value.replace(LONE_SURROGATE_REGEX, '');+}-function isValidJsonString(value: string): boolean {- return (- (value.startsWith('{') && value.endsWith('}')) ||- (value.startsWith('[') && value.endsWith(']'))- );+const OPEN_BRACE = '{'.charCodeAt(0);+const CLOSE_BRACE = '}'.charCodeAt(0);+const OPEN_BRACKET = '['.charCodeAt(0);+const CLOSE_BRACKET = ']'.charCodeAt(0);++/** `json` when the string looks like a whole object/array literal, `malformed` when it only starts like one. */+function classifyString(value: string): 'json' | 'malformed' | 'text' {+ const first = value.charCodeAt(0);+ if (first !== OPEN_BRACE && first !== OPEN_BRACKET) {+ return 'text';+ }+ const last = value.charCodeAt(value.length - 1);+ const expected = first === OPEN_BRACE ? CLOSE_BRACE : CLOSE_BRACKET;+ return last === expected ? 'json' : 'malformed';}-function isMalformedJsonString(value: string): boolean {++/** The shape `JSON.stringify(Buffer)` produces in the Node SDK. */+function isSerializedBuffer(+ value: object+): value is { type: 'Buffer'; data: number[] } {return (- (value.startsWith('{') && !value.endsWith('}')) ||- (value.startsWith('[') && !value.endsWith(']'))+ 'type' in value &&+ value.type === 'Buffer' &&+ 'data' in value &&+ Array.isArray(value.data));}+const utf8Decoder = new TextDecoder();+function decodeSerializedBuffer(value: { data: number[] }): string {+ return utf8Decoder.decode(Uint8Array.from(value.data));+}+export function toDots(obj: Record<string, unknown>,path = ''): Record<string, string> {- // Clickhouse breaks on insert if a property contains invalid surrogate pairs- function removeInvalidSurrogates(value: string): string {- const validSurrogatePairRegex =- /[\uD800-\uDBFF][\uDC00-\uDFFF]|[^\uD800-\uDFFF]/g;- return value.match(validSurrogatePairRegex)?.join('') || '';- }+ const out: Record<string, string> = {};+ let count = 0;- return Object.entries(obj).reduce((acc, [key, value]) => {- if (typeof value === 'object' && value !== null) {- return {- ...acc,- ...toDots(value as Record<string, unknown>, `${path}${key}.`),- };+ const visit = (value: unknown, key: string): void => {+ if (count >= TO_DOTS_MAX_KEYS) {+ return;}if (value === undefined || value === null || value === '') {- return acc;+ return;+ }++ if (typeof value === 'string') {+ const kind = classifyString(value);+ if (kind === 'malformed') {+ return;+ }+ if (kind === 'json') {+ try {+ visit(JSON.parse(value), key);+ } catch {+ // Not JSON after all: dropped like a malformed one.+ }+ return;+ }+ out[key] = removeInvalidSurrogates(value).trim();+ count += 1;+ return;}- if (typeof value === 'string' && isMalformedJsonString(value)) {- return acc;+ if (typeof value === 'object') {+ if (isSerializedBuffer(value)) {+ visit(decodeSerializedBuffer(value), key);+ return;+ }+ visitEntries(value, `${key}.`, TO_DOTS_MAX_KEYS_PER_OBJECT);+ return;}- // Fix nested json strings - but catch parse errors for malformed JSON- if (typeof value === 'string' && isValidJsonString(value)) {- try {- return {- ...acc,- ...toDots(JSON.parse(value), `${path}${key}.`),- };- } catch {- return acc;+ out[key] = String(value);+ count += 1;+ };++ // `for...in` with an early exit, because `Object.entries` on a 20k-key object+ // allocates every tuple before the cap throws them away.+ const visitEntries = (+ value: object,+ prefix: string,+ maxEntries: number+ ): void => {+ if (Array.isArray(value)) {+ const end = Math.min(value.length, maxEntries);+ for (let index = 0; index < end && count < TO_DOTS_MAX_KEYS; index++) {+ visit(value[index], `${prefix}${index}`);+ }+ return;+ }+ let seen = 0;+ for (const childKey in value) {+ if (!Object.hasOwn(value, childKey)) {+ continue;+ }+ if (seen >= maxEntries || count >= TO_DOTS_MAX_KEYS) {+ return;}+ seen += 1;+ visit(+ (value as Record<string, unknown>)[childKey],+ `${prefix}${childKey}`+ );}+ };++ visitEntries(obj, path, TO_DOTS_MAX_KEYS);++ return out;+}- const cleanedValue =- typeof value === 'string'- ? removeInvalidSurrogates(value).trim()- : String(value);+function isRecord(value: unknown): value is Record<string, unknown> {+ return typeof value === 'object' && value !== null && !Array.isArray(value);+}- return {- ...acc,- [`${path}${key}`]: cleanedValue,- };- }, {});+/** Own-property write, so a `__proto__` segment is a key rather than the prototype setter. */+function setOwn(target: Record<string, unknown>, key: string, value: unknown) {+ Object.defineProperty(target, key, {+ value,+ writable: true,+ enumerable: true,+ configurable: true,+ });}+/** The inverse of `toDots`: one mutable walk per key, where an immutable `assocPath` per key copied the growing result each time. */export function toObject(obj: Record<string, string | undefined>): Record<string, unknown> {- let result: Record<string, unknown> = {};- Object.entries(obj).forEach(([key, value]) => {- result = assocPath(key.split('.'), value, result);- });+ const result: Record<string, unknown> = {};+ for (const [key, value] of Object.entries(obj)) {+ const segments = key.split('.');+ const leaf = segments.pop();+ if (leaf === undefined) {+ continue;+ }+ let node = result;+ for (const segment of segments) {+ const child = Object.hasOwn(node, segment) ? node[segment] : undefined;+ if (isRecord(child)) {+ node = child;+ continue;+ }+ const next: Record<string, unknown> = {};+ setOwn(node, segment, next);+ node = next;+ }+ setOwn(node, leaf, value);+ }return result;}diff --git a/self-hosting/cli/src/commands/doctor.ts b/self-hosting/cli/src/commands/doctor.tsindex b0b00c48f..683a2dab9 100644--- a/self-hosting/cli/src/commands/doctor.ts+++ b/self-hosting/cli/src/commands/doctor.ts@@ -1,4 +1,4 @@-import { applyFixes, type Finding, runDoctor } from '../doctor';+import { applyFixes, type Finding, runDoctor, smtpStartTls } from '../doctor';import { checkHost } from '../host';import { isInstallDir, loadInstall, writeInstall } from '../install';import { bold, dim, green, log, red, SEVERITY_LABEL } from '../ui';@@ -56,7 +56,10 @@ export const doctor = async ({}const install = loadInstall(dir);- const findings = runDoctor(install);+ // Network probes stay out of runDoctor, which reload and upgrade call+ // offline and synchronously.+ const smtp = await smtpStartTls(install.env);+ const findings = [...runDoctor(install), ...(smtp ? [smtp] : [])];log(bold(`Install (${dir})`));if (findings.length === 0) {log(`${green('ok')} Nothing to fix.`);diff --git a/self-hosting/cli/src/doctor/index.ts b/self-hosting/cli/src/doctor/index.tsindex 420792727..26351e788 100644--- a/self-hosting/cli/src/doctor/index.ts+++ b/self-hosting/cli/src/doctor/index.ts@@ -1,6 +1,7 @@import { checks } from './checks';import type { Finding, Install } from './types';+export { SMTP_STARTTLS_CHECK_ID, smtpStartTls } from './smtp';export type { Finding, Install, Severity } from './types';export const runDoctor = (install: Install): Finding[] =>diff --git a/self-hosting/cli/src/doctor/smtp.ts b/self-hosting/cli/src/doctor/smtp.tsnew file mode 100644index 000000000..32a8b8c59--- /dev/null+++ b/self-hosting/cli/src/doctor/smtp.ts@@ -0,0 +1,118 @@+import { connect } from 'node:net';+import type { EnvFile } from '../env-file';+import type { Finding } from './types';++export const SMTP_STARTTLS_CHECK_ID = 'env/smtp-starttls';+const DEFAULT_SMTP_PORT = 587;+const PROBE_TIMEOUT_MS = 5000;+// The last line of a (possibly multi-line) SMTP reply has a space after the code.+const FINAL_REPLY_LINE = /^\d{3} /m;+const STARTTLS_EXTENSION = /^250[ -]STARTTLS\b/im;+const SMTP_READY = '220';+const SMTP_OK = '250';++type Probe =+ | { reachable: true; offersStartTls: boolean }+ | { reachable: false; reason: string };++// Reads the greeting, sends EHLO and reports whether STARTTLS is advertised.+// Never starts TLS or authenticates: no credentials leave the machine.+export const probeStartTls = ({+ host,+ port,+ timeoutMs,+}: {+ host: string;+ port: number;+ timeoutMs: number;+}): Promise<Probe> =>+ new Promise((resolve) => {+ const socket = connect({ host, port });+ const finish = (probe: Probe) => {+ clearTimeout(timer);+ socket.destroy();+ resolve(probe);+ };+ const timer = setTimeout(+ () =>+ finish({ reachable: false, reason: `no reply within ${timeoutMs} ms` }),+ timeoutMs+ );+ let reply = '';+ let hasGreeted = false;+ socket.setEncoding('utf8');+ socket.on('error', (error) =>+ finish({ reachable: false, reason: error.message })+ );+ socket.on('close', () =>+ finish({ reachable: false, reason: 'the server closed the connection' })+ );+ socket.on('data', (chunk: string) => {+ reply += chunk;+ if (!FINAL_REPLY_LINE.test(reply)) {+ return;+ }+ if (!hasGreeted) {+ if (!reply.startsWith(SMTP_READY)) {+ finish({+ reachable: false,+ reason: `unexpected greeting: ${reply.trim()}`,+ });+ return;+ }+ hasGreeted = true;+ reply = '';+ socket.write('EHLO openpanel-doctor\r\n');+ return;+ }+ if (!reply.startsWith(SMTP_OK)) {+ finish({ reachable: false, reason: `EHLO failed: ${reply.trim()}` });+ return;+ }+ const offersStartTls = STARTTLS_EXTENSION.test(reply);+ socket.write('QUIT\r\n');+ finish({ reachable: true, offersStartTls });+ });+ });++// The api requires STARTTLS whenever it authenticates without implicit TLS+// (packages/email createSmtpTransport); an unauthenticated relay is exempt.+export const smtpStartTls = async (+ env: EnvFile,+ { timeoutMs = PROBE_TIMEOUT_MS }: { timeoutMs?: number } = {}+): Promise<Finding | null> => {+ const host = env.get('SMTP_HOST');+ const authenticates = Boolean(env.get('SMTP_USER') && env.get('SMTP_PASS'));+ const usesImplicitTls = env.get('SMTP_SECURE') === 'true';+ if (!(host && authenticates) || usesImplicitTls) {+ return null;+ }+ const port = Number(env.get('SMTP_PORT') ?? DEFAULT_SMTP_PORT);+ const address = `${host}:${port}`;+ const probe = Number.isInteger(port)+ ? await probeStartTls({ host, port, timeoutMs })+ : { reachable: false as const, reason: 'SMTP_PORT is not a number' };++ if (!probe.reachable) {+ return {+ id: SMTP_STARTTLS_CHECK_ID,+ severity: 'warn',+ title: `Could not check STARTTLS on ${address}`,+ detail: `${probe.reason}. Doctor connects from this machine, so a host only the containers can reach (a compose service name) cannot be checked here.`,+ manual:+ 'Make sure the server offers STARTTLS on this port, or set SMTP_SECURE=true with SMTP_PORT=465: the api no longer sends SMTP credentials without TLS.',+ };+ }+ if (probe.offersStartTls) {+ return null;+ }+ return {+ id: SMTP_STARTTLS_CHECK_ID,+ severity: 'error',+ title: `${address} does not offer STARTTLS`,+ detail:+ 'With SMTP_USER and SMTP_PASS set and SMTP_SECURE not `true`, the api refuses to send credentials over an unencrypted connection, so every email fails.',+ manual:+ 'Set SMTP_SECURE=true with SMTP_PORT=465, or point SMTP_HOST at a server that supports STARTTLS.',+ };+};diff --git a/self-hosting/cli/src/export/export.ts b/self-hosting/cli/src/export/export.tsindex 3af473b55..3d1591bde 100644--- a/self-hosting/cli/src/export/export.ts+++ b/self-hosting/cli/src/export/export.ts@@ -1,5 +1,5 @@import { createWriteStream } from 'node:fs';-import { appendFile, mkdir, rm, writeFile } from 'node:fs/promises';+import { appendFile, mkdir, readdir, rm, writeFile } from 'node:fs/promises';import { join } from 'node:path';import { Transform } from 'node:stream';import { pipeline } from 'node:stream/promises';@@ -221,12 +221,30 @@ const exportTable = async (return { table, rows, files };};+// The importer reads every file in the directory, not just the manifest, and+// the operator archives the whole directory. Files left by an earlier, broader+// export would ship along with this one.+const assertEmptyOutput = async (out: string): Promise<void> => {+ const entries = await readdir(out).catch((error: NodeJS.ErrnoException) => {+ if (error.code === 'ENOENT') {+ return [];+ }+ throw error;+ });+ if (entries.length > 0) {+ throw new Error(+ `${out} is not empty. Export into a new directory, or remove it first.`+ );+ }+};+export const runExport = async (client: ClickHouse,options: ExportOptions,progress: ExportProgress) => {validateOptions(options);+ await assertEmptyOutput(options.out);await client.query('SELECT 1');await mkdir(options.out, { recursive: true });const manifest = join(options.out, 'manifest.jsonl');diff --git a/self-hosting/cli/src/init/generate.ts b/self-hosting/cli/src/init/generate.tsindex 8dec1c42f..3cc23388a 100644--- a/self-hosting/cli/src/init/generate.ts+++ b/self-hosting/cli/src/init/generate.ts@@ -77,19 +77,23 @@ export const generateSecrets = (existing: EnvFile | null = null): Secrets => ({export const hostOf = (domain: string): string =>domain.replace(/^https?:\/\//, '');-// Values can contain `$` (bcrypt hashes do), so never pass them as the-// replacement string, where `$&` and friends are special.-const fill = (text: string, values: Record<string, string>): string =>- Object.entries(values).reduce(- (result, [name, value]) => result.replaceAll(`$${name}`, () => value),- text+// One pass over the template: an inserted value (an external URL, a bcrypt+// hash) may itself contain `$NAME`, and must not be expanded into another+// secret. Longest names first, so `$DATABASE_URL_DIRECT` is not read as+// `$DATABASE_URL` followed by `_DIRECT`.+const fill = (text: string, values: Record<string, string>): string => {+ const names = Object.keys(values).sort((a, b) => b.length - a.length);+ const placeholder = new RegExp(`\\$(${names.join('|')})`, 'g');+ return text.replace(+ placeholder,+ (_match, name: string) => values[name] ?? '');+};const renderEnv = (answers: InitAnswers, secrets: Secrets): string => {const urlFor = (service: DataService) =>answers.external[service] ?? BUNDLED_URLS[service];const filled = fill(templates.env, {- // Longest names first: `$DATABASE_URL_DIRECT` must not be eaten by `$DATABASE_URL`.DATABASE_URL_DIRECT: urlFor('postgres'),DATABASE_URL: urlFor('postgres'),CLICKHOUSE_URL: urlFor('clickhouse'),diff --git a/self-hosting/cli/test/export.test.ts b/self-hosting/cli/test/export.test.tsindex 3e7a2f16e..958a6a3cc 100644--- a/self-hosting/cli/test/export.test.ts+++ b/self-hosting/cli/test/export.test.ts@@ -5,6 +5,7 @@ import {readdirSync,readFileSync,rmSync,+ writeFileSync,} from 'node:fs';import { tmpdir } from 'node:os';import { join } from 'node:path';@@ -182,6 +183,17 @@ describe('runExport', () => {expect(paged).toContain("created_at < toDateTime64('2026-09-01 12:00', 3)");});+ test('refuses a directory that already holds files', async () => {+ const out = outDir();+ writeFileSync(join(out, 'manifest.jsonl'), '');+ const { client, sql } = fakeClickHouse(['{"id":"1"}']);++ await expect(runExport(client, options(out), NO_PROGRESS)).rejects.toThrow(+ 'is not empty'+ );+ expect(sql).toEqual([]);+ });+test('a failed query removes its partial file instead of leaving it for the importer', async () => {const out = outDir();const { client } = fakeClickHouse(['{"id":"1"}'], true);diff --git a/self-hosting/cli/test/init.test.ts b/self-hosting/cli/test/init.test.tsindex 48a8b3a27..b752ebd79 100644--- a/self-hosting/cli/test/init.test.ts+++ b/self-hosting/cli/test/init.test.ts@@ -69,6 +69,16 @@ describe('init output', () => {expect(files.get('.env')).toContain(`DATABASE_URL_DIRECT="${url}"`);});+ test('placeholder text inside an external url is kept literally', async () => {+ const url = `redis://user:${DOLLAR}[email protected]:6379`;+ const env = (await generate({ external: { redis: url } })).get(+ '.env'+ ) as string;++ expect(env).toContain(`REDIS_URL="${url}"`);+ expect(env).not.toContain(`${secrets.encryptionKey}@`);+ });+test('an external clickhouse skips its config files', async () => {const files = await generate({external: { clickhouse: 'http://ch.internal:8123/op' },diff --git a/self-hosting/cli/test/smtp.test.ts b/self-hosting/cli/test/smtp.test.tsnew file mode 100644index 000000000..d9b07ddd1--- /dev/null+++ b/self-hosting/cli/test/smtp.test.ts@@ -0,0 +1,126 @@+import { afterEach, describe, expect, test } from 'bun:test';+import { type AddressInfo, createServer, type Server } from 'node:net';+import { smtpStartTls } from '../src/doctor';+import { EnvFile } from '../src/env-file';++const SHORT_TIMEOUT_MS = 300;+const servers: Server[] = [];++afterEach(() => {+ for (const server of servers.splice(0)) {+ server.close();+ }+});++// Answers the greeting and EHLO the way a real server does, multi-line replies+// included, and counts connections so a skipped check can prove it never dialed.+const fakeSmtp = async ({+ greeting = '220-mail.test ESMTP\r\n220 ready\r\n',+ extensions,+ ehloReply,+}: {+ greeting?: string | null;+ extensions: string[];+ ehloReply?: string;+}) => {+ const state = { connections: 0, port: 0 };+ const server = createServer((socket) => {+ state.connections += 1;+ if (greeting !== null) {+ socket.write(greeting);+ }+ socket.on('data', (data) => {+ if (!data.toString().startsWith('EHLO')) {+ return;+ }+ if (ehloReply) {+ socket.write(ehloReply);+ return;+ }+ const lines = ['mail.test', ...extensions];+ const reply = lines+ .map((line, index) => {+ const separator = index === lines.length - 1 ? ' ' : '-';+ return `250${separator}${line}\r\n`;+ })+ .join('');+ socket.write(reply);+ });+ });+ servers.push(server);+ await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));+ state.port = (server.address() as AddressInfo).port;+ return state;+};++const envFor = (port: number, extra = '') =>+ new EnvFile(+ `SMTP_HOST=127.0.0.1\nSMTP_PORT=${port}\nSMTP_USER=mailer\nSMTP_PASS=secret\n${extra}`+ );++describe('smtp STARTTLS check', () => {+ test('passes when EHLO advertises STARTTLS', async () => {+ const server = await fakeSmtp({ extensions: ['PIPELINING', 'STARTTLS'] });+ expect(await smtpStartTls(envFor(server.port))).toBeNull();+ expect(server.connections).toBe(1);+ });++ test('is an error when EHLO does not advertise STARTTLS', async () => {+ const server = await fakeSmtp({ extensions: ['PIPELINING', 'AUTH PLAIN'] });+ const finding = await smtpStartTls(envFor(server.port));+ expect(finding?.severity).toBe('error');+ expect(finding?.id).toBe('env/smtp-starttls');+ });++ test('skips without credentials, with implicit TLS, or without SMTP', async () => {+ const server = await fakeSmtp({ extensions: [] });+ const unauthenticated = new EnvFile(+ `SMTP_HOST=127.0.0.1\nSMTP_PORT=${server.port}\nSMTP_USER=mailer\n`+ );+ expect(await smtpStartTls(unauthenticated)).toBeNull();+ expect(+ await smtpStartTls(envFor(server.port, 'SMTP_SECURE=true\n'))+ ).toBeNull();+ expect(await smtpStartTls(new EnvFile('SMTP_USER=a\nSMTP_PASS=b\n'))).toBe(+ null+ );+ expect(server.connections).toBe(0);+ });++ test('still probes when SMTP_SECURE is anything but true', async () => {+ const server = await fakeSmtp({ extensions: [] });+ const finding = await smtpStartTls(+ envFor(server.port, 'SMTP_SECURE=false\n')+ );+ expect(finding?.severity).toBe('error');+ });++ test('warns when the server refuses EHLO', async () => {+ const server = await fakeSmtp({+ extensions: [],+ ehloReply: '502 5.5.2 command not recognized\r\n',+ });+ const finding = await smtpStartTls(envFor(server.port));+ expect(finding?.severity).toBe('warn');+ });++ test('warns when nothing listens on the port', async () => {+ const server = await fakeSmtp({ extensions: [] });+ const { port } = server;+ servers.pop()?.close();+ await Bun.sleep(10);+ const finding = await smtpStartTls(envFor(port), {+ timeoutMs: SHORT_TIMEOUT_MS,+ });+ expect(finding?.severity).toBe('warn');+ });++ test('warns when the server never greets', async () => {+ const server = await fakeSmtp({ greeting: null, extensions: [] });+ const finding = await smtpStartTls(envFor(server.port), {+ timeoutMs: SHORT_TIMEOUT_MS,+ });+ expect(finding?.severity).toBe('warn');+ expect(finding?.detail).toContain(`${SHORT_TIMEOUT_MS} ms`);+ });+});diff --git a/self-hosting/import-to-cloud.mjs b/self-hosting/import-to-cloud.mjsindex 0a023bf7e..75159357f 100755--- a/self-hosting/import-to-cloud.mjs+++ b/self-hosting/import-to-cloud.mjs@@ -42,6 +42,7 @@ import {appendFileSync,createReadStream,existsSync,+ lstatSync,readFileSync,} from 'node:fs';import { readdir } from 'node:fs/promises';@@ -57,6 +58,10 @@ const IMPORT_ORDER = ['session_replay_chunks',];+// What `openpanel export` writes. Anything else in the dump is ignored, which+// also keeps a crafted filename (one with a newline, say) out of the state file.+const EXPORT_FILE_NAME = /^[\w.-]+\.jsonl(\.gz)?$/;+const CH_SETTINGS = {// The self-hosted schema may be slightly ahead of / behind the target.input_format_skip_unknown_fields: 1,@@ -291,6 +296,11 @@ async function main() {.digest('hex').slice(0, 12);const stateFile = path.join(args.dir, `.import-state-${fingerprint}`);+ // The dump comes from someone else's machine; a planted link would make the+ // importer append to whatever file it points at.+ if (lstatSync(stateFile, { throwIfNoEntry: false })?.isSymbolicLink()) {+ throw new Error(`${stateFile} is a symbolic link; refusing to write to it`);+ }const state = new Set(existsSync(stateFile)? readFileSync(stateFile, 'utf8').split('\n').filter(Boolean)@@ -317,7 +327,7 @@ async function main() {}const files = (await readdir(dir))- .filter((f) => f.includes('.jsonl'))+ .filter((f) => EXPORT_FILE_NAME.test(f)).sort();if (files.length === 0) {continue;diff --git a/self-hosting/verify-export.mjs b/self-hosting/verify-export.mjsindex 13ce93159..6ca3fde96 100755--- a/self-hosting/verify-export.mjs+++ b/self-hosting/verify-export.mjs@@ -2,15 +2,22 @@/*** Verify an export directory against its manifest before importing it.*- * Catches the two things that actually go wrong: a file that never finished- * transferring (truncated gzip) and a file that is silently missing.+ * Catches the things that actually go wrong: a file that never finished+ * transferring (truncated gzip), a file that is silently missing, and a file+ * the manifest does not list (left over from an earlier export, and imported+ * anyway, since the importer reads the directory).** node verify-export.mjs ./op-export** Exits non-zero if anything is wrong, so it chains: `node verify-export.mjs* ./op-export && node import-to-cloud.mjs --dir ./op-export ...`*/-import { createReadStream, existsSync, readFileSync } from 'node:fs';+import {+ createReadStream,+ existsSync,+ readdirSync,+ readFileSync,+} from 'node:fs';import path from 'node:path';import { createInterface } from 'node:readline';import { createGunzip } from 'node:zlib';@@ -90,6 +97,18 @@ for (const entry of entries) {checked++;}+const listed = new Set(entries.map((entry) => `${entry.table}/${entry.file}`));+for (const table of readdirSync(dir, { withFileTypes: true })) {+ if (!table.isDirectory()) {+ continue;+ }+ for (const file of readdirSync(path.join(dir, table.name))) {+ if (!listed.has(`${table.name}/${file}`)) {+ problems.push(`UNLISTED ${table.name}/${file} — not in manifest.jsonl`);+ }+ }+}+console.log(`Checked ${checked}/${entries.length} files in ${dir}\n`);for (const [table, rows] of Object.entries(totals)) {console.log(` ${table.padEnd(24)} ${String(rows).padStart(10)} rows`);@@ -107,10 +126,13 @@ if (problems.length > 0) {for (const problem of problems) {console.error(` ${problem}`);}- console.error('\n Re-run your rsync — it will fetch just the bad files.');+ console.error(+ '\n Re-run your rsync — it will fetch just the bad files. Delete UNLISTED'+ );+ console.error(' files, or re-export into an empty directory.');process.exit(1);}console.log(- '\n✅ Export is complete — no missing, truncated or corrupt files.'+ '\n✅ Export is complete — no missing, truncated, corrupt or unlisted files.');diff --git a/tooling/gates/p13-images.sh b/tooling/gates/p13-images.shindex a6e4ef1d4..e2b8bcf8f 100755--- a/tooling/gates/p13-images.sh+++ b/tooling/gates/p13-images.sh@@ -20,7 +20,10 @@ set -uo pipefailREPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"cd "$REPO_ROOT" || exit 1-readonly RUN_DIR="${P13_IMAGES_RUN_DIR:-/tmp/openpanel-p13-images}"+# A fresh private directory by default: a fixed name under /tmp can be created+# first by another local user, who then controls where the logs are written.+RUN_DIR="${P13_IMAGES_RUN_DIR:-$(mktemp -d "${TMPDIR:-/tmp}/openpanel-p13-images.XXXXXX")}"+readonly RUN_DIRreadonly API_BUILD_LOG="$RUN_DIR/api-build.log"readonly DASHBOARD_BUILD_LOG="$RUN_DIR/dashboard-build.log"readonly API_LOG="$RUN_DIR/api.log"