OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix: mcp and ai chat (add limits for not blowing up context)
Carl-Gerhard Lindesvärd77c72aa3f087
diff --git a/apps/api/src/agents/tools/base.ts b/apps/api/src/agents/tools/base.tsindex 60fd68f77..515009855 100644--- a/apps/api/src/agents/tools/base.ts+++ b/apps/api/src/agents/tools/base.ts@@ -53,7 +53,7 @@ export const listEventProperties = chatTool({name: 'list_event_properties',description:- 'List property keys available for an event (or across all events). Call this before using a property as a filter or breakdown in generate_report. NOTE: dotted sub-keys are rolled up to their root — e.g. all `__query.foo`, `__query.bar`, … are represented by a single `__query` entry. Keys are ordered by how many sub-keys roll up under them (most prominent first). To filter on a specific sub-key like `__query.utm_source`, use that exact full key in `set_property_filters` / breakdowns — the rollup is purely a discovery optimization.',+ 'List fields available for filtering or breaking down events. Returns TWO buckets:\n\n- `columns`: top-level event columns (path, referrer, country, device, browser, os, etc.). Apply to every event. Use the bare name in filters/breakdowns (e.g. `{ name: "path", operator: "endsWith", value: ["/cohorts"] }`).\n- `properties`: custom keys from the event\'s JSON `properties` map. Use prefixed as `properties.<key>` in filters/breakdowns (e.g. `properties.plan`).\n\nCall this before using any field as a filter or breakdown in generate_report. NOTE: dotted sub-keys inside `properties` are rolled up to their root — e.g. all `__query.foo`, `__query.bar`, … are represented by a single `__query` entry, ordered by how many sub-keys roll up under them (most prominent first). To filter on a specific sub-key like `properties.__query.utm_source`, use that exact full key — the rollup is purely a discovery optimization.',schema: z.object({eventName: z.string()@@ -215,7 +215,9 @@ export const generateReport = chatTool(z.object({name: z.string()- .describe('Property key — verify with list_event_properties'),+ .describe(+ 'Field to filter on — verify with list_event_properties. Use the bare column name for top-level columns (e.g. `path`, `country`, `device`) or `properties.<key>` for custom JSON properties (e.g. `properties.plan`).',+ ),operator: z.enum(objectToZodEnums(operators)).describe(diff --git a/apps/api/src/agents/tools/events.ts b/apps/api/src/agents/tools/events.tsindex 25acc8055..beacab5b5 100644--- a/apps/api/src/agents/tools/events.ts+++ b/apps/api/src/agents/tools/events.ts@@ -141,7 +141,7 @@ export const listPropertiesForEvent = chatTool({name: 'list_properties_for_event',description:- 'List property keys available for a specific event (or all events). Useful before correlating or filtering. Dotted sub-keys are rolled up to their root (e.g. all `__query.*` become a single `__query`); ordered by how many sub-keys roll up under each root.',+ 'List fields available for filtering / breakdown on a specific event (or all events). Returns `columns` (top-level event columns like `path`, `country`, `device` — use bare name) and `properties` (custom JSON keys — use prefixed as `properties.<key>`). Dotted sub-keys inside `properties` are rolled up to their root (e.g. all `__query.*` become a single `__query`); ordered by how many sub-keys roll up under each root.',schema: z.object({eventName: z.string().optional(),}),diff --git a/apps/api/src/agents/tools/helpers.ts b/apps/api/src/agents/tools/helpers.tsindex eb0f065ac..b27aacf68 100644--- a/apps/api/src/agents/tools/helpers.ts+++ b/apps/api/src/agents/tools/helpers.ts@@ -98,9 +98,15 @@ export function truncateRows<T>(/*** Compact output from `listEventPropertiesCore` for consumption by the LLM.*- * The raw shape is `{ properties: [{ property_key, event_name }, …] }` ordered- * alphabetically and capped at 500 rows. That shape has three token-hungry- * problems when passed back as a tool result:+ * Two buckets come back:+ *+ * - `columns`: top-level event columns (path, referrer, country, …).+ * Apply to every event. Use the bare name in filters/breakdowns.+ * - `properties`: custom keys from the JSON `properties` map. Use+ * prefixed as `properties.<key>` in filters/breakdowns.+ *+ * The raw `properties` rows are ordered alphabetically and capped at 500.+ * That shape has three token-hungry problems when passed back:** 1. Dotted sub-keys explode. A single property like `__query` or `data`* with dynamic sub-paths surfaces as hundreds of rows@@ -117,10 +123,14 @@ export function truncateRows<T>(* "how prominent is this property" signal), and caps the list.*/export function compactEventProperties(- raw: { properties: Array<{ property_key: string; event_name: string }> },+ raw: {+ columns: readonly string[];+ properties: Array<{ property_key: string; event_name: string }>;+ },options: { eventName?: string; max?: number } = {},): {event_name?: string;+ columns: readonly string[];properties: string[];total: number;_truncated: boolean;@@ -138,6 +148,7 @@ export function compactEventProperties(const truncated = sorted.length > max;return {...(eventName ? { event_name: eventName } : {}),+ columns: raw.columns,properties: truncated ? sorted.slice(0, max) : sorted,total: sorted.length,_truncated: truncated,diff --git a/apps/api/src/routes/insights.router.test.ts b/apps/api/src/routes/insights.router.test.tsindex c91a8f087..be2ea05f0 100644--- a/apps/api/src/routes/insights.router.test.ts+++ b/apps/api/src/routes/insights.router.test.ts@@ -173,10 +173,12 @@ describe('GET /insights/:projectId/events', () => {});describe('GET /insights/:projectId/events/properties', () => {- it('returns properties array', async () => {+ it('returns columns + properties arrays', async () => {const res = await get(`/insights/${TEST_PROJECT_ID}/events/properties`);expect(res.statusCode).toBe(200);const body = res.json();+ expect(Array.isArray(body.columns)).toBe(true);+ expect(body.columns).toContain('path');expect(Array.isArray(body.properties)).toBe(true);});});diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.tsindex c295bd3c0..fd559c30f 100644--- a/packages/db/src/services/event.service.ts+++ b/packages/db/src/services/event.service.ts@@ -1171,10 +1171,49 @@ export async function getTopEventNames(projectId: string): Promise<string[]> {export const listEventNamesCore = (projectId: string): Promise<string[]> =>getTopEventNames(projectId);+/**+ * Top-level filterable columns on the `events` table. These apply to+ * every event regardless of name and can be passed straight to+ * `getEventFiltersWhereClause` as filter / breakdown `name` values.+ *+ * Kept as a whitelist because the filter builder splices `name` into+ * SQL verbatim — only these are safe to expose through the AI /+ * MCP discovery surface.+ */+export const EVENT_COLUMNS = [+ 'path',+ 'origin',+ 'referrer',+ 'referrer_name',+ 'referrer_type',+ 'duration',+ 'country',+ 'city',+ 'region',+ 'os',+ 'os_version',+ 'browser',+ 'browser_version',+ 'device',+ 'brand',+ 'model',+ 'sdk_name',+ 'sdk_version',+ 'profile_id',+ 'session_id',+ 'device_id',+ 'revenue',+] as const;++export type IEventColumn = (typeof EVENT_COLUMNS)[number];+export async function listEventPropertiesCore(input: {projectId: string;eventName?: string;-}): Promise<{ properties: Array<{ property_key: string; event_name: string }> }> {+}): Promise<{+ columns: readonly string[];+ properties: Array<{ property_key: string; event_name: string }>;+}> {const builder = clix(ch).select<{ property_key: string; event_name: string }>(['distinct property_key',@@ -1190,7 +1229,7 @@ export async function listEventPropertiesCore(input: {}const rows = await builder.execute();- return { properties: rows };+ return { columns: EVENT_COLUMNS, properties: rows };}export async function getEventPropertyValuesCore(input: {diff --git a/packages/db/src/services/overview.service.ts b/packages/db/src/services/overview.service.tsindex 527ded9a6..9189cac8b 100644--- a/packages/db/src/services/overview.service.ts+++ b/packages/db/src/services/overview.service.ts@@ -78,6 +78,7 @@ export const zGetTopPagesInput = z.object({filters: z.array(z.any()),startDate: z.string(),endDate: z.string(),+ limit: z.number().min(1).max(1000).optional(),});export type IGetTopPagesInput = z.infer<typeof zGetTopPagesInput> & {@@ -90,6 +91,7 @@ export const zGetTopEntryExitInput = z.object({startDate: z.string(),endDate: z.string(),mode: z.enum(['entry', 'exit']),+ limit: z.number().min(1).max(1000).optional(),});export type IGetTopEntryExitInput = z.infer<typeof zGetTopEntryExitInput> & {@@ -648,6 +650,7 @@ export class OverviewService {startDate,endDate,timezone,+ limit,}: IGetTopPagesInput) {const selectColumns: (string | null | undefined | false)[] = ['origin',@@ -679,7 +682,7 @@ export class OverviewService {.rawWhere(this.getRawWhereClause('events', filters)).groupBy(['origin', 'path']).orderBy('sessions', 'DESC')- .limit(MAX_RECORDS_LIMIT);+ .limit(Math.min(limit ?? MAX_RECORDS_LIMIT, MAX_RECORDS_LIMIT));return query.execute();}@@ -691,6 +694,7 @@ export class OverviewService {endDate,mode,timezone,+ limit,}: IGetTopEntryExitInput) {const selectColumns: (string | null | undefined | false)[] = [`${mode}_origin AS origin`,@@ -720,7 +724,7 @@ export class OverviewService {.groupBy([`${mode}_origin`, `${mode}_path`]).having('sum(sign)', '>', 0).orderBy('sessions', 'DESC')- .limit(MAX_RECORDS_LIMIT);+ .limit(Math.min(limit ?? MAX_RECORDS_LIMIT, MAX_RECORDS_LIMIT));const mainQuery = this.withDistinctSessionsIfNeeded(query, {projectId,diff --git a/packages/db/src/services/pages.service.ts b/packages/db/src/services/pages.service.tsindex c8a0796da..40cda4760 100644--- a/packages/db/src/services/pages.service.ts+++ b/packages/db/src/services/pages.service.ts@@ -193,6 +193,7 @@ export async function getTopPagesCore(input: {startDate: input.startDate,endDate: input.endDate,timezone,+ limit: input.limit,});}@@ -201,6 +202,7 @@ export async function getEntryExitPagesCore(input: {startDate: string;endDate: string;mode: 'entry' | 'exit';+ limit?: number;filters?: IChartEventFilter[];}) {const { timezone } = await getSettingsForProject(input.projectId);@@ -211,6 +213,7 @@ export async function getEntryExitPagesCore(input: {endDate: input.endDate,mode: input.mode,timezone,+ limit: input.limit,});}diff --git a/packages/mcp/src/integration/tools.test.ts b/packages/mcp/src/integration/tools.test.tsindex 7af6927e0..337a5dbf2 100644--- a/packages/mcp/src/integration/tools.test.ts+++ b/packages/mcp/src/integration/tools.test.ts@@ -90,12 +90,14 @@ describe('list_event_names', () => {});describe('list_event_properties', () => {- it('returns { properties: array }', async () => {+ it('returns { columns, properties }', async () => {const server = makeServer();registerPropertyValueTools(server as any, CTX);const res = await server.invoke('list_event_properties', {projectId: TEST_PROJECT_ID,});+ expect(Array.isArray(res.columns)).toBe(true);+ expect(res.columns).toContain('path');expect(Array.isArray(res.properties)).toBe(true);});});diff --git a/packages/mcp/src/tools/analytics/pages.ts b/packages/mcp/src/tools/analytics/pages.tsindex 24fae6556..4ca06c828 100644--- a/packages/mcp/src/tools/analytics/pages.ts+++ b/packages/mcp/src/tools/analytics/pages.ts@@ -12,25 +12,42 @@ import {resolveProjectId} from '../shared';+const DEFAULT_PAGE_LIMIT = 50;+const MAX_PAGE_LIMIT = 500;+export function registerPageTools(server: McpServer, context: McpAuthContext) {server.tool('get_top_pages',- 'Get the most visited pages ranked by page views, with unique visitor counts and other engagement metrics.',+ `Get the most visited pages ranked by page views, with unique visitor counts and other engagement metrics. Defaults to the top ${DEFAULT_PAGE_LIMIT} pages.`,{projectId: projectIdSchema(context),...zDateRange,+ limit: z+ .number()+ .int()+ .min(1)+ .max(MAX_PAGE_LIMIT)+ .optional()+ .describe(+ `Max pages to return (default ${DEFAULT_PAGE_LIMIT}, max ${MAX_PAGE_LIMIT}). Lower this if you're hitting response-size limits.`,+ ),},- async ({ projectId: inputProjectId, startDate: sd, endDate: ed }) =>+ async ({ projectId: inputProjectId, startDate: sd, endDate: ed, limit }) =>withErrorHandling(async () => {const projectId = await resolveProjectId(context, inputProjectId);const { startDate, endDate } = resolveDateRange(sd, ed);- return getTopPagesCore({ projectId, startDate, endDate });+ return getTopPagesCore({+ projectId,+ startDate,+ endDate,+ limit: limit ?? DEFAULT_PAGE_LIMIT,+ });}),);server.tool('get_entry_exit_pages',- 'Get the most common entry pages (first page in a session) or exit pages (last page in a session).',+ `Get the most common entry pages (first page in a session) or exit pages (last page in a session). Defaults to the top ${DEFAULT_PAGE_LIMIT} pages.`,{projectId: projectIdSchema(context),...zDateRange,@@ -39,12 +56,33 @@ export function registerPageTools(server: McpServer, context: McpAuthContext) {.describe('"entry" for pages visitors land on first, "exit" for pages they leave from',),+ limit: z+ .number()+ .int()+ .min(1)+ .max(MAX_PAGE_LIMIT)+ .optional()+ .describe(+ `Max pages to return (default ${DEFAULT_PAGE_LIMIT}, max ${MAX_PAGE_LIMIT}). Lower this if you're hitting response-size limits.`,+ ),},- async ({ projectId: inputProjectId, startDate: sd, endDate: ed, mode }) =>+ async ({+ projectId: inputProjectId,+ startDate: sd,+ endDate: ed,+ mode,+ limit,+ }) =>withErrorHandling(async () => {const projectId = await resolveProjectId(context, inputProjectId);const { startDate, endDate } = resolveDateRange(sd, ed);- return getEntryExitPagesCore({ projectId, startDate, endDate, mode });+ return getEntryExitPagesCore({+ projectId,+ startDate,+ endDate,+ mode,+ limit: limit ?? DEFAULT_PAGE_LIMIT,+ });}),);}diff --git a/packages/mcp/src/tools/analytics/property-values.ts b/packages/mcp/src/tools/analytics/property-values.tsindex 9b5dc4ae0..291dff5d0 100644--- a/packages/mcp/src/tools/analytics/property-values.ts+++ b/packages/mcp/src/tools/analytics/property-values.ts@@ -1,4 +1,4 @@-import { TABLE_NAMES, ch, clix } from '@openpanel/db';+import { EVENT_COLUMNS, TABLE_NAMES, ch, clix } from '@openpanel/db';import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';import { z } from 'zod';import type { McpAuthContext } from '../../auth';@@ -15,7 +15,7 @@ export function registerPropertyValueTools() {server.tool('list_event_properties',- 'List all property keys that have been tracked for a specific event (or across all events). Use this to discover what data is available before filtering or breaking down by a property.',+ 'List fields available for filtering or breaking down events. Returns `columns` (top-level event columns like `path`, `country`, `device` — use the bare name in filters) and `properties` (custom JSON keys — use prefixed as `properties.<key>` in filters). Use this to discover what data is available before filtering or breaking down.',{projectId: projectIdSchema(context),eventName: z@@ -41,7 +41,7 @@ export function registerPropertyValueTools(}const rows = await builder.execute();- return { properties: rows };+ return { columns: EVENT_COLUMNS, properties: rows };}),);diff --git a/packages/mcp/src/tools/gsc/cannibalization.ts b/packages/mcp/src/tools/gsc/cannibalization.tsindex cdd79e7ba..a5c570fd1 100644--- a/packages/mcp/src/tools/gsc/cannibalization.ts+++ b/packages/mcp/src/tools/gsc/cannibalization.ts@@ -1,31 +1,62 @@import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';import { getGscCannibalization } from '@openpanel/db';+import { z } from 'zod';import type { McpAuthContext } from '../../auth';import {projectIdSchema,resolveDateRange,-+withErrorHandling,zDateRange,resolveProjectId} from '../shared';+// Cap pages per query in the MCP response. The core function keeps the+// full list for the UI, but feeding every page into an LLM blows the+// response size for sites with heavy cannibalization (100KB+).+const DEFAULT_PAGES_PER_QUERY = 5;+const MAX_PAGES_PER_QUERY = 20;+export function registerGscCannibalizationTools(server: McpServer,context: McpAuthContext) {server.tool('gsc_get_cannibalization',- 'Identify keyword cannibalization: search queries where multiple pages on your site compete against each other in Google. Returns queries where 2+ pages rank, sorted by total impressions. High cannibalization can hurt rankings.',+ `Identify keyword cannibalization: search queries where multiple pages on your site compete against each other in Google. Returns queries where 2+ pages rank, sorted by total impressions (capped at 50 queries). Each query's pages list is truncated to the top ${DEFAULT_PAGES_PER_QUERY} by position. High cannibalization can hurt rankings.`,{projectId: projectIdSchema(context),...zDateRange,+ pagesPerQuery: z+ .number()+ .int()+ .min(1)+ .max(MAX_PAGES_PER_QUERY)+ .optional()+ .describe(+ `Max pages to include per query (default ${DEFAULT_PAGES_PER_QUERY}, max ${MAX_PAGES_PER_QUERY}). Each query can rank many pages; this trims the tail to keep the response small.`,+ ),},- async ({ projectId: inputProjectId, startDate: sd, endDate: ed }) =>+ async ({+ projectId: inputProjectId,+ startDate: sd,+ endDate: ed,+ pagesPerQuery,+ }) =>withErrorHandling(async () => {const projectId = await resolveProjectId(context, inputProjectId);const { startDate, endDate } = resolveDateRange(sd, ed);- return getGscCannibalization(projectId, startDate, endDate);+ const cap = pagesPerQuery ?? DEFAULT_PAGES_PER_QUERY;+ const rows = await getGscCannibalization(projectId, startDate, endDate);+ return rows.map((row) => {+ if (row.pages.length <= cap) return row;+ return {+ ...row,+ pages: row.pages.slice(0, cap),+ pagesTruncated: true,+ totalPages: row.pages.length,+ };+ });}));}