Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

feat: improved insights + weekly email updates

Carl-Gerhard Lindesvärdafd9368261f9
diff --git a/apps/api/package.json b/apps/api/package.json
index d3c73d376..3b845cc46 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -18,7 +18,6 @@
"dependencies": {
"@better-agent/adapters": "^0.1.0-beta.3",
"@better-agent/core": "^0.1.0-beta.3",
- "@better-agent/providers": "^0.1.0-beta.3",
"@fastify/compress": "^8.1.0",
"@fastify/cookie": "^11.0.2",
"@fastify/cors": "^11.1.0",
@@ -28,6 +27,7 @@
"@fastify/websocket": "^11.2.0",
"@hyperdx/node-opentelemetry": "catalog:",
"@node-rs/argon2": "^2.0.2",
+ "@openpanel/ai": "workspace:*",
"@openpanel/auth": "workspace:^",
"@openpanel/common": "workspace:*",
"@openpanel/constants": "workspace:*",
diff --git a/apps/api/src/agents/app.ts b/apps/api/src/agents/app.ts
index 9c0bdf977..a77b4285a 100644
--- a/apps/api/src/agents/app.ts
+++ b/apps/api/src/agents/app.ts
@@ -5,7 +5,7 @@ import {
type ChatModelEntry,
openaiProvider,
resolveModel,
-} from '@openpanel/trpc/src/agents/models';
+} from '@openpanel/ai';
import { prismaConversationStore } from './persistence';
import { buildSystemPrompt } from './prompt';
import { composeChatTools } from './tools';
diff --git a/apps/start/src/components/insights/insight-card.tsx b/apps/start/src/components/insights/insight-card.tsx
index 9c9c8d5a3..6336aae50 100644
--- a/apps/start/src/components/insights/insight-card.tsx
+++ b/apps/start/src/components/insights/insight-card.tsx
@@ -2,12 +2,20 @@ import { countries } from '@/translations/countries';
import type { RouterOutputs } from '@/trpc/client';
import { cn } from '@/utils/cn';
import type { InsightPayload } from '@openpanel/validation';
-import { ArrowDown, ArrowUp, FilterIcon, RotateCcwIcon } from 'lucide-react';
+import { pushModal } from '@/modals';
+import {
+ ArrowDown,
+ ArrowUp,
+ FilterIcon,
+ RotateCcwIcon,
+ SparklesIcon,
+} from 'lucide-react';
import { last } from 'ramda';
import { useState } from 'react';
import { DeltaChip } from '../delta-chip';
import { SerieIcon } from '../report-chart/common/serie-icon';
import { Badge } from '../ui/badge';
+import { Button } from '../ui/button';
function formatWindowKind(windowKind: string): string {
switch (windowKind) {
@@ -165,6 +173,13 @@ export function InsightCard({
{renderTitle()}
</div>
+ {/* AI plain-language summary (Tier-1 enrichment) */}
+ {insight.aiSummary && (
+ <p className="text-xs text-muted-foreground leading-snug line-clamp-3 mt-1">
+ {insight.aiSummary}
+ </p>
+ )}
+
{/* Metric row */}
<div className="mt-auto pt-2">
<div className="flex items-end justify-between gap-3">
@@ -195,6 +210,16 @@ export function InsightCard({
{deltaText}
</DeltaChip>
</div>
+
+ <Button
+ className="-ml-2 mt-2 h-7 self-start px-2 text-muted-foreground text-xs"
+ icon={SparklesIcon}
+ onClick={() => pushModal('InsightDetails', { insight })}
+ size="sm"
+ variant="ghost"
+ >
+ Why did this happen?
+ </Button>
</div>
</div>
);
diff --git a/apps/start/src/modals/index.tsx b/apps/start/src/modals/index.tsx
index 3901989dc..0708a1056 100644
--- a/apps/start/src/modals/index.tsx
+++ b/apps/start/src/modals/index.tsx
@@ -26,6 +26,7 @@ import EditMember from './edit-member';
import EditReference from './edit-reference';
import EditReport from './edit-report';
import EventDetails from './event-details';
+import InsightDetails from './insight-details';
import Instructions from './Instructions';
import OverviewChartDetails from './overview-chart-details';
import OverviewFilters from './overview-filters';
@@ -56,6 +57,7 @@ const modals = {
EditEvent,
EditMember,
EventDetails,
+ InsightDetails,
EditClient,
AddProject,
AddClient,
diff --git a/apps/start/src/modals/insight-details.tsx b/apps/start/src/modals/insight-details.tsx
new file mode 100644
index 000000000..4a414b50c
--- /dev/null
+++ b/apps/start/src/modals/insight-details.tsx
@@ -0,0 +1,223 @@
+import { DeltaChip } from '@/components/delta-chip';
+import { ReportChart } from '@/components/report-chart';
+import { Badge } from '@/components/ui/badge';
+import { SheetContent, SheetHeader, SheetTitle } from '@/components/ui/sheet';
+import { useTRPC } from '@/integrations/trpc/react';
+import type { RouterOutputs } from '@/trpc/client';
+import { cn } from '@/utils/cn';
+import type { IReportInput } from '@openpanel/validation';
+import { useMutation } from '@tanstack/react-query';
+import { SparklesIcon } from 'lucide-react';
+import { useEffect, useState } from 'react';
+
+type Insight = RouterOutputs['insight']['list'][number];
+type Explanation = RouterOutputs['insight']['explain'];
+
+function formatWindow(windowKind: string): string {
+ switch (windowKind) {
+ case 'yesterday':
+ return 'Yesterday';
+ case 'rolling_7d':
+ return '7 Days';
+ case 'rolling_30d':
+ return '30 Days';
+ }
+ return windowKind;
+}
+
+function SeverityDot({ band }: { band: string }) {
+ return (
+ <span className="flex shrink-0 items-center gap-1">
+ <span
+ className={cn(
+ 'h-2 w-2 rounded-full',
+ band === 'severe'
+ ? 'bg-red-500'
+ : band === 'moderate'
+ ? 'bg-yellow-500'
+ : 'bg-blue-500',
+ )}
+ />
+ <span className="text-muted-foreground text-xs capitalize">{band}</span>
+ </span>
+ );
+}
+
+/**
+ * Build a report config for the insight's dimension so we render the same chart
+ * the dashboard uses — filter `screen_view` to the insight's dimension, count
+ * sessions or pageviews per the primary metric, previous period overlaid. Chart
+ * window stays close to the insight's own window.
+ */
+function buildInsightReport(insight: Insight): IReportInput | null {
+ const filters = (insight.payload?.dimensions ?? [])
+ .filter((d) => d.value)
+ .map((d) => ({
+ name: d.key,
+ operator: 'is' as const,
+ value: [d.value],
+ }));
+ if (filters.length === 0) return null;
+
+ const range = insight.windowKind === 'yesterday' ? '7d' : '30d';
+
+ return {
+ projectId: insight.projectId,
+ chartType: 'linear',
+ interval: 'day',
+ range,
+ previous: true,
+ metric: 'sum',
+ breakdowns: [],
+ series: [
+ {
+ type: 'event',
+ id: 'A',
+ name: 'screen_view',
+ displayName: insight.displayName || insight.title,
+ segment:
+ insight.payload?.primaryMetric === 'sessions' ? 'session' : 'event',
+ filters,
+ },
+ ],
+ };
+}
+
+/**
+ * Phase 5: the insight detail sheet — the full insight, its primary metric, a
+ * trend chart, and the on-demand AI "why" explanation. Opened via
+ * `pushModal('InsightDetails', { insight })`; the explanation is fetched once
+ * on open.
+ */
+export default function InsightDetails({ insight }: { insight: Insight }) {
+ const trpc = useTRPC();
+ const [explanation, setExplanation] = useState<Explanation>(null);
+
+ const mutation = useMutation(
+ trpc.insight.explain.mutationOptions({
+ onSuccess: (data) => setExplanation(data),
+ }),
+ );
+
+ // Fetch the explanation once when the sheet opens.
+ // biome-ignore lint/correctness/useExhaustiveDependencies: run once on mount
+ useEffect(() => {
+ mutation.mutate({ insightId: insight.id });
+ }, []);
+
+ // Primary metric (mirrors the card's selection).
+ const metrics = Object.entries(insight.payload?.metrics ?? {});
+ const primaryKey = insight.payload?.primaryMetric;
+ const primary =
+ metrics.find(([k]) => k === primaryKey)?.[1] ?? metrics[0]?.[1];
+ const isRatio = primary?.unit === 'ratio';
+ const fmt = (v?: number | null) =>
+ v == null
+ ? '-'
+ : isRatio
+ ? `${(v * 100).toFixed(1)}%`
+ : Math.round(v).toLocaleString();
+ const deltaText = isRatio
+ ? `${Math.abs((primary?.delta ?? 0) * 100).toFixed(1)}pp`
+ : `${Math.abs((primary?.changePct ?? 0) * 100).toFixed(1)}%`;
+ const direction = primary?.direction ?? 'flat';
+ const report = buildInsightReport(insight);
+
+ return (
+ <SheetContent className="gap-6 sm:max-w-md">
+ <SheetHeader>
+ <div className="flex flex-col gap-2 pr-8">
+ <div className="flex items-center gap-2">
+ <Badge variant="outline">{formatWindow(insight.windowKind)}</Badge>
+ {insight.severityBand && <SeverityDot band={insight.severityBand} />}
+ </div>
+ <SheetTitle className="text-lg leading-snug">
+ {insight.displayName || insight.title}
+ </SheetTitle>
+ </div>
+ </SheetHeader>
+
+ {insight.aiSummary && (
+ <p className="text-muted-foreground leading-normal">
+ {insight.aiSummary}
+ </p>
+ )}
+
+ {primary && (
+ <div className="flex items-end justify-between gap-3 rounded-md border p-4">
+ <div>
+ <div className="mb-1 text-muted-foreground text-xs capitalize">
+ {primaryKey}
+ </div>
+ <div className="font-semibold text-2xl tracking-tight">
+ {fmt(primary.current)}
+ </div>
+ {primary.compare != null && (
+ <div className="text-muted-foreground text-xs">
+ vs {fmt(primary.compare)}
+ </div>
+ )}
+ </div>
+ <DeltaChip
+ size="sm"
+ variant={
+ direction === 'up' ? 'inc' : direction === 'down' ? 'dec' : 'default'
+ }
+ >
+ {deltaText}
+ </DeltaChip>
+ </div>
+ )}
+
+ {report && (
+ <div className="rounded-md border p-3">
+ <ReportChart
+ lazy={false}
+ options={{ hideLegend: true, minHeight: 160, maxHeight: 220 }}
+ report={report}
+ />
+ </div>
+ )}
+
+ <div>
+ <div className="mb-2 flex items-center gap-2 font-medium text-sm">
+ <SparklesIcon className="size-4" /> Why did this happen?
+ </div>
+
+ {mutation.isPending && (
+ <p className="text-muted-foreground text-sm">Analyzing…</p>
+ )}
+
+ {explanation && (
+ <div className="flex flex-col gap-3 text-sm leading-normal">
+ <p>{explanation.summary}</p>
+ {explanation.drivers.length > 0 && (
+ <ul className="flex flex-col gap-2">
+ {explanation.drivers.map((d) => (
+ <li key={d.label}>
+ <span className="font-medium">{d.label}:</span>{' '}
+ <span className="text-muted-foreground">{d.detail}</span>
+ </li>
+ ))}
+ </ul>
+ )}
+ {explanation.relatedReference && (
+ <p className="text-muted-foreground">
+ Possibly related: {explanation.relatedReference}
+ </p>
+ )}
+ <p className="text-[10px] text-muted-foreground uppercase tracking-wide">
+ Confidence: {explanation.confidence}
+ </p>
+ </div>
+ )}
+
+ {!mutation.isPending && mutation.isSuccess && !explanation && (
+ <p className="text-muted-foreground text-sm">
+ Couldn't generate an explanation for this one.
+ </p>
+ )}
+ </div>
+ </SheetContent>
+ );
+}
diff --git a/apps/start/src/routes/_app.$organizationId.$projectId.insights.tsx b/apps/start/src/routes/_app.$organizationId.$projectId.insights.tsx
index a366dfb81..8ea5fda3a 100644
--- a/apps/start/src/routes/_app.$organizationId.$projectId.insights.tsx
+++ b/apps/start/src/routes/_app.$organizationId.$projectId.insights.tsx
@@ -44,6 +44,7 @@ export const Route = createFileRoute(
});
type SortOption =
+ | 'relevance'
| 'impact-desc'
| 'impact-asc'
| 'severity-desc'
@@ -105,12 +106,13 @@ function Component() {
const [sortBy, setSortBy] = useQueryState(
'sort',
parseAsStringEnum<SortOption>([
+ 'relevance',
'impact-desc',
'impact-asc',
'severity-desc',
'severity-asc',
'recent',
- ]).withDefault('impact-desc'),
+ ]).withDefault('relevance'),
);
const filteredAndSorted = useMemo(() => {
@@ -124,10 +126,18 @@ function Component() {
const matchesSummary = insight.summary
?.toLowerCase()
.includes(searchLower);
+ const matchesAiSummary = insight.aiSummary
+ ?.toLowerCase()
+ .includes(searchLower);
const matchesDimension = insight.dimensionKey
.toLowerCase()
.includes(searchLower);
- if (!matchesTitle && !matchesSummary && !matchesDimension) {
+ if (
+ !matchesTitle &&
+ !matchesSummary &&
+ !matchesAiSummary &&
+ !matchesDimension
+ ) {
return false;
}
}
@@ -166,6 +176,12 @@ function Component() {
// Sort (create new array to avoid mutation)
const sorted = [...filtered].sort((a, b) => {
switch (sortBy) {
+ case 'relevance':
+ // AI relevance first (un-enriched sort last), impact as tiebreaker.
+ return (
+ (b.relevanceScore ?? -1) - (a.relevanceScore ?? -1) ||
+ (b.impactScore ?? 0) - (a.impactScore ?? 0)
+ );
case 'impact-desc':
return (b.impactScore ?? 0) - (a.impactScore ?? 0);
case 'impact-asc':
@@ -342,6 +358,7 @@ function Component() {
<SelectValue placeholder="Sort by" />
</SelectTrigger>
<SelectContent>
+ <SelectItem value="relevance">Relevance (AI)</SelectItem>
<SelectItem value="impact-desc">Impact (High → Low)</SelectItem>
<SelectItem value="impact-asc">Impact (Low → High)</SelectItem>
<SelectItem value="severity-desc">Severity (High → Low)</SelectItem>
diff --git a/apps/start/src/routes/_app.$organizationId.account._tabs.email-preferences.tsx b/apps/start/src/routes/_app.$organizationId.account._tabs.email-preferences.tsx
index 9de9f95a4..dfe6eb782 100644
--- a/apps/start/src/routes/_app.$organizationId.account._tabs.email-preferences.tsx
+++ b/apps/start/src/routes/_app.$organizationId.account._tabs.email-preferences.tsx
@@ -93,31 +93,30 @@ function Component() {
</p>
<div className="space-y-4">
- {Object.entries(emailCategories).map(([category, label]) => (
- <Controller
- control={control}
- key={category}
- name={`categories.${category}`}
- render={({ field }) => (
- <div className="flex items-center justify-between gap-4 rounded-md border border-border px-4 py-4 transition-colors hover:bg-def-200">
- <div className="flex-1">
- <div className="font-medium">{label}</div>
- <div className="text-muted-foreground text-sm">
- {category === 'onboarding' &&
- 'Get started tips and guidance emails'}
- {category === 'billing' &&
- 'Subscription updates and payment reminders'}
+ {Object.entries(emailCategories).map(
+ ([category, { label, description }]) => (
+ <Controller
+ control={control}
+ key={category}
+ name={`categories.${category}`}
+ render={({ field }) => (
+ <div className="flex items-center justify-between gap-4 rounded-md border border-border px-4 py-4 transition-colors hover:bg-def-200">
+ <div className="flex-1">
+ <div className="font-medium">{label}</div>
+ <div className="text-muted-foreground text-sm">
+ {description}
+ </div>
</div>
+ <Switch
+ checked={field.value}
+ disabled={mutation.isPending}
+ onCheckedChange={field.onChange}
+ />
</div>
- <Switch
- checked={field.value}
- disabled={mutation.isPending}
- onCheckedChange={field.onChange}
- />
- </div>
- )}
- />
- ))}
+ )}
+ />
+ ),
+ )}
</div>
<Button
diff --git a/apps/start/src/routes/unsubscribe.tsx b/apps/start/src/routes/unsubscribe.tsx
index d33be89f6..e7ddfea9f 100644
--- a/apps/start/src/routes/unsubscribe.tsx
+++ b/apps/start/src/routes/unsubscribe.tsx
@@ -48,7 +48,7 @@ function RouteComponent() {
};
const categoryName =
- emailCategories[category as keyof typeof emailCategories] || category;
+ emailCategories[category as keyof typeof emailCategories]?.label ?? category;
if (isSuccess) {
return (
diff --git a/apps/worker/package.json b/apps/worker/package.json
index f27325fe1..494824189 100644
--- a/apps/worker/package.json
+++ b/apps/worker/package.json
@@ -14,6 +14,7 @@
"@bull-board/api": "6.14.0",
"@bull-board/express": "6.14.0",
"@hyperdx/node-opentelemetry": "catalog:",
+ "@openpanel/ai": "workspace:*",
"@openpanel/common": "workspace:*",
"@openpanel/db": "workspace:*",
"@openpanel/email": "workspace:*",
diff --git a/apps/worker/src/boot-cron.ts b/apps/worker/src/boot-cron.ts
index 4150308dc..9e17b845e 100644
--- a/apps/worker/src/boot-cron.ts
+++ b/apps/worker/src/boot-cron.ts
@@ -103,6 +103,16 @@ export async function bootCron() {
type: 'sessionVacuum',
pattern: '0 4 * * *', // daily at 04:00 UTC — backstop for cleanup leaks
},
+ {
+ name: 'insightCleanup',
+ type: 'insightCleanup',
+ pattern: '30 4 * * *', // daily at 04:30 UTC — prune stale insights/events
+ },
+ {
+ name: 'weeklyDigest',
+ type: 'weeklyDigest',
+ pattern: '0 8 * * 1', // Mondays 08:00 UTC — weekly analytics digest email
+ },
];
if (process.env.SELF_HOSTED && process.env.NODE_ENV === 'production') {
diff --git a/apps/worker/src/boot-debug.ts b/apps/worker/src/boot-debug.ts
index a95c5cac1..28bb2dd1e 100644
--- a/apps/worker/src/boot-debug.ts
+++ b/apps/worker/src/boot-debug.ts
@@ -1,7 +1,15 @@
-import type { CronQueuePayload, CronQueueType } from '@openpanel/queue';
+import { db } from '@openpanel/db';
+import {
+ type CronQueuePayload,
+ type CronQueueType,
+ type InsightsQueuePayloadProject,
+ insightsQueue,
+} from '@openpanel/queue';
import type { Job } from 'bullmq';
import type { Express, Request } from 'express';
import { cronJob } from './jobs/cron';
+import { insightsProjectJob } from './jobs/insights';
+import { previewWeeklyDigestForProject } from './jobs/cron.weekly-digest';
import { logger } from './utils/logger';
const CRON_TYPES = [
@@ -20,6 +28,8 @@ const CRON_TYPES = [
'cohortRefresh',
'sessionReaper',
'sessionVacuum',
+ 'insightCleanup',
+ 'weeklyDigest',
] as const satisfies readonly CronQueueType[];
function escapeHtml(value: string) {
@@ -127,5 +137,121 @@ export function bootDebugRoutes(app: Express) {
}
});
+ // Run the full insights pipeline (engine + AI enrichment) for ONE project,
+ // bypassing the eligibility filter the daily cron applies. For local testing.
+ //
+ // Default: enqueues a real `insightsProject` job on the `insights` queue, so
+ // it shows up in BullBoard and runs through the normal worker path.
+ // `?inline=1`: runs synchronously instead and returns a result summary.
+ app.get('/debug/insights/:projectId', async (req, res) => {
+ const projectId = req.params.projectId;
+ const inline = req.query.inline === '1' || req.query.inline === 'true';
+ const date = new Date().toISOString().slice(0, 10);
+
+ if (!inline) {
+ try {
+ const job = await insightsQueue.add(
+ 'insightsProject',
+ { type: 'insightsProject', payload: { projectId, date } },
+ // Unique id so repeated test runs aren't deduped by BullMQ.
+ { jobId: `debug:${projectId}:${Date.now()}` },
+ );
+ logger.info({ projectId, jobId: job.id }, 'Enqueued insights job');
+ res.json({
+ ok: true,
+ projectId,
+ jobId: job.id,
+ message:
+ 'Enqueued on the "insights" queue — watch it in BullBoard. Add ?inline=1 to run synchronously and get a result summary.',
+ });
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ logger.error({ err: error, projectId }, 'Failed to enqueue insights');
+ res.status(500).json({ ok: false, projectId, error: message });
+ }
+ return;
+ }
+
+ logger.info({ projectId }, 'Manually running insights for project (inline)');
+
+ try {
+ await insightsProjectJob({
+ data: { type: 'insightsProject', payload: { projectId, date } },
+ } as Job<InsightsQueuePayloadProject>);
+
+ const countsByState = await db.projectInsight.groupBy({
+ by: ['state'],
+ where: { projectId },
+ _count: true,
+ });
+
+ const topActive = await db.projectInsight.findMany({
+ where: { projectId, state: 'active' },
+ orderBy: [
+ { relevanceScore: { sort: 'desc', nulls: 'last' } },
+ { impactScore: 'desc' },
+ ],
+ take: 15,
+ select: {
+ title: true,
+ aiSummary: true,
+ aiCategory: true,
+ relevanceScore: true,
+ emailWorthy: true,
+ referenceWorthy: true,
+ enrichedAt: true,
+ },
+ });
+
+ const enrichedCount = topActive.filter((i) => i.enrichedAt).length;
+
+ res.json({
+ ok: true,
+ projectId,
+ countsByState,
+ enrichedInTopActive: `${enrichedCount}/${topActive.length}`,
+ topActive,
+ });
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ logger.error({ err: error, projectId }, 'Manual insights run failed');
+ res.status(500).json({ ok: false, projectId, error: message });
+ }
+ });
+
+ // Weekly digest for ONE project, bypassing eligibility. For local testing.
+ // [email protected] → send only to that address
+ // (no ?to) → return the assembled payload without sending
+ // ?force=1 → build even with 0 visitors this week
+ app.get('/debug/weekly-digest/:projectId', async (req, res) => {
+ const projectId = req.params.projectId;
+ const to = typeof req.query.to === 'string' ? req.query.to : undefined;
+ const force = req.query.force === '1' || req.query.force === 'true';
+ logger.info({ projectId, to, force }, 'Manual weekly digest');
+
+ try {
+ const result = await previewWeeklyDigestForProject(projectId, {
+ to,
+ force,
+ });
+ res.json({
+ ok: true,
+ ...result,
+ hint: to
+ ? `Sent to ${to} (needs RESEND_API_KEY/SMTP_HOST, else logged to worker console).`
+ : 'Preview only — add [email protected] to send. ?force=1 builds even with no traffic this week.',
+ });
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ logger.error({ err: error, projectId }, 'Manual weekly digest failed');
+ res.status(500).json({ ok: false, projectId, error: message });
+ }
+ });
+
logger.info('Debug cron endpoint enabled: /debug/cron/:type');
+ logger.info('Debug insights endpoint enabled: /debug/insights/:projectId');
+ logger.info(
+ 'Debug weekly digest enabled: /debug/weekly-digest/:projectId?to=&force=1',
+ );
}
+
diff --git a/apps/worker/src/jobs/cron.insight-cleanup.ts b/apps/worker/src/jobs/cron.insight-cleanup.ts
new file mode 100644
index 000000000..4dc1fad7d
--- /dev/null
+++ b/apps/worker/src/jobs/cron.insight-cleanup.ts
@@ -0,0 +1,90 @@
+import { db } from '@openpanel/db';
+import { logger as baseLogger } from '@/utils/logger';
+
+const logger = baseLogger.child({ job: 'insight-cleanup' });
+
+const RETENTION_DAYS = Number.parseInt(
+ process.env.INSIGHTS_RETENTION_DAYS || '90',
+ 10,
+);
+
+// Delete in chunks so the first (large) purge never holds a single long
+// transaction / lock on these tables.
+const BATCH_SIZE = 5000;
+
+async function deleteInBatches(
+ label: string,
+ runBatch: () => Promise<number>,
+): Promise<number> {
+ let total = 0;
+ // Cap iterations as a runaway backstop (BATCH_SIZE * 2000 = 10M rows).
+ for (let i = 0; i < 2000; i++) {
+ const deleted = await runBatch();
+ total += deleted;
+ if (deleted < BATCH_SIZE) break;
+ }
+ if (total > 0) {
+ logger.info({ table: label, deleted: total }, 'Deleted stale rows');
+ }
+ return total;
+}
+
+/**
+ * Keeps the insights tables bounded. They otherwise grow forever:
+ * `insight_events` is append-only (one row per detection/material change),
+ * and `project_insights` keeps a row per dimension ever seen.
+ *
+ * Retention rules (default 90 days, override via INSIGHTS_RETENTION_DAYS):
+ * - project_insights `suppressed`: deleted unconditionally. The engine no
+ * longer creates suppressed rows (below-top-N insights are deleted inline);
+ * any remaining ones are legacy leftovers nothing reads or revives.
+ * - project_insights `closed`: dropped once not seen since the cutoff.
+ * `active` rows are the live dataset and are never touched.
+ * - insight_events: drop rows older than the cutoff. Only the latest 20 per
+ * insight are ever read (the AI explain tool), so old events are dead
+ * weight. Deleting the parent insight above already cascades its events;
+ * this sweep also trims old history off still-active insights.
+ */
+export async function insightCleanupCronJob() {
+ const cutoff = new Date(Date.now() - RETENTION_DAYS * 24 * 60 * 60 * 1000);
+
+ const suppressed = await deleteInBatches(
+ 'project_insights:suppressed',
+ () => db.$executeRaw`
+ DELETE FROM "project_insights"
+ WHERE "id" IN (
+ SELECT "id" FROM "project_insights"
+ WHERE "state" = 'suppressed'
+ LIMIT ${BATCH_SIZE}
+ )`,
+ );
+
+ const closed = await deleteInBatches(
+ 'project_insights:closed',
+ () => db.$executeRaw`
+ DELETE FROM "project_insights"
+ WHERE "id" IN (
+ SELECT "id" FROM "project_insights"
+ WHERE "state" = 'closed'
+ AND "lastSeenAt" < ${cutoff}
+ LIMIT ${BATCH_SIZE}
+ )`,
+ );
+ const insights = suppressed + closed;
+
+ const events = await deleteInBatches(
+ 'insight_events',
+ () => db.$executeRaw`
+ DELETE FROM "insight_events"
+ WHERE "id" IN (
+ SELECT "id" FROM "insight_events"
+ WHERE "createdAt" < ${cutoff}
+ LIMIT ${BATCH_SIZE}
+ )`,
+ );
+
+ logger.info(
+ { retentionDays: RETENTION_DAYS, insights, events },
+ 'Insight cleanup complete',
+ );
+}
diff --git a/apps/worker/src/jobs/cron.ts b/apps/worker/src/jobs/cron.ts
index 43504b361..008d2dbd5 100644
--- a/apps/worker/src/jobs/cron.ts
+++ b/apps/worker/src/jobs/cron.ts
@@ -10,6 +10,8 @@ import type { CronQueuePayload } from '@openpanel/queue';
import type { Job } from 'bullmq';
import { cohortRefreshCronJob } from './cron.cohort-refresh';
import { jobDelete } from './cron.delete';
+import { insightCleanupCronJob } from './cron.insight-cleanup';
+import { weeklyDigestCronJob } from './cron.weekly-digest';
import { onboardingJob } from './cron.onboarding';
import { ping } from './cron.ping';
import { salt } from './cron.salt';
@@ -67,5 +69,11 @@ export async function cronJob(job: Job<CronQueuePayload>) {
case 'sessionVacuum': {
return await sessionVacuumCronJob();
}
+ case 'insightCleanup': {
+ return await insightCleanupCronJob();
+ }
+ case 'weeklyDigest': {
+ return await weeklyDigestCronJob();
+ }
}
}
diff --git a/apps/worker/src/jobs/cron.weekly-digest.ts b/apps/worker/src/jobs/cron.weekly-digest.ts
new file mode 100644
index 000000000..54f74fc48
--- /dev/null
+++ b/apps/worker/src/jobs/cron.weekly-digest.ts
@@ -0,0 +1,233 @@
+import { generateWeeklyNarrative } from '@openpanel/ai';
+import { db, getAnalyticsOverviewCore } from '@openpanel/db';
+import { type EmailData, sendEmail } from '@openpanel/email';
+import { logger as baseLogger } from '@/utils/logger';
+
+const logger = baseLogger.child({ job: 'weekly-digest' });
+
+const DAY_MS = 24 * 60 * 60 * 1000;
+const MIN_EVENTS = 10_000;
+const MAX_INSIGHTS = 5;
+
+type DigestData = EmailData<'weekly-digest'>;
+type ProjectRow = { id: string; name: string; organizationId: string };
+
+function formatCount(n: number): string {
+ return Math.round(n).toLocaleString();
+}
+
+function pctDelta(
+ current: number,
+ previous: number,
+): { delta?: string; direction: 'up' | 'down' | 'flat' } {
+ if (previous <= 0) return { direction: 'flat' };
+ const pct = ((current - previous) / previous) * 100;
+ const direction = pct > 0.5 ? 'up' : pct < -0.5 ? 'down' : 'flat';
+ const sign = pct >= 0 ? '+' : '';
+ return { delta: `${sign}${pct.toFixed(0)}%`, direction };
+}
+
+function ppDelta(
+ current: number,
+ previous: number,
+): { delta?: string; direction: 'up' | 'down' | 'flat' } {
+ const diff = current - previous;
+ const direction = diff > 0.5 ? 'up' : diff < -0.5 ? 'down' : 'flat';
+ const sign = diff >= 0 ? '+' : '';
+ return { delta: `${sign}${diff.toFixed(0)}pp`, direction };
+}
+
+function formatRange(startMs: number, endMs: number): string {
+ const opts: Intl.DateTimeFormatOptions = { month: 'short', day: 'numeric' };
+ const start = new Date(startMs).toLocaleDateString('en-US', opts);
+ const end = new Date(endMs).toLocaleDateString('en-US', {
+ ...opts,
+ year: 'numeric',
+ });
+ return `${start} – ${end}`;
+}
+
+/**
+ * Assemble the digest payload for one project (no sending). Returns `skipped`
+ * with a reason when there's nothing worth mailing — unless `force` is set.
+ */
+async function buildDigestData(
+ project: ProjectRow,
+ opts: { force?: boolean } = {},
+): Promise<{ skipped?: string; data?: DigestData }> {
+ const now = Date.now();
+ const curStart = now - 7 * DAY_MS;
+ const prevStart = now - 14 * DAY_MS;
+ const iso = (ms: number) => new Date(ms).toISOString();
+
+ const [cur, prev] = await Promise.all([
+ getAnalyticsOverviewCore({
+ projectId: project.id,
+ startDate: iso(curStart),
+ endDate: iso(now),
+ interval: 'day',
+ }),
+ getAnalyticsOverviewCore({
+ projectId: project.id,
+ startDate: iso(prevStart),
+ endDate: iso(curStart),
+ interval: 'day',
+ }),
+ ]);
+
+ const c = cur.summary;
+ const p = prev.summary;
+
+ if (!opts.force && c.unique_visitors === 0) {
+ return { skipped: 'no visitors in the last 7 days' };
+ }
+
+ const stats = [
+ {
+ label: 'visitors',
+ value: formatCount(c.unique_visitors),
+ ...pctDelta(c.unique_visitors, p.unique_visitors),
+ },
+ {
+ label: 'sessions',
+ value: formatCount(c.total_sessions),
+ ...pctDelta(c.total_sessions, p.total_sessions),
+ },
+ {
+ label: 'pageviews',
+ value: formatCount(c.total_screen_views),
+ ...pctDelta(c.total_screen_views, p.total_screen_views),
+ },
+ {
+ label: 'bounce rate',
+ value: `${Math.round(c.bounce_rate)}%`,
+ ...ppDelta(c.bounce_rate, p.bounce_rate),
+ },
+ ];
+
+ const insightRows = await db.projectInsight.findMany({
+ where: {
+ projectId: project.id,
+ state: 'active',
+ emailWorthy: true,
+ windowKind: { in: ['rolling_7d', 'rolling_30d'] },
+ },
+ orderBy: [
+ { relevanceScore: { sort: 'desc', nulls: 'last' } },
+ { impactScore: 'desc' },
+ ],
+ take: MAX_INSIGHTS,
+ select: { title: true, aiSummary: true, summary: true },
+ });
+
+ const insights = insightRows.map((i) => ({
+ title: i.aiSummary ?? i.title,
+ summary: i.summary ?? undefined,
+ }));
+
+ const dateRange = formatRange(curStart, now);
+
+ let narrative = '';
+ try {
+ narrative = await generateWeeklyNarrative({
+ projectName: project.name,
+ dateRange,
+ stats: [
+ { label: 'visitors', current: c.unique_visitors, previous: p.unique_visitors },
+ { label: 'sessions', current: c.total_sessions, previous: p.total_sessions },
+ { label: 'pageviews', current: c.total_screen_views, previous: p.total_screen_views },
+ { label: 'bounce rate', current: c.bounce_rate, previous: p.bounce_rate, unit: '%' },
+ ],
+ insights,
+ });
+ } catch (err) {
+ logger.warn({ err, projectId: project.id }, 'Narrative generation failed');
+ }
+
+ const dashboardUrl = `${process.env.DASHBOARD_URL ?? 'https://dashboard.openpanel.dev'}/${project.organizationId}/${project.id}`;
+
+ return {
+ data: {
+ projectName: project.name,
+ dashboardUrl,
+ dateRange,
+ narrative: narrative || undefined,
+ stats,
+ insights,
+ },
+ };
+}
+
+async function recipientsForOrg(organizationId: string): Promise<string[]> {
+ const members = await db.member.findMany({
+ where: { organizationId },
+ select: { email: true },
+ });
+ return [...new Set(members.map((m) => m.email).filter(Boolean))];
+}
+
+/**
+ * Weekly digest: per active project, week-over-week stats + the AI's
+ * email-worthy insights + a generated narrative, mailed to org members.
+ * `sendEmail` skips anyone unsubscribed from the `weekly_digest` category.
+ */
+export async function weeklyDigestCronJob() {
+ const projects = await db.project.findMany({
+ where: {
+ deleteAt: null,
+ eventsCount: { gt: MIN_EVENTS },
+ organization: { subscriptionStatus: 'active' },
+ },
+ select: { id: true, name: true, organizationId: true },
+ });
+
+ let sent = 0;
+ for (const project of projects) {
+ try {
+ const { skipped, data } = await buildDigestData(project);
+ if (skipped || !data) continue;
+ const emails = await recipientsForOrg(project.organizationId);
+ if (emails.length === 0) continue;
+ for (const to of emails) {
+ await sendEmail('weekly-digest', { to, data });
+ }
+ sent++;
+ } catch (err) {
+ logger.error({ err, projectId: project.id }, 'Weekly digest failed');
+ }
+ }
+
+ logger.info({ projects: projects.length, sent }, 'Weekly digest complete');
+}
+
+/**
+ * Debug/testing helper for a SINGLE project, bypassing eligibility.
+ * - opts.to → send only to that address (safe for testing)
+ * - no opts.to → assemble and return the payload without sending (preview)
+ * - opts.force → build even if the project had 0 visitors this week
+ */
+export async function previewWeeklyDigestForProject(
+ projectId: string,
+ opts: { to?: string; force?: boolean } = {},
+): Promise<{
+ sent: boolean;
+ to?: string;
+ skipped?: string;
+ data?: DigestData;
+}> {
+ const project = await db.project.findUnique({
+ where: { id: projectId },
+ select: { id: true, name: true, organizationId: true },
+ });
+ if (!project) return { sent: false, skipped: 'project not found' };
+
+ const { skipped, data } = await buildDigestData(project, { force: opts.force });
+ if (skipped || !data) return { sent: false, skipped, data };
+
+ if (opts.to) {
+ await sendEmail('weekly-digest', { to: opts.to, data });
+ return { sent: true, to: opts.to, data };
+ }
+
+ return { sent: false, data };
+}
diff --git a/apps/worker/src/jobs/insights-enrich.ts b/apps/worker/src/jobs/insights-enrich.ts
new file mode 100644
index 000000000..025681fb1
--- /dev/null
+++ b/apps/worker/src/jobs/insights-enrich.ts
@@ -0,0 +1,106 @@
+import {
+ ENRICH_VERSION,
+ type InsightToEnrich,
+ enrichInsights,
+} from '@openpanel/ai';
+import { db } from '@openpanel/db';
+import { logger as baseLogger } from '@/utils/logger';
+
+const logger = baseLogger.child({ job: 'insights-enrich' });
+
+// Modest batches keep the prompt small and the id-mapping reliable. At Tier 3
+// limits (4M TPM / 5k RPM) even the first-run backlog clears comfortably.
+const BATCH_SIZE = 25;
+
+function clamp01(n: number): number {
+ if (Number.isNaN(n)) return 0;
+ return Math.max(0, Math.min(1, n));
+}
+
+/**
+ * Tier-1 enrichment for one project's active insights. Runs after the insight
+ * engine finishes a project, scoring anything new, materially-changed, or
+ * enriched under an older prompt version. Failures are logged and skipped — a
+ * bad enrichment batch must never fail the insights job itself.
+ */
+export async function enrichProjectInsights(projectId: string): Promise<void> {
+ const stale = await db.projectInsight.findMany({
+ where: {
+ projectId,
+ state: 'active',
+ OR: [
+ { enrichedAt: null },
+ { enrichVersion: null },
+ { enrichVersion: { lt: ENRICH_VERSION } },
+ ],
+ },
+ select: {
+ id: true,
+ moduleKey: true,
+ dimensionKey: true,
+ windowKind: true,
+ title: true,
+ summary: true,
+ displayName: true,
+ direction: true,
+ impactScore: true,
+ severityBand: true,
+ payload: true,
+ },
+ });
+
+ if (stale.length === 0) return;
+
+ let enriched = 0;
+ for (let i = 0; i < stale.length; i += BATCH_SIZE) {
+ const batch = stale.slice(i, i + BATCH_SIZE);
+ const input: InsightToEnrich[] = batch.map((r) => ({
+ id: r.id,
+ moduleKey: r.moduleKey,
+ dimensionKey: r.dimensionKey,
+ windowKind: r.windowKind,
+ title: r.title,
+ summary: r.summary,
+ displayName: r.displayName,
+ direction: r.direction,
+ impactScore: r.impactScore,
+ severityBand: r.severityBand,
+ payload: r.payload,
+ }));
+
+ let results: Awaited<ReturnType<typeof enrichInsights>>;
+ try {
+ results = await enrichInsights(input);
+ } catch (err) {
+ logger.error(
+ { err, projectId, batchSize: batch.length },
+ 'Enrichment call failed; skipping batch',
+ );
+ continue;
+ }
+
+ const byId = new Map(results.map((r) => [r.id, r]));
+ for (const row of batch) {
+ const e = byId.get(row.id);
+ if (!e) continue;
+ await db.projectInsight.update({
+ where: { id: row.id },
+ data: {
+ relevanceScore: clamp01(e.relevanceScore),
+ aiSummary: e.summary,
+ aiCategory: e.category,
+ emailWorthy: e.emailWorthy,
+ referenceWorthy: e.referenceWorthy,
+ enrichedAt: new Date(),
+ enrichVersion: ENRICH_VERSION,
+ },
+ });
+ enriched++;
+ }
+ }
+
+ logger.info(
+ { projectId, candidates: stale.length, enriched },
+ 'Insight enrichment complete',
+ );
+}
diff --git a/apps/worker/src/jobs/insights.ts b/apps/worker/src/jobs/insights.ts
index a0156e0d1..66de5f28f 100644
--- a/apps/worker/src/jobs/insights.ts
+++ b/apps/worker/src/jobs/insights.ts
@@ -14,6 +14,8 @@ import type {
} from '@openpanel/queue';
import { insightsQueue } from '@openpanel/queue';
import type { Job } from 'bullmq';
+import { enrichProjectInsights } from './insights-enrich';
+import { logger } from '@/utils/logger';
const defaultEngineConfig = {
keepTopNPerModuleWindow: 20,
@@ -69,4 +71,12 @@ export async function insightsProjectJob(
now: new Date(date),
projectCreatedAt,
});
+
+ // Tier-1 AI enrichment of this project's active insights. Isolated so a
+ // provider outage or rate-limit never fails the insights computation.
+ try {
+ await enrichProjectInsights(projectId);
+ } catch (err) {
+ logger.error({ err, projectId }, 'Insight enrichment failed');
+ }
}
diff --git a/packages/ai/package.json b/packages/ai/package.json
new file mode 100644
index 000000000..22f471ead
--- /dev/null
+++ b/packages/ai/package.json
@@ -0,0 +1,20 @@
+{
+ "name": "@openpanel/ai",
+ "version": "0.0.1",
+ "type": "module",
+ "main": "src/index.ts",
+ "scripts": {
+ "typecheck": "tsc --noEmit"
+ },
+ "dependencies": {
+ "@better-agent/core": "^0.1.0-beta.3",
+ "@better-agent/providers": "^0.1.0-beta.3",
+ "@openpanel/validation": "workspace:*",
+ "zod": "catalog:"
+ },
+ "devDependencies": {
+ "@openpanel/tsconfig": "workspace:*",
+ "@types/node": "catalog:",
+ "typescript": "catalog:"
+ }
+}
diff --git a/packages/ai/src/enrich.ts b/packages/ai/src/enrich.ts
new file mode 100644
index 000000000..513ea3a59
--- /dev/null
+++ b/packages/ai/src/enrich.ts
@@ -0,0 +1,125 @@
+// Tier-1 insight enrichment: a one-shot OpenAI call that scores and summarizes
+// auto-detected analytics insights so the product can filter noise, select
+// email-worthy items, and flag chart-reference candidates. No persistence, no
+// tools — pure transform. The worker queries stale insights, calls this, and
+// writes the results back (this module stays db-free).
+import { betterAgent, defineAgent } from '@better-agent/core';
+import { z } from 'zod';
+import { ALLOWED_MODELS, resolveModel } from './providers';
+
+// Bump when the prompt or output shape changes — the worker re-enriches any
+// insight whose stored enrichVersion is below this.
+export const ENRICH_VERSION = 1;
+
+// Tier-1 is cheap, fast, non-reasoning work — gpt-4.1-mini is the right tier.
+const ENRICH_MODEL_ID = 'gpt-4-1-mini';
+
+function enrichModel() {
+ const entry =
+ ALLOWED_MODELS.find((m) => m.id === ENRICH_MODEL_ID) ??
+ ALLOWED_MODELS.find((m) => m.group === 'OpenAI');
+ if (!entry) {
+ throw new Error('No OpenAI model available for insight enrichment');
+ }
+ return resolveModel(entry);
+}
+
+export interface InsightToEnrich {
+ id: string;
+ moduleKey: string;
+ dimensionKey: string;
+ windowKind: string;
+ title: string;
+ summary: string | null;
+ displayName: string;
+ direction: string | null;
+ impactScore: number;
+ severityBand: string | null;
+ payload: unknown;
+}
+
+export type InsightCategory = 'spike' | 'drop' | 'shift' | 'anomaly' | 'trend';
+
+export interface InsightEnrichment {
+ id: string;
+ relevanceScore: number;
+ summary: string;
+ category: InsightCategory;
+ emailWorthy: boolean;
+ referenceWorthy: boolean;
+}
+
+const enrichmentOutputSchema = z.object({
+ results: z.array(
+ z.object({
+ id: z.string(),
+ // No min/max here — OpenAI strict structured outputs reject numeric
+ // constraints; the caller clamps to 0..1.
+ relevanceScore: z.number(),
+ summary: z.string(),
+ category: z.enum(['spike', 'drop', 'shift', 'anomaly', 'trend']),
+ emailWorthy: z.boolean(),
+ referenceWorthy: z.boolean(),
+ }),
+ ),
+});
+
+const enrichmentOutputJsonSchema = z.toJSONSchema(enrichmentOutputSchema, {
+ target: 'draft-07',
+});
+
+const INSTRUCTION = `You are an analytics insight editor for OpenPanel, a web/product analytics platform.
+
+You receive a JSON array of automatically-detected insights about ONE project's traffic. Each has a moduleKey (referrers, entry-pages, page-trends, geo, devices), a dimension, a time window, a statistically-derived title/summary, a direction, an impactScore, and a payload of metrics.
+
+For EACH input insight, produce:
+- relevanceScore (number 0..1): how much a busy site owner should care RIGHT NOW. Be a HARSH filter — most auto-detected insights are noise. Reserve >0.7 for genuinely notable, actionable changes. Tiny pages, routine wiggles, and statistically-significant-but-boring shifts should score below 0.3.
+- summary (string): one plain-language sentence a non-analyst understands — what changed and why it might matter. No jargon, no "share shifted 0.5pp".
+- category: one of spike | drop | shift | anomaly | trend.
+- emailWorthy (boolean): true ONLY if it belongs in a weekly digest — a clear, interesting weekly-scale change. Usually false.
+- referenceWorthy (boolean): true ONLY if it is a discrete, datable event worth pinning to a chart timeline (e.g. a sudden spike on a specific day), NOT a slow rolling trend. Usually false.
+
+Return exactly one result per input insight, preserving each "id" verbatim. Be calibrated and consistent across the batch.`;
+
+let _app: ReturnType<typeof betterAgent> | null = null;
+function getApp() {
+ if (_app) return _app;
+ const agent = defineAgent({
+ name: 'insight-enrich',
+ description:
+ 'Scores and summarizes analytics insights (one-shot, no persistence).',
+ model: enrichModel(),
+ contextSchema: z.object({}),
+ outputSchema: {
+ schema: enrichmentOutputJsonSchema,
+ name: 'insight_enrichment',
+ strict: true,
+ },
+ instruction: () => INSTRUCTION,
+ tools: () => [],
+ maxSteps: 1,
+ // biome-ignore lint/suspicious/noExplicitAny: same dodge as filter-command.ts
+ } as any);
+ _app = betterAgent({ agents: [agent] });
+ return _app;
+}
+
+/**
+ * Enrich a batch of insights in a single model call. Returns one entry per
+ * insight the model scored (caller maps back by id; missing ids are skipped).
+ * Keep batches modest (~25) so the prompt stays small and the mapping reliable.
+ */
+export async function enrichInsights(
+ insights: InsightToEnrich[],
+): Promise<InsightEnrichment[]> {
+ if (insights.length === 0) return [];
+
+ const input = JSON.stringify(insights);
+ const result = (await getApp().run('insight-enrich', {
+ input,
+ context: {},
+ // biome-ignore lint/suspicious/noExplicitAny: same dodge as filter-command.ts
+ } as any)) as { structured?: { results: InsightEnrichment[] } };
+
+ return result.structured?.results ?? [];
+}
diff --git a/packages/ai/src/explain.ts b/packages/ai/src/explain.ts
new file mode 100644
index 000000000..f37c57c61
--- /dev/null
+++ b/packages/ai/src/explain.ts
@@ -0,0 +1,111 @@
+// Phase 5 — the "why". Given an insight plus a deterministic decomposition of
+// the change (current vs baseline breakdowns across referrer/country/device/
+// utm) and any nearby references, produce an honest explanation: which
+// sub-segment drove the delta, and which off-platform event (reference) might
+// correlate. It explains the *internal decomposition*, not invented external
+// causes. The caller gathers the data (DB/CH); this module just narrates.
+import { betterAgent, defineAgent } from '@better-agent/core';
+import { z } from 'zod';
+import { ALLOWED_MODELS, resolveModel } from './providers';
+
+// gpt-4.1 (the larger non-reasoning model) synthesizes the supplied breakdown
+// well without reasoning-option plumbing. Swap to a reasoning model later if
+// deeper inference is wanted.
+const EXPLAIN_MODEL_ID = 'gpt-4-1';
+
+function explainModel() {
+ const entry =
+ ALLOWED_MODELS.find((m) => m.id === EXPLAIN_MODEL_ID) ??
+ ALLOWED_MODELS.find((m) => m.group === 'OpenAI');
+ if (!entry) {
+ throw new Error('No OpenAI model available for insight explanation');
+ }
+ return resolveModel(entry);
+}
+
+export interface BreakdownComparison {
+ column: string;
+ current: { name: string | null; sessions: number }[];
+ baseline: { name: string | null; sessions: number }[];
+}
+
+export interface ExplainInsightInput {
+ insight: {
+ title: string;
+ dimension: string;
+ window: string;
+ summary?: string;
+ };
+ breakdowns: BreakdownComparison[];
+ references: { title: string; date: string }[];
+}
+
+export interface InsightExplanation {
+ summary: string;
+ drivers: { label: string; detail: string }[];
+ relatedReference: string;
+ confidence: 'low' | 'medium' | 'high';
+}
+
+const explanationSchema = z.object({
+ summary: z.string(),
+ drivers: z.array(
+ z.object({
+ label: z.string(),
+ detail: z.string(),
+ }),
+ ),
+ // '' when nothing correlates — kept required for OpenAI strict mode.
+ relatedReference: z.string(),
+ confidence: z.enum(['low', 'medium', 'high']),
+});
+
+const explanationJsonSchema = z.toJSONSchema(explanationSchema, {
+ target: 'draft-07',
+});
+
+const INSTRUCTION = `You explain WHY an analytics metric changed for a website/product owner.
+
+You receive: the insight (what changed), a decomposition of the change across dimensions (referrer, country, device, utm_source) as current-window vs baseline-window session breakdowns, and any manual references (off-platform events the owner logged) near the window.
+
+Produce:
+- summary: 1-2 plain sentences answering "why did this happen", grounded in the decomposition. Name the sub-segment(s) that account for most of the change (e.g. "most of the lift came from reddit.com referrals").
+- drivers: the concrete contributors, each a short label + a one-line detail with the numbers.
+- relatedReference: the title of a reference that plausibly explains the change, or "" if none fits. Do not force a connection.
+- confidence: low | medium | high — how clearly the decomposition explains the change.
+
+Be honest and precise. You can only see what's in the data: explain the internal decomposition (which segment moved), not external causes you can't observe. If the breakdown doesn't clearly explain it, say so and set confidence low. Never invent numbers.`;
+
+let _app: ReturnType<typeof betterAgent> | null = null;
+function getApp() {
+ if (_app) return _app;
+ const agent = defineAgent({
+ name: 'insight-explain',
+ description: 'Explains why an insight changed (one-shot).',
+ model: explainModel(),
+ contextSchema: z.object({}),
+ outputSchema: {
+ schema: explanationJsonSchema,
+ name: 'insight_explanation',
+ strict: true,
+ },
+ instruction: () => INSTRUCTION,
+ tools: () => [],
+ maxSteps: 1,
+ // biome-ignore lint/suspicious/noExplicitAny: same dodge as enrich.ts
+ } as any);
+ _app = betterAgent({ agents: [agent] });
+ return _app;
+}
+
+export async function generateInsightExplanation(
+ input: ExplainInsightInput,
+): Promise<InsightExplanation | null> {
+ const result = (await getApp().run('insight-explain', {
+ input: JSON.stringify(input),
+ context: {},
+ // biome-ignore lint/suspicious/noExplicitAny: same dodge as enrich.ts
+ } as any)) as { structured?: InsightExplanation };
+
+ return result.structured ?? null;
+}
diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts
new file mode 100644
index 000000000..3100396c1
--- /dev/null
+++ b/packages/ai/src/index.ts
@@ -0,0 +1,4 @@
+export * from './providers';
+export * from './enrich';
+export * from './narrative';
+export * from './explain';
diff --git a/packages/ai/src/narrative.ts b/packages/ai/src/narrative.ts
new file mode 100644
index 000000000..9ec1e0c26
--- /dev/null
+++ b/packages/ai/src/narrative.ts
@@ -0,0 +1,81 @@
+// Weekly-digest narrative: one short OpenAI call that turns a project's
+// week-over-week stats + notable insights into a friendly intro paragraph.
+// Structured output (a single `narrative` string) reuses the same reliable
+// pattern as enrich.ts / filter-command.ts.
+import { betterAgent, defineAgent } from '@better-agent/core';
+import { z } from 'zod';
+import { ALLOWED_MODELS, resolveModel } from './providers';
+
+const NARRATIVE_MODEL_ID = 'gpt-4-1-mini';
+
+function narrativeModel() {
+ const entry =
+ ALLOWED_MODELS.find((m) => m.id === NARRATIVE_MODEL_ID) ??
+ ALLOWED_MODELS.find((m) => m.group === 'OpenAI');
+ if (!entry) {
+ throw new Error('No OpenAI model available for weekly narrative');
+ }
+ return resolveModel(entry);
+}
+
+export interface WeeklyNarrativeInput {
+ projectName: string;
+ dateRange: string;
+ stats: {
+ label: string;
+ current: number;
+ previous: number;
+ unit?: string;
+ }[];
+ insights: { title: string; summary?: string }[];
+}
+
+const narrativeOutputSchema = z.object({ narrative: z.string() });
+const narrativeOutputJsonSchema = z.toJSONSchema(narrativeOutputSchema, {
+ target: 'draft-07',
+});
+
+const INSTRUCTION = `You write the intro paragraph for a weekly analytics email digest sent to a website/product owner.
+
+You receive the project's week-over-week stats and a few notable insights (already filtered as email-worthy). Write 2-4 short, friendly, plain-language sentences summarizing how the week went and what stood out.
+
+Rules:
+- No greeting and no sign-off (those are added around your text).
+- No bullet lists, no markdown, no headings — just prose.
+- Be specific about the numbers that actually matter; do NOT restate every stat.
+- If a notable insight explains a change, connect them.
+- If the week was quiet, say so plainly and briefly. Never invent data.`;
+
+let _app: ReturnType<typeof betterAgent> | null = null;
+function getApp() {
+ if (_app) return _app;
+ const agent = defineAgent({
+ name: 'weekly-narrative',
+ description: 'Writes a weekly digest intro paragraph (one-shot).',
+ model: narrativeModel(),
+ contextSchema: z.object({}),
+ outputSchema: {
+ schema: narrativeOutputJsonSchema,
+ name: 'weekly_narrative',
+ strict: true,
+ },
+ instruction: () => INSTRUCTION,
+ tools: () => [],
+ maxSteps: 1,
+ // biome-ignore lint/suspicious/noExplicitAny: same dodge as enrich.ts
+ } as any);
+ _app = betterAgent({ agents: [agent] });
+ return _app;
+}
+
+export async function generateWeeklyNarrative(
+ input: WeeklyNarrativeInput,
+): Promise<string> {
+ const result = (await getApp().run('weekly-narrative', {
+ input: JSON.stringify(input),
+ context: {},
+ // biome-ignore lint/suspicious/noExplicitAny: same dodge as enrich.ts
+ } as any)) as { structured?: { narrative: string } };
+
+ return result.structured?.narrative ?? '';
+}
diff --git a/packages/trpc/src/agents/models.ts b/packages/ai/src/providers.ts
similarity index 91%
rename from packages/trpc/src/agents/models.ts
rename to packages/ai/src/providers.ts
index 6916db656..6b36ec983 100644
--- a/packages/trpc/src/agents/models.ts
+++ b/packages/ai/src/providers.ts
@@ -1,3 +1,7 @@
+// SERVER-ONLY. This module instantiates `@better-agent/providers` clients and
+// reads provider API keys from `process.env`. Never value-import it from the
+// browser (apps/start) — import the model catalog/types from
+// `@openpanel/validation` instead.
import { createAnthropic } from '@better-agent/providers/anthropic';
import { createOpenAI } from '@better-agent/providers/openai';
import type { ChatModelEntry } from '@openpanel/validation';
diff --git a/packages/ai/tsconfig.json b/packages/ai/tsconfig.json
new file mode 100644
index 000000000..a4a54fb86
--- /dev/null
+++ b/packages/ai/tsconfig.json
@@ -0,0 +1,13 @@
+{
+ "extends": "@openpanel/tsconfig/base.json",
+ "compilerOptions": {
+ "baseUrl": ".",
+ "paths": {
+ "@/*": ["./src/*"]
+ },
+ "tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json",
+ "strictNullChecks": true
+ },
+ "include": ["."],
+ "exclude": ["node_modules"]
+}
diff --git a/packages/constants/index.ts b/packages/constants/index.ts
index a7c2820fb..4a186c8d0 100644
--- a/packages/constants/index.ts
+++ b/packages/constants/index.ts
@@ -595,8 +595,17 @@ export function getCountry(code?: string) {
return countries[code as keyof typeof countries];
}
+// Keys must match the template `category` in @openpanel/email. Each entry's
+// label + description drive the account email-preferences toggles.
export const emailCategories = {
- onboarding: 'Onboarding',
+ onboarding: {
+ label: 'Onboarding',
+ description: 'Get started tips and guidance emails',
+ },
+ weekly_digest: {
+ label: 'Weekly digest',
+ description: 'A weekly summary of your analytics with AI-surfaced insights',
+ },
} as const;
export type EmailCategory = keyof typeof emailCategories;
diff --git a/packages/db/prisma/migrations/20260615075425_insight_ai_enrichment/migration.sql b/packages/db/prisma/migrations/20260615075425_insight_ai_enrichment/migration.sql
new file mode 100644
index 000000000..57bc3ecb6
--- /dev/null
+++ b/packages/db/prisma/migrations/20260615075425_insight_ai_enrichment/migration.sql
@@ -0,0 +1,11 @@
+-- AlterTable
+ALTER TABLE "public"."project_insights" ADD COLUMN "aiCategory" TEXT,
+ADD COLUMN "aiSummary" TEXT,
+ADD COLUMN "emailWorthy" BOOLEAN NOT NULL DEFAULT false,
+ADD COLUMN "enrichVersion" INTEGER,
+ADD COLUMN "enrichedAt" TIMESTAMP(3),
+ADD COLUMN "referenceWorthy" BOOLEAN NOT NULL DEFAULT false,
+ADD COLUMN "relevanceScore" DOUBLE PRECISION;
+
+-- CreateIndex
+CREATE INDEX "project_insights_projectId_state_relevanceScore_idx" ON "public"."project_insights"("projectId", "state", "relevanceScore" DESC);
diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma
index a0c996116..8e9fe979d 100644
--- a/packages/db/prisma/schema.prisma
+++ b/packages/db/prisma/schema.prisma
@@ -653,6 +653,16 @@ model ProjectInsight {
impactScore Float @default(0)
severityBand String? // "low" | "moderate" | "severe"
+ // AI enrichment (Tier-1, written by the OpenAI enrichment job after upsert).
+ // Null until enriched; the dashboard sorts active insights by relevanceScore.
+ relevanceScore Float? // 0..1 — is this worth a human's attention right now
+ aiSummary String? // plain-language one-liner derived from the payload
+ aiCategory String? // "spike" | "drop" | "shift" | "anomaly" | "trend"
+ emailWorthy Boolean @default(false) // include in weekly digest selection
+ referenceWorthy Boolean @default(false) // discrete event worth a chart annotation
+ enrichedAt DateTime?
+ enrichVersion Int? // bump to force re-enrichment when prompt/schema changes
+
version Int @default(1)
threadId String @default(dbgenerated("gen_random_uuid()")) @db.Uuid
@@ -668,6 +678,7 @@ model ProjectInsight {
@@unique([projectId, moduleKey, dimensionKey, windowKind, state])
@@index([projectId, impactScore(sort: Desc)])
@@index([projectId, moduleKey, windowKind, state])
+ @@index([projectId, state, relevanceScore(sort: Desc)])
@@map("project_insights")
}
diff --git a/packages/db/src/services/insights/modules/page-trends.module.ts b/packages/db/src/services/insights/modules/page-trends.module.ts
index 3da83cef6..483630076 100644
--- a/packages/db/src/services/insights/modules/page-trends.module.ts
+++ b/packages/db/src/services/insights/modules/page-trends.module.ts
@@ -165,10 +165,13 @@ export const pageTrendsModule: InsightModule = {
key: 'page-trends',
cadence: ['daily'],
// Share-based thresholds (values in basis points: 100 = 1%)
- // minTotal: require at least 0.5% combined share (current + baseline)
- // minAbsDelta: require at least 0.5 percentage point shift
- // minPct: require at least 25% relative change in share
- thresholds: { minTotal: 50, minAbsDelta: 50, minPct: 0.25, maxDims: 100 },
+ // Tightened to cut noise: page-trends was ~85% of all insight rows because
+ // a 0.5pp share wiggle on up to 100 pages/run crossed the bar.
+ // minTotal: require at least 1% combined share (current + baseline)
+ // minAbsDelta: require at least 1.0 percentage point shift
+ // minPct: require at least 35% relative change in share
+ // maxDims: evaluate at most 30 pages/run (was 100)
+ thresholds: { minTotal: 100, minAbsDelta: 100, minPct: 0.35, maxDims: 30 },
async enumerateDimensions(ctx) {
const { currentMap, baselineMap } = await fetchPageTrendAggregates(ctx);
diff --git a/packages/db/src/services/insights/store.ts b/packages/db/src/services/insights/store.ts
index 9e05316e9..8f57787aa 100644
--- a/packages/db/src/services/insights/store.ts
+++ b/packages/db/src/services/insights/store.ts
@@ -118,6 +118,10 @@ export const insightStore: InsightStore = {
data: {
...baseData,
threadId: existing.threadId, // Preserve threadId
+ // Materially-changed insights need re-enrichment; clearing enrichedAt
+ // re-queues them for the AI pass. Keep the old score/summary as a
+ // fallback until then (don't null those).
+ ...(decision.material ? { enrichedAt: null } : {}),
},
});
} else {
@@ -141,6 +145,8 @@ export const insightStore: InsightStore = {
...baseData,
state: 'active',
threadId: closed.threadId, // Preserve threadId
+ // Reopening is a material event — re-enrich it.
+ enrichedAt: null,
},
});
} else {
@@ -231,113 +237,66 @@ export const insightStore: InsightStore = {
windowKind,
keepTopN,
now,
- }): Promise<{ suppressed: number; unsuppressed: number }> {
- // Get all active insights for this module/window, ordered by impactScore desc
+ }): Promise<{ deleted: number }> {
+ // Below-top-N insights are DELETED rather than persisted as `suppressed`
+ // rows. Nothing reads suppressed insights (every query filters state=active),
+ // yet they were ~75% of the table. A dimension that later climbs back into
+ // the top-N is simply re-created on the next run — it loses thread
+ // continuity, which is cosmetic and not surfaced anywhere.
const insights = await db.projectInsight.findMany({
where: {
projectId,
moduleKey,
windowKind,
- state: { in: ['active', 'suppressed'] },
+ state: 'active',
},
orderBy: { impactScore: 'desc' },
});
if (insights.length === 0) {
- return { suppressed: 0, unsuppressed: 0 };
+ return { deleted: 0 };
}
- let suppressed = 0;
- let unsuppressed = 0;
+ const toDelete: string[] = [];
- // For "yesterday" insights, suppress any that are stale (windowEnd is not actually yesterday)
- // This prevents showing confusing insights like "Yesterday traffic dropped" when it's from 2+ days ago
if (windowKind === 'yesterday') {
+ // Drop stale "yesterday" insights whose windowEnd isn't actually
+ // yesterday (prevents showing "Yesterday traffic dropped" from 2+ days
+ // ago), then apply top-N to the fresh remainder.
const yesterday = new Date(now);
yesterday.setUTCHours(0, 0, 0, 0);
yesterday.setUTCDate(yesterday.getUTCDate() - 1);
const yesterdayTime = yesterday.getTime();
+ const fresh: typeof insights = [];
for (const insight of insights) {
- // If windowEnd is null, consider it stale
const isStale = insight.windowEnd
? new Date(insight.windowEnd).setUTCHours(0, 0, 0, 0) !==
yesterdayTime
: true;
-
- if (isStale && insight.state === 'active') {
- await db.projectInsight.update({
- where: { id: insight.id },
- data: { state: 'suppressed', lastUpdatedAt: now },
- });
- suppressed++;
- }
- }
-
- // Filter to only non-stale insights for top-N logic
- const freshInsights = insights.filter((insight) => {
- if (!insight.windowEnd) return false;
- const windowEndTime = new Date(insight.windowEnd).setUTCHours(
- 0,
- 0,
- 0,
- 0,
- );
- return windowEndTime === yesterdayTime;
- });
-
- const topN = freshInsights.slice(0, keepTopN);
- const belowN = freshInsights.slice(keepTopN);
-
- for (const insight of belowN) {
- if (insight.state === 'active') {
- await db.projectInsight.update({
- where: { id: insight.id },
- data: { state: 'suppressed', lastUpdatedAt: now },
- });
- suppressed++;
+ if (isStale) {
+ toDelete.push(insight.id);
+ } else {
+ fresh.push(insight);
}
}
-
- for (const insight of topN) {
- if (insight.state === 'suppressed') {
- await db.projectInsight.update({
- where: { id: insight.id },
- data: { state: 'active', lastUpdatedAt: now },
- });
- unsuppressed++;
- }
+ // `fresh` preserves the impactScore-desc order; everything past keepTopN goes.
+ for (const insight of fresh.slice(keepTopN)) {
+ toDelete.push(insight.id);
}
-
- return { suppressed, unsuppressed };
- }
-
- // For non-yesterday windows, apply standard top-N suppression
- const topN = insights.slice(0, keepTopN);
- const belowN = insights.slice(keepTopN);
-
- // Suppress those below top N
- for (const insight of belowN) {
- if (insight.state === 'active') {
- await db.projectInsight.update({
- where: { id: insight.id },
- data: { state: 'suppressed', lastUpdatedAt: now },
- });
- suppressed++;
+ } else {
+ for (const insight of insights.slice(keepTopN)) {
+ toDelete.push(insight.id);
}
}
- // Unsuppress those in top N
- for (const insight of topN) {
- if (insight.state === 'suppressed') {
- await db.projectInsight.update({
- where: { id: insight.id },
- data: { state: 'active', lastUpdatedAt: now },
- });
- unsuppressed++;
- }
+ if (toDelete.length === 0) {
+ return { deleted: 0 };
}
- return { suppressed, unsuppressed };
+ const result = await db.projectInsight.deleteMany({
+ where: { id: { in: toDelete } },
+ });
+ return { deleted: result.count };
},
};
diff --git a/packages/db/src/services/insights/types.ts b/packages/db/src/services/insights/types.ts
index 91728fc71..7e6cc1f7e 100644
--- a/packages/db/src/services/insights/types.ts
+++ b/packages/db/src/services/insights/types.ts
@@ -180,12 +180,12 @@ export interface InsightStore {
now: Date;
staleDays: number; // close if not seen for X days
}): Promise<number>; // count closed
- /** Enforce top-N display by suppressing below-threshold insights. */
+ /** Enforce top-N display by deleting below-threshold insights. */
applySuppression(args: {
projectId: string;
moduleKey: string;
windowKind: WindowKind;
keepTopN: number;
now: Date;
- }): Promise<{ suppressed: number; unsuppressed: number }>;
+ }): Promise<{ deleted: number }>;
}
diff --git a/packages/email/src/emails/index.tsx b/packages/email/src/emails/index.tsx
index 34fec96ff..cba787a80 100644
--- a/packages/email/src/emails/index.tsx
+++ b/packages/email/src/emails/index.tsx
@@ -19,6 +19,7 @@ import OnboardingWelcome, { zOnboardingWelcome } from './onboarding-welcome';
import OnboardingWhatToTrack, {
zOnboardingWhatToTrack,
} from './onboarding-what-to-track';
+import WeeklyDigest, { zWeeklyDigest } from './weekly-digest';
export const templates = {
invite: {
@@ -72,6 +73,13 @@ export const templates = {
Component: OnboardingTrialEnded,
schema: zOnboardingTrialEnded,
},
+ 'weekly-digest': {
+ subject: (data: z.infer<typeof zWeeklyDigest>) =>
+ `Your week on ${data.projectName}`,
+ Component: WeeklyDigest,
+ schema: zWeeklyDigest,
+ category: 'weekly_digest' as const,
+ },
} as const;
export type Templates = typeof templates;
diff --git a/packages/email/src/emails/weekly-digest.tsx b/packages/email/src/emails/weekly-digest.tsx
new file mode 100644
index 000000000..6fa22242c
--- /dev/null
+++ b/packages/email/src/emails/weekly-digest.tsx
@@ -0,0 +1,134 @@
+import { Heading, Hr, Link, Section, Text } from '@react-email/components';
+import React from 'react';
+import { z } from 'zod';
+import { Layout } from '../components/layout';
+import { List } from '../components/list';
+
+export const zWeeklyDigest = z.object({
+ projectName: z.string(),
+ dashboardUrl: z.string(),
+ dateRange: z.string(),
+ narrative: z.string().optional(),
+ stats: z.array(
+ z.object({
+ label: z.string(),
+ value: z.string(),
+ delta: z.string().optional(),
+ direction: z.enum(['up', 'down', 'flat']).optional(),
+ }),
+ ),
+ insights: z.array(
+ z.object({
+ title: z.string(),
+ summary: z.string().optional(),
+ }),
+ ),
+});
+
+export type Props = z.infer<typeof zWeeklyDigest>;
+export default WeeklyDigest;
+
+function deltaColor(direction?: 'up' | 'down' | 'flat'): string {
+ if (direction === 'up') return '#16a34a';
+ if (direction === 'down') return '#dc2626';
+ return '#6b7280';
+}
+
+export function WeeklyDigest({
+ projectName,
+ dashboardUrl = 'https://dashboard.openpanel.dev',
+ dateRange,
+ narrative,
+ stats,
+ insights,
+ unsubscribeUrl,
+}: Props & { unsubscribeUrl?: string }) {
+ return (
+ <Layout unsubscribeUrl={unsubscribeUrl}>
+ <Heading as="h2" className="text-xl font-semibold">
+ Your week on {projectName}
+ </Heading>
+ <Text className="text-sm text-gray-500">{dateRange}</Text>
+
+ {narrative && <Text>{narrative}</Text>}
+
+ {stats.length > 0 && (
+ <Section className="my-4">
+ {stats.map((s) => (
+ <Text key={s.label} className="my-1">
+ <strong>{s.value}</strong> {s.label}
+ {s.delta && (
+ <span style={{ color: deltaColor(s.direction) }}>
+ {' '}
+ ({s.delta})
+ </span>
+ )}
+ </Text>
+ ))}
+ </Section>
+ )}
+
+ {insights.length > 0 && (
+ <>
+ <Hr />
+ <Heading as="h3" className="text-base font-semibold">
+ What stood out
+ </Heading>
+ <List
+ items={insights.map((i) => (
+ <span key={i.title}>
+ <strong>{i.title}</strong>
+ {i.summary ? ` — ${i.summary}` : ''}
+ </span>
+ ))}
+ />
+ </>
+ )}
+
+ <Section className="mt-6">
+ <Link href={dashboardUrl}>Open your dashboard →</Link>
+ </Section>
+ </Layout>
+ );
+}
+
+WeeklyDigest.PreviewProps = {
+ projectName: 'Public Web 2',
+ dashboardUrl: 'http://localhost:3000/openpanel-dev/public-web',
+ dateRange: 'Jun 8 – Jun 15, 2026',
+ narrative:
+ "This week on Public Web 2, overall visitor numbers remained steady with a slight dip from 3,898 to 3,889, while sessions saw a modest increase to 4,777. Notably, visits originating from Twitter nearly tripled over the past month, boosting social media engagement considerably. Additionally, user interest in the IP lookup tool has more than doubled, and activity on the dashboard's settings page surged by over 200%, reflecting stronger interaction with site features. However, traffic from Ukraine fell sharply by over 80%, a drop that likely impacted overall traffic and may warrant further attention.",
+ stats: [
+ { label: 'visitors', value: '3 889', delta: '-0%', direction: 'flat' },
+ { label: 'sessions', value: '4 777', delta: '+2%', direction: 'up' },
+ { label: 'pageviews', value: '25 254', delta: '-2%', direction: 'down' },
+ { label: 'bounce rate', value: '49%', delta: '+0pp', direction: 'flat' },
+ ],
+ insights: [
+ {
+ title:
+ 'Visits from Twitter nearly tripled over the last 30 days, a major surge indicating increased social media engagement.',
+ summary: 'Last 30 days. Sessions 277 vs 99.',
+ },
+ {
+ title:
+ 'Visits starting at the IP lookup tool on openpanel.dev more than doubled in the past 30 days, a strong sign of increased user interest in this feature.',
+ summary: 'Last 30 days. Sessions 1168 vs 520.',
+ },
+ {
+ title:
+ 'Settings page on the dashboard experienced a strong surge with 211.5% more sessions, indicating increased user activity or interest.',
+ summary: 'Last 30 days. Sessions 81 vs 26.',
+ },
+ {
+ title:
+ 'Visits from Ukraine fell steeply by over 80% in the last week, a major drop that likely affects overall site traffic and should be investigated.',
+ summary: 'Last 7 days. Traffic change from Ukraine.',
+ },
+ {
+ title:
+ 'Traffic from New Zealand surged by 170% in the last 30 days, a large jump suggesting significant new interest from this market.',
+ summary: 'Last 30 days. Traffic change from New Zealand.',
+ },
+ ],
+} satisfies Props;
diff --git a/packages/queue/src/queues.ts b/packages/queue/src/queues.ts
index f13a70d06..9683374ac 100644
--- a/packages/queue/src/queues.ts
+++ b/packages/queue/src/queues.ts
@@ -152,6 +152,14 @@ export type CronQueuePayloadSessionVacuum = {
type: 'sessionVacuum';
payload: undefined;
};
+export type CronQueuePayloadInsightCleanup = {
+ type: 'insightCleanup';
+ payload: undefined;
+};
+export type CronQueuePayloadWeeklyDigest = {
+ type: 'weeklyDigest';
+ payload: undefined;
+};
export type CronQueuePayload =
| CronQueuePayloadSalt
| CronQueuePayloadFlushEvents
@@ -167,7 +175,9 @@ export type CronQueuePayload =
| CronQueuePayloadGscSync
| CronQueuePayloadCohortRefresh
| CronQueuePayloadSessionReaper
- | CronQueuePayloadSessionVacuum;
+ | CronQueuePayloadSessionVacuum
+ | CronQueuePayloadInsightCleanup
+ | CronQueuePayloadWeeklyDigest;
export type CronQueueType = CronQueuePayload['type'];
diff --git a/packages/trpc/package.json b/packages/trpc/package.json
index a5c0f6aa9..0efb2be59 100644
--- a/packages/trpc/package.json
+++ b/packages/trpc/package.json
@@ -9,7 +9,7 @@
},
"dependencies": {
"@better-agent/core": "^0.1.0-beta.3",
- "@better-agent/providers": "^0.1.0-beta.3",
+ "@openpanel/ai": "workspace:*",
"@openpanel/auth": "workspace:^",
"@openpanel/common": "workspace:*",
"@openpanel/constants": "workspace:*",
diff --git a/packages/trpc/src/agents/filter-command.ts b/packages/trpc/src/agents/filter-command.ts
index 5bac271b9..43952bf92 100644
--- a/packages/trpc/src/agents/filter-command.ts
+++ b/packages/trpc/src/agents/filter-command.ts
@@ -23,7 +23,7 @@ import {
zRange,
} from '@openpanel/validation';
import { z } from 'zod';
-import { resolveModel } from './models';
+import { resolveModel } from '@openpanel/ai';
const operatorEnum = z.enum(objectToZodEnums(operators));
const rangeEnum = zRange;
diff --git a/packages/trpc/src/routers/insight.ts b/packages/trpc/src/routers/insight.ts
index 428cb3049..c02a1f297 100644
--- a/packages/trpc/src/routers/insight.ts
+++ b/packages/trpc/src/routers/insight.ts
@@ -1,9 +1,18 @@
-import { db } from '@openpanel/db';
+import { generateInsightExplanation } from '@openpanel/ai';
+import { db, getTrafficBreakdownCore } from '@openpanel/db';
import { z } from 'zod';
import { getProjectAccess } from '../access';
import { TRPCForbiddenError } from '../errors';
import { createTRPCRouter, protectedProcedure } from '../trpc';
+const DAY_MS = 24 * 60 * 60 * 1000;
+const EXPLAIN_COLUMNS = [
+ 'referrer_name',
+ 'country',
+ 'device',
+ 'utm_source',
+] as const;
+
export const insightRouter = createTRPCRouter({
list: protectedProcedure
.input(
@@ -22,15 +31,18 @@ export const insightRouter = createTRPCRouter({
throw new TRPCForbiddenError('You do not have access to this project');
}
- // Fetch more insights than needed to account for deduplication
+ // Fetch more insights than needed to account for deduplication.
+ // AI relevanceScore leads (un-enriched insights sort last via nulls:last),
+ // with the statistical impactScore as the tiebreaker / fallback.
const allInsights = await db.projectInsight.findMany({
where: {
projectId,
state: 'active',
},
- orderBy: {
- impactScore: 'desc',
- },
+ orderBy: [
+ { relevanceScore: { sort: 'desc', nulls: 'last' } },
+ { impactScore: 'desc' },
+ ],
take: limit * 3, // Fetch 3x to account for deduplication
});
@@ -57,9 +69,14 @@ export const insightRouter = createTRPCRouter({
}
}
- // Convert back to array, sort by impactScore, and limit
+ // Convert back to array, sort by relevanceScore (impactScore fallback),
+ // and limit
const insights = Array.from(deduplicated.values())
- .sort((a, b) => (b.impactScore ?? 0) - (a.impactScore ?? 0))
+ .sort(
+ (a, b) =>
+ (b.relevanceScore ?? -1) - (a.relevanceScore ?? -1) ||
+ (b.impactScore ?? 0) - (a.impactScore ?? 0),
+ )
.slice(0, limit)
.map(({ impactScore, ...rest }) => rest); // Remove impactScore from response
@@ -88,12 +105,104 @@ export const insightRouter = createTRPCRouter({
projectId,
state: 'active',
},
- orderBy: {
- impactScore: 'desc',
- },
+ orderBy: [
+ { relevanceScore: { sort: 'desc', nulls: 'last' } },
+ { impactScore: 'desc' },
+ ],
take: limit,
});
return insights;
}),
+
+ // Phase 5: the "why". Decompose the insight's change across referrer/country/
+ // device/utm (current vs baseline window), pull nearby references, and have
+ // the AI explain which sub-segment drove it. On-demand (a button), not cached.
+ explain: protectedProcedure
+ .input(z.object({ insightId: z.string() }))
+ .mutation(async ({ input: { insightId }, ctx }) => {
+ const insight = await db.projectInsight.findUniqueOrThrow({
+ where: { id: insightId },
+ select: {
+ projectId: true,
+ title: true,
+ aiSummary: true,
+ summary: true,
+ dimensionKey: true,
+ windowKind: true,
+ windowStart: true,
+ windowEnd: true,
+ },
+ });
+
+ const access = await getProjectAccess({
+ userId: ctx.session.userId,
+ projectId: insight.projectId,
+ });
+
+ if (!access) {
+ throw new TRPCForbiddenError('You do not have access to this project');
+ }
+
+ // Current window from the insight; baseline = same span immediately before.
+ const end = insight.windowEnd ?? new Date();
+ const start =
+ insight.windowStart ?? new Date(end.getTime() - 7 * DAY_MS);
+ const spanMs = Math.max(end.getTime() - start.getTime(), DAY_MS);
+ const baseEnd = new Date(start.getTime());
+ const baseStart = new Date(start.getTime() - spanMs);
+ const iso = (d: Date) => d.toISOString();
+
+ const breakdowns = await Promise.all(
+ EXPLAIN_COLUMNS.map(async (column) => {
+ const [cur, base] = await Promise.all([
+ getTrafficBreakdownCore({
+ projectId: insight.projectId,
+ column,
+ startDate: iso(start),
+ endDate: iso(end),
+ }),
+ getTrafficBreakdownCore({
+ projectId: insight.projectId,
+ column,
+ startDate: iso(baseStart),
+ endDate: iso(baseEnd),
+ }),
+ ]);
+ const compact = (rows: typeof cur) =>
+ rows.slice(0, 8).map((r) => ({
+ name: r.name ?? null,
+ sessions: Number(r.sessions ?? 0),
+ }));
+ return { column, current: compact(cur), baseline: compact(base) };
+ }),
+ );
+
+ const references = await db.reference.findMany({
+ where: {
+ projectId: insight.projectId,
+ date: {
+ gte: new Date(start.getTime() - 3 * DAY_MS),
+ lte: new Date(end.getTime() + DAY_MS),
+ },
+ },
+ orderBy: { date: 'desc' },
+ take: 10,
+ select: { title: true, date: true },
+ });
+
+ return generateInsightExplanation({
+ insight: {
+ title: insight.aiSummary ?? insight.title,
+ dimension: insight.dimensionKey,
+ window: insight.windowKind,
+ summary: insight.summary ?? undefined,
+ },
+ breakdowns,
+ references: references.map((r) => ({
+ title: r.title,
+ date: r.date.toISOString().slice(0, 10),
+ })),
+ });
+ }),
});
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 02b77bd76..e85fa9f8e 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -127,9 +127,6 @@ importers:
'@better-agent/core':
specifier: ^0.1.0-beta.3
version: 0.1.0-beta.3
- '@better-agent/providers':
- specifier: ^0.1.0-beta.3
- version: 0.1.0-beta.3(@better-agent/[email protected])
'@fastify/compress':
specifier: ^8.1.0
version: 8.1.0
@@ -157,6 +154,9 @@ importers:
'@node-rs/argon2':
specifier: ^2.0.2
version: 2.0.2
+ '@openpanel/ai':
+ specifier: workspace:*
+ version: link:../../packages/ai
'@openpanel/auth':
specifier: workspace:^
version: link:../../packages/auth
@@ -995,6 +995,9 @@ importers:
'@hyperdx/node-opentelemetry':
specifier: 'catalog:'
version: 0.10.3
+ '@openpanel/ai':
+ specifier: workspace:*
+ version: link:../../packages/ai
'@openpanel/common':
specifier: workspace:*
version: link:../../packages/common
@@ -1087,6 +1090,31 @@ importers:
specifier: 'catalog:'
version: 5.9.3
+ packages/ai:
+ dependencies:
+ '@better-agent/core':
+ specifier: ^0.1.0-beta.3
+ version: 0.1.0-beta.3
+ '@better-agent/providers':
+ specifier: ^0.1.0-beta.3
+ version: 0.1.0-beta.3(@better-agent/[email protected])
+ '@openpanel/validation':
+ specifier: workspace:*
+ version: link:../validation
+ zod:
+ specifier: 'catalog:'
+ version: 4.3.6
+ devDependencies:
+ '@openpanel/tsconfig':
+ specifier: workspace:*
+ version: link:../../tooling/typescript
+ '@types/node':
+ specifier: 'catalog:'
+ version: 24.10.1
+ typescript:
+ specifier: 'catalog:'
+ version: 5.9.3
+
packages/auth:
dependencies:
'@node-rs/argon2':
@@ -1833,9 +1861,9 @@ importers:
'@better-agent/core':
specifier: ^0.1.0-beta.3
version: 0.1.0-beta.3
- '@better-agent/providers':
- specifier: ^0.1.0-beta.3
- version: 0.1.0-beta.3(@better-agent/[email protected])
+ '@openpanel/ai':
+ specifier: workspace:*
+ version: link:../ai
'@openpanel/auth':
specifier: workspace:^
version: link:../auth