Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(docs): add Slack reporting for interesting docs bot prompts (#1091)

Sarina Lid5fc67c0540a
diff --git a/docs/.env.example b/docs/.env.example
index 08edf9c681..64e8122fa2 100644
--- a/docs/.env.example
+++ b/docs/.env.example
@@ -1,3 +1,7 @@
NEXT_PUBLIC_POSTHOG_API_KEY=
NEXT_PUBLIC_POSTHOG_HOST=
-ANTHROPIC_API_KEY=
\ No newline at end of file
+ANTHROPIC_API_KEY=
+SLACK_WEBHOOK_URL=
+POSTHOG_PERSONAL_API_KEY=
+POSTHOG_PROJECT_ID=
+CRON_SECRET=
\ No newline at end of file
diff --git a/docs/src/app/api/copilotkit/route.ts b/docs/src/app/api/copilotkit/route.ts
index 4348d5c410..3554b8035b 100644
--- a/docs/src/app/api/copilotkit/route.ts
+++ b/docs/src/app/api/copilotkit/route.ts
@@ -227,15 +227,19 @@ class AnthropicSafeBuiltInAgent extends BuiltInAgent {
const latestUserMessage = userMessages[userMessages.length - 1];
const userPrompt = this.extractMessageContent(latestUserMessage);
+ const category = userPrompt ? categorizePrompt(userPrompt) : 'other';
+ const questionType = userPrompt ? detectQuestionType(userPrompt) : 'other';
+ const topics = userPrompt ? extractTopics(userPrompt) : [];
+
if (posthog && userPrompt) {
posthog.capture({
distinctId: conversationId,
event: 'copilot_user_prompt',
properties: {
prompt: userPrompt,
- category: categorizePrompt(userPrompt),
- question_type: detectQuestionType(userPrompt),
- topics: extractTopics(userPrompt),
+ category,
+ question_type: questionType,
+ topics,
prompt_length: userPrompt.length,
message_count: filteredMessages.length,
conversation_id: conversationId,
@@ -262,9 +266,9 @@ class AnthropicSafeBuiltInAgent extends BuiltInAgent {
prompt: userPrompt,
response: fullResponse,
// Prompt categorization
- category: categorizePrompt(userPrompt),
- question_type: detectQuestionType(userPrompt),
- topics: extractTopics(userPrompt),
+ category,
+ question_type: questionType,
+ topics,
prompt_length: userPrompt.length,
// Response analysis
response_type: responseAnalysis.response_type,
@@ -322,8 +326,8 @@ class AnthropicSafeBuiltInAgent extends BuiltInAgent {
properties: {
error: err?.message || String(err),
prompt: userPrompt,
- category: categorizePrompt(userPrompt),
- question_type: detectQuestionType(userPrompt),
+ category,
+ question_type: questionType,
conversation_id: conversationId,
timestamp: new Date().toISOString(),
},
diff --git a/docs/src/app/api/cron/prompt-digest/route.ts b/docs/src/app/api/cron/prompt-digest/route.ts
new file mode 100644
index 0000000000..e6b065483f
--- /dev/null
+++ b/docs/src/app/api/cron/prompt-digest/route.ts
@@ -0,0 +1,43 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { fetchRecentPrompts } from '@/lib/posthog-query';
+import { analyzeBatchForSlack } from '@/lib/prompt-analyzer';
+import { postBatchToSlack } from '@/lib/slack-reporter';
+
+export async function GET(req: NextRequest) {
+ // Verify the request is from Vercel Cron
+ const authHeader = req.headers.get('authorization');
+ if (process.env.CRON_SECRET && authHeader !== `Bearer ${process.env.CRON_SECRET}`) {
+ return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
+ }
+
+ try {
+ const hoursBack = Number(req.nextUrl.searchParams.get('hours')) || 1;
+ const prompts = await fetchRecentPrompts(hoursBack);
+
+ if (prompts.length === 0) {
+ return NextResponse.json({ status: 'ok', message: 'No prompts in the last hour' });
+ }
+
+ const interesting = await analyzeBatchForSlack(prompts);
+
+ if (interesting.length === 0) {
+ return NextResponse.json({
+ status: 'ok',
+ message: `Analyzed ${prompts.length} prompts, none met threshold`,
+ });
+ }
+
+ await postBatchToSlack(interesting);
+
+ return NextResponse.json({
+ status: 'ok',
+ message: `Posted ${interesting.length}/${prompts.length} interesting prompts to Slack`,
+ });
+ } catch (err) {
+ console.error('[PromptDigest] Cron failed:', err);
+ return NextResponse.json(
+ { error: 'Internal error', message: err instanceof Error ? err.message : String(err) },
+ { status: 500 }
+ );
+ }
+}
diff --git a/docs/src/lib/posthog-query.ts b/docs/src/lib/posthog-query.ts
new file mode 100644
index 0000000000..a0436c8890
--- /dev/null
+++ b/docs/src/lib/posthog-query.ts
@@ -0,0 +1,72 @@
+import type { PromptContext } from './prompt-analyzer';
+
+interface PostHogEvent {
+ properties: {
+ prompt?: string;
+ category?: string;
+ question_type?: string;
+ topics?: string[];
+ };
+}
+
+interface PostHogEventsResponse {
+ results: PostHogEvent[];
+ next?: string;
+}
+
+export async function fetchRecentPrompts(hoursAgo: number = 1): Promise<PromptContext[]> {
+ const apiKey = process.env.POSTHOG_PERSONAL_API_KEY;
+ const projectId = process.env.POSTHOG_PROJECT_ID;
+ // Derive API host: POSTHOG_HOST if set, otherwise strip the ingestion
+ // subdomain (i.) from NEXT_PUBLIC_POSTHOG_HOST to get the API endpoint
+ const host =
+ process.env.POSTHOG_HOST ||
+ process.env.NEXT_PUBLIC_POSTHOG_HOST?.replace('://eu.i.', '://eu.').replace(
+ '://us.i.',
+ '://us.'
+ ) ||
+ 'https://us.posthog.com';
+
+ if (!apiKey || !projectId) return [];
+
+ const after = new Date(Date.now() - hoursAgo * 60 * 60 * 1000).toISOString();
+ const prompts: PromptContext[] = [];
+ let url: string | null =
+ `${host}/api/projects/${projectId}/events/?event=copilot_user_prompt&after=${after}&limit=100`;
+
+ console.log(`[PostHogQuery] Fetching from: ${url}`);
+ console.log(`[PostHogQuery] Key prefix: ${apiKey.substring(0, 8)}...`);
+
+ while (url) {
+ const response = await fetch(url, {
+ headers: { Authorization: `Bearer ${apiKey}` },
+ });
+
+ if (!response.ok) {
+ const body = await response.text();
+ console.error(
+ `[PostHogQuery] Failed to fetch events: ${response.status} ${response.statusText}`,
+ body
+ );
+ break;
+ }
+
+ const data: PostHogEventsResponse = await response.json();
+
+ for (const event of data.results) {
+ const props = event.properties;
+ if (!props.prompt) continue;
+
+ prompts.push({
+ prompt: props.prompt,
+ category: props.category || 'other',
+ questionType: props.question_type || 'other',
+ topics: Array.isArray(props.topics) ? props.topics : [],
+ });
+ }
+
+ url = data.next || null;
+ }
+
+ return prompts;
+}
diff --git a/docs/src/lib/prompt-analyzer.ts b/docs/src/lib/prompt-analyzer.ts
new file mode 100644
index 0000000000..72241a6a78
--- /dev/null
+++ b/docs/src/lib/prompt-analyzer.ts
@@ -0,0 +1,128 @@
+import Anthropic from '@anthropic-ai/sdk';
+
+export interface PromptAnalysisResult {
+ actionability: { score: number; rationale: string };
+ tractionSignal: { score: number; rationale: string };
+ useCaseInsight: { score: number; rationale: string };
+ summary: string;
+}
+
+export interface PromptContext {
+ prompt: string;
+ category: string;
+ questionType: string;
+ topics: string[];
+}
+
+export interface AnalyzedPrompt {
+ analysis: PromptAnalysisResult;
+ ctx: PromptContext;
+}
+
+const SCORE_THRESHOLD = 7;
+
+let anthropicClient: Anthropic | null = null;
+
+function getClient(): Anthropic | null {
+ if (!process.env.ANTHROPIC_API_KEY) return null;
+ if (!anthropicClient) {
+ anthropicClient = new Anthropic();
+ }
+ return anthropicClient;
+}
+
+function buildBatchPrompt(prompts: PromptContext[]): string {
+ const promptList = prompts
+ .map(
+ (p, i) =>
+ `[${i}] "${p.prompt}" (category: ${p.category}, type: ${p.questionType}, topics: ${p.topics.join(', ') || 'none'})`
+ )
+ .join('\n');
+
+ return `Score each docs chatbot prompt on 3 criteria (1-10). Be selective — most should score low.
+
+Criteria:
+1. Actionability: Can we ship something to improve UX? (8+: clear docs gap or feature request, 1-4: normal question)
+2. Traction Signal: Is someone actively building with the product? (8+: specific impl question or production issue, 1-4: curiosity)
+3. Use Case Insight: Does this reveal how someone uses the product? (8+: names specific workflow/industry, 1-4: generic)
+
+Prompts:
+${promptList}
+
+Respond ONLY with a JSON array, no markdown fences. Keep rationales to 5 words max. Only include prompts scoring 7+ on at least one criterion.
+[{"index":N,"actionability":{"score":N,"rationale":"..."},"traction_signal":{"score":N,"rationale":"..."},"use_case_insight":{"score":N,"rationale":"..."},"summary":"short sentence"}]
+If no prompts are interesting, respond with: []`;
+}
+
+function parseBatchResponse(text: string, prompts: PromptContext[]): AnalyzedPrompt[] {
+ try {
+ const cleaned = text
+ .replace(/```json?\s*/g, '')
+ .replace(/```\s*/g, '')
+ .trim();
+ const parsed = JSON.parse(cleaned);
+
+ if (!Array.isArray(parsed)) return [];
+
+ const results: AnalyzedPrompt[] = [];
+
+ for (const item of parsed) {
+ const index = Number(item.index);
+ if (isNaN(index) || index < 0 || index >= prompts.length) continue;
+
+ const analysis: PromptAnalysisResult = {
+ actionability: {
+ score: Number(item.actionability?.score) || 0,
+ rationale: String(item.actionability?.rationale || ''),
+ },
+ tractionSignal: {
+ score: Number(item.traction_signal?.score) || 0,
+ rationale: String(item.traction_signal?.rationale || ''),
+ },
+ useCaseInsight: {
+ score: Number(item.use_case_insight?.score) || 0,
+ rationale: String(item.use_case_insight?.rationale || ''),
+ },
+ summary: String(item.summary || ''),
+ };
+
+ const maxScore = Math.max(
+ analysis.actionability.score,
+ analysis.tractionSignal.score,
+ analysis.useCaseInsight.score
+ );
+
+ if (maxScore >= SCORE_THRESHOLD) {
+ results.push({ analysis, ctx: prompts[index] });
+ }
+ }
+
+ return results;
+ } catch {
+ return [];
+ }
+}
+
+export async function analyzeBatchForSlack(prompts: PromptContext[]): Promise<AnalyzedPrompt[]> {
+ const client = getClient();
+ if (!client) return [];
+
+ // Filter out trivial prompts
+ const filtered = prompts.filter((p) => p.prompt.trim().length >= 15);
+ if (filtered.length === 0) return [];
+
+ const response = await client.messages.create({
+ model: 'claude-haiku-4-5-20251001',
+ max_tokens: 4096,
+ messages: [
+ {
+ role: 'user',
+ content: buildBatchPrompt(filtered),
+ },
+ ],
+ });
+
+ const text = response.content[0]?.type === 'text' ? response.content[0].text : '';
+
+ return parseBatchResponse(text, filtered);
+}
diff --git a/docs/src/lib/slack-reporter.ts b/docs/src/lib/slack-reporter.ts
new file mode 100644
index 0000000000..0853d48442
--- /dev/null
+++ b/docs/src/lib/slack-reporter.ts
@@ -0,0 +1,81 @@
+import type { AnalyzedPrompt } from './prompt-analyzer';
+
+const SCORE_THRESHOLD = 7;
+
+function formatPromptBlock(item: AnalyzedPrompt): Record<string, unknown>[] {
+ const { analysis, ctx } = item;
+ const triggeredCriteria: string[] = [];
+
+ if (analysis.actionability.score >= SCORE_THRESHOLD) {
+ triggeredCriteria.push(
+ `:wrench: *Actionability* (${analysis.actionability.score}/10): ${analysis.actionability.rationale}`
+ );
+ }
+ if (analysis.tractionSignal.score >= SCORE_THRESHOLD) {
+ triggeredCriteria.push(
+ `:rocket: *Traction Signal* (${analysis.tractionSignal.score}/10): ${analysis.tractionSignal.rationale}`
+ );
+ }
+ if (analysis.useCaseInsight.score >= SCORE_THRESHOLD) {
+ triggeredCriteria.push(
+ `:mag: *Use Case Insight* (${analysis.useCaseInsight.score}/10): ${analysis.useCaseInsight.rationale}`
+ );
+ }
+
+ const topicsText = ctx.topics.length > 0 ? ctx.topics.join(', ') : 'none';
+ const displayPrompt = ctx.prompt.length > 500 ? ctx.prompt.substring(0, 497) + '...' : ctx.prompt;
+
+ return [
+ {
+ type: 'section',
+ text: { type: 'mrkdwn', text: `> ${displayPrompt}` },
+ },
+ {
+ type: 'section',
+ text: { type: 'mrkdwn', text: triggeredCriteria.join('\n') },
+ },
+ {
+ type: 'context',
+ elements: [
+ {
+ type: 'mrkdwn',
+ text: `*Category:* ${ctx.category} | *Type:* ${ctx.questionType} | *Topics:* ${topicsText} | _${analysis.summary}_`,
+ },
+ ],
+ },
+ { type: 'divider' },
+ ];
+}
+
+export async function postBatchToSlack(results: AnalyzedPrompt[]): Promise<void> {
+ const webhookUrl = process.env.SLACK_WEBHOOK_URL;
+ if (!webhookUrl || results.length === 0) return;
+
+ const blocks: Record<string, unknown>[] = [
+ {
+ type: 'header',
+ text: {
+ type: 'plain_text',
+ text: `:speech_balloon: Docs Bot Digest — ${results.length} interesting prompt${results.length === 1 ? '' : 's'}`,
+ emoji: true,
+ },
+ },
+ ];
+
+ for (const result of results) {
+ blocks.push(...formatPromptBlock(result));
+ }
+
+ // Slack blocks limit is 50 — truncate if needed
+ const message = { blocks: blocks.slice(0, 50) };
+
+ const response = await fetch(webhookUrl, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify(message),
+ });
+
+ if (!response.ok) {
+ console.error(`[SlackReporter] Failed to post: ${response.status} ${response.statusText}`);
+ }
+}
diff --git a/docs/vercel.json b/docs/vercel.json
new file mode 100644
index 0000000000..90b43586fe
--- /dev/null
+++ b/docs/vercel.json
@@ -0,0 +1,8 @@
+{
+ "crons": [
+ {
+ "path": "/api/cron/prompt-digest",
+ "schedule": "17 4 * * 2-6"
+ }
+ ]
+}
diff --git a/scripts/typescript-typecheck.js b/scripts/typescript-typecheck.js
index 1a29b73b7d..76b7a27158 100644
--- a/scripts/typescript-typecheck.js
+++ b/scripts/typescript-typecheck.js
@@ -1,9 +1,17 @@
#!/usr/bin/env node
const { execSync } = require('child_process');
+const { readFileSync, writeFileSync, existsSync } = require('fs');
+const { join } = require('path');
// Allow pnpm version mismatch (root uses 9.x, workspace packages declare 10.x)
const env = { ...process.env, COREPACK_ENABLE_STRICT: '0' };
+// Save lockfile before install so we can restore it after — pnpm install
+// may rewrite the lockfile format when the pnpm version differs, which
+// causes pre-commit to report "files were modified by this hook".
+const lockfilePath = join(__dirname, '..', 'libs', 'typescript', 'pnpm-lock.yaml');
+const lockfileBefore = existsSync(lockfilePath) ? readFileSync(lockfilePath) : null;
+
try {
execSync('pnpm -C libs/typescript install', { stdio: 'inherit', env });
// Build core first so its type declarations (dist/) are available
@@ -12,4 +20,9 @@ try {
execSync('pnpm -C libs/typescript -r run typecheck', { stdio: 'inherit', env });
} catch (err) {
process.exit(1);
+} finally {
+ // Restore lockfile to avoid pre-commit "files were modified" failure
+ if (lockfileBefore !== null) {
+ writeFileSync(lockfilePath, lockfileBefore);
+ }
}