Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(docs): add Slack reporting for interesting docs bot prompts (#1091)
Sarina Lid5fc67c0540a
diff --git a/docs/.env.example b/docs/.env.exampleindex 08edf9c681..64e8122fa2 100644--- a/docs/.env.example+++ b/docs/.env.example@@ -1,3 +1,7 @@NEXT_PUBLIC_POSTHOG_API_KEY=NEXT_PUBLIC_POSTHOG_HOST=-ANTHROPIC_API_KEY=\ No newline at end of file+ANTHROPIC_API_KEY=+SLACK_WEBHOOK_URL=+POSTHOG_PERSONAL_API_KEY=+POSTHOG_PROJECT_ID=+CRON_SECRET=\ No newline at end of filediff --git a/docs/src/app/api/copilotkit/route.ts b/docs/src/app/api/copilotkit/route.tsindex 4348d5c410..3554b8035b 100644--- a/docs/src/app/api/copilotkit/route.ts+++ b/docs/src/app/api/copilotkit/route.ts@@ -227,15 +227,19 @@ class AnthropicSafeBuiltInAgent extends BuiltInAgent {const latestUserMessage = userMessages[userMessages.length - 1];const userPrompt = this.extractMessageContent(latestUserMessage);+ const category = userPrompt ? categorizePrompt(userPrompt) : 'other';+ const questionType = userPrompt ? detectQuestionType(userPrompt) : 'other';+ const topics = userPrompt ? extractTopics(userPrompt) : [];+if (posthog && userPrompt) {posthog.capture({distinctId: conversationId,event: 'copilot_user_prompt',properties: {prompt: userPrompt,- category: categorizePrompt(userPrompt),- question_type: detectQuestionType(userPrompt),- topics: extractTopics(userPrompt),+ category,+ question_type: questionType,+ topics,prompt_length: userPrompt.length,message_count: filteredMessages.length,conversation_id: conversationId,@@ -262,9 +266,9 @@ class AnthropicSafeBuiltInAgent extends BuiltInAgent {prompt: userPrompt,response: fullResponse,// Prompt categorization- category: categorizePrompt(userPrompt),- question_type: detectQuestionType(userPrompt),- topics: extractTopics(userPrompt),+ category,+ question_type: questionType,+ topics,prompt_length: userPrompt.length,// Response analysisresponse_type: responseAnalysis.response_type,@@ -322,8 +326,8 @@ class AnthropicSafeBuiltInAgent extends BuiltInAgent {properties: {error: err?.message || String(err),prompt: userPrompt,- category: categorizePrompt(userPrompt),- question_type: detectQuestionType(userPrompt),+ category,+ question_type: questionType,conversation_id: conversationId,timestamp: new Date().toISOString(),},diff --git a/docs/src/app/api/cron/prompt-digest/route.ts b/docs/src/app/api/cron/prompt-digest/route.tsnew file mode 100644index 0000000000..e6b065483f--- /dev/null+++ b/docs/src/app/api/cron/prompt-digest/route.ts@@ -0,0 +1,43 @@+import { NextRequest, NextResponse } from 'next/server';+import { fetchRecentPrompts } from '@/lib/posthog-query';+import { analyzeBatchForSlack } from '@/lib/prompt-analyzer';+import { postBatchToSlack } from '@/lib/slack-reporter';++export async function GET(req: NextRequest) {+ // Verify the request is from Vercel Cron+ const authHeader = req.headers.get('authorization');+ if (process.env.CRON_SECRET && authHeader !== `Bearer ${process.env.CRON_SECRET}`) {+ return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });+ }++ try {+ const hoursBack = Number(req.nextUrl.searchParams.get('hours')) || 1;+ const prompts = await fetchRecentPrompts(hoursBack);++ if (prompts.length === 0) {+ return NextResponse.json({ status: 'ok', message: 'No prompts in the last hour' });+ }++ const interesting = await analyzeBatchForSlack(prompts);++ if (interesting.length === 0) {+ return NextResponse.json({+ status: 'ok',+ message: `Analyzed ${prompts.length} prompts, none met threshold`,+ });+ }++ await postBatchToSlack(interesting);++ return NextResponse.json({+ status: 'ok',+ message: `Posted ${interesting.length}/${prompts.length} interesting prompts to Slack`,+ });+ } catch (err) {+ console.error('[PromptDigest] Cron failed:', err);+ return NextResponse.json(+ { error: 'Internal error', message: err instanceof Error ? err.message : String(err) },+ { status: 500 }+ );+ }+}diff --git a/docs/src/lib/posthog-query.ts b/docs/src/lib/posthog-query.tsnew file mode 100644index 0000000000..a0436c8890--- /dev/null+++ b/docs/src/lib/posthog-query.ts@@ -0,0 +1,72 @@+import type { PromptContext } from './prompt-analyzer';++interface PostHogEvent {+ properties: {+ prompt?: string;+ category?: string;+ question_type?: string;+ topics?: string[];+ };+}++interface PostHogEventsResponse {+ results: PostHogEvent[];+ next?: string;+}++export async function fetchRecentPrompts(hoursAgo: number = 1): Promise<PromptContext[]> {+ const apiKey = process.env.POSTHOG_PERSONAL_API_KEY;+ const projectId = process.env.POSTHOG_PROJECT_ID;+ // Derive API host: POSTHOG_HOST if set, otherwise strip the ingestion+ // subdomain (i.) from NEXT_PUBLIC_POSTHOG_HOST to get the API endpoint+ const host =+ process.env.POSTHOG_HOST ||+ process.env.NEXT_PUBLIC_POSTHOG_HOST?.replace('://eu.i.', '://eu.').replace(+ '://us.i.',+ '://us.'+ ) ||+ 'https://us.posthog.com';++ if (!apiKey || !projectId) return [];++ const after = new Date(Date.now() - hoursAgo * 60 * 60 * 1000).toISOString();+ const prompts: PromptContext[] = [];+ let url: string | null =+ `${host}/api/projects/${projectId}/events/?event=copilot_user_prompt&after=${after}&limit=100`;++ console.log(`[PostHogQuery] Fetching from: ${url}`);+ console.log(`[PostHogQuery] Key prefix: ${apiKey.substring(0, 8)}...`);++ while (url) {+ const response = await fetch(url, {+ headers: { Authorization: `Bearer ${apiKey}` },+ });++ if (!response.ok) {+ const body = await response.text();+ console.error(+ `[PostHogQuery] Failed to fetch events: ${response.status} ${response.statusText}`,+ body+ );+ break;+ }++ const data: PostHogEventsResponse = await response.json();++ for (const event of data.results) {+ const props = event.properties;+ if (!props.prompt) continue;++ prompts.push({+ prompt: props.prompt,+ category: props.category || 'other',+ questionType: props.question_type || 'other',+ topics: Array.isArray(props.topics) ? props.topics : [],+ });+ }++ url = data.next || null;+ }++ return prompts;+}diff --git a/docs/src/lib/prompt-analyzer.ts b/docs/src/lib/prompt-analyzer.tsnew file mode 100644index 0000000000..72241a6a78--- /dev/null+++ b/docs/src/lib/prompt-analyzer.ts@@ -0,0 +1,128 @@+import Anthropic from '@anthropic-ai/sdk';++export interface PromptAnalysisResult {+ actionability: { score: number; rationale: string };+ tractionSignal: { score: number; rationale: string };+ useCaseInsight: { score: number; rationale: string };+ summary: string;+}++export interface PromptContext {+ prompt: string;+ category: string;+ questionType: string;+ topics: string[];+}++export interface AnalyzedPrompt {+ analysis: PromptAnalysisResult;+ ctx: PromptContext;+}++const SCORE_THRESHOLD = 7;++let anthropicClient: Anthropic | null = null;++function getClient(): Anthropic | null {+ if (!process.env.ANTHROPIC_API_KEY) return null;+ if (!anthropicClient) {+ anthropicClient = new Anthropic();+ }+ return anthropicClient;+}++function buildBatchPrompt(prompts: PromptContext[]): string {+ const promptList = prompts+ .map(+ (p, i) =>+ `[${i}] "${p.prompt}" (category: ${p.category}, type: ${p.questionType}, topics: ${p.topics.join(', ') || 'none'})`+ )+ .join('\n');++ return `Score each docs chatbot prompt on 3 criteria (1-10). Be selective — most should score low.++Criteria:+1. Actionability: Can we ship something to improve UX? (8+: clear docs gap or feature request, 1-4: normal question)+2. Traction Signal: Is someone actively building with the product? (8+: specific impl question or production issue, 1-4: curiosity)+3. Use Case Insight: Does this reveal how someone uses the product? (8+: names specific workflow/industry, 1-4: generic)++Prompts:+${promptList}++Respond ONLY with a JSON array, no markdown fences. Keep rationales to 5 words max. Only include prompts scoring 7+ on at least one criterion.+[{"index":N,"actionability":{"score":N,"rationale":"..."},"traction_signal":{"score":N,"rationale":"..."},"use_case_insight":{"score":N,"rationale":"..."},"summary":"short sentence"}]+If no prompts are interesting, respond with: []`;+}++function parseBatchResponse(text: string, prompts: PromptContext[]): AnalyzedPrompt[] {+ try {+ const cleaned = text+ .replace(/```json?\s*/g, '')+ .replace(/```\s*/g, '')+ .trim();+ const parsed = JSON.parse(cleaned);++ if (!Array.isArray(parsed)) return [];++ const results: AnalyzedPrompt[] = [];++ for (const item of parsed) {+ const index = Number(item.index);+ if (isNaN(index) || index < 0 || index >= prompts.length) continue;++ const analysis: PromptAnalysisResult = {+ actionability: {+ score: Number(item.actionability?.score) || 0,+ rationale: String(item.actionability?.rationale || ''),+ },+ tractionSignal: {+ score: Number(item.traction_signal?.score) || 0,+ rationale: String(item.traction_signal?.rationale || ''),+ },+ useCaseInsight: {+ score: Number(item.use_case_insight?.score) || 0,+ rationale: String(item.use_case_insight?.rationale || ''),+ },+ summary: String(item.summary || ''),+ };++ const maxScore = Math.max(+ analysis.actionability.score,+ analysis.tractionSignal.score,+ analysis.useCaseInsight.score+ );++ if (maxScore >= SCORE_THRESHOLD) {+ results.push({ analysis, ctx: prompts[index] });+ }+ }++ return results;+ } catch {+ return [];+ }+}++export async function analyzeBatchForSlack(prompts: PromptContext[]): Promise<AnalyzedPrompt[]> {+ const client = getClient();+ if (!client) return [];++ // Filter out trivial prompts+ const filtered = prompts.filter((p) => p.prompt.trim().length >= 15);+ if (filtered.length === 0) return [];++ const response = await client.messages.create({+ model: 'claude-haiku-4-5-20251001',+ max_tokens: 4096,+ messages: [+ {+ role: 'user',+ content: buildBatchPrompt(filtered),+ },+ ],+ });++ const text = response.content[0]?.type === 'text' ? response.content[0].text : '';++ return parseBatchResponse(text, filtered);+}diff --git a/docs/src/lib/slack-reporter.ts b/docs/src/lib/slack-reporter.tsnew file mode 100644index 0000000000..0853d48442--- /dev/null+++ b/docs/src/lib/slack-reporter.ts@@ -0,0 +1,81 @@+import type { AnalyzedPrompt } from './prompt-analyzer';++const SCORE_THRESHOLD = 7;++function formatPromptBlock(item: AnalyzedPrompt): Record<string, unknown>[] {+ const { analysis, ctx } = item;+ const triggeredCriteria: string[] = [];++ if (analysis.actionability.score >= SCORE_THRESHOLD) {+ triggeredCriteria.push(+ `:wrench: *Actionability* (${analysis.actionability.score}/10): ${analysis.actionability.rationale}`+ );+ }+ if (analysis.tractionSignal.score >= SCORE_THRESHOLD) {+ triggeredCriteria.push(+ `:rocket: *Traction Signal* (${analysis.tractionSignal.score}/10): ${analysis.tractionSignal.rationale}`+ );+ }+ if (analysis.useCaseInsight.score >= SCORE_THRESHOLD) {+ triggeredCriteria.push(+ `:mag: *Use Case Insight* (${analysis.useCaseInsight.score}/10): ${analysis.useCaseInsight.rationale}`+ );+ }++ const topicsText = ctx.topics.length > 0 ? ctx.topics.join(', ') : 'none';+ const displayPrompt = ctx.prompt.length > 500 ? ctx.prompt.substring(0, 497) + '...' : ctx.prompt;++ return [+ {+ type: 'section',+ text: { type: 'mrkdwn', text: `> ${displayPrompt}` },+ },+ {+ type: 'section',+ text: { type: 'mrkdwn', text: triggeredCriteria.join('\n') },+ },+ {+ type: 'context',+ elements: [+ {+ type: 'mrkdwn',+ text: `*Category:* ${ctx.category} | *Type:* ${ctx.questionType} | *Topics:* ${topicsText} | _${analysis.summary}_`,+ },+ ],+ },+ { type: 'divider' },+ ];+}++export async function postBatchToSlack(results: AnalyzedPrompt[]): Promise<void> {+ const webhookUrl = process.env.SLACK_WEBHOOK_URL;+ if (!webhookUrl || results.length === 0) return;++ const blocks: Record<string, unknown>[] = [+ {+ type: 'header',+ text: {+ type: 'plain_text',+ text: `:speech_balloon: Docs Bot Digest — ${results.length} interesting prompt${results.length === 1 ? '' : 's'}`,+ emoji: true,+ },+ },+ ];++ for (const result of results) {+ blocks.push(...formatPromptBlock(result));+ }++ // Slack blocks limit is 50 — truncate if needed+ const message = { blocks: blocks.slice(0, 50) };++ const response = await fetch(webhookUrl, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify(message),+ });++ if (!response.ok) {+ console.error(`[SlackReporter] Failed to post: ${response.status} ${response.statusText}`);+ }+}diff --git a/docs/vercel.json b/docs/vercel.jsonnew file mode 100644index 0000000000..90b43586fe--- /dev/null+++ b/docs/vercel.json@@ -0,0 +1,8 @@+{+ "crons": [+ {+ "path": "/api/cron/prompt-digest",+ "schedule": "17 4 * * 2-6"+ }+ ]+}diff --git a/scripts/typescript-typecheck.js b/scripts/typescript-typecheck.jsindex 1a29b73b7d..76b7a27158 100644--- a/scripts/typescript-typecheck.js+++ b/scripts/typescript-typecheck.js@@ -1,9 +1,17 @@#!/usr/bin/env nodeconst { execSync } = require('child_process');+const { readFileSync, writeFileSync, existsSync } = require('fs');+const { join } = require('path');// Allow pnpm version mismatch (root uses 9.x, workspace packages declare 10.x)const env = { ...process.env, COREPACK_ENABLE_STRICT: '0' };+// Save lockfile before install so we can restore it after — pnpm install+// may rewrite the lockfile format when the pnpm version differs, which+// causes pre-commit to report "files were modified by this hook".+const lockfilePath = join(__dirname, '..', 'libs', 'typescript', 'pnpm-lock.yaml');+const lockfileBefore = existsSync(lockfilePath) ? readFileSync(lockfilePath) : null;+try {execSync('pnpm -C libs/typescript install', { stdio: 'inherit', env });// Build core first so its type declarations (dist/) are available@@ -12,4 +20,9 @@ try {execSync('pnpm -C libs/typescript -r run typecheck', { stdio: 'inherit', env });} catch (err) {process.exit(1);+} finally {+ // Restore lockfile to avoid pre-commit "files were modified" failure+ if (lockfileBefore !== null) {+ writeFileSync(lockfilePath, lockfileBefore);+ }}