Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

chore added events for ai and otter (#1206)

Aman Agarwalb8d80e4b0a32
diff --git a/src/client/package-lock.json b/src/client/package-lock.json
index bdcf6972..f14aaee7 100644
--- a/src/client/package-lock.json
+++ b/src/client/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "openlit",
- "version": "1.21.0",
+ "version": "1.21.1",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "openlit",
- "version": "1.21.0",
+ "version": "1.21.1",
"dependencies": {
"@ai-sdk/anthropic": "^3.0.6",
"@ai-sdk/cohere": "^3.0.3",
diff --git a/src/client/package.json b/src/client/package.json
index c6194b7b..99bcd3ca 100644
--- a/src/client/package.json
+++ b/src/client/package.json
@@ -1,6 +1,6 @@
{
"name": "openlit",
- "version": "1.21.0",
+ "version": "1.21.1",
"private": true,
"scripts": {
"dev": "next dev",
diff --git a/src/client/src/__tests__/constants/events.test.ts b/src/client/src/__tests__/constants/events.test.ts
index 9ecd3b8e..c5a56fde 100644
--- a/src/client/src/__tests__/constants/events.test.ts
+++ b/src/client/src/__tests__/constants/events.test.ts
@@ -60,6 +60,18 @@ describe("CLIENT_EVENTS", () => {
expect(CLIENT_EVENTS.PROMPT_VERSION_ADD_FAILURE).toBe(
"PROMPT_VERSION_ADD_FAILURE"
);
+ expect(CLIENT_EVENTS.PROMPT_IMPROVEMENT_RUN_SUCCESS).toBe(
+ "PROMPT_IMPROVEMENT_RUN_SUCCESS"
+ );
+ expect(CLIENT_EVENTS.PROMPT_IMPROVEMENT_RUN_FAILURE).toBe(
+ "PROMPT_IMPROVEMENT_RUN_FAILURE"
+ );
+ expect(CLIENT_EVENTS.PROMPT_IMPROVEMENT_SUGGESTION_ACCEPTED).toBe(
+ "PROMPT_IMPROVEMENT_SUGGESTION_ACCEPTED"
+ );
+ expect(CLIENT_EVENTS.PROMPT_IMPROVEMENT_SUGGESTION_DECLINED).toBe(
+ "PROMPT_IMPROVEMENT_SUGGESTION_DECLINED"
+ );
});
it("has vault events", () => {
@@ -124,6 +136,40 @@ describe("CLIENT_EVENTS", () => {
);
});
+ it("has otter and ai analysis events", () => {
+ expect(CLIENT_EVENTS.OTTER_CHAT_PAGE_VISITED).toBe(
+ "OTTER_CHAT_PAGE_VISITED"
+ );
+ expect(CLIENT_EVENTS.OTTER_CHAT_MESSAGE_SENT).toBe(
+ "OTTER_CHAT_MESSAGE_SENT"
+ );
+ expect(CLIENT_EVENTS.OTTER_CHAT_MESSAGE_SUCCESS).toBe(
+ "OTTER_CHAT_MESSAGE_SUCCESS"
+ );
+ expect(CLIENT_EVENTS.OTTER_CHAT_MESSAGE_FAILURE).toBe(
+ "OTTER_CHAT_MESSAGE_FAILURE"
+ );
+ expect(CLIENT_EVENTS.OTTER_CHAT_QUERY_EXECUTED).toBe(
+ "OTTER_CHAT_QUERY_EXECUTED"
+ );
+ expect(CLIENT_EVENTS.OTTER_CHAT_QUERY_EXECUTION_FAILURE).toBe(
+ "OTTER_CHAT_QUERY_EXECUTION_FAILURE"
+ );
+ expect(CLIENT_EVENTS.OTTER_USAGE_PAGE_VISITED).toBe(
+ "OTTER_USAGE_PAGE_VISITED"
+ );
+ expect(CLIENT_EVENTS.OTTER_SETTINGS_PAGE_VISITED).toBe(
+ "OTTER_SETTINGS_PAGE_VISITED"
+ );
+ expect(CLIENT_EVENTS.AI_ANALYSIS_VIEWED).toBe("AI_ANALYSIS_VIEWED");
+ expect(CLIENT_EVENTS.AI_ANALYSIS_RUN_SUCCESS).toBe(
+ "AI_ANALYSIS_RUN_SUCCESS"
+ );
+ expect(CLIENT_EVENTS.AI_ANALYSIS_RUN_FAILURE).toBe(
+ "AI_ANALYSIS_RUN_FAILURE"
+ );
+ });
+
it("has organisation events", () => {
expect(CLIENT_EVENTS.ORGANISATION_SWITCHED).toBe("ORGANISATION_SWITCHED");
expect(CLIENT_EVENTS.ORGANISATION_CREATED).toBe("ORGANISATION_CREATED");
@@ -200,6 +246,33 @@ describe("SERVER_EVENTS", () => {
);
});
+ it("has otter server events", () => {
+ expect(SERVER_EVENTS.OTTER_CHAT_MESSAGE_SUCCESS).toBe(
+ "OTTER_CHAT_MESSAGE_SUCCESS"
+ );
+ expect(SERVER_EVENTS.OTTER_CHAT_MESSAGE_FAILURE).toBe(
+ "OTTER_CHAT_MESSAGE_FAILURE"
+ );
+ expect(SERVER_EVENTS.AI_ANALYSIS_GET_SUCCESS).toBe(
+ "AI_ANALYSIS_GET_SUCCESS"
+ );
+ expect(SERVER_EVENTS.AI_ANALYSIS_GET_FAILURE).toBe(
+ "AI_ANALYSIS_GET_FAILURE"
+ );
+ expect(SERVER_EVENTS.AI_ANALYSIS_RUN_SUCCESS).toBe(
+ "AI_ANALYSIS_RUN_SUCCESS"
+ );
+ expect(SERVER_EVENTS.AI_ANALYSIS_RUN_FAILURE).toBe(
+ "AI_ANALYSIS_RUN_FAILURE"
+ );
+ expect(SERVER_EVENTS.PROMPT_IMPROVEMENT_SUCCESS).toBe(
+ "PROMPT_IMPROVEMENT_SUCCESS"
+ );
+ expect(SERVER_EVENTS.PROMPT_IMPROVEMENT_FAILURE).toBe(
+ "PROMPT_IMPROVEMENT_FAILURE"
+ );
+ });
+
it("has folder events", () => {
expect(SERVER_EVENTS.FOLDER_CREATE_SUCCESS).toBe("FOLDER_CREATE_SUCCESS");
expect(SERVER_EVENTS.FOLDER_CREATE_FAILURE).toBe("FOLDER_CREATE_FAILURE");
diff --git a/src/client/src/app/api/chat/improvement/[spanId]/route.ts b/src/client/src/app/api/chat/improvement/[spanId]/route.ts
index 139096f6..67939fee 100644
--- a/src/client/src/app/api/chat/improvement/[spanId]/route.ts
+++ b/src/client/src/app/api/chat/improvement/[spanId]/route.ts
@@ -1,9 +1,11 @@
+import { SERVER_EVENTS } from "@/constants/events";
import { getDBConfigByUser } from "@/lib/db-config";
import {
getTraceImprovement,
streamTraceImprovementAnalysis,
} from "@/lib/platform/chat/improvement";
import { getCurrentUser } from "@/lib/session";
+import PostHogServer from "@/lib/posthog";
import asaw from "@/utils/asaw";
async function getDatabaseConfigId() {
@@ -25,15 +27,26 @@ function getScope(request: Request) {
}
export async function GET(request: Request, context: any) {
+ const startTimestamp = Date.now();
const user = await getCurrentUser();
if (!user) {
logRoute("get_unauthorized", {});
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.AI_ANALYSIS_GET_FAILURE,
+ startTimestamp,
+ properties: { reason: "unauthorized" },
+ });
return Response.json("Unauthorized", { status: 401 });
}
const { spanId } = context.params || {};
if (!spanId) {
logRoute("get_missing_span", {});
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.AI_ANALYSIS_GET_FAILURE,
+ startTimestamp,
+ properties: { reason: "missing_span" },
+ });
return Response.json("No span id provided", { status: 400 });
}
@@ -43,6 +56,11 @@ export async function GET(request: Request, context: any) {
const { data, err } = await getTraceImprovement(spanId, databaseConfigId, scope);
if (err) {
logRoute("get_failed", { spanId, scope, err });
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.AI_ANALYSIS_GET_FAILURE,
+ startTimestamp,
+ properties: { spanId, scope, databaseConfigId, error: err },
+ });
return Response.json(err, { status: 400 });
}
logRoute("get_done", {
@@ -51,20 +69,42 @@ export async function GET(request: Request, context: any) {
rootSpanId: data?.rootSpanId,
runCount: data?.runs?.length || 0,
});
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.AI_ANALYSIS_GET_SUCCESS,
+ startTimestamp,
+ properties: {
+ spanId,
+ scope,
+ databaseConfigId,
+ rootSpanId: data?.rootSpanId,
+ runCount: data?.runs?.length || 0,
+ },
+ });
return Response.json({ data: data || null });
}
export async function POST(request: Request, context: any) {
+ const startTimestamp = Date.now();
const user = await getCurrentUser();
if (!user) {
logRoute("post_unauthorized", {});
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.AI_ANALYSIS_RUN_FAILURE,
+ startTimestamp,
+ properties: { reason: "unauthorized" },
+ });
return Response.json("Unauthorized", { status: 401 });
}
const { spanId } = context.params || {};
if (!spanId) {
logRoute("post_missing_span", {});
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.AI_ANALYSIS_RUN_FAILURE,
+ startTimestamp,
+ properties: { reason: "missing_span" },
+ });
return Response.json("No span id provided", { status: 400 });
}
@@ -78,15 +118,30 @@ export async function POST(request: Request, context: any) {
);
if (err) {
logRoute("post_failed", { spanId, scope, err });
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.AI_ANALYSIS_RUN_FAILURE,
+ startTimestamp,
+ properties: { spanId, scope, databaseConfigId, error: err },
+ });
return Response.json(err, { status: 400 });
}
if (!response) {
logRoute("post_empty_response", { spanId, scope });
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.AI_ANALYSIS_RUN_FAILURE,
+ startTimestamp,
+ properties: { spanId, scope, databaseConfigId, reason: "empty_response" },
+ });
return Response.json("Failed to run AI improvement analysis", {
status: 400,
});
}
logRoute("post_stream_returned", { spanId, scope });
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.AI_ANALYSIS_RUN_SUCCESS,
+ startTimestamp,
+ properties: { spanId, scope, databaseConfigId },
+ });
return response;
}
diff --git a/src/client/src/app/api/chat/message/route.ts b/src/client/src/app/api/chat/message/route.ts
index ac6d9073..7f883dcd 100644
--- a/src/client/src/app/api/chat/message/route.ts
+++ b/src/client/src/app/api/chat/message/route.ts
@@ -1,13 +1,21 @@
+import { SERVER_EVENTS } from "@/constants/events";
import { getCurrentUser } from "@/lib/session";
import { getDBConfigByUser } from "@/lib/db-config";
import { getChatConfigWithApiKey } from "@/lib/platform/chat/config";
import { streamChatMessage, formatStreamError } from "@/lib/platform/chat/stream";
+import PostHogServer from "@/lib/posthog";
import { NextRequest } from "next/server";
import asaw from "@/utils/asaw";
export async function POST(request: NextRequest) {
+ const startTimestamp = Date.now();
const user = await getCurrentUser();
if (!user) {
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.OTTER_CHAT_MESSAGE_FAILURE,
+ startTimestamp,
+ properties: { reason: "unauthorized" },
+ });
return Response.json("Unauthorized", { status: 401 });
}
@@ -15,11 +23,21 @@ export async function POST(request: NextRequest) {
const { conversationId, content } = body;
if (!conversationId || !content) {
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.OTTER_CHAT_MESSAGE_FAILURE,
+ startTimestamp,
+ properties: { reason: "missing_conversation_or_content" },
+ });
return Response.json("Missing conversationId or content", { status: 400 });
}
const { data: config, err: configErr } = await getChatConfigWithApiKey();
if (configErr || !config) {
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.OTTER_CHAT_MESSAGE_FAILURE,
+ startTimestamp,
+ properties: { reason: "missing_config", conversationId },
+ });
return Response.json(
configErr || "Chat not configured. Please set up your AI provider.",
{ status: 400 }
@@ -54,8 +72,32 @@ export async function POST(request: NextRequest) {
text: "**Error:** No response received. Please check your Chat Settings.",
});
}
+ PostHogServer.fireEvent({
+ event: responseText
+ ? SERVER_EVENTS.OTTER_CHAT_MESSAGE_SUCCESS
+ : SERVER_EVENTS.OTTER_CHAT_MESSAGE_FAILURE,
+ startTimestamp,
+ properties: {
+ conversationId,
+ provider: config.provider,
+ model: config.model,
+ dbConfigId,
+ reason: responseText ? undefined : "empty_response",
+ },
+ });
send({ type: "done" });
} catch (e: any) {
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.OTTER_CHAT_MESSAGE_FAILURE,
+ startTimestamp,
+ properties: {
+ conversationId,
+ provider: config.provider,
+ model: config.model,
+ dbConfigId,
+ error: formatStreamError(e),
+ },
+ });
send({ type: "error", error: formatStreamError(e) });
} finally {
controller.close();
@@ -71,6 +113,11 @@ export async function POST(request: NextRequest) {
});
} catch (e: any) {
const errorMsg = formatStreamError(e);
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.OTTER_CHAT_MESSAGE_FAILURE,
+ startTimestamp,
+ properties: { conversationId, error: errorMsg },
+ });
return new Response(`**Error:** ${errorMsg}`, {
headers: { "Content-Type": "text/plain; charset=utf-8" },
});
diff --git a/src/client/src/app/api/prompt/improve/route.ts b/src/client/src/app/api/prompt/improve/route.ts
index da5d3ed5..8562fd9a 100644
--- a/src/client/src/app/api/prompt/improve/route.ts
+++ b/src/client/src/app/api/prompt/improve/route.ts
@@ -1,9 +1,11 @@
import { generateText } from "ai";
+import { SERVER_EVENTS } from "@/constants/events";
import getMessage from "@/constants/messages";
import { getDBConfigByUser } from "@/lib/db-config";
import { getChatConfigWithApiKey } from "@/lib/platform/chat/config";
import { saveOtterRun } from "@/lib/platform/chat/otter-runs";
import { getModelInstance } from "@/lib/platform/chat/stream";
+import PostHogServer from "@/lib/posthog";
import { getCurrentUser } from "@/lib/session";
import asaw from "@/utils/asaw";
@@ -62,8 +64,14 @@ function estimateCost(promptTokens: number, completionTokens: number) {
}
export async function POST(request: Request) {
+ const startTimestamp = Date.now();
const user = await getCurrentUser();
if (!user) {
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.PROMPT_IMPROVEMENT_FAILURE,
+ startTimestamp,
+ properties: { reason: "unauthorized" },
+ });
return Response.json("Unauthorized", { status: 401 });
}
@@ -75,12 +83,22 @@ export async function POST(request: Request) {
: DEFAULT_CRITERIA;
if (!prompt.trim()) {
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.PROMPT_IMPROVEMENT_FAILURE,
+ startTimestamp,
+ properties: { reason: "missing_prompt" },
+ });
return Response.json({ err: m.PROMPT_HUB_CONTENT_REQUIRED }, { status: 400 });
}
const databaseConfigId = await getDatabaseConfigId();
const { data: config, err } = await getChatConfigWithApiKey(databaseConfigId);
if (err || !config) {
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.PROMPT_IMPROVEMENT_FAILURE,
+ startTimestamp,
+ properties: { reason: "missing_config", promptId: promptId || undefined },
+ });
return Response.json(
{ err: err || m.PROMPT_OTTER_CONFIG_NOT_FOUND },
{ status: 400 }
@@ -157,6 +175,21 @@ ${prompt}`,
databaseConfigId
);
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.PROMPT_IMPROVEMENT_SUCCESS,
+ startTimestamp,
+ properties: {
+ promptId: promptId || undefined,
+ provider: config.provider,
+ model: config.model,
+ criteriaCount: criteria.length,
+ suggestionCount: suggestions.length,
+ promptTokens: usageStats.promptTokens,
+ completionTokens: usageStats.completionTokens,
+ cost: usageStats.cost,
+ },
+ });
+
return Response.json({
data: {
suggestions,
@@ -167,6 +200,14 @@ ${prompt}`,
},
});
} catch (error: any) {
+ PostHogServer.fireEvent({
+ event: SERVER_EVENTS.PROMPT_IMPROVEMENT_FAILURE,
+ startTimestamp,
+ properties: {
+ promptId: promptId || undefined,
+ error: error?.message || m.PROMPT_OTTER_ANALYSIS_FAILED,
+ },
+ });
return Response.json(
{ err: error?.message || m.PROMPT_OTTER_ANALYSIS_FAILED },
{ status: 400 }
diff --git a/src/client/src/components/(playground)/chat/chat-layout.tsx b/src/client/src/components/(playground)/chat/chat-layout.tsx
index 8d3a2657..f9eb4a6c 100644
--- a/src/client/src/components/(playground)/chat/chat-layout.tsx
+++ b/src/client/src/components/(playground)/chat/chat-layout.tsx
@@ -2,6 +2,7 @@
import { useEffect, useCallback } from "react";
import { useRouter } from "next/navigation";
+import { usePostHog } from "posthog-js/react";
import { Card } from "@/components/ui/card";
import { ResizeablePanel } from "@/components/ui/resizeable-panel";
import ConversationList from "./conversation-list";
@@ -20,6 +21,7 @@ import {
getChatActions,
} from "@/selectors/chat";
import getMessage from "@/constants/messages";
+import { CLIENT_EVENTS } from "@/constants/events";
import { toast } from "sonner";
interface ChatLayoutProps {
@@ -30,6 +32,7 @@ interface ChatLayoutProps {
export default function ChatLayout({ initialConversationId, initialView = "chat" }: ChatLayoutProps) {
const m = getMessage();
const router = useRouter();
+ const posthog = usePostHog();
const conversations = useRootStore(getChatConversations);
const activeId = useRootStore(getChatActiveId);
@@ -113,6 +116,16 @@ export default function ChatLayout({ initialConversationId, initialView = "chat"
fetchConfig();
}, [fetchConversations, fetchConfig]);
+ useEffect(() => {
+ const event =
+ initialView === "usage"
+ ? CLIENT_EVENTS.OTTER_USAGE_PAGE_VISITED
+ : initialView === "settings"
+ ? CLIENT_EVENTS.OTTER_SETTINGS_PAGE_VISITED
+ : CLIENT_EVENTS.OTTER_CHAT_PAGE_VISITED;
+ posthog?.capture(event);
+ }, [initialView, posthog]);
+
const navigateTo = useCallback(
(id: string | null) => {
if (id) {
diff --git a/src/client/src/components/(playground)/chat/chat-panel.tsx b/src/client/src/components/(playground)/chat/chat-panel.tsx
index dda62840..e2931885 100644
--- a/src/client/src/components/(playground)/chat/chat-panel.tsx
+++ b/src/client/src/components/(playground)/chat/chat-panel.tsx
@@ -1,7 +1,9 @@
"use client";
import { useCallback, useEffect, useRef } from "react";
+import { usePostHog } from "posthog-js/react";
import getMessage from "@/constants/messages";
+import { CLIENT_EVENTS } from "@/constants/events";
import { useRootStore } from "@/store";
import {
getChatMessages,
@@ -26,6 +28,7 @@ export default function ChatPanel({
configInfo,
onNewConversation,
}: ChatPanelProps) {
+ const posthog = usePostHog();
const messages = useRootStore(getChatMessages);
const isStreaming = useRootStore(getChatIsStreaming);
const {
@@ -173,6 +176,10 @@ export default function ChatPanel({
// Add empty assistant message for streaming
addMessage({ role: "assistant", content: "", steps: [], createdAt: new Date().toISOString() });
+ posthog?.capture(CLIENT_EVENTS.OTTER_CHAT_MESSAGE_SENT, {
+ conversationId: currentConvId,
+ hasExistingConversation: Boolean(conversationId),
+ });
try {
const controller = new AbortController();
@@ -253,12 +260,19 @@ export default function ChatPanel({
})
.catch(() => {});
}, 6000);
+ posthog?.capture(CLIENT_EVENTS.OTTER_CHAT_MESSAGE_SUCCESS, {
+ conversationId: currentConvId,
+ });
}
} catch (e: any) {
if (e.name === "AbortError") {
// Remove empty assistant message on cancel
setMessages(messages.filter((m, i) => !(i === messages.length - 1 && m.role === "assistant" && !m.content)));
} else {
+ posthog?.capture(CLIENT_EVENTS.OTTER_CHAT_MESSAGE_FAILURE, {
+ conversationId: currentConvId,
+ error: e.message || getMessage().CHAT_SOMETHING_WENT_WRONG,
+ });
updateLastMessage(`**${getMessage().CHAT_ERROR_PREFIX}** ${e.message || getMessage().CHAT_SOMETHING_WENT_WRONG}`);
}
} finally {
@@ -266,7 +280,7 @@ export default function ChatPanel({
abortControllerRef.current = null;
}
},
- [conversationId, isStreaming, messages, onNewConversation, addMessage, updateLastMessage, updateLastMessageStep, setMessages, setIsStreaming, setInputValue, refreshConversation]
+ [conversationId, isStreaming, messages, onNewConversation, addMessage, posthog, updateLastMessage, updateLastMessageStep, setMessages, setIsStreaming, setInputValue, refreshConversation]
);
const handleExecuteQuery = useCallback(
@@ -279,14 +293,26 @@ export default function ChatPanel({
});
const result = await res.json();
if (!res.ok || result.err) {
+ posthog?.capture(CLIENT_EVENTS.OTTER_CHAT_QUERY_EXECUTION_FAILURE, {
+ messageId,
+ });
return { err: typeof result.err === "string" ? result.err : JSON.stringify(result.err) };
}
+ posthog?.capture(CLIENT_EVENTS.OTTER_CHAT_QUERY_EXECUTED, {
+ messageId,
+ rowsRead: result.stats?.rowsRead,
+ executionTimeMs: result.stats?.executionTimeMs,
+ });
return { data: result.data, stats: result.stats };
} catch (e: any) {
+ posthog?.capture(CLIENT_EVENTS.OTTER_CHAT_QUERY_EXECUTION_FAILURE, {
+ messageId,
+ error: e.message || getMessage().CHAT_QUERY_EXECUTION_FAILED,
+ });
return { err: e.message || getMessage().CHAT_QUERY_EXECUTION_FAILED };
}
},
- []
+ [posthog]
);
const showEmptyState = !conversationId && messages.length === 0 && !isStreaming;
diff --git a/src/client/src/components/(playground)/prompt-hub/prompt-otter-inline-assistant.tsx b/src/client/src/components/(playground)/prompt-hub/prompt-otter-inline-assistant.tsx
index f5433ef3..b186b2b9 100644
--- a/src/client/src/components/(playground)/prompt-hub/prompt-otter-inline-assistant.tsx
+++ b/src/client/src/components/(playground)/prompt-hub/prompt-otter-inline-assistant.tsx
@@ -15,6 +15,8 @@ import {
} from "lucide-react";
import { useCallback, useEffect, useMemo, useState } from "react";
import { toast } from "sonner";
+import { usePostHog } from "posthog-js/react";
+import { CLIENT_EVENTS } from "@/constants/events";
const m = getMessage();
@@ -60,6 +62,7 @@ export default function PromptOtterInlineAssistant({
promptId?: string;
persistState?: boolean;
}) {
+ const posthog = usePostHog();
const [isOpen, setIsOpen] = useState(false);
const [criteria, setCriteria] = useState<string[]>(DEFAULT_CRITERIA);
const [continuation, setContinuation] = useState("");
@@ -155,12 +158,22 @@ export default function PromptOtterInlineAssistant({
if (nextSuggestions.length === 0) {
toast.info(m.PROMPT_OTTER_NO_SUGGESTIONS);
}
+ posthog?.capture(CLIENT_EVENTS.PROMPT_IMPROVEMENT_RUN_SUCCESS, {
+ promptId: promptId || undefined,
+ criteriaCount: effectiveCriteria.length,
+ suggestionCount: nextSuggestions.length,
+ });
} catch (error: any) {
+ posthog?.capture(CLIENT_EVENTS.PROMPT_IMPROVEMENT_RUN_FAILURE, {
+ promptId: promptId || undefined,
+ criteriaCount: effectiveCriteria.length,
+ error: error?.message || m.PROMPT_OTTER_ANALYSIS_FAILED,
+ });
toast.error(error?.message || m.PROMPT_OTTER_ANALYSIS_FAILED);
} finally {
setIsRunning(false);
}
- }, [continuation, criteria, prompt, promptId]);
+ }, [continuation, criteria, m, posthog, prompt, promptId]);
const acceptSuggestion = useCallback(
(suggestion: LocalChange) => {
@@ -178,8 +191,12 @@ export default function PromptOtterInlineAssistant({
)
);
setExpandedSuggestions((prev) => ({ ...prev, [suggestion.id]: false }));
+ posthog?.capture(CLIENT_EVENTS.PROMPT_IMPROVEMENT_SUGGESTION_ACCEPTED, {
+ promptId: promptId || undefined,
+ dimension: suggestion.dimension,
+ });
},
- [onApplyPrompt, prompt]
+ [onApplyPrompt, posthog, prompt, promptId]
);
const declineSuggestion = useCallback((suggestion: LocalChange) => {
@@ -191,7 +208,11 @@ export default function PromptOtterInlineAssistant({
)
);
setExpandedSuggestions((prev) => ({ ...prev, [suggestion.id]: false }));
- }, []);
+ posthog?.capture(CLIENT_EVENTS.PROMPT_IMPROVEMENT_SUGGESTION_DECLINED, {
+ promptId: promptId || undefined,
+ dimension: suggestion.dimension,
+ });
+ }, [posthog, promptId]);
return (
<div className="min-w-0 shrink-0 rounded-md border border-stone-200 bg-stone-50/80 dark:border-stone-800 dark:bg-stone-900/40">
diff --git a/src/client/src/components/(playground)/request/components/trace-improvement-view.tsx b/src/client/src/components/(playground)/request/components/trace-improvement-view.tsx
index 44736e39..1fe9880c 100644
--- a/src/client/src/components/(playground)/request/components/trace-improvement-view.tsx
+++ b/src/client/src/components/(playground)/request/components/trace-improvement-view.tsx
@@ -2,6 +2,7 @@
import { useEffect, useMemo, useRef, useState } from "react";
import { useRouter } from "next/navigation";
+import { usePostHog } from "posthog-js/react";
import {
CheckCircle2,
Circle,
@@ -33,6 +34,7 @@ import {
} from "@/types/trace-analysis";
import { useRequest } from "../request-context";
import getMessage from "@/constants/messages";
+import { CLIENT_EVENTS } from "@/constants/events";
type TraceAnalysisRunResponse = {
id: string;
@@ -518,6 +520,7 @@ export default function TraceImprovementView({
}) {
const m = getMessage();
const router = useRouter();
+ const posthog = usePostHog();
const [, updateRequest] = useRequest();
const [analysis, setAnalysis] = useState<ImprovementResponse | null>(null);
const [streamedAnalysis, setStreamedAnalysis] = useState<TraceAnalysis | null>(null);
@@ -556,6 +559,11 @@ export default function TraceImprovementView({
const result = await res.json();
if (requestKeyRef.current !== requestKey) return;
setAnalysis(result);
+ posthog?.capture(CLIENT_EVENTS.AI_ANALYSIS_VIEWED, {
+ spanId: targetSpanId,
+ scope: targetScope,
+ runCount: result?.data?.runs?.length || 0,
+ });
} catch (err: any) {
if (requestKeyRef.current !== requestKey) return;
toast.error(err?.message || m.TRACE_AI_LOAD_FAILED, {
@@ -603,6 +611,12 @@ export default function TraceImprovementView({
setStreamedAnalysis(null);
setSelectedRunId(latestRun?.id || null);
setSteps((prev) => prev.map((step) => ({ ...step, status: "complete" })));
+ posthog?.capture(CLIENT_EVENTS.AI_ANALYSIS_RUN_SUCCESS, {
+ spanId,
+ scope,
+ runId: latestRun?.id,
+ runCount: runs.length,
+ });
return;
}
if (event.type === "error") {
@@ -654,11 +668,21 @@ export default function TraceImprovementView({
} catch (err: any) {
if (requestKeyRef.current !== requestKey) return;
if (err?.name === "AbortError") {
+ posthog?.capture(CLIENT_EVENTS.AI_ANALYSIS_RUN_FAILURE, {
+ spanId,
+ scope,
+ error: "timeout",
+ });
toast.error(m.TRACE_AI_TIMEOUT, {
id: "trace-improvement",
});
setSelectedRunId(null);
} else {
+ posthog?.capture(CLIENT_EVENTS.AI_ANALYSIS_RUN_FAILURE, {
+ spanId,
+ scope,
+ error: err?.message || m.TRACE_AI_RUN_FAILED,
+ });
toast.error(err?.message || m.TRACE_AI_RUN_FAILED, {
id: "trace-improvement",
});
diff --git a/src/client/src/constants/events.ts b/src/client/src/constants/events.ts
index a39e4d4e..082f0179 100644
--- a/src/client/src/constants/events.ts
+++ b/src/client/src/constants/events.ts
@@ -22,6 +22,10 @@ export const CLIENT_EVENTS = {
PROMPT_ADD_FAILURE: "PROMPT_ADD_FAILURE",
PROMPT_VERSION_ADD_SUCCESS: "PROMPT_VERSION_ADD_SUCCESS",
PROMPT_VERSION_ADD_FAILURE: "PROMPT_VERSION_ADD_FAILURE",
+ PROMPT_IMPROVEMENT_RUN_SUCCESS: "PROMPT_IMPROVEMENT_RUN_SUCCESS",
+ PROMPT_IMPROVEMENT_RUN_FAILURE: "PROMPT_IMPROVEMENT_RUN_FAILURE",
+ PROMPT_IMPROVEMENT_SUGGESTION_ACCEPTED: "PROMPT_IMPROVEMENT_SUGGESTION_ACCEPTED",
+ PROMPT_IMPROVEMENT_SUGGESTION_DECLINED: "PROMPT_IMPROVEMENT_SUGGESTION_DECLINED",
// Vault
VAULT_SECRET_ADD_SUCCESS: "VAULT_SECRET_ADD_SUCCESS",
VAULT_SECRET_UPDATE_SUCCESS: "VAULT_SECRET_UPDATE_SUCCESS",
@@ -78,6 +82,20 @@ export const CLIENT_EVENTS = {
FLEET_HUB_AGENT_VIEWED: "FLEET_HUB_AGENT_VIEWED",
FLEET_HUB_AGENT_CONFIG_SAVED: "FLEET_HUB_AGENT_CONFIG_SAVED",
+ // Otter
+ OTTER_CHAT_MESSAGE_SENT: "OTTER_CHAT_MESSAGE_SENT",
+ OTTER_CHAT_MESSAGE_SUCCESS: "OTTER_CHAT_MESSAGE_SUCCESS",
+ OTTER_CHAT_MESSAGE_FAILURE: "OTTER_CHAT_MESSAGE_FAILURE",
+ OTTER_CHAT_QUERY_EXECUTED: "OTTER_CHAT_QUERY_EXECUTED",
+ OTTER_CHAT_QUERY_EXECUTION_FAILURE: "OTTER_CHAT_QUERY_EXECUTION_FAILURE",
+ OTTER_USAGE_PAGE_VISITED: "OTTER_USAGE_PAGE_VISITED",
+ OTTER_SETTINGS_PAGE_VISITED: "OTTER_SETTINGS_PAGE_VISITED",
+
+ // AI analysis
+ AI_ANALYSIS_VIEWED: "AI_ANALYSIS_VIEWED",
+ AI_ANALYSIS_RUN_SUCCESS: "AI_ANALYSIS_RUN_SUCCESS",
+ AI_ANALYSIS_RUN_FAILURE: "AI_ANALYSIS_RUN_FAILURE",
+
// organisation
ORGANISATION_SWITCHED: "ORGANISATION_SWITCHED",
ORGANISATION_CREATED: "ORGANISATION_CREATED",
@@ -105,6 +123,7 @@ export const CLIENT_EVENTS = {
EVALUATION_TYPE_EDIT_PAGE_VISITED: "EVALUATION_TYPE_EDIT_PAGE_VISITED",
PROMPT_HUB_PAGE_VISITED: "PROMPT_HUB_PAGE_VISITED",
PROMPT_HUB_DETAIL_PAGE_VISITED: "PROMPT_HUB_DETAIL_PAGE_VISITED",
+ OTTER_CHAT_PAGE_VISITED: "OTTER_CHAT_PAGE_VISITED",
VAULT_PAGE_VISITED: "VAULT_PAGE_VISITED",
OPENGROUND_PAGE_VISITED: "OPENGROUND_PAGE_VISITED",
OPENGROUND_NEW_PAGE_VISITED: "OPENGROUND_NEW_PAGE_VISITED",
@@ -135,6 +154,16 @@ export const SERVER_EVENTS = {
EVALUATION_AUTO_SUCCESS: "EVALUATION_AUTO_SUCCESS",
EVALUATION_AUTO_FAILURE: "EVALUATION_AUTO_FAILURE",
+ // Otter
+ OTTER_CHAT_MESSAGE_SUCCESS: "OTTER_CHAT_MESSAGE_SUCCESS",
+ OTTER_CHAT_MESSAGE_FAILURE: "OTTER_CHAT_MESSAGE_FAILURE",
+ AI_ANALYSIS_GET_SUCCESS: "AI_ANALYSIS_GET_SUCCESS",
+ AI_ANALYSIS_GET_FAILURE: "AI_ANALYSIS_GET_FAILURE",
+ AI_ANALYSIS_RUN_SUCCESS: "AI_ANALYSIS_RUN_SUCCESS",
+ AI_ANALYSIS_RUN_FAILURE: "AI_ANALYSIS_RUN_FAILURE",
+ PROMPT_IMPROVEMENT_SUCCESS: "PROMPT_IMPROVEMENT_SUCCESS",
+ PROMPT_IMPROVEMENT_FAILURE: "PROMPT_IMPROVEMENT_FAILURE",
+
// Custom dashboard
FOLDER_CREATE_SUCCESS: "FOLDER_CREATE_SUCCESS",
FOLDER_CREATE_FAILURE: "FOLDER_CREATE_FAILURE",