Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

[MISC] Capture adapter setup and Prompt Studio run results in PostHog (#2327)

Chandrasekharan M0af40dbcc493
diff --git a/frontend/src/components/input-output/configure-ds/ConfigureDs.jsx b/frontend/src/components/input-output/configure-ds/ConfigureDs.jsx
index 5bb0d6f63..49dce7c13 100644
--- a/frontend/src/components/input-output/configure-ds/ConfigureDs.jsx
+++ b/frontend/src/components/input-output/configure-ds/ConfigureDs.jsx
@@ -3,7 +3,7 @@ import PropTypes from "prop-types";
import { createRef, useEffect, useState } from "react";
import { Col, Row } from "@/components/ui/shims/antd-layout";
import { Popover } from "@/components/ui/shims/antd-overlays";
-
+import { saasOnlyErrorProps } from "../../../helpers/PostHogDeployment";
import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate";
import { useExceptionHandler } from "../../../hooks/useExceptionHandler.jsx";
import usePostHogEvents from "../../../hooks/usePostHogEvents.js";
@@ -51,6 +51,19 @@ function ConfigureDs({
} = usePostHogEvents();
const { getUrl } = useRequestUrl();
+ const captureAdapterResult = (eventName, result, errorMessage) => {
+ if (isConnector) {
+ return;
+ }
+ setPostHogCustomEvent(eventName, {
+ adapter_type: type,
+ adapter_name: selectedSourceName,
+ is_edit: Boolean(editItemId),
+ result,
+ ...saasOnlyErrorProps(errorMessage),
+ });
+ };
+
const oauthCacheKey = `oauth-cachekey-${selectedSourceId}`;
const oauthStatusKey = `oauth-status-${selectedSourceId}`;
@@ -232,6 +245,10 @@ function ConfigureDs({
.then((res) => {
const isValid = res?.data?.is_valid;
setIsTcSuccessful(isValid);
+ captureAdapterResult(
+ "adapter_test_connection_result",
+ isValid ? "success" : "invalid",
+ );
if (!isValid) {
setAlertDetails({
type: "error",
@@ -247,7 +264,13 @@ function ConfigureDs({
.catch((err) => {
const TestErrorMessage =
err?.response?.data?.message || "Test connection failed";
- setAlertDetails(handleException(err, TestErrorMessage));
+ const alert = handleException(err, TestErrorMessage);
+ captureAdapterResult(
+ "adapter_test_connection_result",
+ "error",
+ alert.content,
+ );
+ setAlertDetails(alert);
})
.finally(() => {
setIsTcLoading(false);
@@ -338,6 +361,7 @@ function ConfigureDs({
if (!isConnector && method === "POST") {
updateSession(type);
}
+ captureAdapterResult("adapter_save_result", "success");
if (oAuthProvider?.length > 0 && isOAuthMethodSelected()) {
localStorage.removeItem(oauthCacheKey);
@@ -347,7 +371,9 @@ function ConfigureDs({
setOpen(false);
})
.catch((err) => {
- setAlertDetails(handleException(err));
+ const alert = handleException(err);
+ captureAdapterResult("adapter_save_result", "error", alert.content);
+ setAlertDetails(alert);
})
.finally(() => {
setIsSubmitApiLoading(false);
diff --git a/frontend/src/helpers/PostHogDeployment.js b/frontend/src/helpers/PostHogDeployment.js
index ab950193a..bb267f9d4 100644
--- a/frontend/src/helpers/PostHogDeployment.js
+++ b/frontend/src/helpers/PostHogDeployment.js
@@ -12,4 +12,10 @@ const getDeployment = () =>
// PII (email, name) may only leave Unstract-managed SaaS deployments
const isSaasProdDeployment = () => getDeployment() !== "self-hosted";
-export { getDeployment, isSaasProdDeployment };
+// Error text can contain customer endpoints, so it is sent only from SaaS.
+const saasOnlyErrorProps = (errorMessage) =>
+ typeof errorMessage === "string" && errorMessage && isSaasProdDeployment()
+ ? { error: errorMessage.slice(0, 200) }
+ : {};
+
+export { getDeployment, isSaasProdDeployment, saasOnlyErrorProps };
diff --git a/frontend/src/helpers/PostHogDeployment.test.js b/frontend/src/helpers/PostHogDeployment.test.js
new file mode 100644
index 000000000..77fc57ef0
--- /dev/null
+++ b/frontend/src/helpers/PostHogDeployment.test.js
@@ -0,0 +1,29 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import { saasOnlyErrorProps } from "./PostHogDeployment";
+
+const setHost = (hostname) =>
+ vi.spyOn(window, "location", "get").mockReturnValue({ hostname });
+
+describe("saasOnlyErrorProps", () => {
+ afterEach(() => vi.restoreAllMocks());
+
+ it("drops error text on self-hosted deployments", () => {
+ setHost("unstract.customer.internal");
+ expect(saasOnlyErrorProps("connect to https://10.0.0.5 failed")).toEqual(
+ {},
+ );
+ });
+
+ it("keeps error text, capped at 200 chars, on SaaS", () => {
+ setHost("us-central.unstract.com");
+ const props = saasOnlyErrorProps("x".repeat(500));
+ expect(props.error).toHaveLength(200);
+ });
+
+ it("adds nothing when there is no string error", () => {
+ setHost("eu-west.unstract.com");
+ expect(saasOnlyErrorProps(undefined)).toEqual({});
+ expect(saasOnlyErrorProps([{ detail: "limit" }])).toEqual({});
+ });
+});
diff --git a/frontend/src/hooks/usePromptStudioSocket.js b/frontend/src/hooks/usePromptStudioSocket.js
index 1f93cd8e3..4c72fb9d2 100644
--- a/frontend/src/hooks/usePromptStudioSocket.js
+++ b/frontend/src/hooks/usePromptStudioSocket.js
@@ -5,6 +5,7 @@ import { useAlertStore } from "../store/alert-store";
import { useCustomToolStore } from "../store/custom-tool-store";
import { usePromptRunStatusStore } from "../store/prompt-run-status-store";
import { useExceptionHandler } from "./useExceptionHandler";
+import usePostHogEvents from "./usePostHogEvents";
import usePromptOutput from "./usePromptOutput";
const PROMPT_STUDIO_RESULT_EVENT = "prompt_studio_result";
@@ -25,6 +26,7 @@ const usePromptStudioSocket = () => {
const { setAlertDetails } = useAlertStore();
const handleException = useExceptionHandler();
const { updatePromptOutputState } = usePromptOutput();
+ const { setPostHogCustomEvent } = usePostHogEvents();
const clearPromptStatuses = useCallback(
(promptIds, docId, profileId) => {
@@ -144,6 +146,16 @@ const usePromptStudioSocket = () => {
return;
}
+ // Captured before handling so a handler throw doesn't hide the outcome.
+ setPostHogCustomEvent("ps_prompt_run_result", {
+ task_id: extra?.task_id,
+ operation,
+ status,
+ tool_id,
+ prompt_count: extra?.prompt_ids?.length,
+ elapsed_seconds: extra?.elapsed,
+ });
+
if (status === "completed") {
handleCompleted(operation, result, extra);
} else if (status === "failed") {
@@ -155,7 +167,14 @@ const usePromptStudioSocket = () => {
);
}
},
- [handleCompleted, handleFailed, setAlertDetails, handleException, details],
+ [
+ handleCompleted,
+ handleFailed,
+ setAlertDetails,
+ handleException,
+ details,
+ setPostHogCustomEvent,
+ ],
);
useEffect(() => {