LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
[MISC] Capture adapter setup and Prompt Studio run results in PostHog (#2327)
Chandrasekharan M0af40dbcc493
diff --git a/frontend/src/components/input-output/configure-ds/ConfigureDs.jsx b/frontend/src/components/input-output/configure-ds/ConfigureDs.jsxindex 5bb0d6f63..49dce7c13 100644--- a/frontend/src/components/input-output/configure-ds/ConfigureDs.jsx+++ b/frontend/src/components/input-output/configure-ds/ConfigureDs.jsx@@ -3,7 +3,7 @@ import PropTypes from "prop-types";import { createRef, useEffect, useState } from "react";import { Col, Row } from "@/components/ui/shims/antd-layout";import { Popover } from "@/components/ui/shims/antd-overlays";-+import { saasOnlyErrorProps } from "../../../helpers/PostHogDeployment";import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate";import { useExceptionHandler } from "../../../hooks/useExceptionHandler.jsx";import usePostHogEvents from "../../../hooks/usePostHogEvents.js";@@ -51,6 +51,19 @@ function ConfigureDs({} = usePostHogEvents();const { getUrl } = useRequestUrl();+ const captureAdapterResult = (eventName, result, errorMessage) => {+ if (isConnector) {+ return;+ }+ setPostHogCustomEvent(eventName, {+ adapter_type: type,+ adapter_name: selectedSourceName,+ is_edit: Boolean(editItemId),+ result,+ ...saasOnlyErrorProps(errorMessage),+ });+ };+const oauthCacheKey = `oauth-cachekey-${selectedSourceId}`;const oauthStatusKey = `oauth-status-${selectedSourceId}`;@@ -232,6 +245,10 @@ function ConfigureDs({.then((res) => {const isValid = res?.data?.is_valid;setIsTcSuccessful(isValid);+ captureAdapterResult(+ "adapter_test_connection_result",+ isValid ? "success" : "invalid",+ );if (!isValid) {setAlertDetails({type: "error",@@ -247,7 +264,13 @@ function ConfigureDs({.catch((err) => {const TestErrorMessage =err?.response?.data?.message || "Test connection failed";- setAlertDetails(handleException(err, TestErrorMessage));+ const alert = handleException(err, TestErrorMessage);+ captureAdapterResult(+ "adapter_test_connection_result",+ "error",+ alert.content,+ );+ setAlertDetails(alert);}).finally(() => {setIsTcLoading(false);@@ -338,6 +361,7 @@ function ConfigureDs({if (!isConnector && method === "POST") {updateSession(type);}+ captureAdapterResult("adapter_save_result", "success");if (oAuthProvider?.length > 0 && isOAuthMethodSelected()) {localStorage.removeItem(oauthCacheKey);@@ -347,7 +371,9 @@ function ConfigureDs({setOpen(false);}).catch((err) => {- setAlertDetails(handleException(err));+ const alert = handleException(err);+ captureAdapterResult("adapter_save_result", "error", alert.content);+ setAlertDetails(alert);}).finally(() => {setIsSubmitApiLoading(false);diff --git a/frontend/src/helpers/PostHogDeployment.js b/frontend/src/helpers/PostHogDeployment.jsindex ab950193a..bb267f9d4 100644--- a/frontend/src/helpers/PostHogDeployment.js+++ b/frontend/src/helpers/PostHogDeployment.js@@ -12,4 +12,10 @@ const getDeployment = () =>// PII (email, name) may only leave Unstract-managed SaaS deploymentsconst isSaasProdDeployment = () => getDeployment() !== "self-hosted";-export { getDeployment, isSaasProdDeployment };+// Error text can contain customer endpoints, so it is sent only from SaaS.+const saasOnlyErrorProps = (errorMessage) =>+ typeof errorMessage === "string" && errorMessage && isSaasProdDeployment()+ ? { error: errorMessage.slice(0, 200) }+ : {};++export { getDeployment, isSaasProdDeployment, saasOnlyErrorProps };diff --git a/frontend/src/helpers/PostHogDeployment.test.js b/frontend/src/helpers/PostHogDeployment.test.jsnew file mode 100644index 000000000..77fc57ef0--- /dev/null+++ b/frontend/src/helpers/PostHogDeployment.test.js@@ -0,0 +1,29 @@+import { afterEach, describe, expect, it, vi } from "vitest";++import { saasOnlyErrorProps } from "./PostHogDeployment";++const setHost = (hostname) =>+ vi.spyOn(window, "location", "get").mockReturnValue({ hostname });++describe("saasOnlyErrorProps", () => {+ afterEach(() => vi.restoreAllMocks());++ it("drops error text on self-hosted deployments", () => {+ setHost("unstract.customer.internal");+ expect(saasOnlyErrorProps("connect to https://10.0.0.5 failed")).toEqual(+ {},+ );+ });++ it("keeps error text, capped at 200 chars, on SaaS", () => {+ setHost("us-central.unstract.com");+ const props = saasOnlyErrorProps("x".repeat(500));+ expect(props.error).toHaveLength(200);+ });++ it("adds nothing when there is no string error", () => {+ setHost("eu-west.unstract.com");+ expect(saasOnlyErrorProps(undefined)).toEqual({});+ expect(saasOnlyErrorProps([{ detail: "limit" }])).toEqual({});+ });+});diff --git a/frontend/src/hooks/usePromptStudioSocket.js b/frontend/src/hooks/usePromptStudioSocket.jsindex 1f93cd8e3..4c72fb9d2 100644--- a/frontend/src/hooks/usePromptStudioSocket.js+++ b/frontend/src/hooks/usePromptStudioSocket.js@@ -5,6 +5,7 @@ import { useAlertStore } from "../store/alert-store";import { useCustomToolStore } from "../store/custom-tool-store";import { usePromptRunStatusStore } from "../store/prompt-run-status-store";import { useExceptionHandler } from "./useExceptionHandler";+import usePostHogEvents from "./usePostHogEvents";import usePromptOutput from "./usePromptOutput";const PROMPT_STUDIO_RESULT_EVENT = "prompt_studio_result";@@ -25,6 +26,7 @@ const usePromptStudioSocket = () => {const { setAlertDetails } = useAlertStore();const handleException = useExceptionHandler();const { updatePromptOutputState } = usePromptOutput();+ const { setPostHogCustomEvent } = usePostHogEvents();const clearPromptStatuses = useCallback((promptIds, docId, profileId) => {@@ -144,6 +146,16 @@ const usePromptStudioSocket = () => {return;}+ // Captured before handling so a handler throw doesn't hide the outcome.+ setPostHogCustomEvent("ps_prompt_run_result", {+ task_id: extra?.task_id,+ operation,+ status,+ tool_id,+ prompt_count: extra?.prompt_ids?.length,+ elapsed_seconds: extra?.elapsed,+ });+if (status === "completed") {handleCompleted(operation, result, extra);} else if (status === "failed") {@@ -155,7 +167,14 @@ const usePromptStudioSocket = () => {);}},- [handleCompleted, handleFailed, setAlertDetails, handleException, details],+ [+ handleCompleted,+ handleFailed,+ setAlertDetails,+ handleException,+ details,+ setPostHogCustomEvent,+ ],);useEffect(() => {