Skip to content

πŸ™Œ OpenHands: AI-Driven Development

feat: instrument PostHog events for automation lifecycle (#1549)

Hiep Le31ff0fe85de3
diff --git a/__tests__/components/automations/add-automation-modal.test.tsx b/__tests__/components/automations/add-automation-modal.test.tsx
index 84b995af77..6067391ea9 100644
--- a/__tests__/components/automations/add-automation-modal.test.tsx
+++ b/__tests__/components/automations/add-automation-modal.test.tsx
@@ -9,6 +9,14 @@ import {
import { AddAutomationModal } from "#/components/features/automations/add-automation-modal";
import { I18nKey } from "#/i18n/declaration";
+vi.mock("posthog-js/react", () => ({
+ usePostHog: () => undefined,
+}));
+
+vi.mock("#/hooks/query/use-settings", () => ({
+ useSettings: () => ({ data: { user_consents_to_analytics: true } }),
+}));
+
vi.mock("react-i18next", () => ({
useTranslation: () => ({
t: (key: string) => key,
diff --git a/__tests__/components/automations/create-instructions.test.tsx b/__tests__/components/automations/create-instructions.test.tsx
index 0026d52acd..a9bcc70f13 100644
--- a/__tests__/components/automations/create-instructions.test.tsx
+++ b/__tests__/components/automations/create-instructions.test.tsx
@@ -10,6 +10,15 @@ import { CreateInstructions } from "#/components/features/automations/create-ins
import { I18nKey } from "#/i18n/declaration";
import { useConversationStore } from "#/stores/conversation-store";
+const captureMock = vi.fn();
+vi.mock("posthog-js/react", () => ({
+ usePostHog: () => ({ capture: captureMock }),
+}));
+
+vi.mock("#/hooks/query/use-settings", () => ({
+ useSettings: () => ({ data: { user_consents_to_analytics: true } }),
+}));
+
vi.mock("react-i18next", () => ({
useTranslation: () => ({
t: (key: string) => {
@@ -78,9 +87,22 @@ function renderCreateInstructions() {
describe("CreateInstructions", () => {
beforeEach(() => {
+ captureMock.mockClear();
useConversationStore.setState({ messageToSend: null });
});
+ it("captures automation_created with the active backend kind when Create Automation is clicked", async () => {
+ const user = userEvent.setup();
+ renderCreateInstructions();
+
+ await user.click(screen.getByTestId("automations-create-automation"));
+
+ expect(captureMock).toHaveBeenCalledWith(
+ "automation_created",
+ expect.objectContaining({ backend_kind: "local" }),
+ );
+ });
+
it("navigates to conversations with a prefilled prompt when Create Automation is clicked", async () => {
const user = userEvent.setup();
const setMessageToSend = vi.fn();
diff --git a/__tests__/hooks/query/use-automations-backend-switch.test.tsx b/__tests__/hooks/query/use-automations-backend-switch.test.tsx
index 83b44fd9bc..ef91e7b42d 100644
--- a/__tests__/hooks/query/use-automations-backend-switch.test.tsx
+++ b/__tests__/hooks/query/use-automations-backend-switch.test.tsx
@@ -13,6 +13,9 @@ import { ActiveBackendProvider } from "#/contexts/active-backend-context";
import {
useAutomations,
useDispatchAutomation,
+ useDeleteAutomation,
+ useToggleAutomation,
+ useUpdateAutomation,
} from "#/hooks/query/use-automations";
import {
useAutomationDetail,
@@ -33,9 +36,21 @@ vi.mock("#/api/automation-service/automation-service.api", () => ({
getAutomation: vi.fn(),
getAutomationRuns: vi.fn(),
dispatchAutomation: vi.fn(),
+ deleteAutomation: vi.fn(),
+ updateAutomation: vi.fn(),
+ toggleAutomation: vi.fn(),
},
}));
+const captureMock = vi.fn();
+vi.mock("posthog-js/react", () => ({
+ usePostHog: () => ({ capture: captureMock }),
+}));
+
+vi.mock("#/hooks/query/use-settings", () => ({
+ useSettings: () => ({ data: { user_consents_to_analytics: true } }),
+}));
+
const localBackend: Backend = {
id: "local-1",
name: "Local 1",
@@ -102,6 +117,13 @@ beforeEach(() => {
vi.mocked(AutomationService.dispatchAutomation).mockResolvedValue(
automationRun,
);
+ vi.mocked(AutomationService.deleteAutomation).mockReset();
+ vi.mocked(AutomationService.updateAutomation).mockReset();
+ vi.mocked(AutomationService.toggleAutomation).mockReset();
+ vi.mocked(AutomationService.deleteAutomation).mockResolvedValue(undefined);
+ vi.mocked(AutomationService.updateAutomation).mockResolvedValue(automation);
+ vi.mocked(AutomationService.toggleAutomation).mockResolvedValue(automation);
+ captureMock.mockClear();
vi.mocked(AutomationService.getAutomations).mockResolvedValue(listResponse);
vi.mocked(AutomationService.getAutomation).mockResolvedValue(automation);
@@ -230,3 +252,91 @@ describe("useAutomationRuns β€” polling", () => {
15000,
);
});
+
+describe("automation mutation hooks β€” analytics tracking", () => {
+ it("captures automation_executed after a successful dispatch", async () => {
+ const { result } = renderHook(() => useDispatchAutomation(), {
+ wrapper: makeWrapper(),
+ });
+
+ await act(async () => {
+ await result.current.mutateAsync("auto-1");
+ });
+
+ await waitFor(() => {
+ expect(captureMock).toHaveBeenCalledWith(
+ "automation_executed",
+ expect.objectContaining({ backend_kind: "local" }),
+ );
+ });
+ });
+
+ it("captures automation_deleted after a successful delete", async () => {
+ const { result } = renderHook(() => useDeleteAutomation(), {
+ wrapper: makeWrapper(),
+ });
+
+ await act(async () => {
+ await result.current.mutateAsync("auto-1");
+ });
+
+ await waitFor(() => {
+ expect(captureMock).toHaveBeenCalledWith(
+ "automation_deleted",
+ expect.objectContaining({ backend_kind: "local" }),
+ );
+ });
+ });
+
+ it("captures automation_edited after a successful update", async () => {
+ const { result } = renderHook(() => useUpdateAutomation(), {
+ wrapper: makeWrapper(),
+ });
+
+ await act(async () => {
+ await result.current.mutateAsync({
+ id: "auto-1",
+ body: { name: "Renamed" },
+ });
+ });
+
+ await waitFor(() => {
+ expect(captureMock).toHaveBeenCalledWith(
+ "automation_edited",
+ expect.objectContaining({ backend_kind: "local" }),
+ );
+ });
+ });
+
+ it("captures automation_deactivated when an automation is disabled", async () => {
+ const { result } = renderHook(() => useToggleAutomation(), {
+ wrapper: makeWrapper(),
+ });
+
+ await act(async () => {
+ await result.current.mutateAsync({ id: "auto-1", enabled: false });
+ });
+
+ await waitFor(() => {
+ expect(captureMock).toHaveBeenCalledWith(
+ "automation_deactivated",
+ expect.objectContaining({ backend_kind: "local" }),
+ );
+ });
+ });
+
+ it("does not capture automation_deactivated when an automation is enabled", async () => {
+ const { result } = renderHook(() => useToggleAutomation(), {
+ wrapper: makeWrapper(),
+ });
+
+ await act(async () => {
+ await result.current.mutateAsync({ id: "auto-1", enabled: true });
+ });
+
+ expect(captureMock).not.toHaveBeenCalledWith(
+ "automation_deactivated",
+ expect.anything(),
+ );
+ });
+});
diff --git a/src/components/features/automations/create-instructions.tsx b/src/components/features/automations/create-instructions.tsx
index 415457cedb..f6bfe9cc0e 100644
--- a/src/components/features/automations/create-instructions.tsx
+++ b/src/components/features/automations/create-instructions.tsx
@@ -6,6 +6,8 @@ import MessageSquareShareIcon from "#/icons/message-square-share.svg?react";
import { cn } from "#/utils/utils";
import { BrandButton } from "#/components/features/settings/brand-button";
import { useLaunchSkillInChat } from "#/hooks/use-launch-skill-in-chat";
+import { useActiveBackend } from "#/contexts/active-backend-context";
+import { useTracking } from "#/hooks/use-tracking";
const DOCS_URL =
"https://docs.openhands.dev/openhands/usage/automations/overview";
@@ -52,8 +54,11 @@ export function CreateInstructionsContent({
}: CreateInstructionsContentProps = {}) {
const { t } = useTranslation("openhands");
const launchInChat = useLaunchSkillInChat();
+ const active = useActiveBackend();
+ const { trackAutomationCreated } = useTracking();
const handleCreateAutomation = () => {
+ trackAutomationCreated({ backendKind: active.backend.kind });
launchInChat(t(I18nKey.AUTOMATIONS$CREATE_AUTOMATION_PROMPT), onLaunch);
};
diff --git a/src/hooks/query/use-automations.ts b/src/hooks/query/use-automations.ts
index 3d84670a19..aff5d37013 100644
--- a/src/hooks/query/use-automations.ts
+++ b/src/hooks/query/use-automations.ts
@@ -1,6 +1,7 @@
import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query";
import AutomationService from "#/api/automation-service/automation-service.api";
import { useActiveBackend } from "#/contexts/active-backend-context";
+import { useTracking } from "#/hooks/use-tracking";
import type { Automation } from "#/types/automation";
import {
AUTOMATION_DETAIL_QUERY_KEY,
@@ -33,40 +34,53 @@ export function useAutomations(options: UseAutomationsOptions = {}) {
export function useToggleAutomation() {
const queryClient = useQueryClient();
+ const active = useActiveBackend();
+ const { trackAutomationDeactivated } = useTracking();
return useMutation({
mutationFn: ({ id, enabled }: { id: string; enabled: boolean }) =>
AutomationService.toggleAutomation(id, enabled),
- onSuccess: () => {
+ onSuccess: (_data, variables) => {
queryClient.invalidateQueries({ queryKey: AUTOMATIONS_QUERY_KEY });
queryClient.invalidateQueries({ queryKey: AUTOMATION_DETAIL_QUERY_KEY });
+ if (!variables.enabled) {
+ trackAutomationDeactivated({ backendKind: active.backend.kind });
+ }
},
});
}
export function useUpdateAutomation() {
const queryClient = useQueryClient();
+ const active = useActiveBackend();
+ const { trackAutomationEdited } = useTracking();
return useMutation({
mutationFn: ({ id, body }: { id: string; body: Partial<Automation> }) =>
AutomationService.updateAutomation(id, body),
onSuccess: () => {
queryClient.invalidateQueries({ queryKey: AUTOMATIONS_QUERY_KEY });
queryClient.invalidateQueries({ queryKey: AUTOMATION_DETAIL_QUERY_KEY });
+ trackAutomationEdited({ backendKind: active.backend.kind });
},
});
}
export function useDeleteAutomation() {
const queryClient = useQueryClient();
+ const active = useActiveBackend();
+ const { trackAutomationDeleted } = useTracking();
return useMutation({
mutationFn: (id: string) => AutomationService.deleteAutomation(id),
onSuccess: () => {
queryClient.invalidateQueries({ queryKey: AUTOMATIONS_QUERY_KEY });
+ trackAutomationDeleted({ backendKind: active.backend.kind });
},
});
}
export function useDispatchAutomation() {
const queryClient = useQueryClient();
+ const active = useActiveBackend();
+ const { trackAutomationExecuted } = useTracking();
return useMutation({
mutationFn: (id: string) => AutomationService.dispatchAutomation(id),
onSuccess: (_run, id) => {
@@ -75,6 +89,7 @@ export function useDispatchAutomation() {
queryClient.invalidateQueries({
queryKey: [...AUTOMATION_RUNS_QUERY_KEY, id],
});
+ trackAutomationExecuted({ backendKind: active.backend.kind });
},
});
}
diff --git a/src/hooks/use-tracking.ts b/src/hooks/use-tracking.ts
index 438017463e..a5eeef3ccf 100644
--- a/src/hooks/use-tracking.ts
+++ b/src/hooks/use-tracking.ts
@@ -147,6 +147,46 @@ export const useTracking = () => {
track("download_trajectory_button_clicked");
};
+ const trackAutomationCreated = ({
+ backendKind,
+ }: {
+ backendKind: BackendKind;
+ }) => {
+ track("automation_created", { backend_kind: backendKind });
+ };
+
+ const trackAutomationExecuted = ({
+ backendKind,
+ }: {
+ backendKind: BackendKind;
+ }) => {
+ track("automation_executed", { backend_kind: backendKind });
+ };
+
+ const trackAutomationDeleted = ({
+ backendKind,
+ }: {
+ backendKind: BackendKind;
+ }) => {
+ track("automation_deleted", { backend_kind: backendKind });
+ };
+
+ const trackAutomationDeactivated = ({
+ backendKind,
+ }: {
+ backendKind: BackendKind;
+ }) => {
+ track("automation_deactivated", { backend_kind: backendKind });
+ };
+
+ const trackAutomationEdited = ({
+ backendKind,
+ }: {
+ backendKind: BackendKind;
+ }) => {
+ track("automation_edited", { backend_kind: backendKind });
+ };
+
const trackBackendAdded = ({
backendKind,
connectionMethod,
@@ -232,6 +272,11 @@ export const useTracking = () => {
trackSettingsSaved,
trackMcpConfigUpdated,
trackDownloadTrajectoryButtonClicked,
+ trackAutomationCreated,
+ trackAutomationExecuted,
+ trackAutomationDeleted,
+ trackAutomationDeactivated,
+ trackAutomationEdited,
trackBackendAdded,
trackOnboardingStarted,
trackOnboardingStepViewed,