🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { type CaptureResult, type CaptureOptions } from "posthog-js";
import { usePostHog } from "posthog-js/react";
import { useCallback } from "react";
import type { AnnotationEventMap } from "@/src/features/scores/lib/annotationAnalytics";
import type { EvalOnboardingEventMap } from "@/src/features/evals/v2/types/evalOnboardingAnalytics";
import type { AiGatewayEventMap } from "@/src/features/ai-gateway/types/aiGatewayAnalytics";
import type {
ToastInteractionEventProperties,
ToastShownEventProperties,
} from "@/src/features/notifications/toastAnalytics";
export const V4_BETA_ENABLED_POSTHOG_PROPERTY = "v4BetaEnabled";
type ToastEventMap = {
"toast:shown": ToastShownEventProperties;
"toast:report_issue": ToastInteractionEventProperties;
"toast:dismiss": ToastInteractionEventProperties;
};
// resource:action, only use snake_case
const events = {
table: [
"filter_builder_open",
"filter_builder_close",
"search_submit",
"row_height_switch_select",
"pagination_button_click",
"pagination_page_size_select",
"column_visibility_change",
"column_sorting_header_click",
"column_visibility_changed",
],
trace: ["delete_form_open", "delete", "delete_form_submit"],
trace_detail: [
"publish_button_click",
"observation_tree_collapse",
"observation_tree_expand",
"observation_tree_toggle_scores",
"observation_tree_toggle_metrics",
"io_mode_switch",
"io_parser_comparison",
"test_in_playground_button_click",
"display_mode_switch",
"download_button_click",
"view_mode_switch",
"tree_panel_toggle",
"graph_view_toggle",
// Aggregated vs expanded graph build mode (LFE-10676).
"graph_mode_switch",
// `source` distinguishes the inline expand/collapse button, the message
// header control, and the trace settings switch; `collapsed` is the new
// preference value.
"system_prompt_collapse_toggle",
// Fired from the tree, timeline, graph, and search-result click handlers;
// `source` says which surface drove the navigation.
"node_selected",
// Download from the large-string IO fallback (LFE-10991): a top-level
// string over the render limit is shown as a bounded preview + download
// instead of the full Pretty/JSON viewer. Measures how often users hit it.
"large_string_field_download",
// Raw download from the JSON-view fallback shown when a field is too large
// to render in the unvirtualized viewer (LFE-10989).
"json_view_large_field_download",
// Detail-panel tab switch (Preview / Log View / Scores). `tab` is the
// target tab, `target` is trace vs observation. Answers whether the Log
// View and Scores tabs earn their place.
"detail_tab_switch",
// Row actions on the Attributes, Model parameters and Metadata tables in
// the detail panel: `table` is which of the three, `action` is copy,
// include_filter or exclude_filter. Never the key or value text.
"attribute_table_action",
// Controls used *inside* the Log View tab, so a removal decision can weigh
// what people actually do there. `target` is trace vs observation;
// `action` is one of (see `LogViewAction` in TraceLogView.tsx):
// search_focus (once per focus, not per keystroke), indent_toggle,
// milliseconds_toggle, expand_all, collapse_all, row_expand, row_collapse,
// copy_json, json_mode_collapse_toggle, and view_mode_switch (the
// Formatted/JSON toggle, emitted by the hosting detail views).
// Metadata only — never an observation id or the search text.
"log_view_interaction",
// The JSON-view Beta switch (legacy JSON <-> virtualized json-beta).
// `enabled` is the new value. Decides whether json-beta graduates.
"json_beta_toggle",
// Whole-section collapse chevrons on Input/Output/Metadata/Corrected
// Output. `section` is a fixed enum (never content), `collapsed` the new
// state. Informs future default-collapsed decisions.
"io_section_collapse_toggle",
],
// The shared table peek panel (opened via the `peek` URL param). Props carry
// `routePattern` (the Next.js route pattern, never a concrete URL) so opens
// can be sliced by surface without leaking ids.
peek: ["opened", "closed", "expand_toggle", "resized", "open_in_new_tab"],
// Pulse outlier strip above the v4 events table (LFE-14451). Props are
// metadata only — mode/metric/aggregation enums, gesture trigger, bucket
// counts — never bucket values or time-range contents.
pulse: [
"drill_in",
"preview_pinned",
"mode_switch",
"aggregation_switch",
"closed",
"reopened",
],
generations: ["export"],
// Lazy JSON viewer perf telemetry (LFE-14419): learn whether the size gate
// and main-thread assumptions hold on real payloads. Metadata only —
// durations, char counts, tier; never payload content.
json_viewer: ["indexed", "slow_expand"],
saved_views: [
"create",
"update",
"delete",
"update_form_open",
"create_form_open",
"delete_form_open",
"view_selected",
"drawer_open",
"drawer_close",
"update_config",
"permalink_generate",
"permalink_visit",
"update_name",
"search_views",
"system_preset_selected",
"category_chip_open",
"category_chip_apply",
"category_preset_preview",
"category_preset_coming_soon_click",
// Fired when a chip popover closes; carries durationMs + outcome
// ("applied" | "cleared" | "previewed_only" | "no_interaction") so the
// explore → activate funnel and dwell time read from one event.
"category_chip_close",
// A bookmarked/stored system-preset id that the catalog retired — the
// user was shown the one-time notice and landed on the default view.
"retired_view_redirect",
"applied",
],
score: [
"create",
"update",
"delete",
"update_form_open",
"create_form_open",
"update_comment",
"delete_comment",
"form_abandoned",
"value_set",
"level_changed",
"level_added",
],
annotation: ["entry_click"],
annotation_queues: [
"item_added",
"item_removed",
"manage_click",
"score_configs_saved",
],
score_configs: [
"create_form_submit",
"update_form_submit",
"manage_configs_item_click",
"add_category_inline",
"archive_form_open",
"archive_form_submit",
],
models: ["delete_button_click", "new_form_submit", "new_form_open"],
prompts: [
"new_form_submit",
"new_form_open",
"update_form_open",
"update_form_submit",
"bulk_export",
"bulk_import_submit",
],
skills: [
"new_form_open",
"version_create",
"version_download",
"version_compare",
"delete",
"import_open",
"import",
],
prompt_detail: [
"test_in_playground_button_click",
"add_label_submit",
"apply_labels",
"version_delete_open",
"version_delete_submit",
"duplicate_button_click",
"duplicate_form_submit",
],
session_detail: [
"announcement_learn_more_click",
"announcement_feedback_click",
"announcement_dismiss",
"publish_button_click",
"download_button_click",
"copy_session_id_click",
"truncated_observation_open_trace_click",
"truncated_observation_download_click",
"inline_tools_toggled",
"system_prompt_toggled",
"metadata_jsonpath_config_changed",
],
eval_config: [
"new_form_submit",
"new_form_open",
"activate",
"deactivate",
"update",
"delete_form_open",
"delete_evaluator_button_click",
],
eval_templates: [
"view_version",
"new_form_open",
"update_form_open",
"new_form_submit",
"update_form_submit",
"delete_form_open",
"delete_template_button_click",
],
evaluators: [
"create",
"update",
"delete",
"test",
"saved_dialog_submit",
"overview_action_click",
"variable_mapping_configured",
"version_history_interaction",
"default_model_update",
"reactivate",
"gallery_creation_source_select",
"empty_state_template_select",
"empty_state_browse_library",
"empty_state_detect_topics",
"alert_create_clicked",
],
evaluation_rules: [
"create",
"update",
"delete",
"status_change",
"attach_evaluator",
"detach_evaluator",
"filter_reused",
],
// Evaluator creation funnel (gallery -> setup page -> saved dialog). Props
// are typed in EvalOnboardingEventMap: metadata only, never search text,
// prompt content, names or filter values.
eval: [
"onboarding_started",
"onboarding_step_completed",
"onboarding_completed",
"onboarding_gallery_searched",
"onboarding_gallery_section_selected",
"onboarding_evaluator_type_changed",
"onboarding_preview_toggled",
"onboarding_sample_observation_previewed",
"onboarding_prompt_modified",
"onboarding_llm_connection_tab_opened",
"onboarding_model_picker_opened",
"onboarding_model_changed",
"onboarding_ai_generate_requested",
"onboarding_sampling_changed",
"onboarding_historic_eval_toggled",
"onboarding_scope_changed",
"onboarding_create_rule_opened",
"onboarding_execution_skipped",
],
// One-shot batch evaluation from the events / experiments tables.
// Counts and enums only — never mapping contents or observation payloads.
batch_eval: ["run"],
integrations: [
"posthog_form_submitted",
"blob_storage_form_submitted",
"mixpanel_form_submitted",
],
sign_in: ["cloud_region_switch", "button_click"],
sign_up: ["button_click"],
auth: [
"reset_password_email_requested",
"update_password_form_submit",
"set_password_form_submit",
],
playground: [
"execute_button_click",
"save_to_new_prompt_button_click",
"save_to_prompt_version_button_click",
],
dashboard: [
"view",
"widget_saved",
"clone_dashboard",
"home_dashboard_viewed",
"home_dashboard_peeked",
"home_dashboard_set_default",
"home_edit_pencil_click",
"locked_edit_attempt",
"clone_first_cancelled",
"clone_open_existing_click",
"widget_copy_first_open",
"widget_copied_to_project",
"widget_json_downloaded",
"widget_copied_to_clipboard",
"widget_view_as_table",
"widget_high_cardinality_error",
"widget_pasted",
"widget_paste_rejected",
"widget_duplicated",
"dashboard_json_imported",
"add_widget_dialog_open",
"add_widget_tab_switch",
"widget_added",
"chart_tab_switch",
"date_range_changed",
"new_widget_form_open",
"new_dashboard_form_open",
"delete_widget_form_open",
"delete_dashboard_form_open",
"delete_dashboard_button_click",
],
monitors: ["create", "delete_form_open", "delete_monitor_button_click"],
datasets: [
"delete_form_open",
"delete_dataset_button_click",
"update_form_open",
"new_form_open",
"new_form_submit",
"update_form_submit",
"delete_form_submit",
],
organizations: [
"new_form_submit",
"new_form_open",
"demo_project_button_click",
],
projects: ["new_form_submit", "new_form_open"],
dataset_item: [
"archive_toggle",
"new_form_open",
"new_form_submit",
"new_from_trace_form_submit",
"new_from_trace_form_open",
"upload_csv_button_click",
"upload_csv_form_submit",
"select_observations_button_click",
"delete",
],
dataset_run: [
"delete_form_open",
"delete_form_submit",
"new_form_open",
"new_form_submit",
"view_prompt_experiment_docs",
"view_custom_experiment_docs",
"compare_view_click",
"charts_view_added",
"charts_view_removed",
"compare_run_added",
"compare_run_removed",
],
// Experiments UI (v4). Metadata only — counts/enums/booleans/field names;
// never experiment or dataset names, score values, or item content.
// `isV4` + `tableName` on every event. `source` on comparison/baseline
// distinguishes picker vs table-selection vs url (deep link / redirect) vs
// auto — so the auto-selected comparison stays out of "users who compare".
//
experiment: [
"comparison_changed",
"comparison_picker_opened",
"baseline_changed",
"auto_comparison_preference_changed",
"chart_metric_changed",
"chart_added",
"chart_removed",
"chart_type_changed",
"layout_changed",
"diff_mode_changed",
"score_column_scope_toggled",
"item_regression_filter_applied",
],
// Version-update reload notification (LFE-10978). `banner_shown` fires once
// per appearance; the two actions measure the reload-vs-dismiss split. No
// props carry user content.
version_update: ["banner_shown", "reload_clicked", "dismissed"],
notification: ["click_link", "dismiss_notification"],
// User-visible toast denominator. Metadata only: `path` is a static tRPC
// procedure, `operation` is a static action id, and `errorId` is the opaque
// ID shown in the toast. Never send toast text or error payloads.
toast: ["shown", "report_issue", "dismiss"],
tag: [
"add_existing_tag",
"remove_tag",
"modal_open",
"create_new_button_click",
],
onboarding: [
"code_example_tab_switch",
"tracing_check_active",
"tracing_api_key_create_clicked",
"tracing_agent_prompt_copy_clicked",
"tracing_manual_docs_link_clicked",
"signup_survey_submitted",
],
user_settings: ["theme_changed", "feature_preview_toggled"],
project_settings: [
"project_delete",
"rename_form_submit",
"retention_form_submit",
"project_transfer",
"api_key_delete",
"api_key_create",
"llm_api_key_delete",
"llm_api_key_create",
"llm_api_key_update",
"send_membership_invitation",
"delete_membership_invitation",
"delete_membership",
"pricing_dialog_opened",
],
organization_settings: [
"rename_form_submit",
"send_membership_invitation",
"delete_membership_invitation",
"delete_membership",
"api_key_create",
"api_key_delete",
"llm_api_key_create",
"llm_api_key_update",
"llm_api_key_delete",
"pricing_dialog_opened",
"delete_organization",
"ai_features_toggle",
"ai_telemetry_toggle",
"feature_flag_default_toggled",
"personal_feature_previews_opened",
"user_feature_flag_toggled",
],
help_popup: ["opened", "href_clicked"],
navigate_detail_pages: ["button_click_prev_or_next"],
support_chat: [
"initiated",
"opened",
"message_sent",
"community_hours_click",
], // also used on landing page for consistency
topics: [
"results_view_select",
"setup_opened",
"setup_step_saved",
"setup_started",
],
in_app_agent: [
"activity_opened",
"entry_point_click",
"new_chat_started",
"new_chat_turn",
"quick_action_started",
"tool_approval_decided",
],
// AI Gateway setup and adoption. Props are typed in AiGatewayEventMap and
// carry `orgId` on every event: metadata only, never emails, names,
// credentials, key metadata or model ids.
ai_gateway: [
"setup_guide_viewed",
"setup_step_completed",
"ingestion_settings_saved",
"provider_credential_added",
"provider_credential_updated",
"provider_credential_deleted",
"team_invite_sent",
"api_key_created",
"api_key_revoked",
"api_key_secret_copied",
"api_key_secret_revealed",
"model_access_changed",
"model_name_copied",
"agent_setup_instructions_viewed",
"agent_setup_instructions_copied",
],
cmd_k_menu: ["opened", "search_entered", "navigated"],
spend_alert: ["created", "updated", "deleted"],
// Launch surfaces answer "which entry point and state lead people into
// gateway setup or key creation?" Props are state enums and counts only.
ai_gateway_launch: [
"banner_cta_clicked",
"banner_dismissed",
"dialog_setup_clicked",
"dialog_get_key_clicked",
"dialog_learn_more_clicked",
],
sidebar: [
"book_a_call_clicked",
"v4_beta_toggled",
"v4_migration_card_clicked",
],
// Migration-funnel events answer "are people finding the panel, which
// action items do they engage with, and which CTA do they use?"
// panel_opened carries the entry surface; panel_checks_loaded carries the
// amount of work shown (counts only — never keys or SDK payload values).
v4_migration: [
"coding_agent_prompt_copied",
"delay_badge_clicked",
// Discoverability pair for the table delay badge: `shown` is the
// exposure denominator (badge actually rendered), `hovered` counts
// noticed-but-not-clicked (pill expanded long enough to read).
"delay_badge_shown",
"delay_badge_hovered",
"project_chip_clicked",
"contact_book_call_clicked",
"contact_support_clicked",
"status_row_clicked",
"update_required_badge_clicked",
"migrate_evals_with_agent_clicked",
"overview_banner_status_clicked",
"overview_banner_docs_clicked",
// Opt-out users on the legacy-UI top strip → migration status page.
"legacy_ui_banner_status_clicked",
"panel_docs_link_clicked",
"create_project_keys_clicked",
"panel_opened",
"panel_checks_loaded",
"section_expanded",
"evidence_link_clicked",
"section_link_clicked",
"project_keys_copied",
"evals_manual_upgrade_clicked",
"walkthrough_video_clicked",
],
// Filter/search-bar usage analytics (LFE-10781). METADATA ONLY — payloads
// never carry a raw filter value, search text, or AI prompt (PII). Only
// type/column/operator/key(field-name)/counts/lengths/booleans/enums.
// `isV4` on every event reflects fast-mode (v4 events table) at action time.
filters: [
"applied",
"cleared",
"facet_operator_toggled",
"active_only_toggled",
"facet_added",
"facet_search",
"facet_mode_switched",
"expand_all_toggled",
"facet_toggled",
"sidebar_toggled",
"search_submitted",
"search_error",
"ai_generate_requested",
"ai_generate_applied",
"ai_generate_failed",
],
} as const;
// type that represents all possible event names, e.g. "trace:delete"
type EventName = {
[Resource in keyof typeof events]: `${Resource}:${(typeof events)[Resource][number]}`;
}[keyof typeof events];
type TypedEventMap = AnnotationEventMap &
EvalOnboardingEventMap &
AiGatewayEventMap &
ToastEventMap;
type EventProperties = TypedEventMap & {
[E in Exclude<EventName, keyof TypedEventMap>]: Record<string, any> | null;
};
export const usePostHogClientCapture = () => {
const posthog = usePostHog();
// wrapped posthog.capture function that only allows events that are in the
// allowlist; stable identity so it is safe in useCallback/useMemo deps
return useCallback(
function capture<E extends EventName>(
eventName: E,
...[properties, options]: E extends keyof ToastEventMap
? [properties: EventProperties[E], options?: CaptureOptions]
: [properties?: EventProperties[E], options?: CaptureOptions]
): CaptureResult | void {
return posthog.capture(eventName, properties, options);
},
[posthog],
);
};