Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(analytics): record every routine run and time in call (#8864)

Chaitanya Agarwal8c0147b81a24
diff --git a/apps/web/src/features/channel/Call/CallContext.tsx b/apps/web/src/features/channel/Call/CallContext.tsx
index af3941ef84..5173b42c60 100644
--- a/apps/web/src/features/channel/Call/CallContext.tsx
+++ b/apps/web/src/features/channel/Call/CallContext.tsx
@@ -1637,6 +1637,8 @@ function createCallState() {
}
}
+ // When this client joined its current call, for time-in-call analytics.
+ let joinedAt: number | undefined;
const lifecycle = createCallLifecycle({
shouldRequestToken: callSession.shouldRequestToken,
prepareToken: prepareNativeCallJoin,
@@ -1679,6 +1681,7 @@ function createCallState() {
});
}
fireAndForget(invalidateActiveCallQueries(), 'call-membership');
+ joinedAt = Date.now();
analytics.track('call_action', {
action: 'joined',
channelId: call.channelId,
@@ -1686,13 +1689,20 @@ function createCallState() {
trigger: source,
});
},
- onLeft: (call, reason) =>
+ onLeft: (call, reason) => {
+ const secondsInCall =
+ joinedAt === undefined
+ ? undefined
+ : Math.round((Date.now() - joinedAt) / 1000);
+ joinedAt = undefined;
analytics.track('call_action', {
action: 'left',
channelId: call.channelId,
callId: call.callId ?? undefined,
leaveReason: reason,
- }),
+ secondsInCall,
+ });
+ },
reportError: (error) => console.error('call lifecycle failed', error),
});
const [lifecycleSnapshot, setLifecycleSnapshot] = createSignal(
diff --git a/apps/web/src/lib/analytics/app-events.ts b/apps/web/src/lib/analytics/app-events.ts
index 38ede7bb2f..c687600642 100644
--- a/apps/web/src/lib/analytics/app-events.ts
+++ b/apps/web/src/lib/analytics/app-events.ts
@@ -259,6 +259,8 @@ export type AppEvents = {
| 'screen_share_toggled';
channelId: string;
callId?: string;
+ /** On `left`: seconds since this client joined the call. */
+ secondsInCall?: number;
} & Record<string, unknown>;
// Mic audio-processing state transitions (noise suppression attach /
// detach / fallback). `constraintMismatch` flags engines that report
diff --git a/crates/scheduled_action/src/domain/execution.rs b/crates/scheduled_action/src/domain/execution.rs
index e42b7fdba3..e06d1c466f 100644
--- a/crates/scheduled_action/src/domain/execution.rs
+++ b/crates/scheduled_action/src/domain/execution.rs
@@ -18,8 +18,8 @@ use crate::domain::models::{
MAX_ACTION_TIME, ResolvedTaskTarget, ScheduledAction, ScheduledActionUpdate,
};
use crate::domain::ports::{
- RoutineRun, ScheduledActionExecutor, ScheduledActionLiveUpdate, ScheduledActionRepo,
- ScheduledAgentRunner,
+ FinishedRoutineRun, RoutineRun, RoutineRunObserver, RunEnd, RunTarget, RunTrigger,
+ ScheduledActionExecutor, ScheduledActionLiveUpdate, ScheduledActionRepo, ScheduledAgentRunner,
};
/// Run identity allocated before any awaited preparation. Runners can cancel
@@ -54,6 +54,7 @@ pub struct InProcessExecutor<Rpo, Live, Runner> {
live_updates: Arc<Live>,
runner: Arc<Runner>,
admission: Arc<dyn AiAdmissionService>,
+ observer: Arc<dyn RoutineRunObserver>,
tracker: TaskTracker,
cancellation: CancellationToken,
}
@@ -65,6 +66,7 @@ impl<Rpo, Live, Runner> Clone for InProcessExecutor<Rpo, Live, Runner> {
live_updates: self.live_updates.clone(),
runner: self.runner.clone(),
admission: self.admission.clone(),
+ observer: self.observer.clone(),
tracker: self.tracker.clone(),
cancellation: self.cancellation.clone(),
}
@@ -91,6 +93,7 @@ where
runner,
live_updates,
admission: Arc::new(DisabledAiAdmissionService),
+ observer: Arc::new(()),
tracker,
cancellation,
}
@@ -103,6 +106,12 @@ where
self
}
+ /// Tell `observer` about every run that starts and then ends.
+ pub fn with_observer(mut self, observer: Arc<dyn RoutineRunObserver>) -> Self {
+ self.observer = observer;
+ self
+ }
+
async fn prepare(
&self,
action: &ScheduledAction,
@@ -200,6 +209,39 @@ async fn bounded<T>(
}
}
+fn finished_run(
+ action: &ScheduledAction,
+ trigger: RunTrigger,
+ started_at: DateTime<Utc>,
+ result: &Result<()>,
+) -> Option<FinishedRoutineRun> {
+ let target = serde_json::from_value::<AgentTask>(action.task.clone())
+ .ok()
+ .and_then(|task| {
+ task.resolve_target().ok().map(|target| match target {
+ ResolvedTaskTarget::Model { .. } => RunTarget::Model,
+ ResolvedTaskTarget::Agent { .. } => RunTarget::Agent,
+ })
+ })
+ .unwrap_or(RunTarget::Unknown);
+ let end = match result {
+ Ok(()) => RunEnd::Succeeded,
+ Err(error) if error.is::<ExecutionCancelled>() => RunEnd::Interrupted,
+ Err(error) => match error.downcast_ref::<AiAdmissionError>() {
+ Some(AiAdmissionError::Denied(_)) => RunEnd::AiLimitReached,
+ _ => RunEnd::Failed,
+ },
+ };
+ Some(FinishedRoutineRun {
+ owner: action.owner_user().ok()?.clone(),
+ routine_id: action.id?,
+ trigger,
+ target,
+ end,
+ duration: Utc::now() - started_at,
+ })
+}
+
fn execution_record(
action: &ScheduledAction,
resource: Option<ExecutionResource>,
@@ -283,6 +325,13 @@ where
if let Err(error) = &result {
executor.cancel(&action, &handle, error).await;
}
+ let trigger = match firing {
+ RoutineRun::Scheduled { .. } => RunTrigger::Scheduled,
+ _ => RunTrigger::Manual,
+ };
+ if let Some(run) = finished_run(&action, trigger, start_time, &result) {
+ executor.observer.run_finished(run);
+ }
let record = execution_record(&action, handle.resource.clone(), start_time, &result);
let end_time = record.end_time;
let bookkeeping = async {
@@ -356,6 +405,10 @@ where
if let Err(error) = &result {
self.cancel(&run.action, &handle, error).await;
}
+ let trigger = RunTrigger::Event(run.run.pending.event.event_name());
+ if let Some(finished) = finished_run(&run.action, trigger, run.started_at, &result) {
+ self.observer.run_finished(finished);
+ }
let record = execution_record(
&run.action,
handle.resource.clone(),
diff --git a/crates/scheduled_action/src/domain/execution/test.rs b/crates/scheduled_action/src/domain/execution/test.rs
index d721eb5a36..201379e43c 100644
--- a/crates/scheduled_action/src/domain/execution/test.rs
+++ b/crates/scheduled_action/src/domain/execution/test.rs
@@ -2,7 +2,7 @@ use super::*;
use crate::domain::event_runs::{
AuthorizedEventRun, ClaimToken, ConfigurationRevision, PendingEventRun,
};
-use crate::domain::event_trigger::EventReference;
+use crate::domain::event_trigger::{EventName, EventReference};
use crate::domain::models::{ActionKind, ExecutionResourceType, RunVisibility};
use entity_access::domain::models::{EntityAccessReceipt, ViewAccessLevel};
use macro_user_id::user_id::MacroUserIdStr;
@@ -834,3 +834,113 @@ async fn event_cancellation_is_terminal_interruption() {
assert_eq!(result.outcome, EventRunOutcome::Interrupted);
assert_eq!(executor.runner.calls.load(Ordering::SeqCst), 0);
}
+
+#[derive(Default)]
+struct Observer(Mutex<Vec<FinishedRoutineRun>>);
+impl RoutineRunObserver for Observer {
+ fn run_finished(&self, run: FinishedRoutineRun) {
+ self.0.lock().unwrap().push(run);
+ }
+}
+
+fn observed(
+ executor: InProcessExecutor<Repo, Live, Runner>,
+) -> (InProcessExecutor<Repo, Live, Runner>, Arc<Observer>) {
+ let observer = Arc::new(Observer::default());
+ (executor.with_observer(observer.clone()), observer)
+}
+
+fn ends(observer: &Observer) -> Vec<(RunTrigger, RunTarget, RunEnd)> {
+ observer
+ .0
+ .lock()
+ .unwrap()
+ .iter()
+ .map(|run| (run.trigger, run.target, run.end))
+ .collect()
+}
+
+#[tokio::test]
+async fn due_and_manual_runs_are_reported_once_with_their_trigger() {
+ let mut manual = action();
+ manual.next_run_at = Some(Utc::now() + chrono::Duration::hours(1));
+ for (action, trigger) in [
+ (action(), RunTrigger::Scheduled),
+ (manual, RunTrigger::Manual),
+ ] {
+ let (executor, observer) = observed(executor(Repo::default(), Runner::default()));
+ executor.runner.finish.cancel();
+ executor.execute_action(action.clone()).await.unwrap();
+ drain(&executor).await;
+ assert_eq!(
+ ends(&observer),
+ [(trigger, RunTarget::Model, RunEnd::Succeeded)]
+ );
+ let runs = observer.0.lock().unwrap();
+ assert_eq!(runs[0].owner.as_ref(), USER);
+ assert_eq!(Some(runs[0].routine_id), action.id);
+ }
+}
+
+#[tokio::test]
+async fn failed_and_refused_runs_report_how_they_ended() {
+ let (failing, observer) = observed(executor(
+ Repo::default(),
+ Runner {
+ fail_run: true,
+ ..Runner::default()
+ },
+ ));
+ failing.runner.finish.cancel();
+ failing.execute_action(action()).await.unwrap();
+ drain(&failing).await;
+ assert_eq!(ends(&observer)[0].2, RunEnd::Failed);
+
+ for (error, end) in admission_failures()
+ .into_iter()
+ .zip([RunEnd::AiLimitReached, RunEnd::Failed])
+ {
+ let (executor, observer) = observed(
+ executor(Repo::default(), Runner::default()).with_admission(admission(Err(error))),
+ );
+ let _ = executor.execute_action(action()).await;
+ drain(&executor).await;
+ assert_eq!(
+ ends(&observer),
+ [(RunTrigger::Scheduled, RunTarget::Model, end)]
+ );
+ }
+}
+
+#[tokio::test]
+async fn interrupted_runs_are_reported_as_interrupted() {
+ let (executor, observer) = observed(executor(Repo::default(), Runner::default()));
+ executor.execute_action(action()).await.unwrap();
+ executor.cancellation.cancel();
+ drain(&executor).await;
+ assert_eq!(ends(&observer)[0].2, RunEnd::Interrupted);
+}
+
+#[tokio::test]
+async fn event_runs_report_the_event_that_fired_them() {
+ let (executor, observer) = observed(executor(Repo::default(), Runner::default()));
+ executor.runner.finish.cancel();
+ executor.execute(&event_run(), std::future::pending()).await;
+ assert_eq!(
+ ends(&observer),
+ [(
+ RunTrigger::Event(EventName::DocumentUpdated),
+ RunTarget::Model,
+ RunEnd::Succeeded
+ )]
+ );
+}
+
+#[tokio::test]
+async fn runs_that_never_claim_are_not_reported() {
+ let (executor, observer) = observed(executor(Repo::default(), Runner::default()));
+ *executor.repo.claim.lock().unwrap() = Some(ClaimToken::generate());
+ assert!(executor.execute_action(action()).await.is_err());
+ drain(&executor).await;
+ assert!(observer.0.lock().unwrap().is_empty());
+}
diff --git a/crates/scheduled_action/src/domain/ports.rs b/crates/scheduled_action/src/domain/ports.rs
index 142c35e495..0bc098dbe9 100644
--- a/crates/scheduled_action/src/domain/ports.rs
+++ b/crates/scheduled_action/src/domain/ports.rs
@@ -1,5 +1,5 @@
use super::event_runs::{AuthorizedEventRun, ClaimToken, ConfigurationRevision};
-use super::event_trigger::EventReference;
+use super::event_trigger::{EventName, EventReference};
use super::execution::ExecutionHandle;
use super::models::{
ActionExecutionRecord, CreateScheduledAction, DispatchEvent, InProgressExecution,
@@ -261,3 +261,57 @@ pub trait ScheduledAgentRunner: Send + Sync + 'static {
pub trait ScheduledActionLiveUpdate: Send + Sync + 'static {
fn publish_update(&self, update: ScheduledActionUpdate) -> impl Future<Output = ()> + Send;
}
+
+/// What started a finished routine run.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum RunTrigger {
+ /// One of its schedules came due.
+ Scheduled,
+ /// Its owner started it by hand.
+ Manual,
+ /// An event it watches happened.
+ Event(EventName),
+}
+
+/// Who did a routine run's work.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum RunTarget {
+ /// A model, prompted directly.
+ Model,
+ /// One of the owner's agents.
+ Agent,
+ /// The task could not be read.
+ Unknown,
+}
+
+/// How a routine run ended.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum RunEnd {
+ Succeeded,
+ Failed,
+ /// Shutdown or the event worker stopped it.
+ Interrupted,
+ /// The owner's AI allowance refused the run before it started.
+ AiLimitReached,
+}
+
+/// A routine run that started and then ended, whatever the outcome.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct FinishedRoutineRun {
+ pub owner: MacroUserIdStr<'static>,
+ pub routine_id: Uuid,
+ pub trigger: RunTrigger,
+ pub target: RunTarget,
+ pub end: RunEnd,
+ pub duration: chrono::Duration,
+}
+
+/// Told about every finished routine run, e.g. for product analytics.
+/// Called on the run's task, so implementations must be quick or spawn.
+pub trait RoutineRunObserver: Send + Sync + 'static {
+ fn run_finished(&self, run: FinishedRoutineRun);
+}
+
+impl RoutineRunObserver for () {
+ fn run_finished(&self, _run: FinishedRoutineRun) {}
+}
diff --git a/services/agent_harness_service/src/cognition/service/integration_analytics.rs b/services/agent_harness_service/src/cognition/service/integration_analytics.rs
index e1631ccd28..3912a4fbd2 100644
--- a/services/agent_harness_service/src/cognition/service/integration_analytics.rs
+++ b/services/agent_harness_service/src/cognition/service/integration_analytics.rs
@@ -7,19 +7,12 @@
#[cfg(test)]
mod test;
-use std::sync::Arc;
-
-use analytics_client::{
- AnalyticsClient, AnalyticsClientConfig, BackgroundSender, PostHogConfig, ProductEventProperties,
-};
-use entity_access::{domain::ports::AccessRepository, outbound::PgAccessRepository};
-use macro_env::Environment;
use macro_user_id::user_id::MacroUserIdStr;
use mcp_client::domain::models::McpServerRecord;
use serde::Serialize;
use sqlx::PgPool;
-use crate::cognition::config::Config;
+use crate::{cognition::config::Config, product_analytics::ProductAnalytics};
/// Whether a connection was made or removed.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
@@ -86,49 +79,27 @@ impl IntegrationEvent {
}
}
-/// Most integration analytics sends in flight at once; past it, events are
-/// dropped.
-const MAX_IN_FLIGHT_SENDS: usize = 16;
-
/// Records integration changes to PostHog, or nothing when PostHog is not
/// configured.
#[derive(Clone)]
pub struct IntegrationAnalytics {
- sink: Option<Arc<PostHogSink>>,
-}
-
-struct PostHogSink {
- sender: BackgroundSender,
- teams: PgAccessRepository,
- environment: Environment,
+ analytics: ProductAnalytics,
}
impl IntegrationAnalytics {
/// PostHog analytics when `POSTHOG_API_KEY` is set, attributing each event
/// to the user's team; otherwise analytics that record nothing.
pub fn from_config(config: &Config, db: &PgPool) -> Self {
- let Some(api_key) = config.posthog_api_key.value() else {
+ let analytics = ProductAnalytics::new(
+ config.posthog_api_key.value(),
+ config.posthog_host.value(),
+ config.environment,
+ db,
+ );
+ if !analytics.is_enabled() {
tracing::info!("POSTHOG_API_KEY unset; integration analytics disabled");
- return Self { sink: None };
- };
- let client = AnalyticsClient::new(AnalyticsClientConfig {
- posthog: Some(PostHogConfig {
- api_key: api_key.to_owned(),
- host: config
- .posthog_host
- .value()
- .unwrap_or("https://us.i.posthog.com")
- .to_owned(),
- }),
- ..AnalyticsClientConfig::default()
- });
- Self {
- sink: Some(Arc::new(PostHogSink {
- sender: BackgroundSender::new(client, MAX_IN_FLIGHT_SENDS),
- teams: PgAccessRepository::new(db.clone()),
- environment: config.environment,
- })),
}
+ Self { analytics }
}
/// Records `change` in the background so analytics never delays or fails
@@ -139,40 +110,6 @@ impl IntegrationAnalytics {
change: IntegrationChange,
event: IntegrationEvent,
) {
- let Some(sink) = &self.sink else {
- return;
- };
- let task_sink = sink.clone();
- sink.sender.spawn(move |posthog| async move {
- task_sink.capture(&posthog, user_id, change, event).await;
- });
- }
-}
-
-impl PostHogSink {
- async fn capture(
- &self,
- posthog: &AnalyticsClient,
- user_id: MacroUserIdStr<'static>,
- change: IntegrationChange,
- event: IntegrationEvent,
- ) {
- let team_id = self
- .teams
- .get_user_team(&user_id.0)
- .await
- .inspect_err(|error| {
- tracing::warn!(error=?error, "failed to resolve team for integration analytics");
- })
- .ok()
- .flatten()
- .map(|team| team.team_id);
- let properties = ProductEventProperties::new(event, self.environment, team_id);
- let _ = posthog
- .track_posthog(user_id.as_ref(), change.event_name(), &properties)
- .await
- .inspect_err(|error| {
- tracing::warn!(error=?error, event = change.event_name(), "failed to track integration analytics event");
- });
+ self.analytics.track(user_id, change.event_name(), event);
}
}
diff --git a/services/agent_harness_service/src/lib.rs b/services/agent_harness_service/src/lib.rs
index b14cd1b91e..3a4aa489ab 100644
--- a/services/agent_harness_service/src/lib.rs
+++ b/services/agent_harness_service/src/lib.rs
@@ -2,3 +2,4 @@
//! HTTP features composed by the agent harness service.
pub mod cognition;
+pub mod product_analytics;
diff --git a/services/agent_harness_service/src/product_analytics.rs b/services/agent_harness_service/src/product_analytics.rs
new file mode 100644
index 0000000000..f3851ace2a
--- /dev/null
+++ b/services/agent_harness_service/src/product_analytics.rs
@@ -0,0 +1,109 @@
+//! Server-side product events sent to PostHog, attributed to the user's team,
+//! with the same context properties as the web app's events.
+
+use std::sync::Arc;
+
+use analytics_client::{
+ AnalyticsClient, AnalyticsClientConfig, BackgroundSender, PostHogConfig, ProductEventProperties,
+};
+use entity_access::{domain::ports::AccessRepository, outbound::PgAccessRepository};
+use macro_env::Environment;
+use macro_user_id::user_id::MacroUserIdStr;
+use serde::Serialize;
+use sqlx::PgPool;
+
+/// Most sends in flight at once per [`ProductAnalytics`]; past it, events are
+/// dropped.
+const MAX_IN_FLIGHT_SENDS: usize = 16;
+
+/// Records product events to PostHog, or nothing when PostHog is not
+/// configured.
+#[derive(Clone)]
+pub struct ProductAnalytics {
+ sink: Option<Arc<PostHogSink>>,
+}
+
+struct PostHogSink {
+ sender: BackgroundSender,
+ teams: PgAccessRepository,
+ environment: Environment,
+}
+
+impl ProductAnalytics {
+ /// PostHog analytics when `api_key` is set; otherwise analytics that
+ /// record nothing. `host` falls back to the US ingestion origin.
+ pub fn new(
+ api_key: Option<&str>,
+ host: Option<&str>,
+ environment: Environment,
+ db: &PgPool,
+ ) -> Self {
+ let Some(api_key) = api_key else {
+ return Self { sink: None };
+ };
+ let client = AnalyticsClient::new(AnalyticsClientConfig {
+ posthog: Some(PostHogConfig {
+ api_key: api_key.to_owned(),
+ host: host.unwrap_or("https://us.i.posthog.com").to_owned(),
+ }),
+ ..AnalyticsClientConfig::default()
+ });
+ Self {
+ sink: Some(Arc::new(PostHogSink {
+ sender: BackgroundSender::new(client, MAX_IN_FLIGHT_SENDS),
+ teams: PgAccessRepository::new(db.clone()),
+ environment,
+ })),
+ }
+ }
+
+ /// Whether events are sent anywhere.
+ pub fn is_enabled(&self) -> bool {
+ self.sink.is_some()
+ }
+
+ /// Records `event` in the background so analytics never delays or fails
+ /// the work it describes.
+ pub fn track<P>(&self, user_id: MacroUserIdStr<'static>, event: &'static str, properties: P)
+ where
+ P: Serialize + Send + Sync + 'static,
+ {
+ let Some(sink) = &self.sink else {
+ return;
+ };
+ let task_sink = sink.clone();
+ sink.sender.spawn(move |posthog| async move {
+ task_sink
+ .capture(&posthog, user_id, event, properties)
+ .await;
+ });
+ }
+}
+
+impl PostHogSink {
+ async fn capture<P: Serialize>(
+ &self,
+ posthog: &AnalyticsClient,
+ user_id: MacroUserIdStr<'static>,
+ event: &'static str,
+ properties: P,
+ ) {
+ let team_id = self
+ .teams
+ .get_user_team(&user_id.0)
+ .await
+ .inspect_err(|error| {
+ tracing::warn!(error=?error, event, "failed to resolve team for product analytics");
+ })
+ .ok()
+ .flatten()
+ .map(|team| team.team_id);
+ let properties = ProductEventProperties::new(properties, self.environment, team_id);
+ let _ = posthog
+ .track_posthog(user_id.as_ref(), event, &properties)
+ .await
+ .inspect_err(|error| {
+ tracing::warn!(error=?error, event, "failed to track product analytics event");
+ });
+ }
+}
diff --git a/services/agent_harness_service/src/scheduling.rs b/services/agent_harness_service/src/scheduling.rs
index b2dabee685..36baa5e6a7 100644
--- a/services/agent_harness_service/src/scheduling.rs
+++ b/services/agent_harness_service/src/scheduling.rs
@@ -15,6 +15,7 @@ use macro_authorization::{
};
use macro_service_urls::{ConnectionGatewayUrl, LexicalServiceUrl};
pub mod config;
+mod routine_analytics;
use config::Config;
use scheduled_action::domain::event_runs::{
PageSize, admission::EventAdmissionService, condition::ConditionGate,
@@ -131,7 +132,15 @@ pub async fn build(
lifecycle.executions.clone(),
lifecycle.stop_executions.clone(),
)
- .with_admission(ai_admission);
+ .with_admission(ai_admission)
+ .with_observer(Arc::new(routine_analytics::RoutineAnalytics(
+ agent_harness_service::product_analytics::ProductAnalytics::new(
+ config.posthog_api_key.value(),
+ config.posthog_host.value(),
+ environment,
+ &db,
+ ),
+ )));
let service_executor = Arc::new(dispatcher_executor.clone());
let jwt_args = JwtValidationArgs::new_with_secret_manager(environment, &secretsmanager_client)
diff --git a/services/agent_harness_service/src/scheduling/config.rs b/services/agent_harness_service/src/scheduling/config.rs
index b1891f2ab2..1f95b3fa5f 100644
--- a/services/agent_harness_service/src/scheduling/config.rs
+++ b/services/agent_harness_service/src/scheduling/config.rs
@@ -69,6 +69,11 @@ pub struct Config {
/// TypeSafe credential for the Jev classifier behind trigger conditions.
/// Optional: routines cannot save conditions while it is unset.
pub typesafe_api_key: jev::outbound::TypesafeApiKey,
+ /// PostHog project token for routine run analytics. When unset, runs are
+ /// not recorded.
+ pub posthog_api_key: agent_harness_service::cognition::config::PosthogApiKey,
+ /// PostHog ingestion origin. Falls back to the US origin when unset.
+ pub posthog_host: agent_harness_service::cognition::config::PosthogHost,
}
impl Config {
diff --git a/services/agent_harness_service/src/scheduling/routine_analytics.rs b/services/agent_harness_service/src/scheduling/routine_analytics.rs
new file mode 100644
index 0000000000..7b1bd3abf2
--- /dev/null
+++ b/services/agent_harness_service/src/scheduling/routine_analytics.rs
@@ -0,0 +1,65 @@
+//! Product analytics for routine runs: one `routine_run_finished` event per
+//! run that starts, whether a schedule, its owner or an event started it.
+
+#[cfg(test)]
+mod test;
+
+use agent_harness_service::product_analytics::ProductAnalytics;
+use scheduled_action::domain::ports::{
+ FinishedRoutineRun, RoutineRunObserver, RunEnd, RunTarget, RunTrigger,
+};
+use serde::Serialize;
+
+/// Sends each finished routine run to PostHog as its owner.
+pub struct RoutineAnalytics(pub ProductAnalytics);
+
+impl RoutineRunObserver for RoutineAnalytics {
+ fn run_finished(&self, run: FinishedRoutineRun) {
+ let owner = run.owner.clone();
+ self.0
+ .track(owner, "routine_run_finished", RunProperties::from(&run));
+ }
+}
+
+/// Ids, kinds and timings only: never the routine's prompt or output.
+#[derive(Debug, PartialEq, Serialize)]
+struct RunProperties {
+ routine_id: uuid::Uuid,
+ /// `scheduled`, `manual` or `event`.
+ trigger: &'static str,
+ /// The event that fired an event-triggered run, e.g. `email.message_received`.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ trigger_event: Option<&'static str>,
+ /// `model` or `agent`, or `unknown` when the task could not be read.
+ target: &'static str,
+ /// `succeeded`, `failed`, `interrupted` or `ai_limit_reached`.
+ outcome: &'static str,
+ duration_ms: i64,
+}
+
+impl From<&FinishedRoutineRun> for RunProperties {
+ fn from(run: &FinishedRoutineRun) -> Self {
+ let (trigger, trigger_event) = match run.trigger {
+ RunTrigger::Scheduled => ("scheduled", None),
+ RunTrigger::Manual => ("manual", None),
+ RunTrigger::Event(name) => ("event", Some(name.as_str())),
+ };
+ Self {
+ routine_id: run.routine_id,
+ trigger,
+ trigger_event,
+ target: match run.target {
+ RunTarget::Model => "model",
+ RunTarget::Agent => "agent",
+ RunTarget::Unknown => "unknown",
+ },
+ outcome: match run.end {
+ RunEnd::Succeeded => "succeeded",
+ RunEnd::Failed => "failed",
+ RunEnd::Interrupted => "interrupted",
+ RunEnd::AiLimitReached => "ai_limit_reached",
+ },
+ duration_ms: run.duration.num_milliseconds(),
+ }
+ }
+}
diff --git a/services/agent_harness_service/src/scheduling/routine_analytics/test.rs b/services/agent_harness_service/src/scheduling/routine_analytics/test.rs
new file mode 100644
index 0000000000..bc93678fd5
--- /dev/null
+++ b/services/agent_harness_service/src/scheduling/routine_analytics/test.rs
@@ -0,0 +1,44 @@
+use super::*;
+use macro_user_id::user_id::MacroUserIdStr;
+use scheduled_action::domain::event_trigger::EventName;
+use serde_json::json;
+
+fn run(trigger: RunTrigger, end: RunEnd) -> FinishedRoutineRun {
+ FinishedRoutineRun {
+ owner: MacroUserIdStr::parse_from_str("macro|[email protected]").unwrap(),
+ routine_id: uuid::Uuid::nil(),
+ trigger,
+ target: RunTarget::Agent,
+ end,
+ duration: chrono::Duration::milliseconds(1_500),
+ }
+}
+
+#[test]
+fn scheduled_runs_carry_ids_kinds_and_timing_only() {
+ assert_eq!(
+ serde_json::to_value(RunProperties::from(&run(
+ RunTrigger::Scheduled,
+ RunEnd::Succeeded
+ )))
+ .unwrap(),
+ json!({
+ "routine_id": uuid::Uuid::nil(),
+ "trigger": "scheduled",
+ "target": "agent",
+ "outcome": "succeeded",
+ "duration_ms": 1500,
+ })
+ );
+}
+
+#[test]
+fn event_runs_name_the_event_that_fired_them() {
+ let properties = RunProperties::from(&run(
+ RunTrigger::Event(EventName::DocumentUpdated),
+ RunEnd::AiLimitReached,
+ ));
+ assert_eq!(properties.trigger, "event");
+ assert_eq!(properties.trigger_event, Some("document.updated"));
+ assert_eq!(properties.outcome, "ai_limit_reached");
+}