Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
#[cfg(test)]
mod test;
use std::{sync::Arc, time::Duration};
use ai_billing::{AiAdmissionError, AiAdmissionService, DisabledAiAdmissionService};
use ai_usage::AiFeature;
use anyhow::{Context, Result};
use bot_id::BotId;
use chrono::{DateTime, Utc};
use macro_uuid::{Uuid, generate_uuid_v7};
use routine_program::{RoutineProgram, StepRecord};
use tokio_util::{sync::CancellationToken, task::TaskTracker};
use tracing::Instrument;
use crate::domain::event_runs::{
ClaimedEventRun, EventExecutionResult, EventExecutor, EventRunOutcome,
};
use crate::domain::models::{
ActionExecutionRecord, ExecutionResource, ExecutionResult, InProgressExecution,
MAX_ACTION_TIME, ResolvedTaskTarget, RunHistory, RunInfo, ScheduledAction,
ScheduledActionUpdate,
};
use crate::domain::ports::{
FinishedRoutineRun, RoutineRun, RoutineRunObserver, RunEnd, RunTarget, RunTrigger,
ScheduledActionExecutor, ScheduledActionLiveUpdate, ScheduledActionRepo, ScheduledAgentRunner,
};
/// Run identity allocated before any awaited preparation. Runners can cancel
/// partially created remote sessions even when preparation never returns.
#[derive(Debug)]
pub struct ExecutionHandle {
/// Caller-generated agent session ID, available before preparation starts.
pub session_id: Uuid,
/// Caller-generated initial action ID for prompt delivery and status checks.
pub action_id: Uuid,
/// Set as soon as resource existence is established, even if preparation later fails.
/// Runs whose steps never reach an agent have no transcript.
pub resource: Option<ExecutionResource>,
/// The persona whose session this run opened or tried to open, for cleanup.
/// At most one agent runs per run, under the preallocated ids above.
pub agent: Option<BotId>,
/// What each step did so far, kept even when the run fails or is cut short.
pub steps: Vec<StepRecord>,
/// The routine's earlier runs, as its code reads them.
pub history: RunHistory,
/// What started this run and when, once the runner knows.
pub run: Option<RunInfo>,
/// What fired the routine, for the session this run opens.
pub firing: Option<trigger_context::origin::RoutineCause>,
// Cancellation during admission must not contact a runner that never started.
preparation_started: bool,
// Started was published, so Stopped must follow even without a transcript.
announced: bool,
}
impl Default for ExecutionHandle {
fn default() -> Self {
Self {
session_id: generate_uuid_v7(),
action_id: generate_uuid_v7(),
resource: None,
agent: None,
steps: Vec::new(),
history: RunHistory::default(),
run: None,
firing: None,
preparation_started: false,
announced: false,
}
}
}
/// Shared execution path. Cron/manual calls return after tracked preparation;
/// event workers await it and own atomic event/history finalization and release.
pub struct InProcessExecutor<Rpo, Live, Runner> {
repo: Arc<Rpo>,
live_updates: Arc<Live>,
runner: Arc<Runner>,
admission: Arc<dyn AiAdmissionService>,
observer: Arc<dyn RoutineRunObserver>,
tracker: TaskTracker,
cancellation: CancellationToken,
}
impl<Rpo, Live, Runner> Clone for InProcessExecutor<Rpo, Live, Runner> {
fn clone(&self) -> Self {
Self {
repo: self.repo.clone(),
live_updates: self.live_updates.clone(),
runner: self.runner.clone(),
admission: self.admission.clone(),
observer: self.observer.clone(),
tracker: self.tracker.clone(),
cancellation: self.cancellation.clone(),
}
}
}
impl<Rpo, Live, Runner> InProcessExecutor<Rpo, Live, Runner>
where
Rpo: ScheduledActionRepo,
Live: ScheduledActionLiveUpdate,
Runner: ScheduledAgentRunner,
{
/// Admission defaults off for compatibility; production must configure it
/// with `with_admission` using the shared enforcement policy.
pub fn new(
repo: Arc<Rpo>,
runner: Arc<Runner>,
live_updates: Arc<Live>,
tracker: TaskTracker,
cancellation: CancellationToken,
) -> Self {
Self {
repo,
runner,
live_updates,
admission: Arc::new(DisabledAiAdmissionService),
observer: Arc::new(()),
tracker,
cancellation,
}
}
/// Configure model admission. Agent funding and admission belong to the
/// session/harness service, not to the scheduler's model policy.
pub fn with_admission(mut self, admission: Arc<dyn AiAdmissionService>) -> Self {
self.admission = admission;
self
}
/// Tell `observer` about every run that starts and then ends.
pub fn with_observer(mut self, observer: Arc<dyn RoutineRunObserver>) -> Self {
self.observer = observer;
self
}
/// Admit Macro-funded model work (a model agent), open a leading agent's
/// transcript, and announce the run. Programs without a leading agent
/// start with none. Code's own AI calls are admitted by the API they call.
async fn prepare(
&self,
action: &ScheduledAction,
handle: &mut ExecutionHandle,
) -> Result<Option<ExecutionResource>> {
let owner = action.owner_user()?.clone();
let action_id = action.id.context("persisted action required")?;
let program =
RoutineProgram::from_task(&action.task).context("invalid routine task definition")?;
let runs_model = program.agent_steps().into_iter().any(|step| {
matches!(
ResolvedTaskTarget::of(step),
Ok(ResolvedTaskTarget::Model { .. })
)
});
if runs_model {
self.admission.admit(&owner, AiFeature::Automation).await?;
}
// Code reads when the routine last ran; without it, it reads none.
handle.history = self
.repo
.run_history(&action_id)
.await
.inspect_err(|error| tracing::warn!(?error, "routine run history unavailable"))
.unwrap_or_default();
handle.preparation_started = true;
self.runner.prepare(action, handle).await?;
// Retain the resource even if publishing Started stalls or is cancelled.
let resource = handle.resource.clone();
handle.announced = true;
self.live_updates
.publish_update(ScheduledActionUpdate::Started {
owner,
action_id,
chat_id: resource.as_ref().and_then(ExecutionResource::chat_id),
resource: resource.clone(),
})
.await;
Ok(resource)
}
async fn cancel(
&self,
action: &ScheduledAction,
handle: &ExecutionHandle,
error: &anyhow::Error,
) {
if !handle.preparation_started
|| (handle.resource.is_none() && error.is::<AiAdmissionError>())
{
return;
}
match tokio::time::timeout(CANCELLATION_TIMEOUT, self.runner.cancel(action, handle)).await {
Ok(Ok(())) => {}
Ok(Err(error)) => {
tracing::warn!(?error, action_id=?action.id, "runner cancellation failed")
}
Err(error) => {
tracing::warn!(?error, action_id=?action.id, "runner cancellation timed out")
}
}
}
/// Close a run that was announced or established a transcript.
async fn stopped(&self, action: &ScheduledAction, handle: &ExecutionHandle, is_success: bool) {
if !handle.announced && handle.resource.is_none() {
return;
}
if let (Ok(owner), Some(action_id)) = (action.owner_user(), action.id) {
let resource = handle.resource.clone();
self.live_updates
.publish_update(ScheduledActionUpdate::Stopped {
owner: owner.clone(),
action_id,
chat_id: resource.as_ref().and_then(ExecutionResource::chat_id),
resource,
is_success,
})
.await;
}
}
}
const BOOKKEEPING_TIMEOUT: Duration = Duration::from_secs(10);
const CANCELLATION_TIMEOUT: Duration = Duration::from_secs(10);
#[derive(Debug, thiserror::Error)]
#[error("scheduled action cancelled")]
struct ExecutionCancelled;
/// Bound preparation and execution by the claim's original deadline. Dropping
/// the operation triggers local session guards; bounded remote cancellation is
/// attempted separately before bookkeeping.
async fn bounded<T>(
deadline: DateTime<Utc>,
cancellation: impl Future<Output = ()> + Send,
operation: impl Future<Output = Result<T>> + Send,
) -> Result<T> {
let remaining = (deadline - Utc::now()).to_std().unwrap_or_default();
anyhow::ensure!(!remaining.is_zero(), "scheduled action deadline exceeded");
tokio::select! {
biased;
_ = cancellation => Err(ExecutionCancelled.into()),
_ = tokio::time::sleep(remaining) => anyhow::bail!("scheduled action deadline exceeded"),
result = operation => result,
}
}
fn finished_run(
action: &ScheduledAction,
trigger: RunTrigger,
started_at: DateTime<Utc>,
result: &Result<()>,
) -> Option<FinishedRoutineRun> {
let target = match RoutineProgram::from_task(&action.task) {
Ok(program) => match program.agent_steps().first() {
Some(step) => match ResolvedTaskTarget::of(step) {
Ok(ResolvedTaskTarget::Model { .. }) => RunTarget::Model,
Ok(ResolvedTaskTarget::Agent { .. }) => RunTarget::Agent,
Err(_) => RunTarget::Unknown,
},
None => RunTarget::Steps,
},
Err(_) => RunTarget::Unknown,
};
let end = match result {
Ok(()) => RunEnd::Succeeded,
Err(error) if error.is::<ExecutionCancelled>() => RunEnd::Interrupted,
Err(error) => match error.downcast_ref::<AiAdmissionError>() {
Some(AiAdmissionError::Denied(_)) => RunEnd::AiLimitReached,
_ => RunEnd::Failed,
},
};
Some(FinishedRoutineRun {
owner: action.owner_user().ok()?.clone(),
routine_id: action.id?,
routine_name: action.name.clone(),
trigger,
target,
end,
duration: Utc::now() - started_at,
})
}
fn execution_record(
action: &ScheduledAction,
handle: &ExecutionHandle,
started_at: DateTime<Utc>,
result: &Result<()>,
) -> ActionExecutionRecord {
let resource = handle.resource.clone();
let end_time = Utc::now();
ActionExecutionRecord {
id: None,
action_id: action.id.expect("claimed actions are persisted"),
resource_id: resource.as_ref().map(|resource| resource.id.clone()),
start_time: started_at,
end_time,
is_success: result.is_ok(),
result: serde_json::to_value(
ExecutionResult::new(resource, result.as_ref().err().map(ToString::to_string))
.with_steps(handle.steps.clone(), action.configuration_revision)
.with_run(handle.run.clone()),
)
.expect("execution metadata is JSON serializable"),
created_at: end_time,
}
}
impl<Rpo, Live, Runner> ScheduledActionExecutor for InProcessExecutor<Rpo, Live, Runner>
where
Rpo: ScheduledActionRepo,
Live: ScheduledActionLiveUpdate,
Runner: ScheduledAgentRunner,
{
async fn execute_action(&self, action: ScheduledAction) -> Result<InProgressExecution> {
action.owner_user()?;
anyhow::ensure!(!self.cancellation.is_cancelled(), "executor is stopping");
let id = action.id.context("persisted action required")?;
let start_time = Utc::now();
let deadline = start_time + MAX_ACTION_TIME;
// The claim is fenced to `next_run_at`, so a run started once it is
// due consumes that firing, whoever started it.
let firing = match action.next_run_at {
Some(scheduled_for) if scheduled_for <= start_time => {
RoutineRun::Scheduled { scheduled_for }
}
_ => RoutineRun::Manual {
requested_at: start_time,
},
};
// Track preparation too: cancellation of an HTTP request must not leak
// a claim or abandon a run after creating its resource.
let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
let executor = self.clone();
// The run outlives the request that started it; keep it on that trace.
let span = tracing::info_span!("routine.execute", action_id = %id);
let run = async move {
let claim = bounded(
deadline,
executor.cancellation.cancelled(),
executor
.repo
.claim_action(&id, action.configuration_revision, action.next_run_at),
)
.await;
let token = match claim {
Ok(token) => token,
Err(error) => {
let _ = ready_tx.send(Err(error));
return;
}
};
// Consume a scheduled firing before anything runs. If this process
// dies mid-run, its claim goes stale and the poller would otherwise
// find the same firing due and repeat every step's side effects;
// a lost firing beats a doubled one.
if matches!(firing, RoutineRun::Scheduled { .. }) {
let consumed = tokio::time::timeout(
BOOKKEEPING_TIMEOUT,
executor.repo.update_next_run_at(&id),
)
.await;
if !matches!(consumed, Ok(Ok(()))) {
// Running anyway would leave the firing due, so a failed
// run bookkeeping later could repeat every step. Give the
// claim back untouched; the poller retries the firing.
tracing::error!(action_id = %id, "failed to consume the scheduled firing");
let release = executor.repo.release_action(&id, token);
if !matches!(
tokio::time::timeout(BOOKKEEPING_TIMEOUT, release).await,
Ok(Ok(()))
) {
tracing::error!(action_id = %id, "failed to release action claim");
}
let _ = ready_tx.send(Err(anyhow::anyhow!(
"Couldn't start this scheduled run; it will be retried."
)));
return;
}
}
let mut ready_tx = Some(ready_tx);
let mut handle = ExecutionHandle {
firing: Some(firing.cause()),
..ExecutionHandle::default()
};
let result = bounded(deadline, executor.cancellation.cancelled(), async {
let resource = executor.prepare(&action, &mut handle).await?;
if let Some(ready_tx) = ready_tx.take() {
let _ = ready_tx.send(Ok(InProgressExecution {
action_id: id,
chat_id: resource.as_ref().and_then(ExecutionResource::chat_id),
resource,
}));
}
executor.runner.run(&action, &mut handle, firing).await
})
.await;
if let Err(error) = &result {
executor.cancel(&action, &handle, error).await;
}
let trigger = match firing {
RoutineRun::Scheduled { .. } => RunTrigger::Scheduled,
_ => RunTrigger::Manual,
};
if let Some(run) = finished_run(&action, trigger, start_time, &result) {
executor.observer.run_finished(run);
}
let record = execution_record(&action, &handle, start_time, &result);
let end_time = record.end_time;
let bookkeeping = async {
let _ = executor
.repo
.create_execution_record(record)
.await
.inspect_err(|error| {
tracing::error!(?error, action_id=?id, "failed to save execution record");
});
let _ = executor.repo.update_last_executed(&id, end_time).await.inspect_err(|error| {
tracing::error!(?error, action_id=?id, "failed to update last executed time");
});
let _ = executor
.repo
.update_next_run_at(&id)
.await
.inspect_err(|error| {
tracing::error!(?error, action_id=?id, "failed to update next run time");
});
};
if tokio::time::timeout(BOOKKEEPING_TIMEOUT, bookkeeping)
.await
.is_err()
{
tracing::error!(action_id=?id, "execution bookkeeping timed out");
}
// Always attempt fenced release, even after failed preparation or
// stalled history persistence. Never retry model/tool execution.
let release = executor.repo.release_action(&id, token);
match tokio::time::timeout(BOOKKEEPING_TIMEOUT, release).await {
Ok(Ok(())) => {}
Ok(Err(error)) => {
tracing::error!(?error, action_id=?id, "failed to release action claim")
}
Err(error) => tracing::error!(?error, action_id=?id, "claim release timed out"),
}
let _ = tokio::time::timeout(
BOOKKEEPING_TIMEOUT,
executor.stopped(&action, &handle, result.is_ok()),
)
.await;
if let Err(error) = result {
// The error can quote step content; it is kept in the
// owner's run history, not in logs.
tracing::error!(action_id = %id, "scheduled action execution failed");
if let Some(ready_tx) = ready_tx {
let _ = ready_tx.send(Err(error));
}
}
};
self.tracker.spawn(run.instrument(span));
ready_rx
.await
.context("failed to prepare scheduled action resource")?
}
}
impl<Rpo, Live, Runner> EventExecutor for InProcessExecutor<Rpo, Live, Runner>
where
Rpo: ScheduledActionRepo,
Live: ScheduledActionLiveUpdate,
Runner: ScheduledAgentRunner,
{
async fn execute(
&self,
run: &ClaimedEventRun,
cancellation: impl Future<Output = ()> + Send,
) -> EventExecutionResult {
let mut handle = ExecutionHandle {
firing: Some(RoutineRun::Event(&run.run).cause()),
..ExecutionHandle::default()
};
let shutdown = async {
tokio::select! {
_ = self.cancellation.cancelled() => {},
_ = cancellation => {},
}
};
let deadline = run.deadline.min(run.started_at + MAX_ACTION_TIME);
let result = bounded(deadline, shutdown, async {
self.prepare(&run.action, &mut handle).await?;
self.runner
.run(&run.action, &mut handle, RoutineRun::Event(&run.run))
.await
})
.await;
if let Err(error) = &result {
self.cancel(&run.action, &handle, error).await;
}
let trigger = RunTrigger::Event(run.run.pending.event.event_name());
if let Some(finished) = finished_run(&run.action, trigger, run.started_at, &result) {
self.observer.run_finished(finished);
}
let record = execution_record(&run.action, &handle, run.started_at, &result);
let _ = tokio::time::timeout(
BOOKKEEPING_TIMEOUT,
self.stopped(&run.action, &handle, result.is_ok()),
)
.await;
// No claim/release or history writes here: the worker finalizes with
// run.token in the same transaction as the terminal queue transition.
EventExecutionResult {
outcome: match &result {
Ok(()) => EventRunOutcome::Succeeded,
Err(error) if error.is::<ExecutionCancelled>() => EventRunOutcome::Interrupted,
Err(_) => EventRunOutcome::Failed,
},
record: Some(record),
}
}
}