Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

20.3 KB
Raw Blame
#[cfg(test)]
mod test;

use std::{sync::Arc, time::Duration};

use ai_billing::{AiAdmissionError, AiAdmissionService, DisabledAiAdmissionService};
use ai_usage::AiFeature;
use anyhow::{Context, Result};
use bot_id::BotId;
use chrono::{DateTime, Utc};
use macro_uuid::{Uuid, generate_uuid_v7};
use routine_program::{RoutineProgram, StepRecord};
use tokio_util::{sync::CancellationToken, task::TaskTracker};
use tracing::Instrument;

use crate::domain::event_runs::{
    ClaimedEventRun, EventExecutionResult, EventExecutor, EventRunOutcome,
};
use crate::domain::models::{
    ActionExecutionRecord, ExecutionResource, ExecutionResult, InProgressExecution,
    MAX_ACTION_TIME, ResolvedTaskTarget, RunHistory, RunInfo, ScheduledAction,
    ScheduledActionUpdate,
};
use crate::domain::ports::{
    FinishedRoutineRun, RoutineRun, RoutineRunObserver, RunEnd, RunTarget, RunTrigger,
    ScheduledActionExecutor, ScheduledActionLiveUpdate, ScheduledActionRepo, ScheduledAgentRunner,
};

/// Run identity allocated before any awaited preparation. Runners can cancel
/// partially created remote sessions even when preparation never returns.
#[derive(Debug)]
pub struct ExecutionHandle {
    /// Caller-generated agent session ID, available before preparation starts.
    pub session_id: Uuid,
    /// Caller-generated initial action ID for prompt delivery and status checks.
    pub action_id: Uuid,
    /// Set as soon as resource existence is established, even if preparation later fails.
    /// Runs whose steps never reach an agent have no transcript.
    pub resource: Option<ExecutionResource>,
    /// The persona whose session this run opened or tried to open, for cleanup.
    /// At most one agent runs per run, under the preallocated ids above.
    pub agent: Option<BotId>,
    /// What each step did so far, kept even when the run fails or is cut short.
    pub steps: Vec<StepRecord>,
    /// The routine's earlier runs, as its code reads them.
    pub history: RunHistory,
    /// What started this run and when, once the runner knows.
    pub run: Option<RunInfo>,
    /// What fired the routine, for the session this run opens.
    pub firing: Option<trigger_context::origin::RoutineCause>,
    // Cancellation during admission must not contact a runner that never started.
    preparation_started: bool,
    // Started was published, so Stopped must follow even without a transcript.
    announced: bool,
}

impl Default for ExecutionHandle {
    fn default() -> Self {
        Self {
            session_id: generate_uuid_v7(),
            action_id: generate_uuid_v7(),
            resource: None,
            agent: None,
            steps: Vec::new(),
            history: RunHistory::default(),
            run: None,
            firing: None,
            preparation_started: false,
            announced: false,
        }
    }
}

/// Shared execution path. Cron/manual calls return after tracked preparation;
/// event workers await it and own atomic event/history finalization and release.
pub struct InProcessExecutor<Rpo, Live, Runner> {
    repo: Arc<Rpo>,
    live_updates: Arc<Live>,
    runner: Arc<Runner>,
    admission: Arc<dyn AiAdmissionService>,
    observer: Arc<dyn RoutineRunObserver>,
    tracker: TaskTracker,
    cancellation: CancellationToken,
}

impl<Rpo, Live, Runner> Clone for InProcessExecutor<Rpo, Live, Runner> {
    fn clone(&self) -> Self {
        Self {
            repo: self.repo.clone(),
            live_updates: self.live_updates.clone(),
            runner: self.runner.clone(),
            admission: self.admission.clone(),
            observer: self.observer.clone(),
            tracker: self.tracker.clone(),
            cancellation: self.cancellation.clone(),
        }
    }
}

impl<Rpo, Live, Runner> InProcessExecutor<Rpo, Live, Runner>
where
    Rpo: ScheduledActionRepo,
    Live: ScheduledActionLiveUpdate,
    Runner: ScheduledAgentRunner,
{
    /// Admission defaults off for compatibility; production must configure it
    /// with `with_admission` using the shared enforcement policy.
    pub fn new(
        repo: Arc<Rpo>,
        runner: Arc<Runner>,
        live_updates: Arc<Live>,
        tracker: TaskTracker,
        cancellation: CancellationToken,
    ) -> Self {
        Self {
            repo,
            runner,
            live_updates,
            admission: Arc::new(DisabledAiAdmissionService),
            observer: Arc::new(()),
            tracker,
            cancellation,
        }
    }

    /// Configure model admission. Agent funding and admission belong to the
    /// session/harness service, not to the scheduler's model policy.
    pub fn with_admission(mut self, admission: Arc<dyn AiAdmissionService>) -> Self {
        self.admission = admission;
        self
    }

    /// Tell `observer` about every run that starts and then ends.
    pub fn with_observer(mut self, observer: Arc<dyn RoutineRunObserver>) -> Self {
        self.observer = observer;
        self
    }

    /// Admit Macro-funded model work (a model agent), open a leading agent's
    /// transcript, and announce the run. Programs without a leading agent
    /// start with none. Code's own AI calls are admitted by the API they call.
    async fn prepare(
        &self,
        action: &ScheduledAction,
        handle: &mut ExecutionHandle,
    ) -> Result<Option<ExecutionResource>> {
        let owner = action.owner_user()?.clone();
        let action_id = action.id.context("persisted action required")?;
        let program =
            RoutineProgram::from_task(&action.task).context("invalid routine task definition")?;
        let runs_model = program.agent_steps().into_iter().any(|step| {
            matches!(
                ResolvedTaskTarget::of(step),
                Ok(ResolvedTaskTarget::Model { .. })
            )
        });
        if runs_model {
            self.admission.admit(&owner, AiFeature::Automation).await?;
        }
        // Code reads when the routine last ran; without it, it reads none.
        handle.history = self
            .repo
            .run_history(&action_id)
            .await
            .inspect_err(|error| tracing::warn!(?error, "routine run history unavailable"))
            .unwrap_or_default();
        handle.preparation_started = true;
        self.runner.prepare(action, handle).await?;
        // Retain the resource even if publishing Started stalls or is cancelled.
        let resource = handle.resource.clone();
        handle.announced = true;
        self.live_updates
            .publish_update(ScheduledActionUpdate::Started {
                owner,
                action_id,
                chat_id: resource.as_ref().and_then(ExecutionResource::chat_id),
                resource: resource.clone(),
            })
            .await;
        Ok(resource)
    }

    async fn cancel(
        &self,
        action: &ScheduledAction,
        handle: &ExecutionHandle,
        error: &anyhow::Error,
    ) {
        if !handle.preparation_started
            || (handle.resource.is_none() && error.is::<AiAdmissionError>())
        {
            return;
        }
        match tokio::time::timeout(CANCELLATION_TIMEOUT, self.runner.cancel(action, handle)).await {
            Ok(Ok(())) => {}
            Ok(Err(error)) => {
                tracing::warn!(?error, action_id=?action.id, "runner cancellation failed")
            }
            Err(error) => {
                tracing::warn!(?error, action_id=?action.id, "runner cancellation timed out")
            }
        }
    }

    /// Close a run that was announced or established a transcript.
    async fn stopped(&self, action: &ScheduledAction, handle: &ExecutionHandle, is_success: bool) {
        if !handle.announced && handle.resource.is_none() {
            return;
        }
        if let (Ok(owner), Some(action_id)) = (action.owner_user(), action.id) {
            let resource = handle.resource.clone();
            self.live_updates
                .publish_update(ScheduledActionUpdate::Stopped {
                    owner: owner.clone(),
                    action_id,
                    chat_id: resource.as_ref().and_then(ExecutionResource::chat_id),
                    resource,
                    is_success,
                })
                .await;
        }
    }
}

const BOOKKEEPING_TIMEOUT: Duration = Duration::from_secs(10);
const CANCELLATION_TIMEOUT: Duration = Duration::from_secs(10);

#[derive(Debug, thiserror::Error)]
#[error("scheduled action cancelled")]
struct ExecutionCancelled;

/// Bound preparation and execution by the claim's original deadline. Dropping
/// the operation triggers local session guards; bounded remote cancellation is
/// attempted separately before bookkeeping.
async fn bounded<T>(
    deadline: DateTime<Utc>,
    cancellation: impl Future<Output = ()> + Send,
    operation: impl Future<Output = Result<T>> + Send,
) -> Result<T> {
    let remaining = (deadline - Utc::now()).to_std().unwrap_or_default();
    anyhow::ensure!(!remaining.is_zero(), "scheduled action deadline exceeded");
    tokio::select! {
        biased;
        _ = cancellation => Err(ExecutionCancelled.into()),
        _ = tokio::time::sleep(remaining) => anyhow::bail!("scheduled action deadline exceeded"),
        result = operation => result,
    }
}

fn finished_run(
    action: &ScheduledAction,
    trigger: RunTrigger,
    started_at: DateTime<Utc>,
    result: &Result<()>,
) -> Option<FinishedRoutineRun> {
    let target = match RoutineProgram::from_task(&action.task) {
        Ok(program) => match program.agent_steps().first() {
            Some(step) => match ResolvedTaskTarget::of(step) {
                Ok(ResolvedTaskTarget::Model { .. }) => RunTarget::Model,
                Ok(ResolvedTaskTarget::Agent { .. }) => RunTarget::Agent,
                Err(_) => RunTarget::Unknown,
            },
            None => RunTarget::Steps,
        },
        Err(_) => RunTarget::Unknown,
    };
    let end = match result {
        Ok(()) => RunEnd::Succeeded,
        Err(error) if error.is::<ExecutionCancelled>() => RunEnd::Interrupted,
        Err(error) => match error.downcast_ref::<AiAdmissionError>() {
            Some(AiAdmissionError::Denied(_)) => RunEnd::AiLimitReached,
            _ => RunEnd::Failed,
        },
    };
    Some(FinishedRoutineRun {
        owner: action.owner_user().ok()?.clone(),
        routine_id: action.id?,
        routine_name: action.name.clone(),
        trigger,
        target,
        end,
        duration: Utc::now() - started_at,
    })
}

fn execution_record(
    action: &ScheduledAction,
    handle: &ExecutionHandle,
    started_at: DateTime<Utc>,
    result: &Result<()>,
) -> ActionExecutionRecord {
    let resource = handle.resource.clone();
    let end_time = Utc::now();
    ActionExecutionRecord {
        id: None,
        action_id: action.id.expect("claimed actions are persisted"),
        resource_id: resource.as_ref().map(|resource| resource.id.clone()),
        start_time: started_at,
        end_time,
        is_success: result.is_ok(),
        result: serde_json::to_value(
            ExecutionResult::new(resource, result.as_ref().err().map(ToString::to_string))
                .with_steps(handle.steps.clone(), action.configuration_revision)
                .with_run(handle.run.clone()),
        )
        .expect("execution metadata is JSON serializable"),
        created_at: end_time,
    }
}

impl<Rpo, Live, Runner> ScheduledActionExecutor for InProcessExecutor<Rpo, Live, Runner>
where
    Rpo: ScheduledActionRepo,
    Live: ScheduledActionLiveUpdate,
    Runner: ScheduledAgentRunner,
{
    async fn execute_action(&self, action: ScheduledAction) -> Result<InProgressExecution> {
        action.owner_user()?;
        anyhow::ensure!(!self.cancellation.is_cancelled(), "executor is stopping");
        let id = action.id.context("persisted action required")?;
        let start_time = Utc::now();
        let deadline = start_time + MAX_ACTION_TIME;
        // The claim is fenced to `next_run_at`, so a run started once it is
        // due consumes that firing, whoever started it.
        let firing = match action.next_run_at {
            Some(scheduled_for) if scheduled_for <= start_time => {
                RoutineRun::Scheduled { scheduled_for }
            }
            _ => RoutineRun::Manual {
                requested_at: start_time,
            },
        };

        // Track preparation too: cancellation of an HTTP request must not leak
        // a claim or abandon a run after creating its resource.
        let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
        let executor = self.clone();
        // The run outlives the request that started it; keep it on that trace.
        let span = tracing::info_span!("routine.execute", action_id = %id);
        let run = async move {
            let claim = bounded(
                deadline,
                executor.cancellation.cancelled(),
                executor
                    .repo
                    .claim_action(&id, action.configuration_revision, action.next_run_at),
            )
            .await;
            let token = match claim {
                Ok(token) => token,
                Err(error) => {
                    let _ = ready_tx.send(Err(error));
                    return;
                }
            };
            // Consume a scheduled firing before anything runs. If this process
            // dies mid-run, its claim goes stale and the poller would otherwise
            // find the same firing due and repeat every step's side effects;
            // a lost firing beats a doubled one.
            if matches!(firing, RoutineRun::Scheduled { .. }) {
                let consumed = tokio::time::timeout(
                    BOOKKEEPING_TIMEOUT,
                    executor.repo.update_next_run_at(&id),
                )
                .await;
                if !matches!(consumed, Ok(Ok(()))) {
                    // Running anyway would leave the firing due, so a failed
                    // run bookkeeping later could repeat every step. Give the
                    // claim back untouched; the poller retries the firing.
                    tracing::error!(action_id = %id, "failed to consume the scheduled firing");
                    let release = executor.repo.release_action(&id, token);
                    if !matches!(
                        tokio::time::timeout(BOOKKEEPING_TIMEOUT, release).await,
                        Ok(Ok(()))
                    ) {
                        tracing::error!(action_id = %id, "failed to release action claim");
                    }
                    let _ = ready_tx.send(Err(anyhow::anyhow!(
                        "Couldn't start this scheduled run; it will be retried."
                    )));
                    return;
                }
            }
            let mut ready_tx = Some(ready_tx);
            let mut handle = ExecutionHandle {
                firing: Some(firing.cause()),
                ..ExecutionHandle::default()
            };
            let result = bounded(deadline, executor.cancellation.cancelled(), async {
                let resource = executor.prepare(&action, &mut handle).await?;
                if let Some(ready_tx) = ready_tx.take() {
                    let _ = ready_tx.send(Ok(InProgressExecution {
                        action_id: id,
                        chat_id: resource.as_ref().and_then(ExecutionResource::chat_id),
                        resource,
                    }));
                }
                executor.runner.run(&action, &mut handle, firing).await
            })
            .await;
            if let Err(error) = &result {
                executor.cancel(&action, &handle, error).await;
            }
            let trigger = match firing {
                RoutineRun::Scheduled { .. } => RunTrigger::Scheduled,
                _ => RunTrigger::Manual,
            };
            if let Some(run) = finished_run(&action, trigger, start_time, &result) {
                executor.observer.run_finished(run);
            }
            let record = execution_record(&action, &handle, start_time, &result);
            let end_time = record.end_time;
            let bookkeeping = async {
                let _ = executor
                    .repo
                    .create_execution_record(record)
                    .await
                    .inspect_err(|error| {
                        tracing::error!(?error, action_id=?id, "failed to save execution record");
                    });
                let _ = executor.repo.update_last_executed(&id, end_time).await.inspect_err(|error| {
                    tracing::error!(?error, action_id=?id, "failed to update last executed time");
                });
                let _ = executor
                    .repo
                    .update_next_run_at(&id)
                    .await
                    .inspect_err(|error| {
                        tracing::error!(?error, action_id=?id, "failed to update next run time");
                    });
            };
            if tokio::time::timeout(BOOKKEEPING_TIMEOUT, bookkeeping)
                .await
                .is_err()
            {
                tracing::error!(action_id=?id, "execution bookkeeping timed out");
            }
            // Always attempt fenced release, even after failed preparation or
            // stalled history persistence. Never retry model/tool execution.
            let release = executor.repo.release_action(&id, token);
            match tokio::time::timeout(BOOKKEEPING_TIMEOUT, release).await {
                Ok(Ok(())) => {}
                Ok(Err(error)) => {
                    tracing::error!(?error, action_id=?id, "failed to release action claim")
                }
                Err(error) => tracing::error!(?error, action_id=?id, "claim release timed out"),
            }
            let _ = tokio::time::timeout(
                BOOKKEEPING_TIMEOUT,
                executor.stopped(&action, &handle, result.is_ok()),
            )
            .await;
            if let Err(error) = result {
                // The error can quote step content; it is kept in the
                // owner's run history, not in logs.
                tracing::error!(action_id = %id, "scheduled action execution failed");
                if let Some(ready_tx) = ready_tx {
                    let _ = ready_tx.send(Err(error));
                }
            }
        };
        self.tracker.spawn(run.instrument(span));
        ready_rx
            .await
            .context("failed to prepare scheduled action resource")?
    }
}

impl<Rpo, Live, Runner> EventExecutor for InProcessExecutor<Rpo, Live, Runner>
where
    Rpo: ScheduledActionRepo,
    Live: ScheduledActionLiveUpdate,
    Runner: ScheduledAgentRunner,
{
    async fn execute(
        &self,
        run: &ClaimedEventRun,
        cancellation: impl Future<Output = ()> + Send,
    ) -> EventExecutionResult {
        let mut handle = ExecutionHandle {
            firing: Some(RoutineRun::Event(&run.run).cause()),
            ..ExecutionHandle::default()
        };
        let shutdown = async {
            tokio::select! {
                _ = self.cancellation.cancelled() => {},
                _ = cancellation => {},
            }
        };
        let deadline = run.deadline.min(run.started_at + MAX_ACTION_TIME);
        let result = bounded(deadline, shutdown, async {
            self.prepare(&run.action, &mut handle).await?;
            self.runner
                .run(&run.action, &mut handle, RoutineRun::Event(&run.run))
                .await
        })
        .await;
        if let Err(error) = &result {
            self.cancel(&run.action, &handle, error).await;
        }
        let trigger = RunTrigger::Event(run.run.pending.event.event_name());
        if let Some(finished) = finished_run(&run.action, trigger, run.started_at, &result) {
            self.observer.run_finished(finished);
        }
        let record = execution_record(&run.action, &handle, run.started_at, &result);
        let _ = tokio::time::timeout(
            BOOKKEEPING_TIMEOUT,
            self.stopped(&run.action, &handle, result.is_ok()),
        )
        .await;
        // No claim/release or history writes here: the worker finalizes with
        // run.token in the same transaction as the terminal queue transition.
        EventExecutionResult {
            outcome: match &result {
                Ok(()) => EventRunOutcome::Succeeded,
                Err(error) if error.is::<ExecutionCancelled>() => EventRunOutcome::Interrupted,
                Err(_) => EventRunOutcome::Failed,
            },
            record: Some(record),
        }
    }
}