Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

9.6 KB
Raw Blame
//! Internal HTTP surface for every search-event backfill.
//!
//! Each POST handler kicks the orchestrator onto a background tokio task
//! and returns `202 Accepted` with a job id right away — prod-scale corpora
//! can take many minutes to drain, well past the ALB idle timeout. Clients
//! poll `GET /internal/backfill/{job_id}` for progress; the orchestrator
//! updates the shared progress counter in DynamoDB as each page lands. Any
//! SPS instance can answer the GET because state lives in DynamoDB, not
//! in process memory. On shutdown the registry's local cancellation tokens
//! fire so drains stop between pages instead of being killed mid-publish.
//!
//! Adding a new entity is one POST handler + one `.route(...)`; the status
//! handler is generic.

use std::sync::Arc;

use axum::{
    Router,
    extract::{self, Path, State},
    http::StatusCode,
    response::{IntoResponse, Response},
    routing::{get, post},
};
use macro_authorization::{InternalOnly, MacroAuthorizationExtractor};
use serde::Serialize;

use crate::BackfillServiceImpl;
use crate::api::context::{ApiContext, AuthorizationService};
use crate::domain::jobs::{BackfillJobs, JobId};
use crate::domain::models::{
    CalendarEventBackfillRequest, CallBackfillRequest, ChannelBackfillRequest, ChatBackfillRequest,
    DocumentBackfillRequest, EmailBackfillRequest, ProjectBackfillRequest,
    PropertiesBackfillRequest,
};
use crate::domain::service::BackfillService;

pub fn router() -> Router<ApiContext> {
    Router::new()
        .route("/agent-sessions", post(agent_sessions))
        .route("/calls", post(calls))
        .route("/chats", post(chats))
        .route("/channels", post(channels))
        .route("/documents", post(documents))
        .route("/emails", post(emails))
        .route("/properties", post(properties))
        .route("/projects", post(projects))
        .route("/calendar-events", post(calendar_events))
        .route("/{job_id}", get(status))
}

#[derive(Debug, Serialize)]
struct AcceptedReceipt {
    job_id: JobId,
}

#[tracing::instrument(skip(service, jobs, _internal_authorization, req))]
async fn agent_sessions(
    State(service): State<Arc<crate::AgentSessionIndexer>>,
    State(jobs): State<BackfillJobs>,
    _internal_authorization: MacroAuthorizationExtractor<AuthorizationService, InternalOnly>,
    extract::Json(req): extract::Json<
        crate::domain::agent_session_index::AgentSessionBackfillRequest,
    >,
) -> Response {
    spawn_backfill(
        service,
        jobs,
        "agent-sessions",
        move |svc, progress, cancel| async move { svc.backfill(req, progress, cancel).await },
    )
    .await
}

#[tracing::instrument(skip(service, jobs, _internal_authorization, req))]
async fn calls(
    State(service): State<Arc<BackfillServiceImpl>>,
    State(jobs): State<BackfillJobs>,
    _internal_authorization: MacroAuthorizationExtractor<AuthorizationService, InternalOnly>,
    extract::Json(req): extract::Json<CallBackfillRequest>,
) -> Response {
    spawn_backfill(
        service,
        jobs,
        "calls",
        move |svc, progress, cancel| async move { svc.backfill_calls(req, progress, cancel).await },
    )
    .await
}

#[tracing::instrument(skip(service, jobs, _internal_authorization, req))]
async fn chats(
    State(service): State<Arc<BackfillServiceImpl>>,
    State(jobs): State<BackfillJobs>,
    _internal_authorization: MacroAuthorizationExtractor<AuthorizationService, InternalOnly>,
    extract::Json(req): extract::Json<ChatBackfillRequest>,
) -> Response {
    spawn_backfill(
        service,
        jobs,
        "chats",
        move |svc, progress, cancel| async move { svc.backfill_chats(req, progress, cancel).await },
    )
    .await
}

#[tracing::instrument(skip(service, jobs, _internal_authorization, req))]
async fn channels(
    State(service): State<Arc<BackfillServiceImpl>>,
    State(jobs): State<BackfillJobs>,
    _internal_authorization: MacroAuthorizationExtractor<AuthorizationService, InternalOnly>,
    extract::Json(req): extract::Json<ChannelBackfillRequest>,
) -> Response {
    spawn_backfill(
        service,
        jobs,
        "channels",
        move |svc, progress, cancel| async move {
            svc.backfill_channels(req, progress, cancel).await
        },
    )
    .await
}

#[tracing::instrument(skip(service, jobs, _internal_authorization, req))]
async fn documents(
    State(service): State<Arc<BackfillServiceImpl>>,
    State(jobs): State<BackfillJobs>,
    _internal_authorization: MacroAuthorizationExtractor<AuthorizationService, InternalOnly>,
    extract::Json(req): extract::Json<DocumentBackfillRequest>,
) -> Response {
    spawn_backfill(
        service,
        jobs,
        "documents",
        move |svc, progress, cancel| async move {
            svc.backfill_documents(req, progress, cancel).await
        },
    )
    .await
}

#[tracing::instrument(skip(service, jobs, _internal_authorization, req))]
async fn emails(
    State(service): State<Arc<BackfillServiceImpl>>,
    State(jobs): State<BackfillJobs>,
    _internal_authorization: MacroAuthorizationExtractor<AuthorizationService, InternalOnly>,
    extract::Json(req): extract::Json<EmailBackfillRequest>,
) -> Response {
    spawn_backfill(
        service,
        jobs,
        "emails",
        move |svc, progress, cancel| async move {
            svc.backfill_emails(req, progress, cancel).await
        },
    )
    .await
}

#[tracing::instrument(skip(service, jobs, _internal_authorization, req))]
async fn properties(
    State(service): State<Arc<BackfillServiceImpl>>,
    State(jobs): State<BackfillJobs>,
    _internal_authorization: MacroAuthorizationExtractor<AuthorizationService, InternalOnly>,
    extract::Json(req): extract::Json<PropertiesBackfillRequest>,
) -> Response {
    spawn_backfill(
        service,
        jobs,
        "properties",
        move |svc, progress, cancel| async move {
            svc.backfill_entity_properties(req, progress, cancel).await
        },
    )
    .await
}

#[tracing::instrument(skip(service, jobs, _internal_authorization, req))]
async fn projects(
    State(service): State<Arc<BackfillServiceImpl>>,
    State(jobs): State<BackfillJobs>,
    _internal_authorization: MacroAuthorizationExtractor<AuthorizationService, InternalOnly>,
    extract::Json(req): extract::Json<ProjectBackfillRequest>,
) -> Response {
    spawn_backfill(
        service,
        jobs,
        "projects",
        move |svc, progress, cancel| async move {
            svc.backfill_projects(req, progress, cancel).await
        },
    )
    .await
}

#[tracing::instrument(skip(service, jobs, _internal_authorization, req))]
async fn calendar_events(
    State(service): State<Arc<BackfillServiceImpl>>,
    State(jobs): State<BackfillJobs>,
    _internal_authorization: MacroAuthorizationExtractor<AuthorizationService, InternalOnly>,
    extract::Json(req): extract::Json<CalendarEventBackfillRequest>,
) -> Response {
    spawn_backfill(
        service,
        jobs,
        "calendar-events",
        move |svc, progress, cancel| async move {
            svc.backfill_calendar_events(req, progress, cancel).await
        },
    )
    .await
}

#[tracing::instrument(skip(jobs, _internal_authorization))]
async fn status(
    State(jobs): State<BackfillJobs>,
    _internal_authorization: MacroAuthorizationExtractor<AuthorizationService, InternalOnly>,
    Path(job_id): Path<String>,
) -> Response {
    match jobs.snapshot(&JobId::from(job_id)).await {
        Ok(Some(snap)) => axum::Json(snap).into_response(),
        Ok(None) => (StatusCode::NOT_FOUND, "unknown job id").into_response(),
        Err(e) => {
            tracing::error!(error=?e, "failed to read backfill job snapshot");
            (StatusCode::INTERNAL_SERVER_ERROR, "registry unavailable").into_response()
        }
    }
}

/// Allocate a job slot in the registry, hand the progress + cancel token
/// to the worker future the caller built, spawn it, and return
/// `202 Accepted` with the job id. The worker future captures the request
/// body so the HTTP body reference doesn't outlive the handler.
async fn spawn_backfill<S, F, Fut>(
    service: Arc<S>,
    jobs: BackfillJobs,
    entity: &'static str,
    run: F,
) -> Response
where
    S: Send + Sync + 'static,
    F: FnOnce(
            Arc<S>,
            Arc<crate::domain::jobs::JobProgress>,
            tokio_util::sync::CancellationToken,
        ) -> Fut
        + Send
        + 'static,
    Fut: std::future::Future<
            Output = Result<
                crate::domain::models::BackfillReceipt,
                crate::domain::models::BackfillError,
            >,
        > + Send,
{
    let handle = match jobs.start(entity).await {
        Ok(h) => h,
        Err(e) => {
            tracing::error!(error=?e, entity, "failed to allocate backfill job in registry");
            return (StatusCode::INTERNAL_SERVER_ERROR, "registry unavailable").into_response();
        }
    };
    let id = handle.id.clone();
    let id_for_task = handle.id.clone();
    let jobs_for_task = jobs.clone();

    tracing::info!(job_id = %id, entity, "spawning backfill");

    tokio::spawn(async move {
        let result = run(service, handle.progress, handle.cancel).await;
        if let Err(e) = &result {
            tracing::error!(job_id = %id_for_task, entity, error = ?e, "backfill failed");
        } else {
            tracing::info!(job_id = %id_for_task, entity, "backfill completed");
        }
        if let Err(e) = jobs_for_task.finish(&id_for_task, result).await {
            tracing::error!(job_id = %id_for_task, error=?e, "failed to record backfill terminal status in registry");
        }
    });

    (
        StatusCode::ACCEPTED,
        axum::Json(AcceptedReceipt { job_id: id }),
    )
        .into_response()
}