Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
//! Internal HTTP surface for every search-event backfill.
//!
//! Each POST handler kicks the orchestrator onto a background tokio task
//! and returns `202 Accepted` with a job id right away — prod-scale corpora
//! can take many minutes to drain, well past the ALB idle timeout. Clients
//! poll `GET /internal/backfill/{job_id}` for progress; the orchestrator
//! updates the shared progress counter in DynamoDB as each page lands. Any
//! SPS instance can answer the GET because state lives in DynamoDB, not
//! in process memory. On shutdown the registry's local cancellation tokens
//! fire so drains stop between pages instead of being killed mid-publish.
//!
//! Adding a new entity is one POST handler + one `.route(...)`; the status
//! handler is generic.
use std::sync::Arc;
use axum::{
Router,
extract::{self, Path, State},
http::StatusCode,
response::{IntoResponse, Response},
routing::{get, post},
};
use macro_authorization::{InternalOnly, MacroAuthorizationExtractor};
use serde::Serialize;
use crate::BackfillServiceImpl;
use crate::api::context::{ApiContext, AuthorizationService};
use crate::domain::jobs::{BackfillJobs, JobId};
use crate::domain::models::{
CalendarEventBackfillRequest, CallBackfillRequest, ChannelBackfillRequest, ChatBackfillRequest,
DocumentBackfillRequest, EmailBackfillRequest, ProjectBackfillRequest,
PropertiesBackfillRequest,
};
use crate::domain::service::BackfillService;
pub fn router() -> Router<ApiContext> {
Router::new()
.route("/agent-sessions", post(agent_sessions))
.route("/calls", post(calls))
.route("/chats", post(chats))
.route("/channels", post(channels))
.route("/documents", post(documents))
.route("/emails", post(emails))
.route("/properties", post(properties))
.route("/projects", post(projects))
.route("/calendar-events", post(calendar_events))
.route("/{job_id}", get(status))
}
#[derive(Debug, Serialize)]
struct AcceptedReceipt {
job_id: JobId,
}
#[tracing::instrument(skip(service, jobs, _internal_authorization, req))]
async fn agent_sessions(
State(service): State<Arc<crate::AgentSessionIndexer>>,
State(jobs): State<BackfillJobs>,
_internal_authorization: MacroAuthorizationExtractor<AuthorizationService, InternalOnly>,
extract::Json(req): extract::Json<
crate::domain::agent_session_index::AgentSessionBackfillRequest,
>,
) -> Response {
spawn_backfill(
service,
jobs,
"agent-sessions",
move |svc, progress, cancel| async move { svc.backfill(req, progress, cancel).await },
)
.await
}
#[tracing::instrument(skip(service, jobs, _internal_authorization, req))]
async fn calls(
State(service): State<Arc<BackfillServiceImpl>>,
State(jobs): State<BackfillJobs>,
_internal_authorization: MacroAuthorizationExtractor<AuthorizationService, InternalOnly>,
extract::Json(req): extract::Json<CallBackfillRequest>,
) -> Response {
spawn_backfill(
service,
jobs,
"calls",
move |svc, progress, cancel| async move { svc.backfill_calls(req, progress, cancel).await },
)
.await
}
#[tracing::instrument(skip(service, jobs, _internal_authorization, req))]
async fn chats(
State(service): State<Arc<BackfillServiceImpl>>,
State(jobs): State<BackfillJobs>,
_internal_authorization: MacroAuthorizationExtractor<AuthorizationService, InternalOnly>,
extract::Json(req): extract::Json<ChatBackfillRequest>,
) -> Response {
spawn_backfill(
service,
jobs,
"chats",
move |svc, progress, cancel| async move { svc.backfill_chats(req, progress, cancel).await },
)
.await
}
#[tracing::instrument(skip(service, jobs, _internal_authorization, req))]
async fn channels(
State(service): State<Arc<BackfillServiceImpl>>,
State(jobs): State<BackfillJobs>,
_internal_authorization: MacroAuthorizationExtractor<AuthorizationService, InternalOnly>,
extract::Json(req): extract::Json<ChannelBackfillRequest>,
) -> Response {
spawn_backfill(
service,
jobs,
"channels",
move |svc, progress, cancel| async move {
svc.backfill_channels(req, progress, cancel).await
},
)
.await
}
#[tracing::instrument(skip(service, jobs, _internal_authorization, req))]
async fn documents(
State(service): State<Arc<BackfillServiceImpl>>,
State(jobs): State<BackfillJobs>,
_internal_authorization: MacroAuthorizationExtractor<AuthorizationService, InternalOnly>,
extract::Json(req): extract::Json<DocumentBackfillRequest>,
) -> Response {
spawn_backfill(
service,
jobs,
"documents",
move |svc, progress, cancel| async move {
svc.backfill_documents(req, progress, cancel).await
},
)
.await
}
#[tracing::instrument(skip(service, jobs, _internal_authorization, req))]
async fn emails(
State(service): State<Arc<BackfillServiceImpl>>,
State(jobs): State<BackfillJobs>,
_internal_authorization: MacroAuthorizationExtractor<AuthorizationService, InternalOnly>,
extract::Json(req): extract::Json<EmailBackfillRequest>,
) -> Response {
spawn_backfill(
service,
jobs,
"emails",
move |svc, progress, cancel| async move {
svc.backfill_emails(req, progress, cancel).await
},
)
.await
}
#[tracing::instrument(skip(service, jobs, _internal_authorization, req))]
async fn properties(
State(service): State<Arc<BackfillServiceImpl>>,
State(jobs): State<BackfillJobs>,
_internal_authorization: MacroAuthorizationExtractor<AuthorizationService, InternalOnly>,
extract::Json(req): extract::Json<PropertiesBackfillRequest>,
) -> Response {
spawn_backfill(
service,
jobs,
"properties",
move |svc, progress, cancel| async move {
svc.backfill_entity_properties(req, progress, cancel).await
},
)
.await
}
#[tracing::instrument(skip(service, jobs, _internal_authorization, req))]
async fn projects(
State(service): State<Arc<BackfillServiceImpl>>,
State(jobs): State<BackfillJobs>,
_internal_authorization: MacroAuthorizationExtractor<AuthorizationService, InternalOnly>,
extract::Json(req): extract::Json<ProjectBackfillRequest>,
) -> Response {
spawn_backfill(
service,
jobs,
"projects",
move |svc, progress, cancel| async move {
svc.backfill_projects(req, progress, cancel).await
},
)
.await
}
#[tracing::instrument(skip(service, jobs, _internal_authorization, req))]
async fn calendar_events(
State(service): State<Arc<BackfillServiceImpl>>,
State(jobs): State<BackfillJobs>,
_internal_authorization: MacroAuthorizationExtractor<AuthorizationService, InternalOnly>,
extract::Json(req): extract::Json<CalendarEventBackfillRequest>,
) -> Response {
spawn_backfill(
service,
jobs,
"calendar-events",
move |svc, progress, cancel| async move {
svc.backfill_calendar_events(req, progress, cancel).await
},
)
.await
}
#[tracing::instrument(skip(jobs, _internal_authorization))]
async fn status(
State(jobs): State<BackfillJobs>,
_internal_authorization: MacroAuthorizationExtractor<AuthorizationService, InternalOnly>,
Path(job_id): Path<String>,
) -> Response {
match jobs.snapshot(&JobId::from(job_id)).await {
Ok(Some(snap)) => axum::Json(snap).into_response(),
Ok(None) => (StatusCode::NOT_FOUND, "unknown job id").into_response(),
Err(e) => {
tracing::error!(error=?e, "failed to read backfill job snapshot");
(StatusCode::INTERNAL_SERVER_ERROR, "registry unavailable").into_response()
}
}
}
/// Allocate a job slot in the registry, hand the progress + cancel token
/// to the worker future the caller built, spawn it, and return
/// `202 Accepted` with the job id. The worker future captures the request
/// body so the HTTP body reference doesn't outlive the handler.
async fn spawn_backfill<S, F, Fut>(
service: Arc<S>,
jobs: BackfillJobs,
entity: &'static str,
run: F,
) -> Response
where
S: Send + Sync + 'static,
F: FnOnce(
Arc<S>,
Arc<crate::domain::jobs::JobProgress>,
tokio_util::sync::CancellationToken,
) -> Fut
+ Send
+ 'static,
Fut: std::future::Future<
Output = Result<
crate::domain::models::BackfillReceipt,
crate::domain::models::BackfillError,
>,
> + Send,
{
let handle = match jobs.start(entity).await {
Ok(h) => h,
Err(e) => {
tracing::error!(error=?e, entity, "failed to allocate backfill job in registry");
return (StatusCode::INTERNAL_SERVER_ERROR, "registry unavailable").into_response();
}
};
let id = handle.id.clone();
let id_for_task = handle.id.clone();
let jobs_for_task = jobs.clone();
tracing::info!(job_id = %id, entity, "spawning backfill");
tokio::spawn(async move {
let result = run(service, handle.progress, handle.cancel).await;
if let Err(e) = &result {
tracing::error!(job_id = %id_for_task, entity, error = ?e, "backfill failed");
} else {
tracing::info!(job_id = %id_for_task, entity, "backfill completed");
}
if let Err(e) = jobs_for_task.finish(&id_for_task, result).await {
tracing::error!(job_id = %id_for_task, error=?e, "failed to record backfill terminal status in registry");
}
});
(
StatusCode::ACCEPTED,
axum::Json(AcceptedReceipt { job_id: id }),
)
.into_response()
}