Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
//! Kafka worker that turns committed posts into agent-session events.
// The consumer loop is generic over the trigger topic, and the select! it
// awaits in nests the concrete service types past the default query depth.
#![recursion_limit = "256"]
mod config;
use agent_session::outbound::postgres::PgAgentSessionRepo;
use agent_trigger::domain::context::{DiscussionReader, PeopleDirectory};
use agent_trigger::domain::processing::process_message_event;
use agent_trigger::domain::project_assignment::ProjectAssignmentService;
use agent_trigger::domain::service::AgentTriggerService;
use agent_trigger::domain::sources::{MessageTriggerEvents, TriggerEvents};
use agent_trigger::domain::task_assignment::{
ProjectTaskAssignmentContext, TaskAssignmentContext, process_task_assignment,
};
use agent_trigger::outbound::{
BotRepoAgentLookup, ChannelRepoTypeLookup, DssTaskAssignmentContext, JevTriggerJudge,
LexicalExplicitReplyExtractor, MessageDiscussionReader, MessageThreadHistory, ParentNameReader,
PgPeopleDirectory, VisionImageCaptioner,
};
use anyhow::Context as _;
use bots::outbound::pg_bots_repo::PgBotsRepo;
use channels::outbound::pg_channels_repo::PgChannelsRepo;
use config::Config;
use connection_gateway_client::client::ConnectionGatewayClient;
use document_storage_service_client::DocumentStorageServiceClient;
use entity_access::domain::{ports::EntityAccessService, service::EntityAccessServiceImpl};
use entity_access::outbound::PgAccessRepository;
use entity_registry::OwnerGrantPolicy;
use entity_registry_db_utils::OwnedEntityRegistrar;
use jev::domain::JevClassifier;
use jev::outbound::SystemOneJev;
use kafka_util::{GroupName, InitialOffset, KafkaEventConsumer, consumer_span, record_span_error};
use lexical_client::LexicalClient;
use macro_entrypoint::{MacroEntrypoint, shutdown_signal};
use macro_event_broker::{
KafkaConsumerAdapter, KafkaEventPublisher, MacroEventBrokerService, MacroEventCollection,
MacroEventConsumerService,
};
use macro_service_urls::{
ConnectionGatewayUrl, DocumentStorageServiceUrl, LexicalServiceUrl, StaticFileServiceUrl,
};
use messages::domain::api::MessageCommands;
use messages::outbound::pg_message_repo::PgMessageRepository;
use properties::{
PermissionServiceImpl, PropertiesPgRepo, PropertiesService, PropertiesServiceImpl,
};
use rdkafka::consumer::CommitMode;
use rdkafka::message::{BorrowedMessage, Message as _};
use sqlx::postgres::PgPoolOptions;
use std::sync::Arc;
use system_properties::{
PgSystemPropertiesRepository, SystemPropertiesService, SystemPropertiesServiceImpl,
};
use tracing::Instrument as _;
struct AgentTriggerConsumerGroup;
impl GroupName for AgentTriggerConsumerGroup {
const GROUP_NAME: &'static str = "agent-trigger-service";
// Existing partitions resume committed offsets. Newly subscribed topics
// start with future events, without assigning agents to historical tasks.
const INITIAL_OFFSET: InitialOffset = InitialOffset::Latest;
}
/// The concrete trigger service this binary composes.
type Trigger = AgentTriggerService<
PgAgentSessionRepo<PgBotsRepo>,
BotRepoAgentLookup<PgBotsRepo>,
BotRepoAgentLookup<PgBotsRepo>,
BotRepoAgentLookup<PgBotsRepo>,
LexicalExplicitReplyExtractor,
Option<JevTriggerJudge<JevClassifier<SystemOneJev>>>,
MessageThreadHistory<
entity_access::domain::service::EntityAccessServiceImpl<
entity_access::outbound::PgAccessRepository,
>,
>,
>;
type Publisher = MacroEventBrokerService<KafkaEventPublisher, macro_event_broker::GlobalSpawner>;
type ChannelTypes = ChannelRepoTypeLookup<PgChannelsRepo>;
type TriggerKafkaAdapter<M> = KafkaConsumerAdapter<AgentTriggerConsumerGroup, M>;
type TriggerConsumer<M> = MacroEventConsumerService<M, TriggerKafkaAdapter<M>>;
// This worker only posts silent assignment roots. Session replies and their
// notifications are delivered by the harness's full message service.
struct SilentAssignmentNotifications;
impl messages::domain::delivery::DiscussionNotifier for SilentAssignmentNotifications {
async fn send(
&self,
_: messages::domain::delivery::DiscussionNotification<'_>,
) -> Result<(), rootcause::Report> {
Ok(())
}
}
impl properties::NotificationService for SilentAssignmentNotifications {
type Err = anyhow::Error;
async fn send_task_assigned<'a>(
&self,
_: properties::domain::model::TaskAssignedNotification<'a>,
) -> Result<(), Self::Err> {
Ok(())
}
}
fn commit_message<M: MacroEventCollection + 'static>(
consumer: &TriggerConsumer<M>,
message: &BorrowedMessage<'_>,
) -> anyhow::Result<()> {
consumer
.inner()
.commit_message(message, CommitMode::Sync)
.map_err(|error| anyhow::anyhow!("failed to commit trigger event offset: {error:?}"))
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let entrypoint = MacroEntrypoint::default().init();
let result = run().await;
entrypoint.shutdown();
result
}
async fn run() -> anyhow::Result<()> {
let config = Config::from_env()?;
let pool = PgPoolOptions::new()
.min_connections(1)
.max_connections(5)
.connect(config.database_url.as_ref())
.await
.context("failed to connect to macrodb")?;
let lexical = LexicalClient::new(
config.internal_api_key.clone(),
LexicalServiceUrl::new()?.to_string(),
);
let recorder =
ai_usage::pg_recorder_with_enforcement(pool.clone(), config.enable_ai_usage_enforcement);
let admission = ai_billing::composition::pg_admission_service(
pool.clone(),
config.enable_ai_usage_enforcement,
config.ai_pricing(),
);
let documents = DocumentStorageServiceClient::new(
config.document_storage_service_auth_key.clone(),
DocumentStorageServiceUrl::new()?.to_string(),
);
let task_context = ProjectTaskAssignmentContext::new(
DssTaskAssignmentContext::new(
documents.clone(),
lexical.clone(),
PropertiesServiceImpl::new(
PropertiesPgRepo::new(pool.clone()),
Some(PermissionServiceImpl::new(
pool.clone(),
Arc::new(EntityAccessServiceImpl::new(PgAccessRepository::new(
pool.clone(),
))),
)),
None::<SilentAssignmentNotifications>,
),
),
SystemPropertiesServiceImpl::new(PgSystemPropertiesRepository::new(pool.clone())),
EntityAccessServiceImpl::new(PgAccessRepository::new(pool.clone())),
initiative::domain::lookup::InitiativeLookup::new(
initiative::outbound::PgInitiativeRepo::new(pool.clone()),
),
);
let message_reader = Arc::new(messages::domain::service::MessageService::new(
PgMessageRepository::new(pool.clone())
.with_initiatives(initiative::domain::lookup::InitiativeLookup::new(
initiative::outbound::PgInitiativeRepo::new(pool.clone()),
))
.with_crm(crm::outbound::lookup::PgCrmParentReader::new(pool.clone())),
messages::domain::ports::NoMessageEventPublisher,
));
let people = PgPeopleDirectory::new(pool.clone(), PgBotsRepo::new(pool.clone()));
let discussions = MessageDiscussionReader::new(
message_reader.clone(),
PgPeopleDirectory::new(pool.clone(), PgBotsRepo::new(pool.clone())),
ParentNameReader::new(
PgChannelsRepo::new(pool.clone()),
documents,
initiative::domain::lookup::InitiativeLookup::new(
initiative::outbound::PgInitiativeRepo::new(pool.clone()),
),
crm::outbound::lookup::PgCrmParentReader::new(pool.clone()),
call::outbound::pg_call_repo::PgCallRepo::new(pool.clone()),
),
lexical.clone(),
);
let images = VisionImageCaptioner::new(
static_file::outbound::CdnStaticFileRepo::new(StaticFileServiceUrl::new()?.to_string()),
recorder.clone(),
);
let classifier = config
.typesafe_api_key
.value()
.map(SystemOneJev::typesafe)
.transpose()
.context("invalid TYPESAFE_API_KEY")?
.map(|provider| Arc::new(JevClassifier::new(provider, recorder)));
let trigger = AgentTriggerService::new(
PgAgentSessionRepo::new(
pool.clone(),
OwnedEntityRegistrar::new(OwnerGrantPolicy::new(PgBotsRepo::new(pool.clone()))),
),
BotRepoAgentLookup::new(PgBotsRepo::new(pool.clone())),
BotRepoAgentLookup::new(PgBotsRepo::new(pool.clone())),
BotRepoAgentLookup::new(PgBotsRepo::new(pool.clone())),
LexicalExplicitReplyExtractor::new(lexical),
classifier.map(|classifier| JevTriggerJudge::new(classifier, images)),
MessageThreadHistory::new(
message_reader,
entity_access::domain::service::EntityAccessServiceImpl::new(
entity_access::outbound::PgAccessRepository::new(pool.clone()),
),
),
)
.with_admission(admission);
let channel_types = ChannelRepoTypeLookup::new(PgChannelsRepo::new(pool.clone()));
let publisher = MacroEventBrokerService::new(
KafkaEventPublisher::new(config.kafka_brokers.as_ref())?,
macro_event_broker::GlobalSpawner,
);
let project_assignments = ProjectAssignmentService::new(
PropertiesServiceImpl::new(
PropertiesPgRepo::new(pool.clone()),
Some(PermissionServiceImpl::new(
pool.clone(),
Arc::new(EntityAccessServiceImpl::new(PgAccessRepository::new(
pool.clone(),
))),
)),
None::<SilentAssignmentNotifications>,
)
.with_event_broker(publisher.clone()),
SystemPropertiesServiceImpl::new(PgSystemPropertiesRepository::new(pool.clone())),
EntityAccessServiceImpl::new(PgAccessRepository::new(pool.clone())),
);
let discussion_delivery = messages::domain::delivery::DiscussionDelivery::new(
messages::outbound::pg_discussion_context::PgDiscussionContext(pool.clone()),
messages::outbound::entity_access_audience::EntityAccessMessageAudience(
entity_access::domain::service::EntityAccessServiceImpl::new(
entity_access::outbound::PgAccessRepository::new(pool.clone()),
),
),
messages::outbound::connection_gateway::ConnectionGatewayMessages(Arc::new(
ConnectionGatewayClient::new(
config.internal_api_key.clone(),
ConnectionGatewayUrl::new()?.to_string(),
),
)),
SilentAssignmentNotifications,
);
let messages = messages::domain::service::MessageService::new(
PgMessageRepository::new(pool),
messages::domain::effects::MessageEffects::new(
messages::outbound::broker::BrokerMessagePublisher::new(publisher.clone()),
messages::domain::ports::NoMessageEventPublisher,
discussion_delivery,
),
);
let consumer =
KafkaEventConsumer::<AgentTriggerConsumerGroup>::from_env(config.kafka_brokers.as_ref())?;
let consumer = KafkaConsumerAdapter::<AgentTriggerConsumerGroup, ()>::new(consumer);
consume::<MessageTriggerEvents>(
consumer,
&trigger,
&publisher,
&ContextReaders {
channel_types,
discussions,
people,
},
&messages,
&task_context,
&project_assignments,
)
.await
}
/// What the trigger reads beside the post to describe the agents it calls.
struct ContextReaders<Discussions, People> {
channel_types: ChannelTypes,
discussions: Discussions,
people: People,
}
/// Read the trigger topic until shutdown, evaluating every committed post.
async fn consume<Events: TriggerEvents>(
consumer: KafkaConsumerAdapter<AgentTriggerConsumerGroup, ()>,
trigger: &Trigger,
publisher: &Publisher,
readers: &ContextReaders<impl DiscussionReader, impl PeopleDirectory>,
messages: &dyn MessageCommands,
task_context: &impl TaskAssignmentContext,
project_assignments: &ProjectAssignmentService<
impl PropertiesService,
impl SystemPropertiesService,
impl EntityAccessService,
>,
) -> anyhow::Result<()> {
let consumer = consumer
.subscribe::<Events>()
.map_err(|error| anyhow::anyhow!("failed to subscribe to trigger events: {error:?}"))?;
let consumer = TriggerConsumer::<Events>::new(consumer);
tracing::info!(
topics = ?Events::topics(),
group = AgentTriggerConsumerGroup::GROUP_NAME,
"agent trigger service listening"
);
let mut shutdown = std::pin::pin!(shutdown_signal());
let mut run_error = None;
loop {
tokio::select! {
() = &mut shutdown => {
tracing::info!("agent trigger service shutting down");
break;
}
result = consumer.recv() => {
let message = match result {
Ok(message) => message,
Err(error) => {
tracing::error!(error = ?error, "failed to receive trigger event");
continue;
}
};
let span = consumer_span(message.inner(), AgentTriggerConsumerGroup::GROUP_NAME);
let result = async {
let kafka_message = message.inner();
let decoded = match message.decode_payload() {
Ok(decoded) => decoded.into_trigger(),
Err(error) => {
record_span_error(&tracing::Span::current(), &error);
tracing::error!(
error = ?error,
partition = kafka_message.partition(),
offset = kafka_message.offset(),
"dropping undecodable trigger event"
);
commit_message(&consumer, kafka_message)?;
return Ok::<(), anyhow::Error>(());
}
};
tracing::Span::current().record("macro.event.id", tracing::field::display(decoded.event_id));
tracing::Span::current().record("macro.event.type", decoded.event_type);
if let Some(posted) = &decoded.posted {
process_message_event(trigger, publisher, &readers.channel_types, &readers.discussions, posted).await?;
}
if let Some(assignment) = &decoded.assignment {
process_task_assignment(trigger, publisher, messages, task_context, &readers.people, assignment).await?;
}
if let Some(added) = &decoded.project_task {
project_assignments.process(added).await?;
}
commit_message(&consumer, kafka_message)?;
Ok(())
}
.instrument(span.clone())
.await;
if let Err(error) = &result {
record_span_error(&span, error);
}
if let Err(error) = result {
run_error = Some(error);
break;
}
}
}
}
match run_error {
Some(error) => Err(error),
None => Ok(()),
}
}