Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

15.4 KB
Raw Blame
//! Kafka worker that turns committed posts into agent-session events.

// The consumer loop is generic over the trigger topic, and the select! it
// awaits in nests the concrete service types past the default query depth.
#![recursion_limit = "256"]

mod config;

use agent_session::outbound::postgres::PgAgentSessionRepo;
use agent_trigger::domain::context::{DiscussionReader, PeopleDirectory};
use agent_trigger::domain::processing::process_message_event;
use agent_trigger::domain::project_assignment::ProjectAssignmentService;
use agent_trigger::domain::service::AgentTriggerService;
use agent_trigger::domain::sources::{MessageTriggerEvents, TriggerEvents};
use agent_trigger::domain::task_assignment::{
    ProjectTaskAssignmentContext, TaskAssignmentContext, process_task_assignment,
};
use agent_trigger::outbound::{
    BotRepoAgentLookup, ChannelRepoTypeLookup, DssTaskAssignmentContext, JevTriggerJudge,
    LexicalExplicitReplyExtractor, MessageDiscussionReader, MessageThreadHistory, ParentNameReader,
    PgPeopleDirectory, VisionImageCaptioner,
};
use anyhow::Context as _;
use bots::outbound::pg_bots_repo::PgBotsRepo;
use channels::outbound::pg_channels_repo::PgChannelsRepo;
use config::Config;
use connection_gateway_client::client::ConnectionGatewayClient;
use document_storage_service_client::DocumentStorageServiceClient;
use entity_access::domain::{ports::EntityAccessService, service::EntityAccessServiceImpl};
use entity_access::outbound::PgAccessRepository;
use entity_registry::OwnerGrantPolicy;
use entity_registry_db_utils::OwnedEntityRegistrar;
use jev::domain::JevClassifier;
use jev::outbound::SystemOneJev;
use kafka_util::{GroupName, InitialOffset, KafkaEventConsumer, consumer_span, record_span_error};
use lexical_client::LexicalClient;
use macro_entrypoint::{MacroEntrypoint, shutdown_signal};
use macro_event_broker::{
    KafkaConsumerAdapter, KafkaEventPublisher, MacroEventBrokerService, MacroEventCollection,
    MacroEventConsumerService,
};
use macro_service_urls::{
    ConnectionGatewayUrl, DocumentStorageServiceUrl, LexicalServiceUrl, StaticFileServiceUrl,
};
use messages::domain::api::MessageCommands;
use messages::outbound::pg_message_repo::PgMessageRepository;
use properties::{
    PermissionServiceImpl, PropertiesPgRepo, PropertiesService, PropertiesServiceImpl,
};
use rdkafka::consumer::CommitMode;
use rdkafka::message::{BorrowedMessage, Message as _};
use sqlx::postgres::PgPoolOptions;
use std::sync::Arc;
use system_properties::{
    PgSystemPropertiesRepository, SystemPropertiesService, SystemPropertiesServiceImpl,
};
use tracing::Instrument as _;

struct AgentTriggerConsumerGroup;

impl GroupName for AgentTriggerConsumerGroup {
    const GROUP_NAME: &'static str = "agent-trigger-service";
    // Existing partitions resume committed offsets. Newly subscribed topics
    // start with future events, without assigning agents to historical tasks.
    const INITIAL_OFFSET: InitialOffset = InitialOffset::Latest;
}

/// The concrete trigger service this binary composes.
type Trigger = AgentTriggerService<
    PgAgentSessionRepo<PgBotsRepo>,
    BotRepoAgentLookup<PgBotsRepo>,
    BotRepoAgentLookup<PgBotsRepo>,
    BotRepoAgentLookup<PgBotsRepo>,
    LexicalExplicitReplyExtractor,
    Option<JevTriggerJudge<JevClassifier<SystemOneJev>>>,
    MessageThreadHistory<
        entity_access::domain::service::EntityAccessServiceImpl<
            entity_access::outbound::PgAccessRepository,
        >,
    >,
>;
type Publisher = MacroEventBrokerService<KafkaEventPublisher, macro_event_broker::GlobalSpawner>;
type ChannelTypes = ChannelRepoTypeLookup<PgChannelsRepo>;

type TriggerKafkaAdapter<M> = KafkaConsumerAdapter<AgentTriggerConsumerGroup, M>;
type TriggerConsumer<M> = MacroEventConsumerService<M, TriggerKafkaAdapter<M>>;

// This worker only posts silent assignment roots. Session replies and their
// notifications are delivered by the harness's full message service.
struct SilentAssignmentNotifications;

impl messages::domain::delivery::DiscussionNotifier for SilentAssignmentNotifications {
    async fn send(
        &self,
        _: messages::domain::delivery::DiscussionNotification<'_>,
    ) -> Result<(), rootcause::Report> {
        Ok(())
    }
}

impl properties::NotificationService for SilentAssignmentNotifications {
    type Err = anyhow::Error;

    async fn send_task_assigned<'a>(
        &self,
        _: properties::domain::model::TaskAssignedNotification<'a>,
    ) -> Result<(), Self::Err> {
        Ok(())
    }
}

fn commit_message<M: MacroEventCollection + 'static>(
    consumer: &TriggerConsumer<M>,
    message: &BorrowedMessage<'_>,
) -> anyhow::Result<()> {
    consumer
        .inner()
        .commit_message(message, CommitMode::Sync)
        .map_err(|error| anyhow::anyhow!("failed to commit trigger event offset: {error:?}"))
}

#[tokio::main]
async fn main() -> anyhow::Result<()> {
    let entrypoint = MacroEntrypoint::default().init();
    let result = run().await;
    entrypoint.shutdown();
    result
}

async fn run() -> anyhow::Result<()> {
    let config = Config::from_env()?;
    let pool = PgPoolOptions::new()
        .min_connections(1)
        .max_connections(5)
        .connect(config.database_url.as_ref())
        .await
        .context("failed to connect to macrodb")?;

    let lexical = LexicalClient::new(
        config.internal_api_key.clone(),
        LexicalServiceUrl::new()?.to_string(),
    );
    let recorder =
        ai_usage::pg_recorder_with_enforcement(pool.clone(), config.enable_ai_usage_enforcement);
    let admission = ai_billing::composition::pg_admission_service(
        pool.clone(),
        config.enable_ai_usage_enforcement,
        config.ai_pricing(),
    );
    let documents = DocumentStorageServiceClient::new(
        config.document_storage_service_auth_key.clone(),
        DocumentStorageServiceUrl::new()?.to_string(),
    );
    let task_context = ProjectTaskAssignmentContext::new(
        DssTaskAssignmentContext::new(
            documents.clone(),
            lexical.clone(),
            PropertiesServiceImpl::new(
                PropertiesPgRepo::new(pool.clone()),
                Some(PermissionServiceImpl::new(
                    pool.clone(),
                    Arc::new(EntityAccessServiceImpl::new(PgAccessRepository::new(
                        pool.clone(),
                    ))),
                )),
                None::<SilentAssignmentNotifications>,
            ),
        ),
        SystemPropertiesServiceImpl::new(PgSystemPropertiesRepository::new(pool.clone())),
        EntityAccessServiceImpl::new(PgAccessRepository::new(pool.clone())),
        initiative::domain::lookup::InitiativeLookup::new(
            initiative::outbound::PgInitiativeRepo::new(pool.clone()),
        ),
    );
    let message_reader = Arc::new(messages::domain::service::MessageService::new(
        PgMessageRepository::new(pool.clone())
            .with_initiatives(initiative::domain::lookup::InitiativeLookup::new(
                initiative::outbound::PgInitiativeRepo::new(pool.clone()),
            ))
            .with_crm(crm::outbound::lookup::PgCrmParentReader::new(pool.clone())),
        messages::domain::ports::NoMessageEventPublisher,
    ));
    let people = PgPeopleDirectory::new(pool.clone(), PgBotsRepo::new(pool.clone()));
    let discussions = MessageDiscussionReader::new(
        message_reader.clone(),
        PgPeopleDirectory::new(pool.clone(), PgBotsRepo::new(pool.clone())),
        ParentNameReader::new(
            PgChannelsRepo::new(pool.clone()),
            documents,
            initiative::domain::lookup::InitiativeLookup::new(
                initiative::outbound::PgInitiativeRepo::new(pool.clone()),
            ),
            crm::outbound::lookup::PgCrmParentReader::new(pool.clone()),
            call::outbound::pg_call_repo::PgCallRepo::new(pool.clone()),
        ),
        lexical.clone(),
    );
    let images = VisionImageCaptioner::new(
        static_file::outbound::CdnStaticFileRepo::new(StaticFileServiceUrl::new()?.to_string()),
        recorder.clone(),
    );
    let classifier = config
        .typesafe_api_key
        .value()
        .map(SystemOneJev::typesafe)
        .transpose()
        .context("invalid TYPESAFE_API_KEY")?
        .map(|provider| Arc::new(JevClassifier::new(provider, recorder)));
    let trigger = AgentTriggerService::new(
        PgAgentSessionRepo::new(
            pool.clone(),
            OwnedEntityRegistrar::new(OwnerGrantPolicy::new(PgBotsRepo::new(pool.clone()))),
        ),
        BotRepoAgentLookup::new(PgBotsRepo::new(pool.clone())),
        BotRepoAgentLookup::new(PgBotsRepo::new(pool.clone())),
        BotRepoAgentLookup::new(PgBotsRepo::new(pool.clone())),
        LexicalExplicitReplyExtractor::new(lexical),
        classifier.map(|classifier| JevTriggerJudge::new(classifier, images)),
        MessageThreadHistory::new(
            message_reader,
            entity_access::domain::service::EntityAccessServiceImpl::new(
                entity_access::outbound::PgAccessRepository::new(pool.clone()),
            ),
        ),
    )
    .with_admission(admission);
    let channel_types = ChannelRepoTypeLookup::new(PgChannelsRepo::new(pool.clone()));
    let publisher = MacroEventBrokerService::new(
        KafkaEventPublisher::new(config.kafka_brokers.as_ref())?,
        macro_event_broker::GlobalSpawner,
    );
    let project_assignments = ProjectAssignmentService::new(
        PropertiesServiceImpl::new(
            PropertiesPgRepo::new(pool.clone()),
            Some(PermissionServiceImpl::new(
                pool.clone(),
                Arc::new(EntityAccessServiceImpl::new(PgAccessRepository::new(
                    pool.clone(),
                ))),
            )),
            None::<SilentAssignmentNotifications>,
        )
        .with_event_broker(publisher.clone()),
        SystemPropertiesServiceImpl::new(PgSystemPropertiesRepository::new(pool.clone())),
        EntityAccessServiceImpl::new(PgAccessRepository::new(pool.clone())),
    );
    let discussion_delivery = messages::domain::delivery::DiscussionDelivery::new(
        messages::outbound::pg_discussion_context::PgDiscussionContext(pool.clone()),
        messages::outbound::entity_access_audience::EntityAccessMessageAudience(
            entity_access::domain::service::EntityAccessServiceImpl::new(
                entity_access::outbound::PgAccessRepository::new(pool.clone()),
            ),
        ),
        messages::outbound::connection_gateway::ConnectionGatewayMessages(Arc::new(
            ConnectionGatewayClient::new(
                config.internal_api_key.clone(),
                ConnectionGatewayUrl::new()?.to_string(),
            ),
        )),
        SilentAssignmentNotifications,
    );
    let messages = messages::domain::service::MessageService::new(
        PgMessageRepository::new(pool),
        messages::domain::effects::MessageEffects::new(
            messages::outbound::broker::BrokerMessagePublisher::new(publisher.clone()),
            messages::domain::ports::NoMessageEventPublisher,
            discussion_delivery,
        ),
    );
    let consumer =
        KafkaEventConsumer::<AgentTriggerConsumerGroup>::from_env(config.kafka_brokers.as_ref())?;
    let consumer = KafkaConsumerAdapter::<AgentTriggerConsumerGroup, ()>::new(consumer);

    consume::<MessageTriggerEvents>(
        consumer,
        &trigger,
        &publisher,
        &ContextReaders {
            channel_types,
            discussions,
            people,
        },
        &messages,
        &task_context,
        &project_assignments,
    )
    .await
}

/// What the trigger reads beside the post to describe the agents it calls.
struct ContextReaders<Discussions, People> {
    channel_types: ChannelTypes,
    discussions: Discussions,
    people: People,
}

/// Read the trigger topic until shutdown, evaluating every committed post.
async fn consume<Events: TriggerEvents>(
    consumer: KafkaConsumerAdapter<AgentTriggerConsumerGroup, ()>,
    trigger: &Trigger,
    publisher: &Publisher,
    readers: &ContextReaders<impl DiscussionReader, impl PeopleDirectory>,
    messages: &dyn MessageCommands,
    task_context: &impl TaskAssignmentContext,
    project_assignments: &ProjectAssignmentService<
        impl PropertiesService,
        impl SystemPropertiesService,
        impl EntityAccessService,
    >,
) -> anyhow::Result<()> {
    let consumer = consumer
        .subscribe::<Events>()
        .map_err(|error| anyhow::anyhow!("failed to subscribe to trigger events: {error:?}"))?;
    let consumer = TriggerConsumer::<Events>::new(consumer);

    tracing::info!(
        topics = ?Events::topics(),
        group = AgentTriggerConsumerGroup::GROUP_NAME,
        "agent trigger service listening"
    );

    let mut shutdown = std::pin::pin!(shutdown_signal());
    let mut run_error = None;
    loop {
        tokio::select! {
            () = &mut shutdown => {
                tracing::info!("agent trigger service shutting down");
                break;
            }
            result = consumer.recv() => {
                let message = match result {
                    Ok(message) => message,
                    Err(error) => {
                        tracing::error!(error = ?error, "failed to receive trigger event");
                        continue;
                    }
                };
                let span = consumer_span(message.inner(), AgentTriggerConsumerGroup::GROUP_NAME);
                let result = async {
                    let kafka_message = message.inner();
                    let decoded = match message.decode_payload() {
                        Ok(decoded) => decoded.into_trigger(),
                        Err(error) => {
                            record_span_error(&tracing::Span::current(), &error);
                            tracing::error!(
                                error = ?error,
                                partition = kafka_message.partition(),
                                offset = kafka_message.offset(),
                                "dropping undecodable trigger event"
                            );
                            commit_message(&consumer, kafka_message)?;
                            return Ok::<(), anyhow::Error>(());
                        }
                    };
                    tracing::Span::current().record("macro.event.id", tracing::field::display(decoded.event_id));
                    tracing::Span::current().record("macro.event.type", decoded.event_type);

                    if let Some(posted) = &decoded.posted {
                        process_message_event(trigger, publisher, &readers.channel_types, &readers.discussions, posted).await?;
                    }
                    if let Some(assignment) = &decoded.assignment {
                        process_task_assignment(trigger, publisher, messages, task_context, &readers.people, assignment).await?;
                    }
                    if let Some(added) = &decoded.project_task {
                        project_assignments.process(added).await?;
                    }
                    commit_message(&consumer, kafka_message)?;
                    Ok(())
                }
                .instrument(span.clone())
                .await;
                if let Err(error) = &result {
                    record_span_error(&span, error);
                }
                if let Err(error) = result {
                    run_error = Some(error);
                    break;
                }
            }
        }
    }

    match run_error {
        Some(error) => Err(error),
        None => Ok(()),
    }
}