Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

7.3 KB
Raw Blame
//! Kafka consumer example for the `macro.documents` topic.
//!
//! Connects with the same `KAFKA_BROKERS` env var and environment-driven
//! transport as [`macro_event_broker::KafkaEventPublisher`] (plaintext for
//! `ENVIRONMENT=local`, TLS + SASL/OAUTHBEARER with MSK IAM otherwise),
//! subscribes to the [`DocumentMacroEvent`] topic and prints every document
//! lifecycle event it receives.
//!
//! The consumer follows the standard poll-loop / worker split: a Kafka poll
//! loop decodes messages and hands them to a processing task over a bounded
//! tokio mpsc channel (backpressure applies when the processor falls behind),
//! offsets are committed only after a successful handoff (at-least-once), and
//! ctrl-c drains the channel before exiting.
//!
//! Run against the local docker broker:
//! ```sh
//! ENVIRONMENT=local KAFKA_BROKERS=localhost:9092 \
//!     cargo run -p documents --example documents_consumer
//! ```
//! or against the dev MSK cluster (needs AWS credentials and network access):
//! ```sh
//! ENVIRONMENT=develop AWS_REGION=us-east-1 KAFKA_BROKERS=<bootstrapBrokersSaslIam> \
//!     cargo run -p documents --example documents_consumer
//! ```

use anyhow::Context as _;
use documents::domain::events::{DocumentMacroEvent, DocumentTopicEvent};
use kafka_util::{GroupName, KafkaEventConsumer};
use macro_event_broker::{
    KafkaConsumerAdapter, MacroEvent as _, MacroEventCollection as _, MacroEventConsumerService,
};
use rdkafka::consumer::CommitMode;
use rdkafka::message::Message;
use tokio::sync::mpsc;

macro_env_var::env_var! {
    struct ConsumerEnvVars {
        KafkaBrokers,
    }
}

/// Consumer group for this example. Offsets are committed under this group, so
/// re-runs resume where the previous run left off.
struct DocumentsConsumerGroup;

impl GroupName for DocumentsConsumerGroup {
    const GROUP_NAME: &'static str = "documents-consumer-example";
}

/// Bounded capacity of the channel between the poll loop and the processor.
const CHANNEL_CAPACITY: usize = 128;

macro_event_broker::declare_topics!(DeclaredMacroEvent: DocumentMacroEvent);

type DocumentsKafkaAdapter = KafkaConsumerAdapter<DocumentsConsumerGroup, DeclaredMacroEvent>;
type DocumentsConsumerService =
    MacroEventConsumerService<DeclaredMacroEvent, DocumentsKafkaAdapter>;

/// A decoded event plus the Kafka coordinates it came from, as handed from the
/// poll loop to the processor over the channel.
struct ReceivedEvent {
    partition: i32,
    offset: i64,
    event: DocumentMacroEvent,
}

fn event_type(event: &DocumentTopicEvent) -> &'static str {
    match event {
        DocumentTopicEvent::Created(_) => "document.created",
        DocumentTopicEvent::Updated(_) => "document.updated",
        DocumentTopicEvent::Deleted(_) => "document.deleted",
        DocumentTopicEvent::ContentUploaded(_) => "document.content_uploaded",
        DocumentTopicEvent::SyncContentUpdated(_) => "document.sync_content_updated",
        DocumentTopicEvent::Purged(_) => "document.purged",
        DocumentTopicEvent::Copied(_) => "document.copied",
        DocumentTopicEvent::Interaction(_) => "document.interaction",
    }
}

/// Print each received event. This stands in for real event handling — swap
/// the body for indexing, notifications, etc.
async fn process_events(mut events: mpsc::Receiver<ReceivedEvent>) {
    while let Some(received) = events.recv().await {
        let event = received.event;
        let envelope = event.event();

        println!(
            "[partition {} offset {}] {} key={} event_id={} schema_version={}",
            received.partition,
            received.offset,
            event_type(&envelope.event),
            event.key(),
            envelope.event_id,
            envelope.schema_version,
        );
        match serde_json::to_string_pretty(envelope) {
            Ok(json) => println!("{json}"),
            Err(e) => eprintln!("failed to re-serialize event: {e}"),
        }
    }
}

#[tokio::main]
async fn main() -> anyhow::Result<()> {
    // Surface tracing from macro_event_broker (e.g. MSK IAM token refreshes)
    // and librdkafka's internal connection/auth logs (bridged from the `log`
    // crate). Tune with RUST_LOG, e.g. RUST_LOG=debug for librdkafka detail.
    tracing_subscriber::fmt()
        .with_env_filter(
            tracing_subscriber::EnvFilter::try_from_default_env()
                .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
        )
        .init();

    let env = ConsumerEnvVars::new().context("KAFKA_BROKERS must be set")?;

    let consumer =
        KafkaEventConsumer::<DocumentsConsumerGroup>::from_env(env.kafka_brokers.as_ref())?;
    let consumer = KafkaConsumerAdapter::<DocumentsConsumerGroup, ()>::new(consumer)
        .subscribe::<DeclaredMacroEvent>()
        .map_err(|error| anyhow::anyhow!("failed to subscribe to documents topic: {error:?}"))?;
    let consumer = DocumentsConsumerService::new(consumer);
    println!(
        "listening on topic={} group={} brokers={} (ctrl-c to stop)",
        DeclaredMacroEvent::topics()[0],
        DocumentsConsumerGroup::GROUP_NAME,
        env.kafka_brokers.as_ref(),
    );

    let (events_tx, events_rx) = mpsc::channel::<ReceivedEvent>(CHANNEL_CAPACITY);
    let processor = tokio::spawn(process_events(events_rx));

    let mut shutdown = std::pin::pin!(tokio::signal::ctrl_c());
    loop {
        tokio::select! {
            _ = &mut shutdown => {
                println!("received ctrl-c, shutting down");
                break;
            }
            result = consumer.recv() => {
                let message = match result {
                    Ok(message) => message,
                    Err(e) => {
                        eprintln!("kafka receive error: {e}");
                        continue;
                    }
                };

                let kafka_message = message.inner();
                match message.decode_payload() {
                    Ok(DeclaredMacroEvent::DocumentMacroEvent(event)) => {
                        events_tx
                            .send(ReceivedEvent {
                                partition: kafka_message.partition(),
                                offset: kafka_message.offset(),
                                event,
                            })
                            .await
                            .context("processor task closed unexpectedly")?;
                    }
                    // Undecodable messages are logged and skipped rather than
                    // wedging the partition on a poison message.
                    Err(e) => eprintln!(
                        "failed to decode message at partition {} offset {}: {e}",
                        kafka_message.partition(),
                        kafka_message.offset(),
                    ),
                }

                // Commit only after the event was handed to the processor:
                // at-least-once delivery (up to the channel buffer on a crash).
                let _ = consumer
                    .inner()
                    .commit_message(kafka_message, CommitMode::Async)
                    .inspect_err(|e| {
                        eprintln!("failed to commit offset {}: {e:?}", kafka_message.offset());
                    });
            }
        }
    }

    // Close the channel and let the processor drain what it already received.
    drop(events_tx);
    processor.await.context("processor task panicked")?;

    Ok(())
}