Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

9.3 KB
Raw Blame
use std::{future::ready, sync::Arc, time::Instant};

use axum::{
    extract::{Json, State},
    http::StatusCode,
    routing::get,
    Router,
};
use chrono::{DateTime, Utc};
use common_kafka::kafka_consumer::RecvErr;
use common_metrics::{serve, setup_metrics_routes};
use common_types::embedding::{EmbeddingRecord, EmbeddingRequest};
use embedding_worker::{
    ad_hoc::{handle_ad_hoc_request, AdHocEmbeddingRequest, AdHocEmbeddingResponse},
    app_context::AppContext,
    config::Config,
    handle_batch,
    metrics_utils::{
        DROPPED_REQUESTS, RECENTLY_SEEN_DOCUMENTS, RECENTLY_SEEN_OPERATIONS,
        RECENTLY_SEEN_OPERATION_TIME,
    },
    recently_seen::{dedup_seen, DocumentKey, SeenRecord},
};

use metrics::{counter, histogram};
use serde::{Deserialize, Serialize};
use tokio::task::JoinHandle;
use tracing::level_filters::LevelFilter;
use tracing::{error, info};
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer};
use uuid::Uuid;

common_alloc::used!();

fn setup_tracing() {
    let log_layer = tracing_subscriber::fmt::layer().with_filter(
        EnvFilter::builder()
            .with_default_directive(LevelFilter::INFO.into())
            .from_env_lossy()
            .add_directive("pyroscope=warn".parse().unwrap()),
    );
    tracing_subscriber::registry().with(log_layer).init();
}

pub async fn index() -> &'static str {
    "error tracking embedding service"
}

async fn ad_hoc_handler(
    State(context): State<Arc<AppContext>>,
    Json(request): Json<AdHocEmbeddingRequest>,
) -> Result<Json<AdHocEmbeddingResponse>, StatusCode> {
    handle_ad_hoc_request(context, request).await
}

#[derive(Deserialize)]
struct RecentlySeenRequest {
    team_id: i32,
    documents: Vec<DocumentKey>,
}

#[derive(Serialize)]
struct RecentlySeenResult {
    #[serde(flatten)]
    document: DocumentKey,
    emitted_at: Option<DateTime<Utc>>,
}

async fn recently_seen_handler(
    State(context): State<Arc<AppContext>>,
    Json(request): Json<RecentlySeenRequest>,
) -> Json<Vec<RecentlySeenResult>> {
    let started_at = Instant::now();
    let lookup = context
        .recently_seen
        .lookup(request.team_id, request.documents)
        .await;
    histogram!(RECENTLY_SEEN_OPERATION_TIME, "operation" => "read")
        .record(started_at.elapsed().as_secs_f64());
    counter!(RECENTLY_SEEN_OPERATIONS, "operation" => "read").increment(1);
    let hit_count = lookup
        .values()
        .filter(|emitted_at| emitted_at.is_some())
        .count();
    counter!(RECENTLY_SEEN_DOCUMENTS, "operation" => "read", "result" => "hit")
        .increment(hit_count as u64);
    counter!(RECENTLY_SEEN_DOCUMENTS, "operation" => "read", "result" => "miss")
        .increment((lookup.len() - hit_count) as u64);

    Json(
        lookup
            .into_iter()
            .map(|(document, emitted_at)| RecentlySeenResult {
                document,
                emitted_at,
            })
            .collect(),
    )
}

fn start_health_liveness_server(config: &Config, context: Arc<AppContext>) -> JoinHandle<()> {
    let config = config.clone();
    let liveness_context = context.clone();
    let router = Router::new()
        .route("/", get(index))
        .route("/_readiness", get(index))
        .route(
            "/_liveness",
            get(move || ready(liveness_context.health_registry.get_status())),
        )
        .route("/generate/ad_hoc", axum::routing::post(ad_hoc_handler))
        .route("/recently_seen", axum::routing::post(recently_seen_handler))
        .with_state(context);
    let router = setup_metrics_routes(router);
    let bind = format!("{}:{}", config.host, config.port);
    tokio::task::spawn(async move {
        serve(router, &bind)
            .await
            .expect("failed to start serving metrics");
    })
}

#[tokio::main]
async fn main() {
    setup_tracing();
    info!("Starting up...");

    let config = Config::init_with_defaults().unwrap();

    // Start continuous profiling if enabled (keep _agent alive for the duration of the program)
    let _profiling_agent = match config.continuous_profiling.start_agent() {
        Ok(agent) => agent,
        Err(e) => {
            error!("Failed to start continuous profiling agent: {e:?}");
            None
        }
    };

    common_posthog::init(
        "embedding-worker",
        config.posthog_api_key.as_deref(),
        &config.posthog_endpoint,
    )
    .await
    .unwrap();

    let context = Arc::new(AppContext::new(config.clone()).await.unwrap());

    start_health_liveness_server(&config, context.clone());

    let batch_wait_time = std::time::Duration::from_secs(config.max_event_batch_wait_seconds);
    let batch_size = config.max_events_per_batch;

    loop {
        context.worker_liveness.report_healthy().await;
        // Just grab the event as a serde_json::Value and immediately drop it,
        // we can work out a real type for it later (once we're deployed etc)
        let received: Vec<Result<(EmbeddingRequest, _), _>> = context
            .kafka_consumer
            .json_recv_batch(batch_size, batch_wait_time)
            .await;

        let mut transactional_producer = context.transactional_producer.lock().await;

        let mut to_process = Vec::with_capacity(received.len());
        let mut offsets = Vec::with_capacity(received.len());

        for message in received {
            match message {
                Ok((event, offset)) => {
                    to_process.push(event);
                    offsets.push(offset);
                }
                Err(RecvErr::Kafka(e)) => {
                    panic!("Kafka error: {e}")
                }
                Err(err) => {
                    // If we failed to parse the message, or it was empty, just log and continue, our
                    // consumer has already stored the offset for us.
                    error!("Error receiving message: {:?}", err);
                    counter!(DROPPED_REQUESTS, &[("cause", "recv_err")]).increment(1);
                    continue;
                }
            };
        }

        let responses = match handle_batch(to_process, &offsets, context.clone()).await {
            Ok(embeddings) => embeddings,
            Err(failure) => {
                error!("Error handling batch: {failure:?}");
                panic!("Unhandled error: {failure:?}");
            }
        };

        let txn = match transactional_producer.begin() {
            Ok(txn) => txn,
            Err(e) => {
                error!("Failed to start kafka transaction, {:?}", e);
                panic!("Failed to start kafka transaction: {e:?}");
            }
        };

        // Write the callback messages
        let emit_results = txn
            .send_keyed_iter_to_kafka(
                &context.config.response_topic,
                |_| Some(Uuid::now_v7().to_string()),
                &responses,
            )
            .await;

        for res in emit_results.into_iter() {
            res.expect("We can emit to kafka");
        }

        // Write the embedding records to CH
        let records: Vec<EmbeddingRecord> = responses
            .into_iter()
            .flat_map(Vec::<EmbeddingRecord>::from)
            .collect();

        // Capture immediately before Kafka assigns ClickHouse inserted_at.
        let emitted_at = Utc::now();
        let emit_results = txn
            .send_keyed_iter_to_kafka(
                &context.config.output_topic,
                |_| Some(Uuid::now_v7().to_string()),
                &records,
            )
            .await;

        for res in emit_results.into_iter() {
            res.expect("We can emit to kafka");
        }

        let metadata = context.kafka_consumer.metadata();

        // Associate the embedding request records with the offsets
        match txn.associate_offsets(offsets, &metadata) {
            Ok(_) => {}
            Err(e) => {
                error!(
                    "Failed to associate offsets with kafka transaction, {:?}",
                    e
                );
                panic!("Failed to associate offsets with kafka transaction, {e:?}");
            }
        }

        // Commit the transaction
        match txn.commit() {
            Ok(_) => {}
            Err(e) => {
                error!("Failed to commit kafka transaction, {:?}", e);
                panic!("Failed to commit kafka transaction, {e:?}");
            }
        }

        // Store only after commit so a hit never advertises uncommitted output.
        let seen = dedup_seen(records.iter().map(|record| {
            let key = DocumentKey {
                product: record.product.clone(),
                document_type: record.document_type.clone(),
                rendering: record.rendering.clone(),
                document_id: record.document_id.clone(),
            };
            SeenRecord {
                team_id: record.team_id,
                key,
                emitted_at,
            }
        }));
        if !seen.is_empty() {
            let started_at = Instant::now();
            context.recently_seen.record(&seen).await;
            histogram!(RECENTLY_SEEN_OPERATION_TIME, "operation" => "write")
                .record(started_at.elapsed().as_secs_f64());
            counter!(RECENTLY_SEEN_OPERATIONS, "operation" => "write").increment(1);
            counter!(RECENTLY_SEEN_DOCUMENTS, "operation" => "write", "result" => "attempted")
                .increment(seen.len() as u64);
        }
    }
}