:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::{future::ready, sync::Arc, time::Instant};
use axum::{
extract::{Json, State},
http::StatusCode,
routing::get,
Router,
};
use chrono::{DateTime, Utc};
use common_kafka::kafka_consumer::RecvErr;
use common_metrics::{serve, setup_metrics_routes};
use common_types::embedding::{EmbeddingRecord, EmbeddingRequest};
use embedding_worker::{
ad_hoc::{handle_ad_hoc_request, AdHocEmbeddingRequest, AdHocEmbeddingResponse},
app_context::AppContext,
config::Config,
handle_batch,
metrics_utils::{
DROPPED_REQUESTS, RECENTLY_SEEN_DOCUMENTS, RECENTLY_SEEN_OPERATIONS,
RECENTLY_SEEN_OPERATION_TIME,
},
recently_seen::{dedup_seen, DocumentKey, SeenRecord},
};
use metrics::{counter, histogram};
use serde::{Deserialize, Serialize};
use tokio::task::JoinHandle;
use tracing::level_filters::LevelFilter;
use tracing::{error, info};
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer};
use uuid::Uuid;
common_alloc::used!();
fn setup_tracing() {
let log_layer = tracing_subscriber::fmt::layer().with_filter(
EnvFilter::builder()
.with_default_directive(LevelFilter::INFO.into())
.from_env_lossy()
.add_directive("pyroscope=warn".parse().unwrap()),
);
tracing_subscriber::registry().with(log_layer).init();
}
pub async fn index() -> &'static str {
"error tracking embedding service"
}
async fn ad_hoc_handler(
State(context): State<Arc<AppContext>>,
Json(request): Json<AdHocEmbeddingRequest>,
) -> Result<Json<AdHocEmbeddingResponse>, StatusCode> {
handle_ad_hoc_request(context, request).await
}
#[derive(Deserialize)]
struct RecentlySeenRequest {
team_id: i32,
documents: Vec<DocumentKey>,
}
#[derive(Serialize)]
struct RecentlySeenResult {
#[serde(flatten)]
document: DocumentKey,
emitted_at: Option<DateTime<Utc>>,
}
async fn recently_seen_handler(
State(context): State<Arc<AppContext>>,
Json(request): Json<RecentlySeenRequest>,
) -> Json<Vec<RecentlySeenResult>> {
let started_at = Instant::now();
let lookup = context
.recently_seen
.lookup(request.team_id, request.documents)
.await;
histogram!(RECENTLY_SEEN_OPERATION_TIME, "operation" => "read")
.record(started_at.elapsed().as_secs_f64());
counter!(RECENTLY_SEEN_OPERATIONS, "operation" => "read").increment(1);
let hit_count = lookup
.values()
.filter(|emitted_at| emitted_at.is_some())
.count();
counter!(RECENTLY_SEEN_DOCUMENTS, "operation" => "read", "result" => "hit")
.increment(hit_count as u64);
counter!(RECENTLY_SEEN_DOCUMENTS, "operation" => "read", "result" => "miss")
.increment((lookup.len() - hit_count) as u64);
Json(
lookup
.into_iter()
.map(|(document, emitted_at)| RecentlySeenResult {
document,
emitted_at,
})
.collect(),
)
}
fn start_health_liveness_server(config: &Config, context: Arc<AppContext>) -> JoinHandle<()> {
let config = config.clone();
let liveness_context = context.clone();
let router = Router::new()
.route("/", get(index))
.route("/_readiness", get(index))
.route(
"/_liveness",
get(move || ready(liveness_context.health_registry.get_status())),
)
.route("/generate/ad_hoc", axum::routing::post(ad_hoc_handler))
.route("/recently_seen", axum::routing::post(recently_seen_handler))
.with_state(context);
let router = setup_metrics_routes(router);
let bind = format!("{}:{}", config.host, config.port);
tokio::task::spawn(async move {
serve(router, &bind)
.await
.expect("failed to start serving metrics");
})
}
#[tokio::main]
async fn main() {
setup_tracing();
info!("Starting up...");
let config = Config::init_with_defaults().unwrap();
// Start continuous profiling if enabled (keep _agent alive for the duration of the program)
let _profiling_agent = match config.continuous_profiling.start_agent() {
Ok(agent) => agent,
Err(e) => {
error!("Failed to start continuous profiling agent: {e:?}");
None
}
};
common_posthog::init(
"embedding-worker",
config.posthog_api_key.as_deref(),
&config.posthog_endpoint,
)
.await
.unwrap();
let context = Arc::new(AppContext::new(config.clone()).await.unwrap());
start_health_liveness_server(&config, context.clone());
let batch_wait_time = std::time::Duration::from_secs(config.max_event_batch_wait_seconds);
let batch_size = config.max_events_per_batch;
loop {
context.worker_liveness.report_healthy().await;
// Just grab the event as a serde_json::Value and immediately drop it,
// we can work out a real type for it later (once we're deployed etc)
let received: Vec<Result<(EmbeddingRequest, _), _>> = context
.kafka_consumer
.json_recv_batch(batch_size, batch_wait_time)
.await;
let mut transactional_producer = context.transactional_producer.lock().await;
let mut to_process = Vec::with_capacity(received.len());
let mut offsets = Vec::with_capacity(received.len());
for message in received {
match message {
Ok((event, offset)) => {
to_process.push(event);
offsets.push(offset);
}
Err(RecvErr::Kafka(e)) => {
panic!("Kafka error: {e}")
}
Err(err) => {
// If we failed to parse the message, or it was empty, just log and continue, our
// consumer has already stored the offset for us.
error!("Error receiving message: {:?}", err);
counter!(DROPPED_REQUESTS, &[("cause", "recv_err")]).increment(1);
continue;
}
};
}
let responses = match handle_batch(to_process, &offsets, context.clone()).await {
Ok(embeddings) => embeddings,
Err(failure) => {
error!("Error handling batch: {failure:?}");
panic!("Unhandled error: {failure:?}");
}
};
let txn = match transactional_producer.begin() {
Ok(txn) => txn,
Err(e) => {
error!("Failed to start kafka transaction, {:?}", e);
panic!("Failed to start kafka transaction: {e:?}");
}
};
// Write the callback messages
let emit_results = txn
.send_keyed_iter_to_kafka(
&context.config.response_topic,
|_| Some(Uuid::now_v7().to_string()),
&responses,
)
.await;
for res in emit_results.into_iter() {
res.expect("We can emit to kafka");
}
// Write the embedding records to CH
let records: Vec<EmbeddingRecord> = responses
.into_iter()
.flat_map(Vec::<EmbeddingRecord>::from)
.collect();
// Capture immediately before Kafka assigns ClickHouse inserted_at.
let emitted_at = Utc::now();
let emit_results = txn
.send_keyed_iter_to_kafka(
&context.config.output_topic,
|_| Some(Uuid::now_v7().to_string()),
&records,
)
.await;
for res in emit_results.into_iter() {
res.expect("We can emit to kafka");
}
let metadata = context.kafka_consumer.metadata();
// Associate the embedding request records with the offsets
match txn.associate_offsets(offsets, &metadata) {
Ok(_) => {}
Err(e) => {
error!(
"Failed to associate offsets with kafka transaction, {:?}",
e
);
panic!("Failed to associate offsets with kafka transaction, {e:?}");
}
}
// Commit the transaction
match txn.commit() {
Ok(_) => {}
Err(e) => {
error!("Failed to commit kafka transaction, {:?}", e);
panic!("Failed to commit kafka transaction, {e:?}");
}
}
// Store only after commit so a hit never advertises uncommitted output.
let seen = dedup_seen(records.iter().map(|record| {
let key = DocumentKey {
product: record.product.clone(),
document_type: record.document_type.clone(),
rendering: record.rendering.clone(),
document_id: record.document_id.clone(),
};
SeenRecord {
team_id: record.team_id,
key,
emitted_at,
}
}));
if !seen.is_empty() {
let started_at = Instant::now();
context.recently_seen.record(&seen).await;
histogram!(RECENTLY_SEEN_OPERATION_TIME, "operation" => "write")
.record(started_at.elapsed().as_secs_f64());
counter!(RECENTLY_SEEN_OPERATIONS, "operation" => "write").increment(1);
counter!(RECENTLY_SEEN_DOCUMENTS, "operation" => "write", "result" => "attempted")
.increment(seen.len() as u64);
}
}
}