:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::time::Duration;
use std::{borrow::Cow, sync::Arc};
use anyhow::Result;
use common_kafka::kafka_consumer::Offset;
use common_types::embedding::{
EmbeddingModel, EmbeddingRequest, EmbeddingResponse, EmbeddingResult, ModelResult,
};
use metrics::counter;
use rand::Rng;
use reqwest::{Client, Method, Request, RequestBuilder};
use tracing::{error, warn};
use crate::{
app_context::AppContext,
metrics_utils::{
RequestLabels, DROPPED_REQUESTS, EMBEDDINGS_GENERATED, EMBEDDING_FAILED,
EMBEDDING_REQUEST_TIME, EMBEDDING_TOTAL_TIME, EMBEDDING_TOTAL_TOKENS, MESSAGES_RECEIVED,
MESSAGE_TRUNCATED, REQUESTS_SENT, RESPONSES_RECEIVED,
},
organization::apply_ai_opt_in,
};
static CL100K_ENCODER: std::sync::LazyLock<tiktoken_rs::CoreBPE> = std::sync::LazyLock::new(|| {
tiktoken_rs::cl100k_base().expect("Failed to initialize cl100k_base encoder")
});
const MAX_RETRY_ATTEMPTS: usize = 4; // 1 initial + 3 retries
const RETRY_BASE_SECS: u64 = 2;
const RETRY_JITTER_RANGE: std::ops::RangeInclusive<i64> = -1000..=1000;
pub mod ad_hoc;
pub mod app_context;
pub mod config;
pub mod metrics_utils;
pub mod organization;
pub mod recently_seen;
pub async fn handle_batch(
requests: Vec<EmbeddingRequest>,
_offsets: &[Offset], // TODO - tie errors to offsets
context: Arc<AppContext>,
) -> Result<Vec<EmbeddingResponse>> {
let mut handles = vec![];
for request in requests.into_iter() {
let team_id = request.team_id;
let labels = RequestLabels::from(&request);
let Some(request) = apply_ai_opt_in(&context, request, team_id).await? else {
counter!(
DROPPED_REQUESTS,
labels.and([("cause", "ai_opt_in")]).render()
)
.increment(1);
continue;
};
let ctx = context.clone();
handles.push(async move {
let mut results = vec![];
for model in &request.models {
let (model, embedding) =
handle_single(ctx.clone(), *model, request.clone()).await?;
results.push(ModelResult {
model,
outcome: EmbeddingResult::Success { embedding },
});
}
Ok::<_, anyhow::Error>(EmbeddingResponse { request, results })
});
}
let results: Result<Vec<_>> = futures::future::join_all(handles)
.await
.into_iter()
.collect();
results
}
pub async fn handle_single(
context: Arc<AppContext>,
model: EmbeddingModel,
request: EmbeddingRequest,
) -> Result<(EmbeddingModel, Vec<f64>)> {
let labels = RequestLabels::from(&request)
.and_model(model)
.and([("from", "kafka")]);
counter!(MESSAGES_RECEIVED, labels.render()).increment(1);
let (embedding, _) = match generate_embedding(context, model, &request.content, &labels).await {
Ok(r) => r,
Err(e) => {
counter!(EMBEDDING_FAILED, labels.render()).increment(1);
error!("Failed to handle request: {request:?}");
return Err(e);
}
};
counter!(EMBEDDINGS_GENERATED, labels.render()).increment(1);
Ok((model, embedding))
}
// Exponential backoff (base 2) with jitter, in milliseconds, for retry `attempt`.
fn retry_backoff_ms(attempt: usize) -> u64 {
let base_ms = RETRY_BASE_SECS.pow(attempt as u32 + 1) * 1000;
let jitter_ms = rand::thread_rng().gen_range(RETRY_JITTER_RANGE);
(base_ms as i64 + jitter_ms).max(0) as u64
}
pub async fn generate_embedding(
context: Arc<AppContext>,
model: EmbeddingModel,
content: &str,
labels: &RequestLabels,
) -> Result<(Vec<f64>, usize)> {
let total_time = common_metrics::timing_guard(EMBEDDING_TOTAL_TIME, labels.render());
// Generate the text to actually send to the embedding provider
let (text, token_count) = generate_embedding_text(content, &model, labels)?;
context.respect_rate_limits(model, token_count).await;
let request_time = common_metrics::timing_guard(EMBEDDING_REQUEST_TIME, labels.render());
let mut last_status = None;
let mut last_error_body = None;
let mut last_transport_error = None;
for attempt in 0..MAX_RETRY_ATTEMPTS {
let api_req = construct_request(
&text,
model,
&context.config.openai_api_key,
context.client.clone(),
);
counter!(REQUESTS_SENT, labels.render()).increment(1);
let response = match context.client.execute(api_req).await {
Ok(response) => response,
Err(e) => {
// Transport errors (timeouts, connection resets, etc.) are transient.
// Retry them with backoff like a 5xx rather than aborting the whole
// batch, which would panic and restart the worker.
if attempt < MAX_RETRY_ATTEMPTS - 1 {
let sleep_ms = retry_backoff_ms(attempt);
warn!(
"Request to embedding provider failed ({}), retrying in {}ms (attempt {}/{})",
e,
sleep_ms,
attempt + 1,
MAX_RETRY_ATTEMPTS - 1
);
tokio::time::sleep(Duration::from_millis(sleep_ms)).await;
}
last_status = None;
last_transport_error = Some(e);
continue;
}
};
let status = response.status();
let response_labels = labels
.clone()
.and([("status_code", status.as_u16().to_string())]);
counter!(RESPONSES_RECEIVED, response_labels.render()).increment(1);
if status.is_success() {
context.update_rate_limits(model, &response).await;
let embedding = model
.extract_embedding_from_response_body(&response.json().await?)
.ok_or_else(|| anyhow::anyhow!("Failed to extract embedding"))?;
request_time.label("outcome", "success").fin();
total_time.label("outcome", "success").fin();
counter!(EMBEDDING_TOTAL_TOKENS, labels.render()).increment(token_count as u64);
return Ok((embedding, token_count));
}
last_status = Some(status);
last_error_body = response.text().await.ok();
// Only retry on 5xx - for stuff like 429's, we want to crash and restart as a backoff
if !status.is_server_error() {
break;
}
if attempt < MAX_RETRY_ATTEMPTS - 1 {
let sleep_ms = retry_backoff_ms(attempt);
warn!(
"Got {} from embedding provider, retrying in {}ms (attempt {}/{})",
status,
sleep_ms,
attempt + 1,
MAX_RETRY_ATTEMPTS - 1
);
tokio::time::sleep(Duration::from_millis(sleep_ms)).await;
}
}
// All attempts exhausted or non-retryable error
match last_status {
Some(status) => {
error!(
"Failed to generate embeddings, got {} from {}",
status,
model.provider()
);
if let Some(error_message) = last_error_body {
error!("Error message from {}: {}", model.provider(), error_message);
}
}
None => {
error!(
"Failed to generate embeddings, no response from {}: {}",
model.provider(),
last_transport_error
.map(|e| e.to_string())
.unwrap_or_else(|| "unknown error".to_string())
);
}
}
Err(anyhow::anyhow!("Failed to generate embeddings"))
}
// This is here, rather than on the embedding model, to avoid taking a dep on tiktoken in common/types. We
// can reconsider it later if we want
pub fn generate_embedding_text<'a>(
content: &'a str,
model: &EmbeddingModel,
labels: &RequestLabels,
) -> Result<(Cow<'a, str>, usize)> {
let content = model.escape_input(content);
let (text, count) = match model {
EmbeddingModel::OpenAITextEmbeddingSmall
| EmbeddingModel::OpenAITextEmbeddingLarge
| EmbeddingModel::OpenAITextEmbeddingSmall256 => {
let encoder = &*CL100K_ENCODER;
let mut tokens: Vec<_> = encoder
.encode_with_special_tokens(&content)
.into_iter()
.take(model.model_input_window())
.collect();
if tokens.len() < model.model_input_window() {
return Ok((content, tokens.len()));
}
// Truncation can split a multi-byte character's token sequence,
// producing bytes that aren't valid UTF-8 on decode. Drop trailing
// tokens until we land on a clean boundary.
let text = loop {
match encoder.decode(tokens.clone()) {
Ok(text) => break text,
Err(_) => {
tokens.pop();
}
}
};
let token_count = tokens.len();
(text, token_count)
}
};
if text.len() < content.len() {
counter!(MESSAGE_TRUNCATED, labels.render()).increment(1);
}
Ok((Cow::Owned(text), count))
}
pub fn construct_request(
content: &str,
model: EmbeddingModel,
api_key: &str,
client: Client,
) -> Request {
let req = Request::new(
Method::POST,
model
.model_url()
.parse()
.expect("The models enum only produces valid urls"),
);
let req = RequestBuilder::from_parts(client, req)
.header("Authorization", format!("Bearer {api_key}"))
.header("Content-Type", "application/json")
.json(&model.construct_request_body(content));
// This expect is fine, because we have total control over everything in the
// request, except the string of input content, which will serialize correctly.
req.build().expect("we manage to build the request")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_generate_embedding_text_truncation_at_multibyte_boundary() {
// Emojis like 🔥 encode to 3 tokens in cl100k_base. When content exceeds
// the 8192 token window, truncation can split an emoji's token sequence,
// producing bytes that aren't valid UTF-8 on decode.
let padding = "word ".repeat(8180);
let content = format!("{padding}{}", "🔥".repeat(100));
let model = EmbeddingModel::default();
let (text, _count) =
generate_embedding_text(&content, &model, &RequestLabels::default()).unwrap();
assert!(content.starts_with(&*text));
}
}