:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(embedding-worker): serve text-embedding-3-small at 256 dimensions (#121286)
Daniel Viscaea1b57fc7983
diff --git a/rust/common/types/src/embeddings.rs b/rust/common/types/src/embeddings.rsindex 1c8b36eb65ee..cd9d55bf8d61 100644--- a/rust/common/types/src/embeddings.rs+++ b/rust/common/types/src/embeddings.rs@@ -70,6 +70,9 @@ pub enum EmbeddingModel {OpenAITextEmbeddingSmall,#[serde(rename = "text-embedding-3-large-3072")]OpenAITextEmbeddingLarge,+ // Served at 256 dimensions through the OpenAI `dimensions` parameter, for high-volume documents.+ #[serde(rename = "text-embedding-3-small-256")]+ OpenAITextEmbeddingSmall256,}#[derive(Debug, Clone, Eq, PartialEq)]@@ -81,7 +84,8 @@ pub struct ApiLimits {impl EmbeddingModel {pub fn name(&self) -> &'static str {match self {- EmbeddingModel::OpenAITextEmbeddingSmall => "text-embedding-3-small",+ EmbeddingModel::OpenAITextEmbeddingSmall+ | EmbeddingModel::OpenAITextEmbeddingSmall256 => "text-embedding-3-small",EmbeddingModel::OpenAITextEmbeddingLarge => "text-embedding-3-large",}}@@ -90,20 +94,33 @@ impl EmbeddingModel {match self {EmbeddingModel::OpenAITextEmbeddingSmall => 1536,EmbeddingModel::OpenAITextEmbeddingLarge => 3072,+ EmbeddingModel::OpenAITextEmbeddingSmall256 => 256,+ }+ }++ pub fn requested_dimensions(&self) -> Option<usize> {+ match self {+ EmbeddingModel::OpenAITextEmbeddingSmall | EmbeddingModel::OpenAITextEmbeddingLarge => {+ None+ }+ EmbeddingModel::OpenAITextEmbeddingSmall256 => Some(256),}}pub fn model_input_window(&self) -> usize {match self {- EmbeddingModel::OpenAITextEmbeddingSmall => 8192,- EmbeddingModel::OpenAITextEmbeddingLarge => 8192,+ EmbeddingModel::OpenAITextEmbeddingSmall+ | EmbeddingModel::OpenAITextEmbeddingLarge+ | EmbeddingModel::OpenAITextEmbeddingSmall256 => 8192,}}/// Apply model-specific escaping/sanitisation to input text before tokenisation.pub fn escape_input<'a>(&self, content: &'a str) -> std::borrow::Cow<'a, str> {match self {- EmbeddingModel::OpenAITextEmbeddingSmall | EmbeddingModel::OpenAITextEmbeddingLarge => {+ EmbeddingModel::OpenAITextEmbeddingSmall+ | EmbeddingModel::OpenAITextEmbeddingLarge+ | EmbeddingModel::OpenAITextEmbeddingSmall256 => {const OPENAI_SPECIAL_TOKENS: [(&str, &str); 6] = [("<|endoftext|>", ">|endoftext|<"),("<|fim_prefix|>", ">|fim_prefix|<"),@@ -128,37 +145,38 @@ impl EmbeddingModel {pub fn model_url(&self) -> &'static str {match self {- EmbeddingModel::OpenAITextEmbeddingSmall | EmbeddingModel::OpenAITextEmbeddingLarge => {- "https://api.openai.com/v1/embeddings"- }+ EmbeddingModel::OpenAITextEmbeddingSmall+ | EmbeddingModel::OpenAITextEmbeddingLarge+ | EmbeddingModel::OpenAITextEmbeddingSmall256 => "https://api.openai.com/v1/embeddings",}}pub fn limits_key(&self) -> &'static str {// Based on our openai usage limits, these are distinct bucketsmatch self {- EmbeddingModel::OpenAITextEmbeddingSmall => "openai_text_embedding_small",+ EmbeddingModel::OpenAITextEmbeddingSmall+ | EmbeddingModel::OpenAITextEmbeddingSmall256 => "openai_text_embedding_small",EmbeddingModel::OpenAITextEmbeddingLarge => "openai_text_embedding_large",}}pub fn provider(&self) -> &'static str {match self {- EmbeddingModel::OpenAITextEmbeddingSmall | EmbeddingModel::OpenAITextEmbeddingLarge => {- "openai"- }+ EmbeddingModel::OpenAITextEmbeddingSmall+ | EmbeddingModel::OpenAITextEmbeddingLarge+ | EmbeddingModel::OpenAITextEmbeddingSmall256 => "openai",}}pub fn api_limits(&self) -> ApiLimits {// TODO - these are VERY conservative, but keep in mind each pod will max out at this.match self {- EmbeddingModel::OpenAITextEmbeddingSmall | EmbeddingModel::OpenAITextEmbeddingLarge => {- ApiLimits {- requests_per_minute: 3_000,- tokens_per_minute: 1_000_000,- }- }+ EmbeddingModel::OpenAITextEmbeddingSmall+ | EmbeddingModel::OpenAITextEmbeddingLarge+ | EmbeddingModel::OpenAITextEmbeddingSmall256 => ApiLimits {+ requests_per_minute: 3_000,+ tokens_per_minute: 1_000_000,+ },}}@@ -168,7 +186,9 @@ impl EmbeddingModel {headers: &dyn Fn(&str) -> Option<String>,) -> Option<ApiLimits> {match self {- EmbeddingModel::OpenAITextEmbeddingSmall | EmbeddingModel::OpenAITextEmbeddingLarge => {+ EmbeddingModel::OpenAITextEmbeddingSmall+ | EmbeddingModel::OpenAITextEmbeddingLarge+ | EmbeddingModel::OpenAITextEmbeddingSmall256 => {let rpm = headers("x-ratelimit-limit-requests")?.parse().ok()?;let tpm = headers("x-ratelimit-limit-tokens")?.parse().ok()?;Some(ApiLimits {@@ -181,10 +201,13 @@ impl EmbeddingModel {pub fn construct_request_body(&self, text: &str) -> Value {match self {- EmbeddingModel::OpenAITextEmbeddingSmall | EmbeddingModel::OpenAITextEmbeddingLarge => {+ EmbeddingModel::OpenAITextEmbeddingSmall+ | EmbeddingModel::OpenAITextEmbeddingLarge+ | EmbeddingModel::OpenAITextEmbeddingSmall256 => {let request = OAIEmbeddingRequest {input: text.to_string(),model: self.name().to_string(),+ dimensions: self.requested_dimensions(),};serde_json::to_value(request).expect("We are able to serialize the request")}@@ -193,7 +216,9 @@ impl EmbeddingModel {pub fn extract_embedding_from_response_body(&self, response: &Value) -> Option<Vec<f64>> {match self {- EmbeddingModel::OpenAITextEmbeddingSmall | EmbeddingModel::OpenAITextEmbeddingLarge => {+ EmbeddingModel::OpenAITextEmbeddingSmall+ | EmbeddingModel::OpenAITextEmbeddingLarge+ | EmbeddingModel::OpenAITextEmbeddingSmall256 => {let response: OAIEmbeddingResponse =serde_json::from_value(response.clone()).ok()?;response.data.into_iter().next().map(|d| d.embedding)@@ -206,6 +231,8 @@ impl EmbeddingModel {struct OAIEmbeddingRequest {input: String,model: String,+ #[serde(skip_serializing_if = "Option::is_none")]+ dimensions: Option<usize>,}#[derive(Deserialize)]@@ -255,6 +282,49 @@ impl From<EmbeddingResponse> for Vec<EmbeddingRecord> {mod tests {use super::*;+ #[test]+ fn small_256_round_trips_through_its_table_name() {+ let json = serde_json::to_string(&EmbeddingModel::OpenAITextEmbeddingSmall256).unwrap();+ assert_eq!(json, "\"text-embedding-3-small-256\"");+ let parsed: EmbeddingModel = serde_json::from_str(&json).unwrap();+ assert!(matches!(+ parsed,+ EmbeddingModel::OpenAITextEmbeddingSmall256+ ));+ assert_eq!(parsed.model_dimension(), 256);+ assert_eq!(parsed.name(), "text-embedding-3-small");+ }++ #[test]+ fn small_256_request_body_asks_openai_for_256_dimensions() {+ let body = EmbeddingModel::OpenAITextEmbeddingSmall256.construct_request_body("hi");+ assert_eq!(body["model"], "text-embedding-3-small");+ assert_eq!(body["dimensions"], 256);+ assert_eq!(body["input"], "hi");+ }++ #[test]+ fn full_size_models_do_not_send_dimensions() {+ for model in [+ EmbeddingModel::OpenAITextEmbeddingSmall,+ EmbeddingModel::OpenAITextEmbeddingLarge,+ ] {+ let body = model.construct_request_body("hi");+ assert!(+ body.get("dimensions").is_none(),+ "{model:?} must not send dimensions"+ );+ }+ }++ #[test]+ fn small_256_shares_the_small_model_rate_limit_bucket() {+ assert_eq!(+ EmbeddingModel::OpenAITextEmbeddingSmall256.limits_key(),+ EmbeddingModel::OpenAITextEmbeddingSmall.limits_key()+ );+ }+#[test]fn test_escape_input() {let model = EmbeddingModel::default();diff --git a/rust/embedding-worker/src/ad_hoc.rs b/rust/embedding-worker/src/ad_hoc.rsindex a8bd1a68b29b..68a2ffb4fe3d 100644--- a/rust/embedding-worker/src/ad_hoc.rs+++ b/rust/embedding-worker/src/ad_hoc.rs@@ -84,7 +84,9 @@ pub async fn handle_ad_hoc_request(pub fn check_would_truncate(content: &str, model: &EmbeddingModel) -> bool {match model {- EmbeddingModel::OpenAITextEmbeddingSmall | EmbeddingModel::OpenAITextEmbeddingLarge => {+ EmbeddingModel::OpenAITextEmbeddingSmall+ | EmbeddingModel::OpenAITextEmbeddingLarge+ | EmbeddingModel::OpenAITextEmbeddingSmall256 => {// Count every token: capping the count at the input window first would make the// comparison below unsatisfiable.let token_count = CL100K_ENCODER.encode_with_special_tokens(content).len();diff --git a/rust/embedding-worker/src/lib.rs b/rust/embedding-worker/src/lib.rsindex 4dbc363cb518..6f10dfba4f1e 100644--- a/rust/embedding-worker/src/lib.rs+++ b/rust/embedding-worker/src/lib.rs@@ -236,7 +236,9 @@ pub fn generate_embedding_text<'a>() -> Result<(Cow<'a, str>, usize)> {let content = model.escape_input(content);let (text, count) = match model {- EmbeddingModel::OpenAITextEmbeddingSmall | EmbeddingModel::OpenAITextEmbeddingLarge => {+ EmbeddingModel::OpenAITextEmbeddingSmall+ | EmbeddingModel::OpenAITextEmbeddingLarge+ | EmbeddingModel::OpenAITextEmbeddingSmall256 => {let encoder = &*CL100K_ENCODER;let mut tokens: Vec<_> = encoder.encode_with_special_tokens(&content)