:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
perf(cymbal): cache hermes frames without a chunk id for the resolved ttl (#120959)
James Gillb047cd0d701d
diff --git a/rust/cymbal/src/core/symbolication/symbol/local.rs b/rust/cymbal/src/core/symbolication/symbol/local.rsindex 796bc48e18b8..b79fa12c5264 100644--- a/rust/cymbal/src/core/symbolication/symbol/local.rs+++ b/rust/cymbal/src/core/symbolication/symbol/local.rs@@ -46,7 +46,7 @@ const FRAME_EXPIRY_FALLBACK_SECONDS: u64 = 300;#[derive(Clone)]pub struct LocalSymbolResolver {catalog: Arc<Catalog>,- cache: Cache<RawFrameId, Vec<ErrorTrackingStackFrame>>,+ cache: Cache<RawFrameId, CachedFrames>,// (team_id, sorted truncated refs) -> latest release id, including None results. Without// this, every exception with symbol-set refs costs one Postgres query per resolve, even// when frame resolution itself is fully served from the frame cache above. Refs are@@ -54,40 +54,47 @@ pub struct LocalSymbolResolver {// negative cache in the Saving layer.release_id_cache: Cache<(TeamId, Vec<String>), Option<Uuid>>,pool: PgPool,- ttl_policy: FrameResultTtlPolicy,+ base_ttl_policy: FrameResultTtlPolicy,// Lines of pre/post source context to attach per resolved frame.context_lines: usize,skip_unchanged_rewrites: bool,}-impl Expiry<RawFrameId, Vec<ErrorTrackingStackFrame>> for FrameResultTtlPolicy {+// Each entry carries the per-frame policy that judged its Postgres freshness, so the+// in-memory expiry and the stored-row freshness check cannot disagree.+#[derive(Clone)]+struct CachedFrames {+ records: Vec<ErrorTrackingStackFrame>,+ ttl_policy: FrameResultTtlPolicy,+}++struct FrameCacheExpiry;++impl Expiry<RawFrameId, CachedFrames> for FrameCacheExpiry {fn expire_after_create(&self,key: &RawFrameId,- value: &Vec<ErrorTrackingStackFrame>,+ value: &CachedFrames,_created_at: Instant,) -> Option<Duration> {- Some(expiration_duration(self, key, value))+ Some(expiration_duration(key, value))}fn expire_after_update(&self,key: &RawFrameId,- value: &Vec<ErrorTrackingStackFrame>,+ value: &CachedFrames,_updated_at: Instant,_duration_until_expiry: Option<Duration>,) -> Option<Duration> {- Some(expiration_duration(self, key, value))+ Some(expiration_duration(key, value))}}-fn expiration_duration(- policy: &FrameResultTtlPolicy,- key: &RawFrameId,- value: &[ErrorTrackingStackFrame],-) -> Duration {- policy- .ttl_for_records(key, value)+fn expiration_duration(key: &RawFrameId, value: &CachedFrames) -> Duration {+ value+ .ttl_policy+ .ttl_for_records(key, &value.records).to_std().unwrap_or_else(|err| {tracing::warn!(error = %err, "invalid frame cache ttl, using fallback");@@ -97,12 +104,12 @@ fn expiration_duration(impl LocalSymbolResolver {pub fn new(config: &ResolverConfig, catalog: Arc<Catalog>, pool: PgPool) -> Self {- let ttl_policy = FrameResultTtlPolicy::new(+ let base_ttl_policy = FrameResultTtlPolicy::new(chrono::Duration::seconds(config.frame_resolved_ttl_seconds as i64),chrono::Duration::seconds(config.frame_unresolved_ttl_seconds as i64),);let cache = CacheBuilder::new(config.frame_cache_size)- .expire_after(ttl_policy)+ .expire_after(FrameCacheExpiry).build();let release_id_cache = CacheBuilder::new(config.release_id_cache_max_bytes)@@ -122,7 +129,7 @@ impl LocalSymbolResolver {pool,cache,release_id_cache,- ttl_policy,+ base_ttl_policy,context_lines: config.context_line_count,skip_unchanged_rewrites: config.skip_unchanged_frame_rewrites,}@@ -138,12 +145,19 @@ impl LocalSymbolResolver {metrics::counter!(SUSPICIOUS_FRAMES_DETECTED, "frame_type" => "raw").increment(1);}let raw_id = frame.raw_id(team_id, debug_images);+ let ttl_policy = self.base_ttl_policy.for_frame(frame);let mut cache_miss = false;- let frames = self+ let cached = self.cache.try_get_with(raw_id.clone(), async {cache_miss = true;- self.resolve_impl(frame, raw_id.clone(), debug_images).await+ let records = self+ .resolve_impl(frame, raw_id.clone(), debug_images, ttl_policy)+ .await?;+ Ok::<_, UnhandledError>(CachedFrames {+ records,+ ttl_policy,+ })}).await.map_err(|e| UnhandledError::Other(e.to_string()))?;@@ -154,7 +168,7 @@ impl LocalSymbolResolver {metrics::counter!(FRAME_CACHE_HITS).increment(1);}- Ok(frames.into_iter().map(|f| f.contents).collect())+ Ok(cached.records.into_iter().map(|f| f.contents).collect())}async fn resolve_impl(@@ -162,9 +176,10 @@ impl LocalSymbolResolver {frame: &RawFrame,raw_id: RawFrameId,debug_images: &[DebugImage],+ ttl_policy: FrameResultTtlPolicy,) -> Result<Vec<ErrorTrackingStackFrame>, UnhandledError> {let stored =- ErrorTrackingStackFrame::load_snapshot(&self.pool, &raw_id, self.ttl_policy).await?;+ ErrorTrackingStackFrame::load_snapshot(&self.pool, &raw_id, ttl_policy).await?;if stored.fresh {metrics::counter!(FRAME_DB_HITS).increment(1);return Ok(stored.records);@@ -209,13 +224,18 @@ impl LocalSymbolResolver {let persisted = self.skip_unchanged_rewrites&& write_outcome == FrameWriteOutcome::Unchanged&& self- .persist_unchanged_snapshot(&raw_id, &stored.records, records.len() as u64)+ .persist_unchanged_snapshot(+ &raw_id,+ &stored.records,+ records.len() as u64,+ ttl_policy,+ ).await?;if !persisted {let refresh_if_older_than = self.skip_unchanged_rewrites- .then(|| Utc::now() - self.ttl_policy.refresh_age(&raw_id, &stored.records));+ .then(|| Utc::now() - ttl_policy.refresh_age(&raw_id, &stored.records));for record in &records {let rows_affected = match record.save_with_refresh_guard(&self.pool, refresh_if_older_than)@@ -251,6 +271,7 @@ impl LocalSymbolResolver {raw_id: &RawFrameId,stored: &[ErrorTrackingStackFrame],part_count: u64,+ ttl_policy: FrameResultTtlPolicy,) -> Result<bool, UnhandledError> {// The per-part upsert loop is not transactional, so a racing writer can// leave a multi-part snapshot half written. A timestamp-only refresh@@ -262,7 +283,7 @@ impl LocalSymbolResolver {}let now = Utc::now();- if !unchanged_snapshot_needs_refresh(&self.ttl_policy, raw_id, stored, now) {+ if !unchanged_snapshot_needs_refresh(&ttl_policy, raw_id, stored, now) {record_frame_write(FrameWriteOutcome::Skipped, 0);return Ok(true);}@@ -386,6 +407,7 @@ mod test {core::config::ResolverConfig,core::types::Stacktrace,frames::RawFrame,+ langs::{hermes::RawHermesFrame, CommonFrameMetadata},symbolication::symbol::records::ErrorTrackingStackFrame,symbolication::symbol::{local::LocalSymbolResolver, SymbolResolver},symbolication::symbol_store::{@@ -593,7 +615,7 @@ mod test {// get the framelet frame_id = frame.raw_id(0, &[]);- let frame = ErrorTrackingStackFrame::load_all(&pool, &frame_id, resolver.ttl_policy)+ let frame = ErrorTrackingStackFrame::load_all(&pool, &frame_id, resolver.base_ttl_policy).await.unwrap().pop()@@ -683,6 +705,54 @@ mod test {assert!(rows[0].1 > created_1);}+ #[sqlx::test(migrations = "./tests/test_migrations")]+ pub async fn hermes_frame_without_chunk_id_keeps_its_failure_for_the_resolved_ttl(+ pool: PgPool,+ ) {+ let (mut config, catalog, _server) = setup_test_context(pool.clone(), |_, cl| cl).await;+ config.frame_unresolved_ttl_seconds = 0;+ let resolver = LocalSymbolResolver::new(&config, Arc::new(catalog), pool.clone());++ for (chunk_id, expect_long_ttl) in [(None, true), (Some("never-uploaded"), false)] {+ let frame = RawFrame::Hermes(RawHermesFrame {+ column: Some(1000),+ source: "index.android.bundle".to_string(),+ fn_name: "anonymous".to_string(),+ chunk_id: chunk_id.map(str::to_string),+ meta: CommonFrameMetadata::default(),+ });+ let raw_id = frame.raw_id(0, &[]);+ let created_at = || async {+ sqlx::query_scalar::<_, chrono::DateTime<chrono::Utc>>(+ "SELECT created_at FROM posthog_errortrackingstackframe WHERE raw_id = $1",+ )+ .bind(&raw_id.hash_id)+ .fetch_one(&pool)+ .await+ .unwrap()+ };++ let resolved = resolver.resolve_raw_frame(0, &frame, &[]).await.unwrap();+ assert!(!resolved[0].resolved, "{chunk_id:?}");+ resolver.cache.run_pending_tasks().await;+ assert_eq!(+ resolver.cache.contains_key(&raw_id),+ expect_long_ttl,+ "{chunk_id:?}"+ );++ let first_write = created_at().await;+ resolver.cache.invalidate_all();+ resolver.cache.run_pending_tasks().await;+ resolver.resolve_raw_frame(0, &frame, &[]).await.unwrap();+ assert_eq!(+ created_at().await == first_write,+ expect_long_ttl,+ "{chunk_id:?}"+ );+ }+ }+async fn insert_release(pool: &PgPool, team_id: i32, days_ago: i32) -> Uuid {let id = Uuid::new_v4();sqlx::query(diff --git a/rust/cymbal/src/core/symbolication/symbol/records.rs b/rust/cymbal/src/core/symbolication/symbol/records.rsindex ae507664489a..fa85b93b74aa 100644--- a/rust/cymbal/src/core/symbolication/symbol/records.rs+++ b/rust/cymbal/src/core/symbolication/symbol/records.rs@@ -7,7 +7,7 @@ use uuid::Uuid;use crate::core::write_attribution::FrameWriteOutcome;use crate::error::UnhandledError;-use crate::frames::{Context, Frame};+use crate::frames::{Context, Frame, RawFrame};const FRAME_TTL_JITTER_PERCENT: u32 = 10;@@ -25,6 +25,17 @@ impl FrameResultTtlPolicy {}}+ pub fn for_frame(&self, frame: &RawFrame) -> Self {+ if frame.can_never_resolve() {+ Self {+ unresolved_ttl: self.resolved_ttl,+ ..*self+ }+ } else {+ *self+ }+ }+pub fn ttl_for_records(&self,id: &RawFrameId,diff --git a/rust/cymbal/src/core/types/frames/mod.rs b/rust/cymbal/src/core/types/frames/mod.rsindex 5b06b43a60f4..2306c84ea1ef 100644--- a/rust/cymbal/src/core/types/frames/mod.rs+++ b/rust/cymbal/src/core/types/frames/mod.rs@@ -124,6 +124,13 @@ impl RawFrame {self.raw_id(team_id, debug_images).to_full(index as i32)}+ pub fn can_never_resolve(&self) -> bool {+ match self {+ RawFrame::Hermes(frame) => frame.can_never_resolve(),+ _ => false,+ }+ }+pub fn is_suspicious(&self) -> bool {match self {RawFrame::JavaScriptWeb(frame) => frame.is_suspicious(),diff --git a/rust/cymbal/src/core/types/langs/hermes.rs b/rust/cymbal/src/core/types/langs/hermes.rsindex 566b87f4c2e5..710e57a89251 100644--- a/rust/cymbal/src/core/types/langs/hermes.rs+++ b/rust/cymbal/src/core/types/langs/hermes.rs@@ -117,6 +117,13 @@ impl RawHermesFrame {self.get_ref().ok().map(|r| r.to_string())}+ // No upload can resolve a frame without a chunk id, because the symbol set lookup+ // keys on it. The chunk id is part of `frame_id`, so every frame with this raw id+ // fails the same way.+ pub fn can_never_resolve(&self) -> bool {+ self.chunk_id.is_none()+ }+fn get_ref(&self) -> Result<OrChunkId<HermesRef>, HermesError> {self.chunk_id.as_ref()