Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

8.9 KB
Raw Blame
use std::{collections::HashMap, time::Duration as StdDuration};

use aws_config::retry::RetryConfig;
use aws_sdk_dynamodb::types::{AttributeValue, KeysAndAttributes, PutRequest, WriteRequest};
use axum::async_trait;
use chrono::{DateTime, Duration, Utc};
use metrics::counter;
use rand::Rng;
use tokio::time::sleep;
use tracing::{error, warn};

use crate::{
    config::Config,
    metrics_utils::{RECENTLY_SEEN_READ_ERRORS, RECENTLY_SEEN_RETRIES, RECENTLY_SEEN_WRITE_ERRORS},
    recently_seen::{DocumentKey, RecentlySeenStore, SeenRecord},
};
const PK: &str = "pk";
const SK: &str = "sk";
const EMITTED_AT: &str = "emitted_at";
const TTL_ATTR: &str = "expires_at";
// DynamoDB caps BatchWriteItem at 25 items and BatchGetItem at 100 keys per request.
const BATCH_WRITE_CHUNK: usize = 25;
const BATCH_GET_CHUNK: usize = 100;
const AWS_MAX_ATTEMPTS: u32 = 5;
const UNPROCESSED_MAX_ATTEMPTS: usize = 5;
const UNPROCESSED_INITIAL_BACKOFF_MS: u64 = 50;

async fn backoff_before_retry(operation: &'static str, attempt: usize) {
    counter!(RECENTLY_SEEN_RETRIES, "operation" => operation).increment(1);
    let max_delay_ms = UNPROCESSED_INITIAL_BACKOFF_MS * 2_u64.pow((attempt - 1) as u32);
    let delay_ms = rand::thread_rng().gen_range(max_delay_ms / 2..=max_delay_ms);
    sleep(StdDuration::from_millis(delay_ms)).await;
}

fn partition_key(team_id: i32, key: &DocumentKey) -> String {
    format!(
        "{}#{}#{}#{}",
        team_id, key.product, key.document_type, key.rendering
    )
}

pub struct DynamoDbStore {
    client: aws_sdk_dynamodb::Client,
    table: String,
    ttl: Duration,
}

pub async fn build_dynamodb_store(config: &Config) -> DynamoDbStore {
    let ttl = Duration::seconds(config.recent_ids_ttl_seconds);
    let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest())
        .retry_config(RetryConfig::standard().with_max_attempts(AWS_MAX_ATTEMPTS));
    if let Some(region) = &config.aws_region {
        loader = loader.region(aws_sdk_dynamodb::config::Region::new(region.clone()));
    }
    let shared = loader.load().await;
    let client = aws_sdk_dynamodb::Client::new(&shared);

    DynamoDbStore {
        client,
        table: config.recent_ids_dynamodb_table.clone(),
        ttl,
    }
}

#[async_trait]
impl RecentlySeenStore for DynamoDbStore {
    async fn record(&self, documents: &[SeenRecord]) {
        let expires_at = Utc::now().timestamp() + self.ttl.num_seconds();
        for chunk in documents.chunks(BATCH_WRITE_CHUNK) {
            let mut write_requests = Vec::with_capacity(chunk.len());
            for doc in chunk {
                let item = HashMap::from([
                    (
                        PK.to_string(),
                        AttributeValue::S(partition_key(doc.team_id, &doc.key)),
                    ),
                    (
                        SK.to_string(),
                        AttributeValue::S(doc.key.document_id.clone()),
                    ),
                    (
                        EMITTED_AT.to_string(),
                        AttributeValue::S(doc.emitted_at.to_rfc3339()),
                    ),
                    (
                        TTL_ATTR.to_string(),
                        AttributeValue::N(expires_at.to_string()),
                    ),
                ]);
                let put = PutRequest::builder()
                    .set_item(Some(item))
                    .build()
                    .expect("item is set");
                write_requests.push(WriteRequest::builder().put_request(put).build());
            }

            let mut pending = write_requests;
            for attempt in 1..=UNPROCESSED_MAX_ATTEMPTS {
                let pending_count = pending.len();
                let mut response = match self
                    .client
                    .batch_write_item()
                    .request_items(self.table.clone(), pending)
                    .send()
                    .await
                {
                    Ok(response) => response,
                    Err(e) => {
                        error!("Failed to write recently-seen records to DynamoDB: {e:?}");
                        counter!(RECENTLY_SEEN_WRITE_ERRORS).increment(pending_count as u64);
                        break;
                    }
                };

                pending = response
                    .unprocessed_items
                    .as_mut()
                    .and_then(|items| items.remove(&self.table))
                    .unwrap_or_default();
                if pending.is_empty() {
                    break;
                }
                if attempt == UNPROCESSED_MAX_ATTEMPTS {
                    warn!(
                        "DynamoDB left {} recently-seen records unprocessed after {attempt} attempts",
                        pending.len()
                    );
                    counter!(RECENTLY_SEEN_WRITE_ERRORS).increment(pending.len() as u64);
                    break;
                }

                backoff_before_retry("write", attempt).await;
            }
        }
    }

    async fn lookup(
        &self,
        team_id: i32,
        keys: Vec<DocumentKey>,
    ) -> HashMap<DocumentKey, Option<DateTime<Utc>>> {
        let mut results: HashMap<DocumentKey, Option<DateTime<Utc>>> =
            keys.iter().cloned().map(|k| (k, None)).collect();

        for chunk in keys.chunks(BATCH_GET_CHUNK) {
            // BatchGetItem rejects duplicate keys, so the reverse index also deduplicates.
            let mut index: HashMap<(String, String), DocumentKey> = HashMap::new();
            let mut request_keys = Vec::with_capacity(chunk.len());
            for key in chunk {
                let pk = partition_key(team_id, key);
                let sk = key.document_id.clone();
                if index
                    .insert((pk.clone(), sk.clone()), key.clone())
                    .is_some()
                {
                    continue;
                }
                request_keys.push(HashMap::from([
                    (PK.to_string(), AttributeValue::S(pk)),
                    (SK.to_string(), AttributeValue::S(sk)),
                ]));
            }

            let mut pending = KeysAndAttributes::builder()
                .set_keys(Some(request_keys))
                .projection_expression(format!("{PK}, {SK}, {EMITTED_AT}"))
                .build()
                .expect("keys are set");

            for attempt in 1..=UNPROCESSED_MAX_ATTEMPTS {
                let pending_count = pending.keys().len();
                let mut response = match self
                    .client
                    .batch_get_item()
                    .request_items(self.table.clone(), pending)
                    .send()
                    .await
                {
                    Ok(response) => response,
                    Err(e) => {
                        error!("Failed to read recently-seen records from DynamoDB: {e:?}");
                        counter!(RECENTLY_SEEN_READ_ERRORS).increment(pending_count as u64);
                        break;
                    }
                };

                if let Some(items) = response
                    .responses
                    .as_mut()
                    .and_then(|responses| responses.remove(&self.table))
                {
                    for item in items {
                        let (Some(Ok(pk)), Some(Ok(sk))) = (
                            item.get(PK).map(AttributeValue::as_s),
                            item.get(SK).map(AttributeValue::as_s),
                        ) else {
                            continue;
                        };
                        if let Some(doc_key) = index.get(&(pk.clone(), sk.clone())) {
                            let emitted = item
                                .get(EMITTED_AT)
                                .and_then(|v| v.as_s().ok())
                                .and_then(|s| DateTime::parse_from_rfc3339(s).ok())
                                .map(|dt| dt.with_timezone(&Utc));
                            results.insert(doc_key.clone(), emitted);
                        }
                    }
                }

                let Some(next_pending) = response
                    .unprocessed_keys
                    .as_mut()
                    .and_then(|keys| keys.remove(&self.table))
                else {
                    break;
                };
                if next_pending.keys().is_empty() {
                    break;
                }
                pending = next_pending;

                if attempt == UNPROCESSED_MAX_ATTEMPTS {
                    warn!(
                        "DynamoDB left {} recently-seen lookups unprocessed after {attempt} attempts",
                        pending.keys().len()
                    );
                    counter!(RECENTLY_SEEN_READ_ERRORS).increment(pending.keys().len() as u64);
                    break;
                }

                backoff_before_retry("read", attempt).await;
            }
        }

        results
    }
}