:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use crate::v0_request::ProcessedEvent;
use async_trait::async_trait;
use aws_sdk_s3::config::Builder;
use aws_sdk_s3::Client as S3Client;
use chrono::{DateTime, Datelike, Utc};
use metrics::{counter, histogram};
use std::env;
use std::sync::Arc;
use std::time::{Duration, SystemTime};
use tokio::sync::broadcast;
use tokio::sync::broadcast::Sender;
use tokio::sync::Mutex;
use tokio::task;
use tokio::time::sleep;
use tokio::time::Instant;
use tracing::instrument;
use tracing::log::{debug, error, info};
use crate::api::CaptureError;
use crate::outputs::{PreparedEvent, PublishEvents, PublishPrepared};
use crate::sinks::sink::SinkResult;
const FLUSH_INTERVAL: Duration = Duration::from_secs(1);
const HEALTH_INTERVAL: Duration = Duration::from_secs(10);
const MAX_BUFFER_SIZE: usize = 4 * 1024 * 1024; // 4MB
struct Inner {
client: S3Client,
bucket: String,
prefix: String,
buffer: Arc<Mutex<EventBuffer>>,
liveness: lifecycle::Handle,
}
pub struct S3Sink {
inner: Arc<Inner>,
}
struct EventBuffer {
event_bytes: Vec<u8>,
event_count: usize,
time_elapsed: Instant,
tx: Sender<Result<(), CaptureError>>,
}
impl EventBuffer {
fn new() -> Self {
let (tx, _) = broadcast::channel(32);
Self {
event_bytes: Vec::new(),
event_count: 0,
time_elapsed: Instant::now(),
tx,
}
}
fn add_line(&mut self, body: &[u8]) {
self.event_bytes.extend_from_slice(body);
self.event_bytes.push(b'\n');
self.event_count += 1;
}
fn should_flush(&self) -> bool {
!self.event_bytes.is_empty()
&& (self.event_bytes.len() >= MAX_BUFFER_SIZE
|| self.time_elapsed.elapsed() >= FLUSH_INTERVAL)
}
}
impl S3Sink {
pub async fn new(
bucket: String,
prefix: String,
s3_endpoint: Option<String>,
liveness: lifecycle::Handle,
) -> anyhow::Result<S3Sink> {
info!("Initializing S3 sink with bucket: {bucket}");
// Load base config
let mut config_loader = aws_config::defaults(aws_config::BehaviorVersion::latest());
if let Some(s3_endpoint) = s3_endpoint.clone() {
config_loader = config_loader.endpoint_url(s3_endpoint);
}
let mut config = Builder::from(&config_loader.load().await);
if s3_endpoint.is_some() {
// custom s3 endpoints need force_path_style set
config = config.force_path_style(true);
}
let client = S3Client::from_conf(config.build());
let buffer = Arc::new(Mutex::new(EventBuffer::new()));
let inner = Arc::new(Inner {
client,
bucket,
prefix,
buffer,
liveness,
});
// Do initial healthcheck
inner.healthcheck().await;
// Create weak reference for background task
let inner_weak = Arc::downgrade(&inner);
let shutdown_handle = inner.liveness.clone();
// Spawn background task with shutdown handling
task::spawn(async move {
let mut last_healthcheck = Instant::now();
loop {
tokio::select! {
_ = sleep(Duration::from_millis(10)) => {
// Try to upgrade weak reference - if it fails, the S3Sink has been dropped
let inner = match inner_weak.upgrade() {
Some(inner) => inner,
None => break,
};
let mut buffer = inner.buffer.lock().await;
if buffer.should_flush() {
let mut old_buffer = {
// Replace the current buffer with a brand-new one
std::mem::replace(&mut *buffer, EventBuffer::new())
};
// Drop the lock so writers can continue while we flush
drop(buffer);
let result = inner.flush_buffer(&mut old_buffer).await;
if result.is_ok() {
last_healthcheck = Instant::now();
}
drop(old_buffer.tx.send(result));
}
// If we haven't written any events the healthcheck will stall;
// force healthcheck at least every HEALTH_INTERVAL
if last_healthcheck.elapsed() >= HEALTH_INTERVAL {
inner.healthcheck().await;
last_healthcheck = Instant::now();
}
}
_ = shutdown_handle.shutdown_recv() => {
// Final flush of any buffered events before exit
if let Some(inner) = inner_weak.upgrade() {
let mut buffer = inner.buffer.lock().await;
if buffer.event_count > 0 {
let mut old_buffer =
std::mem::replace(&mut *buffer, EventBuffer::new());
drop(buffer);
let result = inner.flush_buffer(&mut old_buffer).await;
drop(old_buffer.tx.send(result));
}
}
break;
}
}
}
});
Ok(S3Sink { inner })
}
}
impl Inner {
async fn healthcheck(&self) {
// Verify bucket exists and is accessible
if self
.client
.head_bucket()
.bucket(&self.bucket)
.send()
.await
.is_ok()
{
self.liveness.report_healthy();
};
}
async fn flush_buffer(&self, buffer: &mut EventBuffer) -> Result<(), CaptureError> {
let start = Instant::now();
let events_count = buffer.event_count;
let batch_size = buffer.event_bytes.len();
match self.do_flush(buffer).await {
Ok(_) => {
let duration = start.elapsed();
histogram!("capture_s3_flush_duration_ms").record(duration.as_millis() as f64);
histogram!("capture_s3_batch_size_events").record(events_count as f64);
histogram!("capture_s3_batch_size_bytes").record(batch_size as f64);
Ok(())
}
Err(e) => {
counter!("capture_s3_flush_errors_total", "error" => e.to_string()).increment(1);
Err(e)
}
}
}
async fn do_flush(&self, buffer: &mut EventBuffer) -> Result<(), CaptureError> {
let mut retries = 3;
loop {
match self.try_flush(buffer).await {
Ok(_) => return Ok(()),
Err(e) => {
retries -= 1;
if retries == 0 {
return Err(e);
}
sleep(Duration::from_millis(100 * (4 - retries))).await;
}
}
}
}
async fn try_flush(&self, buffer: &mut EventBuffer) -> Result<(), CaptureError> {
if buffer.event_bytes.is_empty() {
return Ok(());
}
let hostname = env::var("HOSTNAME").unwrap_or("unknown".to_string());
let now: DateTime<Utc> = SystemTime::now().into();
let path = format!(
"{}{}/{:02}/{:02}/events_{}_{}.jsonl.gz",
self.prefix,
now.year(),
now.month(),
now.day(),
now.timestamp_millis(),
hostname
);
debug!(
"Flushing {} events to S3 path: {}",
buffer.event_count, path
);
let event_count = buffer.event_count;
let written_bytes = buffer.event_bytes.len();
// take the event_bytes - this will implicitly reset to a new Vec
let event_bytes = std::mem::take(&mut buffer.event_bytes);
buffer.event_count = 0;
buffer.time_elapsed = Instant::now();
match self
.client
.put_object()
.bucket(&self.bucket)
.key(&path)
.body(event_bytes.into())
.send()
.await
{
Ok(_) => {
counter!("capture_s3_events_written_total").increment(event_count as u64);
counter!("capture_s3_bytes_written_total").increment(written_bytes as u64);
histogram!("capture_s3_batch_size").record(event_count as f64);
self.liveness.report_healthy();
Ok(())
}
Err(err) => {
error!("Failed to write to S3: {err:#}");
counter!("capture_s3_write_errors_total").increment(1);
Err(CaptureError::RetryableSinkError)
}
}
}
}
impl S3Sink {
async fn write_lines<B: AsRef<[u8]>>(
&self,
bodies: impl ExactSizeIterator<Item = B> + Send,
) -> Result<(), CaptureError> {
// An idle buffer never flushes, so waiting on it would never return.
if bodies.len() == 0 {
return Ok(());
}
let mut buffer = self.inner.buffer.lock().await;
for body in bodies {
buffer.add_line(body.as_ref());
}
let mut rx = buffer.tx.subscribe();
drop(buffer);
rx.recv()
.await
.unwrap_or(Err(CaptureError::NonRetryableSinkError))
}
}
#[async_trait]
impl PublishEvents for S3Sink {
#[instrument(skip_all)]
async fn publish_events(&self, events: Vec<ProcessedEvent>) -> Result<(), CaptureError> {
let bodies = events
.iter()
.map(|event| serde_json::to_vec(&event.event))
.collect::<Result<Vec<_>, _>>()?;
self.write_lines(bodies.iter()).await
}
}
#[async_trait]
impl PublishPrepared for S3Sink {
#[instrument(skip_all)]
async fn publish_prepared(&self, events: Vec<PreparedEvent>) -> Vec<SinkResult> {
let flushed = self
.write_lines(events.iter().map(|event| &event.payload))
.await;
events
.iter()
.map(|event| match &flushed {
Ok(()) => SinkResult::published(event.uuid),
Err(err) => SinkResult::failed(event.uuid, err.clone()),
})
.collect()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::utils::uuid_v7_from_datetime;
use crate::v0_request::{DataType, ProcessedEventMetadata};
use common_types::CapturedEvent;
use tokio_util::sync::CancellationToken;
async fn setup_test_sink() -> S3Sink {
let shutdown_token = CancellationToken::new();
let mut manager = lifecycle::Manager::builder("test")
.with_trap_signals(false)
.with_prestop_check(false)
.with_shutdown_token(shutdown_token)
.build();
let handle = manager.register(
"s3",
lifecycle::ComponentOptions::new().with_liveness_deadline(Duration::from_secs(30)),
);
let _monitor = manager.monitor_background();
// Use environment variables for test configuration
env::set_var("AWS_ACCESS_KEY_ID", "object_storage_root_user");
env::set_var("AWS_SECRET_ACCESS_KEY", "object_storage_root_password");
S3Sink::new(
"capture".to_string(),
"".to_string(),
Some("http://localhost:19000".to_string()),
handle,
)
.await
.expect("Failed to create S3 sink")
}
fn create_test_event() -> ProcessedEvent {
let timestamp = chrono::DateTime::parse_from_rfc3339("2024-01-01T00:00:00Z")
.unwrap()
.with_timezone(&chrono::Utc);
ProcessedEvent {
event: CapturedEvent {
uuid: uuid_v7_from_datetime(timestamp),
distinct_id: "test_id".to_string(),
session_id: None,
ip: "127.0.0.1".to_string(),
data: "test data".to_string(),
now: "2024-01-01T00:00:00Z".to_string(),
sent_at: None,
token: "test_token".to_string(),
event: "test_event".to_string(),
timestamp,
is_cookieless_mode: false,
historical_migration: false,
},
metadata: ProcessedEventMetadata {
data_type: DataType::AnalyticsMain,
session_id: None,
computed_timestamp: None,
event_name: "test_event".to_string(),
force_overflow: false,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: None,
skip_heatmap_processing: false,
overflow_reason: None,
distinct_id_truncated_from: None,
},
}
}
#[tokio::test]
async fn empty_prepared_batch_returns_without_waiting_for_a_flush() {
let sink = setup_test_sink().await;
let results = tokio::time::timeout(Duration::from_secs(5), sink.publish_prepared(vec![]))
.await
.expect("an empty batch must not wait on the idle buffer");
assert!(results.is_empty());
}
#[tokio::test]
async fn test_basic_functionality() {
let sink = setup_test_sink().await;
// Test single event
let event = create_test_event();
sink.publish_events(vec![event.clone()])
.await
.expect("Failed to publish event");
// Test batch
let batch = vec![event.clone(), event.clone()];
sink.publish_events(batch)
.await
.expect("Failed to publish batch");
}
#[tokio::test]
async fn test_large_event() {
let sink = setup_test_sink().await;
// Create event that will exceed MAX_BUFFER_SIZE
let large_data = "x".repeat(MAX_BUFFER_SIZE + 1000);
let event = ProcessedEvent {
event: CapturedEvent {
data: large_data,
..create_test_event().event
},
metadata: create_test_event().metadata,
};
sink.publish_events(vec![event])
.await
.expect("Failed to publish large event");
}
}