:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::{collections::HashMap, path::PathBuf, sync::Arc, time::Duration};
use anyhow::{Context, Error};
use async_trait::async_trait;
use chrono::{DateTime, Duration as ChronoDuration, Utc};
use reqwest::header::HeaderMap;
use reqwest::{Client, Error as ReqwestError};
use tempfile::TempDir;
use tokio::{fs::File, io::AsyncWriteExt, sync::Mutex};
use tracing::{debug, info, warn};
use super::{read_prepared_chunk, remove_prepared_key, DataSource, PreparedPart, RemoteStaging};
use crate::error::{RateLimitedError, ToUserError};
use crate::extractor::PartExtractor;
use crate::staging::StagingGuard;
// Extract a user friendly error message
// from status code of request to the export source endpoint
fn extract_status_error(error: &ReqwestError) -> String {
if let Some(status) = error.status() {
match status.as_u16() {
400 => "Export endpoint returned 400 -- check your date range, credentials, and API key permissions".to_string(),
401 => "Authentication failed, check your credentials".to_string(),
403 => "Access denied -- check your credentials".to_string(),
408 => "Request timed out -- data export may be too large to support exporting straight from source".to_string(),
429 => "Rate limit exceeded -- pause the job and try again later".to_string(),
500 => "Remote server error".to_string(),
_ => "Unknown error -- try the job again or use a different source".to_string(),
}
} else {
"Unknown error -- try the job again or use a different source".to_string()
}
}
fn extract_client_request_error(error: &ReqwestError) -> String {
if error.is_timeout() {
"Request timed out -- data export may be too large to support exporting from this source"
.to_string()
} else {
"Unknown error -- try the job again or use a different source".to_string()
}
}
// Parse Retry-After header per RFC7231: either delta-seconds or HTTP-date
pub(crate) fn parse_retry_after_header(headers: &HeaderMap) -> Option<Duration> {
let value = headers.get(reqwest::header::RETRY_AFTER)?;
let s = value.to_str().ok()?;
if let Ok(seconds) = s.trim().parse::<u64>() {
return Some(Duration::from_secs(seconds));
}
if let Ok(date) = httpdate::parse_http_date(s) {
let now = std::time::SystemTime::now();
if let Ok(diff) = date.duration_since(now) {
// clamp to zero if in past
return Some(diff);
}
}
None
}
#[derive(Clone)]
pub enum AuthConfig {
None,
ApiKey { header_name: String, key: String },
BearerToken { token: String },
BasicAuth { username: String, password: String },
// Annoying special case for Mixpanel
MixpanelAuth { secret_key: String },
}
pub struct DateRangeExportSourceBuilder {
base_url: String,
start: DateTime<Utc>,
end: DateTime<Utc>,
interval_duration: i64,
extractor: Arc<dyn PartExtractor>,
staging_dir: PathBuf,
// Optional with defaults
start_qp: String,
end_qp: String,
timeout: Duration,
retries: usize,
retry_delay: Duration,
auth_config: AuthConfig,
date_format: String,
headers: HashMap<String, String>,
staging_max_bytes: u64,
remote_staging: Option<RemoteStaging>,
}
impl DateRangeExportSourceBuilder {
pub fn new(
base_url: String,
start: DateTime<Utc>,
end: DateTime<Utc>,
interval_duration: i64,
extractor: Arc<dyn PartExtractor>,
staging_dir: PathBuf,
) -> Self {
Self {
base_url,
start,
end,
interval_duration,
extractor,
staging_dir,
start_qp: "start".to_string(),
end_qp: "end".to_string(),
timeout: Duration::from_secs(30),
retries: 3,
retry_delay: Duration::from_secs(30),
auth_config: AuthConfig::None,
date_format: "%Y-%m-%dT%H:%M:%SZ".to_string(),
headers: HashMap::new(),
staging_max_bytes: 0,
remote_staging: None,
}
}
pub fn with_staging_max_bytes(mut self, staging_max_bytes: u64) -> Self {
self.staging_max_bytes = staging_max_bytes;
self
}
/// Stage decompressed part plaintext in a remote backend (temp bucket) instead of
/// serving reads from the local streaming machinery. `None` (default) keeps the
/// local path unchanged.
pub fn with_remote_staging(mut self, remote_staging: Option<RemoteStaging>) -> Self {
self.remote_staging = remote_staging;
self
}
pub fn with_query_params(mut self, start_qp: String, end_qp: String) -> Self {
self.start_qp = start_qp;
self.end_qp = end_qp;
self
}
pub fn with_timeout(mut self, timeout: Duration) -> Self {
self.timeout = timeout;
self
}
pub fn with_retries(mut self, retries: usize) -> Self {
self.retries = retries;
self
}
pub fn with_retry_delay(mut self, retry_delay: Duration) -> Self {
self.retry_delay = retry_delay;
self
}
pub fn with_auth(mut self, auth_config: AuthConfig) -> Self {
self.auth_config = auth_config;
self
}
pub fn with_date_format(mut self, date_format: String) -> Self {
self.date_format = date_format;
self
}
pub fn with_headers(mut self, headers: HashMap<String, String>) -> Self {
self.headers = headers;
self
}
pub fn build(self) -> Result<DateRangeExportSource, Error> {
let mut intervals = Vec::new();
let mut current = self.start;
while current < self.end {
let next = std::cmp::min(
current + ChronoDuration::seconds(self.interval_duration),
self.end,
);
intervals.push((current, next));
current = next;
}
let client = reqwest::Client::builder()
.timeout(self.timeout)
.build()
.map_err(|e| Error::msg(e.to_string()))?;
Ok(DateRangeExportSource {
base_url: self.base_url,
intervals,
interval_duration: self.interval_duration,
start_qp: self.start_qp,
end_qp: self.end_qp,
headers: self.headers,
auth_config: self.auth_config,
date_format: self.date_format,
client,
retries: self.retries,
retry_delay: self.retry_delay,
extractor: self.extractor,
staging_dir: self.staging_dir,
staging_max_bytes: self.staging_max_bytes,
remote_staging: self.remote_staging,
temp_dir: Arc::new(Mutex::new(None)),
prepared_keys: Arc::new(Mutex::new(HashMap::new())),
})
}
}
pub struct DateRangeExportSource {
pub base_url: String,
pub intervals: Vec<(DateTime<Utc>, DateTime<Utc>)>,
/// The duration of each interval in seconds. Used to adjust the end date
/// query parameter for APIs with inclusive date ranges.
pub interval_duration: i64,
pub client: Client,
pub retries: usize,
pub retry_delay: Duration,
pub extractor: Arc<dyn PartExtractor>,
pub start_qp: String,
pub end_qp: String,
staging_dir: PathBuf,
staging_max_bytes: u64,
remote_staging: Option<RemoteStaging>,
temp_dir: Arc<Mutex<Option<TempDir>>>,
prepared_keys: Arc<Mutex<HashMap<String, PreparedPart>>>,
auth_config: AuthConfig,
date_format: String,
headers: HashMap<String, String>,
}
/*
* Support for importing data directly from another analytics service's batch export endpoint.
* A date range is specified for the chunk of data to be exported and usually comes to the client
* in a compressed format.
*
* Handling different compression formats is supported by supplying a PartExtractor trait that can be used to extract
* data from a compressed file into a byte seekable interface that can be processed in chunks
*
* The tricky part about this source is that the export endpoints have some minimum time range that they can export over.
* Because a customer may have more data in the minimum time range than we can processs and commit in one go, we need to cache the data
* on the worker's file system so we don't have to download the entire part of data for an interval
* for each chunk that we process from it.
*/
impl DateRangeExportSource {
pub fn builder(
base_url: String,
start: DateTime<Utc>,
end: DateTime<Utc>,
interval_duration: i64,
extractor: Arc<dyn PartExtractor>,
staging_dir: PathBuf,
) -> DateRangeExportSourceBuilder {
DateRangeExportSourceBuilder::new(
base_url,
start,
end,
interval_duration,
extractor,
staging_dir,
)
}
fn interval_key((start, end): (DateTime<Utc>, DateTime<Utc>)) -> String {
format!("{}_{}", start.to_rfc3339(), end.to_rfc3339())
}
fn interval_from_key(&self, key: &str) -> Option<(DateTime<Utc>, DateTime<Utc>)> {
let parts: Vec<&str> = key.split('_').collect();
if parts.len() != 2 {
return None;
}
let start = DateTime::parse_from_rfc3339(parts[0])
.ok()?
.with_timezone(&Utc);
let end = DateTime::parse_from_rfc3339(parts[1])
.ok()?
.with_timezone(&Utc);
Some((start, end))
}
pub fn format_date_range_from_key(&self, key: &str) -> Option<String> {
let (start, end) = self.interval_from_key(key)?;
Some(format!(
"{} to {}",
start.format("%Y-%m-%d %H:%M UTC"),
end.format("%Y-%m-%d %H:%M UTC")
))
}
async fn get_temp_dir_path(&self) -> Result<PathBuf, Error> {
let temp_dir_guard = self.temp_dir.lock().await;
Ok(temp_dir_guard
.as_ref()
.ok_or_else(|| Error::msg("Temp directory not initialized"))?
.path()
.to_path_buf())
}
async fn cleanup_temp_files(&self) -> Result<(), Error> {
let temp_dir = self.get_temp_dir_path().await?;
let mut entries = tokio::fs::read_dir(temp_dir)
.await
.with_context(|| "Failed to read temp directory while cleaning up files")?;
while let Some(entry) = entries.next_entry().await.with_context(|| {
"Failed to read next entry from temp directory, while cleaning up files"
})? {
let path = entry.path();
if let Some(extension) = path.extension() {
if extension == "raw" || extension == "data" {
if let Err(e) = tokio::fs::remove_file(&path).await {
warn!("Failed to remove temp file {}: {:?}", path.display(), e);
} else {
debug!("Cleaned up temp file: {}", path.display());
}
}
}
}
Ok(())
}
// Keys for this source need their parts to to be "prepared" before we can consume from them
// To support exporting directly from sources that do not provide a byte seekable interface, we need to create
// that byte seekable interface ourselves
// This method streams the compressed data from the source into a temp `.raw` file on
// disk and returns its path (`None` for an empty part, e.g. a 404 interval). What
// happens to the `.raw` depends on the staging mode: the local path opens a streaming
// decoder over it (disk bounded by compressed size); the remote path ingests it into
// the temp bucket and deletes it.
async fn download_part_raw(&self, key: &str) -> Result<Option<(PathBuf, u64)>, Error> {
let (start, end) = self
.interval_from_key(key)
.ok_or_else(|| Error::msg("Invalid interval key"))?;
// clean up any orphaned .raw and .data files for this job before starting download
if let Err(e) = self.cleanup_temp_files().await {
warn!("Failed to cleanup temp files: {:?}", e);
}
info!("Downloading and preparing key: {}", key);
// Let errors (including 429 wrapped as RateLimitedError) bubble up to job-level backoff
self.download_part_raw_inner(key, start, end).await
}
async fn download_part_raw_inner(
&self,
key: &str,
start: DateTime<Utc>,
end: DateTime<Utc>,
) -> Result<Option<(PathBuf, u64)>, Error> {
// All date range export APIs (Mixpanel, Amplitude, etc.) use inclusive date ranges,
// meaning both start and end dates/times are included in the results. Our intervals
// are created as semi-open [start, end), so we subtract one interval unit from the
// end date to avoid fetching overlapping data between adjacent intervals.
//
// For example, with hourly intervals:
// - Internal interval: [14:00, 15:00)
// - Without adjustment: start=14:00, end=15:00 → returns hours 14 AND 15 (overlap!)
// - With adjustment: start=14:00, end=14:00 → returns only hour 14 (correct)
//
// When the interval is shorter than interval_duration (e.g. a partial last interval),
// we clamp end_for_query to start so the query never has end < start.
let end_for_query =
std::cmp::max(end - ChronoDuration::seconds(self.interval_duration), start);
let mut request = self.client.get(&self.base_url).query(&[
(&self.start_qp, start.format(&self.date_format).to_string()),
(
&self.end_qp,
end_for_query.format(&self.date_format).to_string(),
),
]);
request = match &self.auth_config {
AuthConfig::None => request,
AuthConfig::ApiKey { header_name, key } => request.header(header_name, key),
AuthConfig::BearerToken { token } => request.bearer_auth(token),
AuthConfig::BasicAuth { username, password } => {
request.basic_auth(username, Some(password))
}
AuthConfig::MixpanelAuth { secret_key } => request.basic_auth(secret_key, None::<&str>),
};
let mut headers = reqwest::header::HeaderMap::new();
for (key, value) in &self.headers {
if let (Ok(header_name), Ok(header_value)) = (
reqwest::header::HeaderName::from_bytes(key.as_bytes()),
reqwest::header::HeaderValue::from_str(value),
) {
headers.insert(header_name, header_value);
}
}
request = request.headers(headers);
let response = request.send().await.or_else(|req_error| {
let friendly_msg = extract_client_request_error(&req_error);
Err(req_error).user_error(friendly_msg)
})?;
// If there isn't data for this interval, create an empty file with size 0
// We want to return something to the .process() thread so that we end up making a commit
// for this key/part of the overall job
if response.status() == 404 {
info!(
"No data available for key: {} (404 response), sleeping for 2 seconds",
key
);
tokio::time::sleep(Duration::from_secs(2)).await;
return Ok(None);
}
if response.status().as_u16() == 429 {
let headers_clone = response.headers().clone();
let http_err = response.error_for_status().unwrap_err();
let retry_after = parse_retry_after_header(&headers_clone);
let rl = RateLimitedError {
retry_after,
source: http_err,
};
let err = anyhow::Error::from(rl).context(crate::error::UserError::new(
"Rate limit exceeded -- pause the job and try again later",
));
return Err(err);
}
let response = response.error_for_status().or_else(|status_error| {
let friendly_msg = extract_status_error(&status_error);
Err(status_error).user_error(friendly_msg)
})?;
let temp_dir = self.get_temp_dir_path().await?;
// Pause the job if staging is already over budget (e.g. leftover from a
// prior part) before we add to it, and again as the `.raw` grows.
let mut guard = StagingGuard::new(self.staging_dir.clone(), self.staging_max_bytes);
guard.check().await?;
let raw_file_path = temp_dir.join(format!("{}.raw", key.replace(':', "_")));
let mut raw_file = File::create(&raw_file_path)
.await
.with_context(|| format!("Failed to create raw file : {}", raw_file_path.display()))?;
let mut stream = response.bytes_stream();
let mut total_bytes = 0;
use futures_util::StreamExt;
while let Some(result) = stream.next().await {
let chunk: bytes::Bytes = result?;
raw_file.write_all(&chunk).await.with_context(|| {
format!(
"Failed to write chunk to raw file: {}",
raw_file_path.display()
)
})?;
total_bytes += chunk.len();
guard.record(chunk.len() as u64).await?;
}
raw_file.sync_all().await.with_context(|| {
format!(
"Failed to sync raw file to disk: {}",
raw_file_path.display()
)
})?;
// Final check once the whole `.raw` is on disk: the per-chunk `record` calls are
// throttled, so a part smaller than the check interval could otherwise exceed the
// limit without ever being measured. This enforces the limit at the part boundary.
guard.check().await?;
info!(
"Streamed {} bytes to file {} for key: {}",
total_bytes,
raw_file_path.display(),
key
);
// A 200 with a zero-byte body is an export interval with no data (some
// export APIs return this instead of a 404). Report it as an empty part
// with its size known immediately - mirroring the 404 branch above and
// s3_gzip's zero-byte handling - rather than staging a streaming part
// for a file we already know is empty.
if total_bytes == 0 {
info!("No data available for key: {} (empty response body)", key);
if let Err(e) = tokio::fs::remove_file(&raw_file_path).await {
warn!(
"Failed to remove empty raw file {}: {e}",
raw_file_path.display()
);
}
return Ok(None);
}
Ok(Some((raw_file_path, total_bytes as u64)))
}
async fn get_chunk_from_prepared_key(
&self,
key: &str,
offset: u64,
size: u64,
) -> Result<Vec<u8>, Error> {
read_prepared_chunk(&self.prepared_keys, key, offset, size).await
}
/// Free pod-local state: prepared readers/`.raw` files and the job temp dir.
/// Clear refs first, then explicitly `close()` the temp dir to surface removal
/// errors. Never touches remote staging.
async fn cleanup_local_resources(&self) {
{
let mut prepared_keys = self.prepared_keys.lock().await;
prepared_keys.clear();
}
{
let mut temp_dir_guard = self.temp_dir.lock().await;
if let Some(temp_dir) = temp_dir_guard.take() {
let path = temp_dir.path().to_path_buf();
if let Err(e) = temp_dir.close() {
warn!("Failed to remove temp directory {}: {e}", path.display());
} else {
debug!("Cleaned up temp directory: {}", path.display());
}
}
}
}
}
#[async_trait]
impl DataSource for DateRangeExportSource {
async fn keys(&self) -> Result<Vec<String>, Error> {
Ok(self
.intervals
.iter()
.map(|&interval| Self::interval_key(interval))
.collect())
}
async fn size(&self, key: &str) -> Result<Option<u64>, Error> {
if let Some(remote) = &self.remote_staging {
return remote.backend.size(key).await;
}
let prepared_keys = self.prepared_keys.lock().await;
Ok(prepared_keys.get(key).and_then(|part| part.total_size))
}
async fn get_chunk(&self, key: &str, offset: u64, size: u64) -> Result<Vec<u8>, Error> {
if let Some(remote) = &self.remote_staging {
return remote.read_chunk(key, offset, size).await;
}
let mut retries = self.retries;
loop {
match self.get_chunk_from_prepared_key(key, offset, size).await {
Ok(chunk) => return Ok(chunk),
Err(e) => {
if retries == 0 {
return Err(e);
}
warn!(
"Error reading prepared chunk: {:?}, remaining retries: {}",
e, retries
);
tokio::time::sleep(Duration::from_secs(1)).await;
retries -= 1;
}
}
}
}
async fn prepare_for_job(&self) -> Result<(), Error> {
// Temp dir lifetime is tied to the job via self.temp_dir
let temp_dir = tempfile::Builder::new()
.prefix("job-")
.tempdir_in(&self.staging_dir)
.with_context(|| {
format!(
"Failed to create temp directory in staging dir: {}",
self.staging_dir.display()
)
})?;
debug!("Created temp directory for job: {:?}", temp_dir.path());
{
let mut temp_dir_guard = self.temp_dir.lock().await;
*temp_dir_guard = Some(temp_dir);
}
Ok(())
}
async fn cleanup_after_job(&self) -> Result<(), Error> {
// Terminal: sweep any staged objects this job left in the remote backend
// (best-effort; the bucket TTL is the final backstop), then release local
// resources.
if let Some(remote) = &self.remote_staging {
remote.sweep_job().await;
}
self.cleanup_local_resources().await;
debug!("Job cleanup complete");
Ok(())
}
async fn release_job_resources(&self) -> Result<(), Error> {
// Transient interruption: keep staged remote objects for the resume to
// re-attach to; free only pod-local disk and in-memory state (the disk is
// shared with whatever job this pod claims next).
self.cleanup_local_resources().await;
debug!("Job resources released (remote staging kept for resume)");
Ok(())
}
async fn cleanup_after_data_error(&self) -> Result<(), Error> {
// Data-error pause: quarantine staged plaintext for post-mortem (it is
// the exact byte stream the failing offset points into), so the resume
// re-downloads a clean copy while support can still inspect the bytes
// that failed to parse.
let quarantine_result = match &self.remote_staging {
Some(remote) => remote.quarantine_job().await,
None => Ok(()),
};
self.cleanup_local_resources().await;
debug!("Job data-error cleanup complete (staged parts quarantined)");
quarantine_result
}
// We call this every time we process a chunk from a key/part
// So this needs to be idempotent/a no-op when the key/part is already prepared
async fn prepare_key(&self, key: &str) -> Result<(), Error> {
if let Some(remote) = &self.remote_staging {
return remote
.prepare_key(key, || self.download_part_raw(key))
.await;
}
{
let prepared_keys = self.prepared_keys.lock().await;
if prepared_keys.contains_key(key) {
debug!("Key already prepared: {}", key);
return Ok(());
}
}
let prepared_part = match self.download_part_raw(key).await? {
None => PreparedPart::empty(),
Some((raw_file_path, total_bytes)) => {
// Keep the `.raw` file and decompress on demand as the job reads forward.
let reader = self.extractor.open_reader(raw_file_path.clone());
info!("Prepared key {key} ({total_bytes} compressed bytes, streaming decode)");
PreparedPart::streaming(raw_file_path, reader)
}
};
{
let mut prepared_keys = self.prepared_keys.lock().await;
prepared_keys.insert(key.to_string(), prepared_part);
}
Ok(())
}
// Should be called after we've read the last of a key/part into memory and attempt to commit it
async fn cleanup_key(&self, key: &str) -> Result<(), Error> {
if let Some(remote) = &self.remote_staging {
return remote.backend.cleanup_key(key).await;
}
remove_prepared_key(&self.prepared_keys, key).await;
Ok(())
}
fn get_date_range_for_key(&self, key: &str) -> Option<String> {
self.format_date_range_from_key(key)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::extractor::StreamingReader;
use chrono::{TimeZone, Utc};
use httpmock::MockServer;
use std::path::Path;
use tempfile::TempDir;
/// Test extractor that streams the downloaded body back verbatim (no
/// decompression, no newline normalization), so assertions can compare
/// against the exact plaintext body served by the mock server.
struct MockExtractor;
impl PartExtractor for MockExtractor {
fn open_reader(&self, raw_file_path: PathBuf) -> StreamingReader {
StreamingReader::open_verbatim(raw_file_path)
}
}
/// Read a key to completion through the public source API in `chunk`-sized
/// forward reads, returning the reconstructed bytes. Mirrors how the job's
/// chunker consumes a key (monotonic offsets), and drives lazy size discovery.
async fn read_key_to_end(source: &DateRangeExportSource, key: &str, chunk: u64) -> Vec<u8> {
let mut out = Vec::new();
let mut offset = 0u64;
loop {
let bytes = source.get_chunk(key, offset, chunk).await.unwrap();
if bytes.is_empty() {
break;
}
offset += bytes.len() as u64;
out.extend_from_slice(&bytes);
}
out
}
/// Count `.raw` staging files anywhere under `dir`. Peak/residual `.raw` count
/// is how we assert staging disk is freed (see the free-on-EOF tests below).
fn count_raw_files(dir: &Path) -> usize {
let mut count = 0;
let mut stack = vec![dir.to_path_buf()];
while let Some(d) = stack.pop() {
let Ok(entries) = std::fs::read_dir(&d) else {
continue;
};
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
stack.push(path);
} else if path.extension().and_then(|e| e.to_str()) == Some("raw") {
count += 1;
}
}
}
count
}
const TEST_DATA: &str = r#"{"event": "test1", "timestamp": "2023-01-01T00:00:00Z"}
{"event": "test2", "timestamp": "2023-01-01T01:00:00Z"}
{"event": "test3", "timestamp": "2023-01-01T02:00:00Z"}"#;
fn create_test_source(
base_url: String,
interval_duration: i64,
staging_dir: PathBuf,
) -> DateRangeExportSource {
let start = Utc.with_ymd_and_hms(2023, 1, 1, 0, 0, 0).unwrap();
let end = Utc.with_ymd_and_hms(2023, 1, 1, 6, 0, 0).unwrap();
DateRangeExportSource::builder(
base_url,
start,
end,
interval_duration,
Arc::new(MockExtractor),
staging_dir,
)
.with_auth(AuthConfig::None)
.with_date_format("%Y-%m-%dT%H:%M:%SZ".to_string())
.with_headers(HashMap::new())
.build()
.unwrap()
}
/// A source decoding real gzip (the production PlainGzip extractor), for tests
/// that exercise decode failures and empty bodies end to end.
fn create_gzip_source(base_url: String, staging_dir: PathBuf) -> DateRangeExportSource {
let start = Utc.with_ymd_and_hms(2023, 1, 1, 0, 0, 0).unwrap();
let end = Utc.with_ymd_and_hms(2023, 1, 1, 1, 0, 0).unwrap();
DateRangeExportSource::builder(
base_url,
start,
end,
3600,
crate::extractor::ExtractorType::PlainGzip.create_extractor(),
staging_dir,
)
.with_auth(AuthConfig::None)
.with_date_format("%Y-%m-%dT%H:%M:%SZ".to_string())
.with_headers(HashMap::new())
.build()
.unwrap()
}
#[tokio::test]
async fn test_corrupt_gzip_errors_instead_of_silently_truncating() {
// A gzip that decodes some blocks then fails mid-stream. get_chunk's retry
// loop must surface the decode error - not have a retried read observe the
// dead producer's closed channel as a clean EOF, record the partial decode
// length as the part's total size, and let the job commit a truncated part
// as complete (silent data loss).
let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
std::io::Write::write_all(&mut gz, "line\n".repeat(100_000).as_bytes()).unwrap();
let mut body = gz.finish().unwrap();
body.truncate(body.len() - 5);
let server = MockServer::start();
let _mock = server.mock(|when, then| {
when.method(httpmock::Method::GET).path("/export");
then.status(200).body(body.clone());
});
let staging = TempDir::new().unwrap();
let source = create_gzip_source(server.url("/export"), staging.path().to_path_buf());
source.prepare_for_job().await.unwrap();
let key = source.keys().await.unwrap().remove(0);
source.prepare_key(&key).await.unwrap();
// Read forward exactly like the job's chunker until the source reports
// either an error (correct) or end-of-part (must not happen).
let mut offset = 0u64;
let outcome = loop {
match source.get_chunk(&key, offset, 64 * 1024).await {
Ok(chunk) if chunk.is_empty() => break Ok(()),
Ok(chunk) => offset += chunk.len() as u64,
Err(e) => break Err(e),
}
};
assert!(
outcome.is_err(),
"mid-stream decode failure must error the read, not end the part early"
);
assert_eq!(
source.size(&key).await.unwrap(),
None,
"a truncated decode must never be recorded as the part's total size"
);
}
#[tokio::test]
async fn test_empty_200_body_is_an_empty_part_by_design() {
// A 200 with a zero-byte body (an export day with no events, without gzip
// framing) must become an empty part with its size known immediately after
// prepare - mirroring the 404 branch and s3_gzip's zero-byte handling -
// not a streaming part.
let server = MockServer::start();
let _mock = server.mock(|when, then| {
when.method(httpmock::Method::GET).path("/export");
then.status(200).body(Vec::<u8>::new());
});
let staging = TempDir::new().unwrap();
let source = create_gzip_source(server.url("/export"), staging.path().to_path_buf());
source.prepare_for_job().await.unwrap();
let key = source.keys().await.unwrap().remove(0);
source.prepare_key(&key).await.unwrap();
assert_eq!(
source.size(&key).await.unwrap(),
Some(0),
"empty body must prepare as an empty part with a known size"
);
assert!(source.get_chunk(&key, 0, 1024).await.unwrap().is_empty());
assert_eq!(
count_raw_files(staging.path()),
0,
"the zero-byte .raw must not be kept"
);
}
#[tokio::test]
async fn test_interval_generation() {
let server = MockServer::start();
let _staging = TempDir::new().unwrap();
let source = create_test_source(server.url("/export"), 3600, _staging.path().to_path_buf()); // 1 hour intervals
let keys = source.keys().await.unwrap();
assert_eq!(keys.len(), 6); // 6 hours with 1-hour intervals
assert!(keys[0].starts_with("2023-01-01T00:00:00"));
assert!(keys[0].contains("2023-01-01T01:00:00"));
}
#[tokio::test]
async fn test_parse_retry_after_seconds() {
// delta-seconds parse
let mut headers = reqwest::header::HeaderMap::new();
headers.insert(
reqwest::header::RETRY_AFTER,
reqwest::header::HeaderValue::from_static("120"),
);
let d = super::parse_retry_after_header(&headers).unwrap();
assert_eq!(d.as_secs(), 120);
}
#[tokio::test]
async fn test_parse_retry_after_http_date() {
// HTTP-date parse
let future = httpdate::fmt_http_date(
std::time::SystemTime::now() + std::time::Duration::from_secs(90),
);
let mut headers = reqwest::header::HeaderMap::new();
headers.insert(
reqwest::header::RETRY_AFTER,
reqwest::header::HeaderValue::from_str(&future).unwrap(),
);
let d = super::parse_retry_after_header(&headers).unwrap();
assert!(d.as_secs() <= 90 && d.as_secs() > 0);
}
#[tokio::test]
async fn test_interval_key_parsing() {
let server = MockServer::start();
let _staging = TempDir::new().unwrap();
let source = create_test_source(server.url("/export"), 3600, _staging.path().to_path_buf());
let key = "2023-01-01T00:00:00+00:00_2023-01-01T01:00:00+00:00";
let interval = source.interval_from_key(key).unwrap();
assert_eq!(
interval.0,
Utc.with_ymd_and_hms(2023, 1, 1, 0, 0, 0).unwrap()
);
assert_eq!(
interval.1,
Utc.with_ymd_and_hms(2023, 1, 1, 1, 0, 0).unwrap()
);
}
#[tokio::test]
async fn test_successful_data_download_and_prepare() {
let server = MockServer::start();
// With the inclusive end date fix, the end query param is adjusted by subtracting
// interval_duration (3600 seconds = 1 hour), so end becomes the same as start.
let _mock = server.mock(|when, then| {
when.method(httpmock::Method::GET)
.path("/export")
.query_param("start", "2023-01-01T00:00:00Z")
.query_param("end", "2023-01-01T00:00:00Z");
then.status(200)
.header("content-type", "application/json")
.body(TEST_DATA);
});
let _staging = TempDir::new().unwrap();
let source = create_test_source(server.url("/export"), 3600, _staging.path().to_path_buf());
source.prepare_for_job().await.unwrap();
let keys = source.keys().await.unwrap();
let key = &keys[0];
source.prepare_key(key).await.unwrap();
// Size is discovered lazily: unknown until the stream has been read to EOF.
assert_eq!(source.size(key).await.unwrap(), None);
let data = read_key_to_end(&source, key, 8).await;
assert_eq!(data, TEST_DATA.as_bytes());
source.cleanup_after_job().await.unwrap();
}
#[tokio::test]
async fn test_404_handling() {
let server = MockServer::start();
let _mock = server.mock(|when, then| {
when.method(httpmock::Method::GET).path("/export");
then.status(404);
});
let _staging = TempDir::new().unwrap();
let source = create_test_source(server.url("/export"), 3600, _staging.path().to_path_buf());
source.prepare_for_job().await.unwrap();
let keys = source.keys().await.unwrap();
let key = &keys[0];
source.prepare_key(key).await.unwrap();
let size = source.size(key).await.unwrap();
assert_eq!(size, Some(0)); // Should create empty file for 404
source.cleanup_after_job().await.unwrap();
}
#[tokio::test]
async fn test_auth_configurations() {
let server = MockServer::start();
let _mock = server.mock(|when, then| {
when.method(httpmock::Method::GET)
.path("/export")
.header("X-API-Key", "test-key");
then.status(200).body(TEST_DATA);
});
let start = Utc.with_ymd_and_hms(2023, 1, 1, 0, 0, 0).unwrap();
let end = Utc.with_ymd_and_hms(2023, 1, 1, 1, 0, 0).unwrap();
let _staging = TempDir::new().unwrap();
let source = DateRangeExportSource::builder(
server.url("/export"),
start,
end,
3600,
Arc::new(MockExtractor),
_staging.path().to_path_buf(),
)
.with_auth(AuthConfig::ApiKey {
header_name: "X-API-Key".to_string(),
key: "test-key".to_string(),
})
.with_date_format("%Y-%m-%dT%H:%M:%SZ".to_string())
.with_headers(HashMap::new())
.build()
.unwrap();
source.prepare_for_job().await.unwrap();
let keys = source.keys().await.unwrap();
source.prepare_key(&keys[0]).await.unwrap();
source.cleanup_after_job().await.unwrap();
}
// Mixpanel service accounts need project_id, which the job config passes inside base_url.
#[tokio::test]
async fn test_basic_auth_keeps_query_params_in_base_url() {
let server = MockServer::start();
let mock = server.mock(|when, then| {
when.method(httpmock::Method::GET)
.path("/export")
.query_param("project_id", "123")
.query_param("from_date", "2023-01-01")
.query_param("to_date", "2023-01-01")
.header("Authorization", "Basic c2EtdXNlcjpzYS1zZWNyZXQ=");
then.status(200).body(TEST_DATA);
});
let start = Utc.with_ymd_and_hms(2023, 1, 1, 0, 0, 0).unwrap();
let end = Utc.with_ymd_and_hms(2023, 1, 2, 0, 0, 0).unwrap();
let _staging = TempDir::new().unwrap();
let source = DateRangeExportSource::builder(
server.url("/export?project_id=123"),
start,
end,
86400,
Arc::new(MockExtractor),
_staging.path().to_path_buf(),
)
.with_query_params("from_date".to_string(), "to_date".to_string())
.with_auth(AuthConfig::BasicAuth {
username: "sa-user".to_string(),
password: "sa-secret".to_string(),
})
.with_date_format("%Y-%m-%d".to_string())
.with_headers(HashMap::new())
.build()
.unwrap();
source.prepare_for_job().await.unwrap();
let keys = source.keys().await.unwrap();
source.prepare_key(&keys[0]).await.unwrap();
source.cleanup_after_job().await.unwrap();
mock.assert();
}
#[tokio::test]
async fn test_bearer_token_auth() {
let server = MockServer::start();
let _mock = server.mock(|when, then| {
when.method(httpmock::Method::GET)
.path("/export")
.header("Authorization", "Bearer test-token");
then.status(200).body(TEST_DATA);
});
let start = Utc.with_ymd_and_hms(2023, 1, 1, 0, 0, 0).unwrap();
let end = Utc.with_ymd_and_hms(2023, 1, 1, 1, 0, 0).unwrap();
let _staging = TempDir::new().unwrap();
let source = DateRangeExportSource::builder(
server.url("/export"),
start,
end,
3600,
Arc::new(MockExtractor),
_staging.path().to_path_buf(),
)
.with_auth(AuthConfig::BearerToken {
token: "test-token".to_string(),
})
.with_date_format("%Y-%m-%dT%H:%M:%SZ".to_string())
.with_headers(HashMap::new())
.build()
.unwrap();
source.prepare_for_job().await.unwrap();
let keys = source.keys().await.unwrap();
source.prepare_key(&keys[0]).await.unwrap();
source.cleanup_after_job().await.unwrap();
}
#[tokio::test]
async fn test_get_chunk_functionality() {
let server = MockServer::start();
let _mock = server.mock(|when, then| {
when.method(httpmock::Method::GET).path("/export");
then.status(200).body(TEST_DATA);
});
let _staging = TempDir::new().unwrap();
let source = create_test_source(server.url("/export"), 3600, _staging.path().to_path_buf());
source.prepare_for_job().await.unwrap();
let keys = source.keys().await.unwrap();
let key = &keys[0];
source.prepare_key(key).await.unwrap();
let chunk = source.get_chunk(key, 0, 10).await.unwrap();
assert_eq!(chunk.len(), 10);
assert_eq!(&chunk, &TEST_DATA.as_bytes()[0..10]);
let chunk = source.get_chunk(key, 10, 20).await.unwrap();
assert_eq!(chunk.len(), 20);
assert_eq!(&chunk, &TEST_DATA.as_bytes()[10..30]);
source.cleanup_after_job().await.unwrap();
}
#[tokio::test]
async fn test_get_chunk_beyond_file_size() {
let server = MockServer::start();
let _mock = server.mock(|when, then| {
when.method(httpmock::Method::GET).path("/export");
then.status(200).body(TEST_DATA);
});
let _staging = TempDir::new().unwrap();
let source = create_test_source(server.url("/export"), 3600, _staging.path().to_path_buf());
source.prepare_for_job().await.unwrap();
let keys = source.keys().await.unwrap();
let key = &keys[0];
source.prepare_key(key).await.unwrap();
let file_size = TEST_DATA.len() as u64;
// Reads are forward-only. A window that straddles EOF returns just the
// available tail; a subsequent read at EOF returns empty.
let chunk = source.get_chunk(key, file_size - 5, 20).await.unwrap();
assert_eq!(chunk.len(), 5);
let chunk = source.get_chunk(key, file_size, 10).await.unwrap();
assert!(chunk.is_empty());
source.cleanup_after_job().await.unwrap();
}
#[tokio::test]
async fn test_error_handling_401() {
use crate::error::get_user_message;
let server = MockServer::start();
let _mock = server.mock(|when, then| {
when.method(httpmock::Method::GET).path("/export");
then.status(401);
});
let _staging = TempDir::new().unwrap();
let source = create_test_source(server.url("/export"), 3600, _staging.path().to_path_buf());
source.prepare_for_job().await.unwrap();
let keys = source.keys().await.unwrap();
let key = &keys[0];
let result = source.prepare_key(key).await;
assert!(result.is_err());
let err = result.unwrap_err();
let user_msg = get_user_message(&err);
assert_eq!(user_msg, "Authentication failed, check your credentials");
source.cleanup_after_job().await.unwrap();
}
#[tokio::test]
async fn test_error_handling_500() {
use crate::error::get_user_message;
let server = MockServer::start();
let _mock = server.mock(|when, then| {
when.method(httpmock::Method::GET).path("/export");
then.status(500);
});
let _staging = TempDir::new().unwrap();
let source = create_test_source(server.url("/export"), 3600, _staging.path().to_path_buf());
source.prepare_for_job().await.unwrap();
let keys = source.keys().await.unwrap();
let key = &keys[0];
let result = source.prepare_key(key).await;
assert!(result.is_err());
let err = result.unwrap_err();
let user_msg = get_user_message(&err);
assert_eq!(user_msg, "Remote server error");
source.cleanup_after_job().await.unwrap();
}
#[tokio::test]
async fn test_custom_headers() {
let server = MockServer::start();
let _mock = server.mock(|when, then| {
when.method(httpmock::Method::GET)
.path("/export")
.header("X-Custom-Header", "custom-value");
then.status(200).body(TEST_DATA);
});
let start = Utc.with_ymd_and_hms(2023, 1, 1, 0, 0, 0).unwrap();
let end = Utc.with_ymd_and_hms(2023, 1, 1, 1, 0, 0).unwrap();
let _staging = TempDir::new().unwrap();
let mut headers = HashMap::new();
headers.insert("X-Custom-Header".to_string(), "custom-value".to_string());
let source = DateRangeExportSource::builder(
server.url("/export"),
start,
end,
3600,
Arc::new(MockExtractor),
_staging.path().to_path_buf(),
)
.with_auth(AuthConfig::None)
.with_date_format("%Y-%m-%dT%H:%M:%SZ".to_string())
.with_headers(headers)
.build()
.unwrap();
source.prepare_for_job().await.unwrap();
let keys = source.keys().await.unwrap();
source.prepare_key(&keys[0]).await.unwrap();
source.cleanup_after_job().await.unwrap();
}
#[tokio::test]
async fn test_key_cleanup() {
let server = MockServer::start();
let _mock = server.mock(|when, then| {
when.method(httpmock::Method::GET).path("/export");
then.status(200).body(TEST_DATA);
});
let _staging = TempDir::new().unwrap();
let source = create_test_source(server.url("/export"), 3600, _staging.path().to_path_buf());
source.prepare_for_job().await.unwrap();
let keys = source.keys().await.unwrap();
let key = &keys[0];
source.prepare_key(key).await.unwrap();
// The key is prepared and readable.
let chunk = source.get_chunk(key, 0, 8).await.unwrap();
assert!(!chunk.is_empty());
source.cleanup_key(key).await.unwrap();
// After cleanup the key is torn down, so reading it errors.
assert!(source.get_chunk(key, 0, 8).await.is_err());
source.cleanup_after_job().await.unwrap();
}
#[tokio::test]
async fn test_prepare_key_idempotent() {
let server = MockServer::start();
let mock = server.mock(|when, then| {
when.method(httpmock::Method::GET).path("/export");
then.status(200).body(TEST_DATA);
});
let _staging = TempDir::new().unwrap();
let source = create_test_source(server.url("/export"), 3600, _staging.path().to_path_buf());
source.prepare_for_job().await.unwrap();
let keys = source.keys().await.unwrap();
let key = &keys[0];
source.prepare_key(key).await.unwrap();
source.prepare_key(key).await.unwrap();
assert_eq!(mock.hits(), 1);
source.cleanup_after_job().await.unwrap();
}
#[tokio::test]
async fn test_reading_entire_file_frees_raw_and_keeps_size() {
let server = MockServer::start();
let _mock = server.mock(|when, then| {
when.method(httpmock::Method::GET).path("/export");
then.status(200).body(TEST_DATA);
});
let staging = TempDir::new().unwrap();
let source = create_test_source(server.url("/export"), 3600, staging.path().to_path_buf());
source.prepare_for_job().await.unwrap();
let keys = source.keys().await.unwrap();
let key = &keys[0];
source.prepare_key(key).await.unwrap();
assert_eq!(
count_raw_files(staging.path()),
1,
"prepared key stages one .raw"
);
// Reading the key to completion reconstructs the body, and the source frees
// the compressed `.raw` on the EOF read — no `cleanup_after_job` needed.
let data = read_key_to_end(&source, key, 8).await;
assert_eq!(data, TEST_DATA.as_bytes());
assert_eq!(
count_raw_files(staging.path()),
0,
"the .raw must be deleted once the key is fully read"
);
// The bookkeeping entry is retained so the now-known size is still reportable.
assert_eq!(
source.size(key).await.unwrap(),
Some(TEST_DATA.len() as u64)
);
source.cleanup_after_job().await.unwrap();
}
/// Robustness proof for the hex-security finding / the A1 class of regression:
/// staging cleanup happens on the EOF read itself, so a caller that consumes
/// the final chunk and immediately treats the part as done (issuing no further
/// read) still frees the `.raw`. A single read larger than the body reaches EOF
/// in one shot; we assert the file is gone without any terminal `offset>=total`
/// read, and that the retained entry keeps `prepare_key` from re-downloading it.
#[tokio::test]
async fn test_raw_freed_on_single_eof_read_without_terminal_read() {
let server = MockServer::start();
let mock = server.mock(|when, then| {
when.method(httpmock::Method::GET).path("/export");
then.status(200).body(TEST_DATA);
});
let staging = TempDir::new().unwrap();
let source = create_test_source(server.url("/export"), 3600, staging.path().to_path_buf());
source.prepare_for_job().await.unwrap();
let keys = source.keys().await.unwrap();
let key = &keys[0];
source.prepare_key(key).await.unwrap();
assert_eq!(mock.hits(), 1);
assert_eq!(count_raw_files(staging.path()), 1);
// One read past the end returns the whole body and reaches EOF.
let bytes = source
.get_chunk(key, 0, TEST_DATA.len() as u64 + 4096)
.await
.unwrap();
assert_eq!(bytes, TEST_DATA.as_bytes());
// Cleanup fired on that read alone — no second (terminal) get_chunk issued.
assert_eq!(
count_raw_files(staging.path()),
0,
"the .raw must be freed on the EOF read, without a terminal read"
);
assert_eq!(
source.size(key).await.unwrap(),
Some(TEST_DATA.len() as u64)
);
// The retained entry keeps prepare_key a no-op, so the freed file is not
// re-downloaded (which would re-stage the .raw we just reclaimed).
source.prepare_key(key).await.unwrap();
assert_eq!(
mock.hits(),
1,
"prepare_key must not re-download a freed key"
);
assert_eq!(count_raw_files(staging.path()), 0);
source.cleanup_after_job().await.unwrap();
}
#[tokio::test]
async fn test_format_date_range_from_key() {
let server = MockServer::start();
let _staging = TempDir::new().unwrap();
let source = create_test_source(server.url("/export"), 3600, _staging.path().to_path_buf());
let key = "2023-01-01T00:00:00+00:00_2023-01-01T01:00:00+00:00";
let formatted = source.format_date_range_from_key(key).unwrap();
assert_eq!(formatted, "2023-01-01 00:00 UTC to 2023-01-01 01:00 UTC");
}
#[tokio::test]
async fn test_format_date_range_from_key_different_dates() {
let server = MockServer::start();
let _staging = TempDir::new().unwrap();
let source = create_test_source(server.url("/export"), 3600, _staging.path().to_path_buf());
let key = "2023-12-31T23:30:00+00:00_2024-01-01T00:30:00+00:00";
let formatted = source.format_date_range_from_key(key).unwrap();
assert_eq!(formatted, "2023-12-31 23:30 UTC to 2024-01-01 00:30 UTC");
}
#[tokio::test]
async fn test_format_date_range_from_invalid_key() {
let server = MockServer::start();
let _staging = TempDir::new().unwrap();
let source = create_test_source(server.url("/export"), 3600, _staging.path().to_path_buf());
let key = "invalid-key-format";
let formatted = source.format_date_range_from_key(key);
assert!(formatted.is_none());
}
#[tokio::test]
async fn test_get_date_range_for_key_trait_method() {
let server = MockServer::start();
let _staging = TempDir::new().unwrap();
let source = create_test_source(server.url("/export"), 3600, _staging.path().to_path_buf());
let key = "2023-06-15T12:00:00+00:00_2023-06-15T13:00:00+00:00";
let date_range = source.get_date_range_for_key(key).unwrap();
assert_eq!(date_range, "2023-06-15 12:00 UTC to 2023-06-15 13:00 UTC");
}
#[tokio::test]
async fn test_format_date_range_edge_cases() {
let server = MockServer::start();
let _staging = TempDir::new().unwrap();
let source = create_test_source(server.url("/export"), 3600, _staging.path().to_path_buf());
let key_with_ms = "2023-01-01T00:00:00.123+00:00_2023-01-01T01:30:45.456+00:00";
let formatted = source.format_date_range_from_key(key_with_ms).unwrap();
assert_eq!(formatted, "2023-01-01 00:00 UTC to 2023-01-01 01:30 UTC");
let invalid_key = "2023-01-01T00:00:00+00:00";
assert!(source.format_date_range_from_key(invalid_key).is_none());
let invalid_key2 = "2023-01-01T00:00:00+00:00_2023-01-01T01:00:00+00:00_extra";
assert!(source.format_date_range_from_key(invalid_key2).is_none());
let invalid_date_key = "invalid-date_2023-01-01T01:00:00+00:00";
assert!(source
.format_date_range_from_key(invalid_date_key)
.is_none());
}
#[tokio::test]
async fn test_429_surfaces_immediately() {
let server = MockServer::start();
let mock = server.mock(|when, then| {
when.method(httpmock::Method::GET).path("/export");
then.status(429); // Always return 429
});
let _staging = TempDir::new().unwrap();
let source = DateRangeExportSource::builder(
server.url("/export"),
Utc.with_ymd_and_hms(2023, 1, 1, 0, 0, 0).unwrap(),
Utc.with_ymd_and_hms(2023, 1, 1, 1, 0, 0).unwrap(),
3600,
Arc::new(MockExtractor),
_staging.path().to_path_buf(),
)
.build()
.unwrap();
source.prepare_for_job().await.unwrap();
let keys = source.keys().await.unwrap();
let key = &keys[0];
let result = source.prepare_key(key).await;
assert!(result.is_err());
let err = result.unwrap_err();
// Our code wraps 429 as RateLimitedError (with reqwest::Error as source). Use helper.
assert!(crate::error::is_rate_limited_error(&err));
// No internal retry loop anymore
assert_eq!(
mock.hits(),
1,
"Single request, surfaced to job-level backoff"
);
source.cleanup_after_job().await.unwrap();
}
/// Test that verifies no duplicate events are fetched when using adjacent date intervals.
///
/// All date range export APIs (Mixpanel, Amplitude, etc.) use inclusive date ranges:
/// - from_date and to_date are both INCLUSIVE
/// - from_date=2025-06-05&to_date=2025-06-06 returns events from June 5 AND June 6
///
/// The implementation automatically adjusts the end date query parameter by subtracting
/// one interval_duration, so that each interval fetches only its own data without overlap.
///
/// The mock server handles all possible date combinations (both buggy and correct queries).
/// The test passes only if NO duplicate $insert_id values appear across all fetched data.
#[tokio::test]
async fn test_adjacent_intervals_should_not_produce_duplicate_events() {
// Test events for each day, with unique $insert_id values
let june_5_events = r#"{"event":"Event A","properties":{"time":1749085200,"distinct_id":"user1","$insert_id":"id_june5_event1"}}
{"event":"Event B","properties":{"time":1749088800,"distinct_id":"user2","$insert_id":"id_june5_event2"}}"#;
let june_6_events = r#"{"event":"Event C","properties":{"time":1749171600,"distinct_id":"user1","$insert_id":"id_june6_event1"}}
{"event":"Event D","properties":{"time":1749175200,"distinct_id":"user2","$insert_id":"id_june6_event2"}}"#;
let june_7_events = r#"{"event":"Event E","properties":{"time":1749258000,"distinct_id":"user1","$insert_id":"id_june7_event1"}}
{"event":"Event F","properties":{"time":1749261600,"distinct_id":"user2","$insert_id":"id_june7_event2"}}"#;
let server = MockServer::start();
// Mock all possible date query combinations, simulating the API's INCLUSIVE behavior
// Buggy queries (old implementation would send these without adjustment):
let _mock_buggy_1 = server.mock(|when, then| {
when.method(httpmock::Method::GET)
.path("/export")
.query_param("from_date", "2025-06-05")
.query_param("to_date", "2025-06-06");
then.status(200)
.body(format!("{}\n{}", june_5_events, june_6_events));
});
let _mock_buggy_2 = server.mock(|when, then| {
when.method(httpmock::Method::GET)
.path("/export")
.query_param("from_date", "2025-06-06")
.query_param("to_date", "2025-06-07");
then.status(200)
.body(format!("{}\n{}", june_6_events, june_7_events));
});
// Correct queries (implementation automatically subtracts interval_duration from end):
let _mock_fixed_1 = server.mock(|when, then| {
when.method(httpmock::Method::GET)
.path("/export")
.query_param("from_date", "2025-06-05")
.query_param("to_date", "2025-06-05");
then.status(200).body(june_5_events);
});
let _mock_fixed_2 = server.mock(|when, then| {
when.method(httpmock::Method::GET)
.path("/export")
.query_param("from_date", "2025-06-06")
.query_param("to_date", "2025-06-06");
then.status(200).body(june_6_events);
});
// Create source with Mixpanel-style config: daily intervals, %Y-%m-%d format
// The interval_duration (86400 seconds = 1 day) is automatically used to adjust
// the end date query parameter for inclusive APIs.
let start = Utc.with_ymd_and_hms(2025, 6, 5, 0, 0, 0).unwrap();
let end = Utc.with_ymd_and_hms(2025, 6, 7, 0, 0, 0).unwrap();
let _staging = TempDir::new().unwrap();
let source = DateRangeExportSource::builder(
server.url("/export"),
start,
end,
86400, // 1 day intervals - also used to adjust end date for inclusive APIs
Arc::new(MockExtractor),
_staging.path().to_path_buf(),
)
.with_query_params("from_date".to_string(), "to_date".to_string())
.with_date_format("%Y-%m-%d".to_string())
.build()
.unwrap();
source.prepare_for_job().await.unwrap();
let keys = source.keys().await.unwrap();
assert_eq!(keys.len(), 2, "Should have 2 daily intervals");
// Fetch data for both intervals
source.prepare_key(&keys[0]).await.unwrap();
let chunk0 = source.get_chunk(&keys[0], 0, 100000).await.unwrap();
source.prepare_key(&keys[1]).await.unwrap();
let chunk1 = source.get_chunk(&keys[1], 0, 100000).await.unwrap();
// Combine all fetched data and extract $insert_id values
let data0 = String::from_utf8(chunk0).unwrap();
let data1 = String::from_utf8(chunk1).unwrap();
let combined = format!("{}\n{}", data0, data1);
// Extract all $insert_id values from the combined data
let mut insert_ids: Vec<&str> = Vec::new();
for line in combined.lines() {
if line.trim().is_empty() {
continue;
}
if let Some(start_idx) = line.find("\"$insert_id\":\"") {
let rest = &line[start_idx + 14..];
if let Some(end_idx) = rest.find('"') {
insert_ids.push(&rest[..end_idx]);
}
}
}
// Check for duplicates
let mut seen = std::collections::HashSet::new();
let mut duplicates = Vec::new();
for id in &insert_ids {
if !seen.insert(*id) {
duplicates.push(*id);
}
}
assert!(
duplicates.is_empty(),
"DUPLICATE EVENTS DETECTED! The following $insert_id values appeared multiple times: {:?}\n\
This indicates that adjacent date intervals are fetching overlapping data.\n\
Total events fetched: {}, Unique events: {}\n\
Chunk 0 data:\n{}\n\
Chunk 1 data:\n{}",
duplicates,
insert_ids.len(),
seen.len(),
data0,
data1
);
source.cleanup_after_job().await.unwrap();
}
}
/// Remote (temp-bucket) staging mode: the source ingests each downloaded part into a
/// StagingBackend via the decompress pipeline and serves reads from it. These tests use
/// the real backend over `object_store::memory::InMemory` and real gzip bodies over
/// httpmock, exercising the exact production code path minus the network store.
#[cfg(test)]
mod remote_staging_tests {
use super::*;
use crate::extractor::ExtractorType;
use crate::staging::TempBucketBackend;
use chrono::{TimeZone, Utc};
use flate2::{write::GzEncoder, Compression};
use httpmock::{Mock, MockServer};
use object_store::memory::InMemory;
use object_store::ObjectStore;
use std::io::Write;
use std::path::Path;
use tempfile::TempDir;
const BODY: &[u8] = b"{\"event\":\"a\"}\n{\"event\":\"b\"}\n{\"event\":\"c\"}\n";
fn gzip(data: &[u8]) -> Vec<u8> {
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
encoder.write_all(data).unwrap();
encoder.finish().unwrap()
}
fn mock_export<'a>(server: &'a MockServer, body: &[u8]) -> Mock<'a> {
let gz = gzip(body);
server.mock(move |when, then| {
when.method(httpmock::Method::GET).path("/export");
then.status(200).body(gz.clone());
})
}
fn remote_staging(
store: Arc<dyn ObjectStore>,
max_plaintext_bytes: u64,
) -> Option<RemoteStaging> {
Some(RemoteStaging {
backend: Arc::new(TempBucketBackend::new(store, "staging/", "job-TEST")),
extractor_type: ExtractorType::PlainGzip,
max_plaintext_bytes,
})
}
/// One-hour-interval source over `hours` keys, staging remotely into `store`
/// (or locally when `remote` is None), decoding real gzip.
fn build_source(
base_url: String,
staging: &Path,
hours: u32,
remote: Option<RemoteStaging>,
) -> DateRangeExportSource {
let start = Utc.with_ymd_and_hms(2023, 1, 1, 0, 0, 0).unwrap();
let end = Utc.with_ymd_and_hms(2023, 1, 1, hours, 0, 0).unwrap();
DateRangeExportSource::builder(
base_url,
start,
end,
3600,
ExtractorType::PlainGzip.create_extractor(),
staging.to_path_buf(),
)
.with_auth(AuthConfig::None)
.with_date_format("%Y-%m-%dT%H:%M:%SZ".to_string())
.with_headers(HashMap::new())
.with_remote_staging(remote)
.build()
.unwrap()
}
/// Read a key back through `get_chunk` in small record-misaligned slices.
async fn read_all(source: &DateRangeExportSource, key: &str, slice: u64) -> Vec<u8> {
let total = source.size(key).await.unwrap().expect("size must be known");
let mut out = Vec::new();
let mut offset = 0u64;
while offset < total {
let chunk = source.get_chunk(key, offset, slice).await.unwrap();
assert!(!chunk.is_empty(), "read stalled before total");
offset += chunk.len() as u64;
out.extend_from_slice(&chunk);
}
out
}
#[tokio::test]
async fn remote_round_trip_multi_part_byte_identical_to_local() {
let server = MockServer::start();
let _mock = mock_export(&server, BODY);
let staging = TempDir::new().unwrap();
let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
let remote_src = build_source(
server.url("/export"),
staging.path(),
2,
remote_staging(Arc::clone(&store), 0),
);
remote_src.prepare_for_job().await.unwrap();
let keys = remote_src.keys().await.unwrap();
assert_eq!(keys.len(), 2);
// Local-mode source over the same origin is the byte-identity reference.
let local_staging = TempDir::new().unwrap();
let local_src = build_source(server.url("/export"), local_staging.path(), 2, None);
local_src.prepare_for_job().await.unwrap();
for key in &keys {
remote_src.prepare_key(key).await.unwrap();
local_src.prepare_key(key).await.unwrap();
// Size is known immediately after remote staging (no lazy-size pass).
assert_eq!(remote_src.size(key).await.unwrap(), Some(BODY.len() as u64));
let remote_bytes = read_all(&remote_src, key, 7).await;
assert_eq!(remote_bytes, BODY);
// Local streaming mode discovers size lazily; drive it with the same slices.
let mut local_bytes = Vec::new();
let mut offset = 0u64;
loop {
let chunk = local_src.get_chunk(key, offset, 7).await.unwrap();
if chunk.is_empty() {
break;
}
offset += chunk.len() as u64;
local_bytes.extend_from_slice(&chunk);
}
assert_eq!(remote_bytes, local_bytes, "backends must be byte-identical");
// The .raw is deleted right after a successful stage: staging disk holds
// nothing for this part while it is being read.
assert_eq!(count_raw_files_in(staging.path()), 0);
}
}
fn count_raw_files_in(dir: &Path) -> usize {
let mut count = 0;
let mut stack = vec![dir.to_path_buf()];
while let Some(d) = stack.pop() {
let Ok(entries) = std::fs::read_dir(&d) else {
continue;
};
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
stack.push(path);
} else if path.extension().and_then(|e| e.to_str()) == Some("raw") {
count += 1;
}
}
}
count
}
#[tokio::test]
async fn remote_resume_attaches_without_redownload() {
let server = MockServer::start();
let mock = mock_export(&server, BODY);
let staging = TempDir::new().unwrap();
let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
let src_a = build_source(
server.url("/export"),
staging.path(),
1,
remote_staging(Arc::clone(&store), 0),
);
src_a.prepare_for_job().await.unwrap();
let key = &src_a.keys().await.unwrap()[0];
src_a.prepare_key(key).await.unwrap();
assert_eq!(mock.hits(), 1);
// A fresh source (simulating a pod restart: empty in-memory state) over the same
// backend store must attach via head and never re-hit the origin.
let staging_b = TempDir::new().unwrap();
let src_b = build_source(
server.url("/export"),
staging_b.path(),
1,
remote_staging(Arc::clone(&store), 0),
);
src_b.prepare_for_job().await.unwrap();
src_b.prepare_key(key).await.unwrap();
assert_eq!(mock.hits(), 1, "resume must not re-download from origin");
assert_eq!(src_b.size(key).await.unwrap(), Some(BODY.len() as u64));
assert_eq!(read_all(&src_b, key, 7).await, BODY);
}
#[tokio::test]
async fn flip_mid_part_resumes_byte_identical_from_offset() {
// The no-Postgres-migration guarantee: a part half-read under local staging can
// be resumed at the same byte offset under temp-bucket staging (and vice versa)
// because both decode the identical plaintext stream.
let server = MockServer::start();
let _mock = mock_export(&server, BODY);
// Read a prefix under local streaming mode.
let local_staging = TempDir::new().unwrap();
let local_src = build_source(server.url("/export"), local_staging.path(), 1, None);
local_src.prepare_for_job().await.unwrap();
let key = &local_src.keys().await.unwrap()[0];
local_src.prepare_key(key).await.unwrap();
let offset = 17u64; // deliberately mid-record
let prefix = local_src.get_chunk(key, 0, offset).await.unwrap();
assert_eq!(prefix.len() as u64, offset);
// Flip: a fresh remote-mode source resumes from the persisted offset.
let staging = TempDir::new().unwrap();
let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
let remote_src = build_source(
server.url("/export"),
staging.path(),
1,
remote_staging(store, 0),
);
remote_src.prepare_for_job().await.unwrap();
remote_src.prepare_key(key).await.unwrap();
let suffix = remote_src
.get_chunk(key, offset, BODY.len() as u64)
.await
.unwrap();
let mut reassembled = prefix;
reassembled.extend_from_slice(&suffix);
assert_eq!(reassembled, BODY, "no gap or overlap across the flip");
}
#[tokio::test]
async fn flip_back_to_local_mid_part_resumes_byte_identical_from_offset() {
// The rollback direction of the flip guarantee: a part half-read under
// temp-bucket staging resumes at the same persisted byte offset under local
// streaming mode (STAGING_BACKEND flipped back), because both modes decode
// the identical plaintext stream.
let server = MockServer::start();
let _mock = mock_export(&server, BODY);
// Read a prefix under temp-bucket mode.
let staging = TempDir::new().unwrap();
let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
let remote_src = build_source(
server.url("/export"),
staging.path(),
1,
remote_staging(store, 0),
);
remote_src.prepare_for_job().await.unwrap();
let key = &remote_src.keys().await.unwrap()[0];
remote_src.prepare_key(key).await.unwrap();
let offset = 17u64; // deliberately mid-record
let prefix = remote_src.get_chunk(key, 0, offset).await.unwrap();
assert_eq!(prefix.len() as u64, offset);
// Flip back: a fresh local-mode source resumes from the persisted offset.
let local_staging = TempDir::new().unwrap();
let local_src = build_source(server.url("/export"), local_staging.path(), 1, None);
local_src.prepare_for_job().await.unwrap();
local_src.prepare_key(key).await.unwrap();
let suffix = local_src
.get_chunk(key, offset, BODY.len() as u64)
.await
.unwrap();
let mut reassembled = prefix;
reassembled.extend_from_slice(&suffix);
assert_eq!(
reassembled, BODY,
"no gap or overlap across the rollback flip"
);
}
#[tokio::test]
async fn remote_empty_part_stages_empty_object_and_skips_redownload() {
let server = MockServer::start();
let mock = server.mock(|when, then| {
when.method(httpmock::Method::GET).path("/export");
then.status(404);
});
let staging = TempDir::new().unwrap();
let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
let src = build_source(
server.url("/export"),
staging.path(),
1,
remote_staging(store, 0),
);
src.prepare_for_job().await.unwrap();
let key = &src.keys().await.unwrap()[0];
src.prepare_key(key).await.unwrap();
assert_eq!(src.size(key).await.unwrap(), Some(0));
assert!(src.get_chunk(key, 0, 100).await.unwrap().is_empty());
// The staged empty object makes prepare_key idempotent: no second origin hit.
src.prepare_key(key).await.unwrap();
assert_eq!(mock.hits(), 1);
}
#[tokio::test]
async fn remote_ceiling_breach_pauses_and_stages_nothing() {
let server = MockServer::start();
let _mock = mock_export(&server, BODY);
let staging = TempDir::new().unwrap();
let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
let src = build_source(
server.url("/export"),
staging.path(),
1,
remote_staging(Arc::clone(&store), 16), // ceiling far below BODY.len()
);
src.prepare_for_job().await.unwrap();
let key = &src.keys().await.unwrap()[0];
let err = src.prepare_key(key).await.unwrap_err();
// Public message is actionable with no internal config names; the env-var
// detail lives in the internal chain for status_message/logs.
let user_msg = crate::error::get_user_message(&err);
assert!(
user_msg.contains("Split the import"),
"unexpected message: {user_msg}"
);
assert!(
!user_msg.contains("STAGED_PLAINTEXT_MAX_BYTES"),
"public message must not leak env var names: {user_msg}"
);
assert!(
format!("{err:#}").contains("STAGED_PLAINTEXT_MAX_BYTES=16"),
"internal chain must carry the limit detail: {err:#}"
);
// Failed-stage atomicity: no readable object was published.
assert_eq!(src.size(key).await.unwrap(), None);
// And the downloaded .raw is not left behind on the failure path (a paused job
// must not hold staging disk while awaiting resume).
assert_eq!(
count_raw_files_in(staging.path()),
0,
"a failed stage must not leak the downloaded .raw"
);
}
#[tokio::test]
async fn remote_cleanup_key_and_job_sweep() {
let server = MockServer::start();
let _mock = mock_export(&server, BODY);
let staging = TempDir::new().unwrap();
let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
let src = build_source(
server.url("/export"),
staging.path(),
2,
remote_staging(Arc::clone(&store), 0),
);
src.prepare_for_job().await.unwrap();
let keys = src.keys().await.unwrap();
src.prepare_key(&keys[0]).await.unwrap();
src.prepare_key(&keys[1]).await.unwrap();
// cleanup_key removes exactly one part; double-delete is a no-op.
src.cleanup_key(&keys[0]).await.unwrap();
assert_eq!(src.size(&keys[0]).await.unwrap(), None);
assert_eq!(src.size(&keys[1]).await.unwrap(), Some(BODY.len() as u64));
src.cleanup_key(&keys[0]).await.unwrap();
// cleanup_after_job sweeps the remainder.
src.cleanup_after_job().await.unwrap();
let fresh = build_source(
server.url("/export"),
staging.path(),
2,
remote_staging(store, 0),
);
assert_eq!(fresh.size(&keys[1]).await.unwrap(), None);
}
#[tokio::test]
async fn release_job_resources_keeps_staged_parts_for_resume() {
// The transient-interruption path (backoff / sink rollback / shutdown):
// local resources are freed but staged remote parts survive, so the resume
// re-attaches — no origin re-hit — and re-reads byte-identical data at any
// offset (the property a sink-rollback retry depends on).
let server = MockServer::start();
let mock = mock_export(&server, BODY);
let staging = TempDir::new().unwrap();
let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
let src = build_source(
server.url("/export"),
staging.path(),
1,
remote_staging(Arc::clone(&store), 0),
);
src.prepare_for_job().await.unwrap();
let key = src.keys().await.unwrap().remove(0);
src.prepare_key(&key).await.unwrap();
assert_eq!(mock.hits(), 1);
let before = src.get_chunk(&key, 5, 12).await.unwrap();
src.release_job_resources().await.unwrap();
// Staged object survived; the job temp dir did not.
assert_eq!(src.size(&key).await.unwrap(), Some(BODY.len() as u64));
assert_eq!(
std::fs::read_dir(staging.path()).unwrap().count(),
0,
"local job temp dir must be freed"
);
// A fresh source (the resuming pod) attaches without re-downloading and
// re-reads the identical bytes at the same offset.
let staging_b = TempDir::new().unwrap();
let resumed = build_source(
server.url("/export"),
staging_b.path(),
1,
remote_staging(store, 0),
);
resumed.prepare_for_job().await.unwrap();
resumed.prepare_key(&key).await.unwrap();
assert_eq!(mock.hits(), 1, "resume must not re-hit the origin");
assert_eq!(resumed.get_chunk(&key, 5, 12).await.unwrap(), before);
}
#[tokio::test]
async fn cleanup_after_job_forces_clean_redownload_on_resume() {
// The source-pause path: a human may fix the source file in place, so the
// sweep guarantees the resume re-downloads a clean copy — attaching to a
// stale staged copy of a corrupt file would just re-fail the job.
let server = MockServer::start();
let mock = mock_export(&server, BODY);
let staging = TempDir::new().unwrap();
let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
let src = build_source(
server.url("/export"),
staging.path(),
1,
remote_staging(Arc::clone(&store), 0),
);
src.prepare_for_job().await.unwrap();
let key = src.keys().await.unwrap().remove(0);
src.prepare_key(&key).await.unwrap();
assert_eq!(mock.hits(), 1);
src.cleanup_after_job().await.unwrap();
let staging_b = TempDir::new().unwrap();
let resumed = build_source(
server.url("/export"),
staging_b.path(),
1,
remote_staging(store, 0),
);
resumed.prepare_for_job().await.unwrap();
resumed.prepare_key(&key).await.unwrap();
assert_eq!(
mock.hits(),
2,
"resume after a terminal sweep must re-download from origin"
);
}
}