:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use anyhow::Error;
use flate2::read::MultiGzDecoder;
use serde::{Deserialize, Serialize};
use std::panic::AssertUnwindSafe;
use std::{
io::{Read, Seek, SeekFrom},
path::PathBuf,
sync::Arc,
};
use tokio::sync::mpsc;
use zip::ZipArchive;
/// Magic-byte prefixes of the compression formats we might be handed by mistake.
/// Each entry maps a constant header prefix to the format name.
///
/// A newline-delimited JSON import only ever expects plaintext JSON, whose values
/// begin with `{ [ " -`, a digit, `t`/`f`/`n`, or whitespace. None of the first
/// bytes below (`0x1f 0x28 0x50 0x42 0xfd 0x78`) collide with those, so a match at
/// the *start of a part* is an unambiguous "still compressed" signal rather than a
/// coincidence - see [`detect_compression_magic`].
const COMPRESSION_MAGICS: &[(&[u8], &str)] = &[
(&[0x1f, 0x8b], "gzip"),
(&[0x28, 0xb5, 0x2f, 0xfd], "zstd"),
(&[0x50, 0x4b, 0x03, 0x04], "zip"),
(&[0x50, 0x4b, 0x05, 0x06], "zip"), // empty archive
(&[0x50, 0x4b, 0x07, 0x08], "zip"), // spanned archive
(&[0x42, 0x5a, 0x68], "bzip2"),
(&[0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00], "xz"),
// zlib streams start with CM/CINFO byte 0x78 followed by an FLG byte chosen so
// that (CMF*256 + FLG) % 31 == 0. These four FLG values cover the compression
// levels every common zlib encoder emits.
(&[0x78, 0x01], "zlib"),
(&[0x78, 0x5e], "zlib"),
(&[0x78, 0x9c], "zlib"),
(&[0x78, 0xda], "zlib"),
];
/// Return the name of the compression format whose magic bytes prefix `data`, or
/// `None` if `data` does not start with a recognized compression header. Data
/// shorter than a candidate prefix simply does not match it.
pub(crate) fn detect_compression_magic(data: &[u8]) -> Option<&'static str> {
COMPRESSION_MAGICS
.iter()
.find(|(magic, _)| data.starts_with(magic))
.map(|(_, name)| *name)
}
/// Read the first bytes of `file` into `buf`, rewind to the start, and return
/// the compression format those bytes identify (if any). `buf` must be at least
/// as long as the longest entry in [`COMPRESSION_MAGICS`]; a file shorter than a
/// magic simply does not match it.
fn sniff_magic(
file: &mut std::fs::File,
buf: &mut [u8; 8],
) -> std::io::Result<Option<&'static str>> {
let n = file.read(buf)?;
file.seek(SeekFrom::Start(0))?;
Ok(detect_compression_magic(&buf[..n]))
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(rename_all = "snake_case")]
pub enum ExtractorType {
ZipGzipJson,
#[default]
PlainGzip,
}
impl ExtractorType {
pub fn create_extractor(&self) -> Arc<dyn PartExtractor> {
match self {
ExtractorType::ZipGzipJson => Arc::new(ZipGzipJsonExtractor),
ExtractorType::PlainGzip => Arc::new(PlainGzipExtractor),
}
}
}
/// The data source trait reads parts in forward, monotonically increasing byte
/// ranges. For sources that hand us a compressed file (export endpoints, gzipped
/// S3 objects), we previously decompressed the whole thing to a second on-disk
/// file so the range reads could `seek`. That made worst-case disk usage scale
/// with the *decompressed* size — a single multi-GB compressed export could
/// expand to tens of GB and exhaust the staging volume.
///
/// Instead we stream: a `PartExtractor` opens a [`StreamingReader`] over the
/// compressed file that decompresses on demand as the job reads forward, so disk
/// usage is bounded by the compressed `.raw` file alone. Decompression runs on a
/// dedicated producer thread (off the async runtime) feeding a bounded channel;
/// the reader keeps only a small carry buffer (~one chunk) of decompressed bytes.
pub trait PartExtractor: Send + Sync {
/// Open a forward streaming reader over the compressed file at `raw_file_path`.
fn open_reader(&self, raw_file_path: PathBuf) -> StreamingReader;
}
/// Block size the producer thread reads/sends at. Larger blocks keep channel
/// overhead low for multi-GB files.
const PRODUCER_BLOCK_SIZE: usize = 64 * 1024;
/// Bounded channel capacity (in blocks). Caps in-flight decompressed bytes and
/// applies backpressure to the producer thread, so memory stays bounded.
const PRODUCER_CHANNEL_CAPACITY: usize = 32;
pub(crate) type Block = Result<Vec<u8>, String>;
/// Spawn a decompression producer on a dedicated OS thread (not the tokio blocking
/// pool — the decoder owns the thread for the lifetime of the read) feeding a bounded
/// channel. The thread exits when the receiver is dropped.
///
/// Guards against a producer panic (e.g. an internal flate2/zip panic): without this, a
/// panic would drop `tx` and consumers would read the closed channel as a clean EOF,
/// recording a truncated part as complete (silent data loss). The panic is converted
/// into a channel error instead. Shared by [`StreamingReader`] and the staging
/// pipeline so both consumers get identical bytes and identical panic semantics.
pub(crate) fn spawn_producer_thread<F>(producer: F) -> mpsc::Receiver<Block>
where
F: FnOnce(mpsc::Sender<Block>) + Send + 'static,
{
let (tx, rx) = mpsc::channel(PRODUCER_CHANNEL_CAPACITY);
std::thread::spawn(move || {
let err_tx = tx.clone();
if std::panic::catch_unwind(AssertUnwindSafe(move || producer(tx))).is_err() {
let _unused =
err_tx.blocking_send(Err("decompression producer thread panicked".to_string()));
}
});
rx
}
/// A decompressed range returned by [`StreamingReader::read_at`].
pub struct ReadChunk {
pub bytes: Vec<u8>,
/// The total decompressed size, populated only once end-of-stream has been
/// observed. `None` while more data may follow — callers must not treat the
/// size as known until this is `Some`.
pub total: Option<u64>,
}
/// A forward-only, resumable reader over a decompressed stream.
///
/// The decompressed size is discovered lazily (only once the producer hits EOF),
/// because we never materialize the decompressed data and therefore cannot know
/// its length up front. Reads must be monotonically non-decreasing in `offset`;
/// small rewinds within the most recently returned span are supported (the job's
/// chunker can re-request the tail of a partially-consumed chunk), but seeking
/// backwards beyond the retained carry buffer is not.
pub struct StreamingReader {
rx: mpsc::Receiver<Block>,
/// Decompressed bytes for the half-open range `[carry_start, decoded_end)`.
carry: Vec<u8>,
carry_start: u64,
decoded_end: u64,
eof: bool,
/// A decode/producer error, latched so every subsequent read keeps failing.
/// The producer thread exits after sending an error, and without the latch a
/// retried read would see the closed channel as a clean EOF and report the
/// partially decoded length as the stream total - callers that retry reads
/// would then treat a truncated stream as complete (silent data loss).
error: Option<String>,
}
impl StreamingReader {
fn spawn<F>(producer: F) -> Self
where
F: FnOnce(mpsc::Sender<Block>) + Send + 'static,
{
Self {
rx: spawn_producer_thread(producer),
carry: Vec::new(),
carry_start: 0,
decoded_end: 0,
eof: false,
error: None,
}
}
/// Return decompressed bytes in `[offset, offset + max_len)`, clamped to the
/// end of the stream. `offset` must be >= the start of the retained window.
pub async fn read_at(&mut self, offset: u64, max_len: usize) -> Result<ReadChunk, Error> {
if let Some(e) = &self.error {
return Err(Error::msg(e.clone()));
}
if offset < self.carry_start {
return Err(Error::msg(format!(
"streaming reader received non-monotonic read: offset {offset} precedes retained window start {}",
self.carry_start
)));
}
let target = offset.saturating_add(max_len as u64);
while !self.eof && self.decoded_end < target {
match self.rx.recv().await {
Some(Ok(block)) => {
self.decoded_end += block.len() as u64;
self.carry.extend_from_slice(&block);
// Drop everything before `offset` to bound memory to ~one chunk.
// Forward-only access guarantees those bytes are never re-read.
if self.carry_start < offset {
let drop_to = offset.min(self.decoded_end);
let drop_len = (drop_to - self.carry_start) as usize;
self.carry.drain(0..drop_len);
self.carry_start = drop_to;
}
}
Some(Err(e)) => {
self.error = Some(e.clone());
return Err(Error::msg(e));
}
None => self.eof = true,
}
}
let avail_end = self.decoded_end.min(target);
let bytes = if avail_end <= offset {
Vec::new()
} else {
let start = (offset - self.carry_start) as usize;
let end = (avail_end - self.carry_start) as usize;
self.carry[start..end].to_vec()
};
let total = self.eof.then_some(self.decoded_end);
Ok(ReadChunk { bytes, total })
}
}
pub struct ZipGzipJsonExtractor;
impl PartExtractor for ZipGzipJsonExtractor {
fn open_reader(&self, raw_file_path: PathBuf) -> StreamingReader {
StreamingReader::spawn(move |tx| run_zip_gzip_json_producer(raw_file_path, tx))
}
}
pub struct PlainGzipExtractor;
impl PartExtractor for PlainGzipExtractor {
fn open_reader(&self, raw_file_path: PathBuf) -> StreamingReader {
StreamingReader::spawn(move |tx| run_plain_gzip_producer(raw_file_path, tx))
}
}
/// Decompress a gzip stream, streaming blocks to `tx`. Appends a trailing
/// newline if the decompressed content is non-empty and didn't already end with
/// one, so the downstream JSONL parser always sees a complete final line.
///
/// The part is dispatched on its magic bytes rather than assumed to be gzip:
/// export APIs that compress via content negotiation skip compression below a
/// size threshold (Mixpanel sends bodies <= 1400 bytes uncompressed even when
/// `Accept-Encoding: gzip` was requested), so a part with no recognized
/// compression header is streamed as already-plaintext JSONL. A recognized
/// *non-gzip* compression header is a compression-setting mismatch worth naming
/// up front, not corruption.
///
/// `MultiGzDecoder` decodes every gzip member in the file, not just the first.
/// PostHog's own S3 batch exports compress each record as its own gzip member
/// and concatenate them, so a single-member decoder would stop after the first
/// event and report a clean EOF, importing a truncated part as complete.
/// Single-member streams (Mixpanel, Amplitude) decode identically.
pub(crate) fn run_plain_gzip_producer(raw_file_path: PathBuf, tx: mpsc::Sender<Block>) {
let mut file = match std::fs::File::open(&raw_file_path) {
Ok(f) => f,
Err(e) => {
let _unused = tx.blocking_send(Err(format!(
"Failed to open gzip file {}: {e}",
raw_file_path.display()
)));
return;
}
};
let mut magic = [0u8; 8];
let sniffed = match sniff_magic(&mut file, &mut magic) {
Ok(sniffed) => sniffed,
Err(e) => {
let _unused = tx.blocking_send(Err(format!(
"Failed to read gzip file {}: {e}",
raw_file_path.display()
)));
return;
}
};
let mut reader: Box<dyn Read> = match sniffed {
Some("gzip") => Box::new(MultiGzDecoder::new(file)),
Some(fmt) => {
let _unused = tx.blocking_send(Err(format!(
"The file appears to be {fmt}-compressed, but this import expects gzip \
or plaintext - check the source's compression setting."
)));
return;
}
None => Box::new(file),
};
let mut buffer = vec![0u8; PRODUCER_BLOCK_SIZE];
let mut last_byte: Option<u8> = None;
let mut produced_any = false;
loop {
// Fill the block across multiple reads before sending: one decoder read
// never crosses a gzip member boundary, so per-record members would
// otherwise produce one allocation and one channel send per record.
let mut filled = 0usize;
let mut eof = false;
while filled < buffer.len() {
match reader.read(&mut buffer[filled..]) {
Ok(0) => {
eof = true;
break;
}
Ok(n) => filled += n,
Err(e) => {
// In passthrough mode the reader is the plain file, so a read
// failure is an I/O problem (permissions, disk), not corruption.
let msg = match sniffed {
Some("gzip") => format!("Failed to decompress gzip data: {e}"),
_ => format!(
"Failed to read plaintext file {}: {e}",
raw_file_path.display()
),
};
let _unused = tx.blocking_send(Err(msg));
return;
}
}
}
if filled > 0 {
produced_any = true;
last_byte = Some(buffer[filled - 1]);
if tx.blocking_send(Ok(buffer[..filled].to_vec())).is_err() {
return;
}
}
if eof {
break;
}
}
if produced_any && last_byte != Some(b'\n') {
let _unused = tx.blocking_send(Ok(vec![b'\n']));
}
}
/// Decompress every `.json.gz` entry of a zip archive in natural-sorted order,
/// streaming blocks to `tx`. A trailing newline is appended after each non-empty
/// entry that didn't end with one, so the downstream JSONL parser sees complete
/// lines at every entry boundary.
pub(crate) fn run_zip_gzip_json_producer(raw_file_path: PathBuf, tx: mpsc::Sender<Block>) {
let file = match std::fs::File::open(&raw_file_path) {
Ok(f) => f,
Err(e) => {
let _unused = tx.blocking_send(Err(format!(
"Failed to open zip file {}: {e}",
raw_file_path.display()
)));
return;
}
};
let mut archive = match ZipArchive::new(file) {
Ok(a) => a,
Err(e) => {
let _unused = tx.blocking_send(Err(format!("Failed to read zip archive: {e}")));
return;
}
};
let mut file_names: Vec<String> = (0..archive.len())
.filter_map(|i| {
archive.by_index(i).ok().and_then(|f| {
let name = f.name().to_string();
name.ends_with(".json.gz").then_some(name)
})
})
.collect();
file_names.sort_by(|a, b| natord::compare(a, b));
let mut buffer = vec![0u8; PRODUCER_BLOCK_SIZE];
for name in file_names {
let zip_file = match archive.by_name(&name) {
Ok(zf) => zf,
Err(e) => {
let _unused = tx.blocking_send(Err(format!(
"Failed to find file {name} in zip archive: {e}"
)));
return;
}
};
let mut decoder = MultiGzDecoder::new(zip_file);
let mut last_byte: Option<u8> = None;
let mut entry_bytes: u64 = 0;
loop {
// Fill the block across multiple reads, as in run_plain_gzip_producer.
let mut filled = 0usize;
let mut eof = false;
while filled < buffer.len() {
match decoder.read(&mut buffer[filled..]) {
Ok(0) => {
eof = true;
break;
}
Ok(n) => filled += n,
Err(e) => {
let _unused = tx.blocking_send(Err(format!(
"Failed to decompress gzip data from {name}: {e}"
)));
return;
}
}
}
if filled > 0 {
entry_bytes += filled as u64;
last_byte = Some(buffer[filled - 1]);
if tx.blocking_send(Ok(buffer[..filled].to_vec())).is_err() {
return;
}
}
if eof {
break;
}
}
if entry_bytes > 0 && last_byte != Some(b'\n') && tx.blocking_send(Ok(vec![b'\n'])).is_err()
{
return;
}
}
}
#[cfg(test)]
fn run_verbatim_producer(raw_file_path: PathBuf, tx: mpsc::Sender<Block>) {
let mut file = match std::fs::File::open(&raw_file_path) {
Ok(f) => f,
Err(e) => {
let _unused = tx.blocking_send(Err(format!(
"Failed to open file {}: {e}",
raw_file_path.display()
)));
return;
}
};
let mut buffer = vec![0u8; PRODUCER_BLOCK_SIZE];
loop {
match file.read(&mut buffer) {
Ok(0) => break,
Ok(n) => {
if tx.blocking_send(Ok(buffer[..n].to_vec())).is_err() {
return;
}
}
Err(e) => {
let _unused = tx.blocking_send(Err(format!("Failed to read file: {e}")));
return;
}
}
}
}
/// Producer that emits one block then panics, to simulate an internal decoder panic.
#[cfg(test)]
fn run_panicking_producer(tx: mpsc::Sender<Block>) {
let _unused = tx.blocking_send(Ok(b"partial".to_vec()));
panic!("simulated producer panic");
}
#[cfg(test)]
impl StreamingReader {
/// Stream the file at `raw_file_path` verbatim (no decompression, no newline
/// normalization). Used by tests that exercise the source plumbing with
/// already-plaintext bodies.
pub(crate) fn open_verbatim(raw_file_path: PathBuf) -> StreamingReader {
StreamingReader::spawn(move |tx| run_verbatim_producer(raw_file_path, tx))
}
/// A reader whose producer panics mid-stream. Used to prove a panic surfaces as an
/// error rather than a silently truncated EOF.
pub(crate) fn open_panicking() -> StreamingReader {
StreamingReader::spawn(run_panicking_producer)
}
/// Drive the reader forward to EOF in `chunk`-sized reads, returning the full
/// reconstructed stream and its discovered total size.
pub(crate) async fn read_to_end_for_test(&mut self, chunk: usize) -> (Vec<u8>, u64) {
let mut out = Vec::new();
let mut offset = 0u64;
loop {
let rc = self.read_at(offset, chunk).await.expect("read_at failed");
offset += rc.bytes.len() as u64;
out.extend_from_slice(&rc.bytes);
if let Some(total) = rc.total {
if offset >= total {
return (out, total);
}
} else if rc.bytes.is_empty() {
// Not at EOF yet but nothing produced — avoid spinning.
return (out, offset);
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use anyhow::Result;
use flate2::{write::GzEncoder, Compression};
use std::{fs::File as StdFile, io::Write};
use tempfile::TempDir;
use zip::{write::SimpleFileOptions, ZipWriter};
fn create_test_gzip_file(content: &str, path: &std::path::Path) -> Result<()> {
create_multi_member_gzip_file(&[content], path)
}
fn create_multi_member_gzip_file(members: &[&str], path: &std::path::Path) -> Result<()> {
let mut file = StdFile::create(path)?;
for content in members {
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
encoder.write_all(content.as_bytes())?;
file.write_all(&encoder.finish()?)?;
}
Ok(())
}
fn create_test_zip_with_gzip_json(
json_files: Vec<(&str, &str)>,
zip_path: &std::path::Path,
) -> Result<()> {
let file = StdFile::create(zip_path)?;
let mut zip = ZipWriter::new(file);
for (filename, content) in json_files {
let options = SimpleFileOptions::default();
zip.start_file(filename, options)?;
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
encoder.write_all(content.as_bytes())?;
let compressed = encoder.finish()?;
zip.write_all(&compressed)?;
}
zip.finish()?;
Ok(())
}
#[test]
fn test_extractory_type_default() {
let default_type = ExtractorType::default();
assert!(matches!(default_type, ExtractorType::PlainGzip));
}
#[test]
fn test_detect_compression_magic_positive() {
// Real headers emitted by each encoder must be recognized by name.
let cases: [(&[u8], &str); 8] = [
(&[0x1f, 0x8b, 0x08, 0x00], "gzip"),
(&[0x28, 0xb5, 0x2f, 0xfd, 0x00], "zstd"),
(b"PK\x03\x04rest", "zip"),
(b"PK\x05\x06", "zip"),
(b"BZh9blah", "bzip2"),
(&[0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00, 0x00], "xz"),
(&[0x78, 0x9c, 0xcb, 0x48], "zlib"),
(&[0x78, 0xda, 0x01], "zlib"),
];
for (bytes, expected) in cases {
assert_eq!(
detect_compression_magic(bytes),
Some(expected),
"bytes {bytes:02x?} should be detected as {expected}"
);
}
}
#[test]
fn test_detect_compression_magic_negative() {
// Plaintext JSONL and other non-magic inputs must never be flagged: this is
// what keeps the offset-0 job-loop guard from ever tripping on real data.
let cases: [&[u8]; 7] = [
b"{\"event\":\"x\"}\n", // bare JSON object line
b"[1,2,3]\n", // JSON array line
&[0xef, 0xbb, 0xbf, b'{'], // UTF-8 BOM then JSON
&[0x1f], // gzip first byte only - too short to match
&[0x78], // zlib first byte only - too short to match
&[0xff, 0xff, 0xff], // invalid-utf8 binary that is not a magic
b"", // empty
];
for bytes in cases {
assert_eq!(
detect_compression_magic(bytes),
None,
"bytes {bytes:02x?} must not be detected as compression"
);
}
}
#[tokio::test]
async fn test_plain_gzip_producer_enriches_wrong_format_error() {
// A zstd file reaching the gzip extractor is a compression-setting mismatch,
// not corruption. The decode error must name the actual format so the pause
// message is actionable, instead of a bare "failed to decompress" string.
let temp_dir = TempDir::new().unwrap();
let zstd_file = temp_dir.path().join("actually.zst");
// A minimal zstd magic-led body is enough: the gzip decoder rejects it at the header.
std::fs::write(&zstd_file, [0x28, 0xb5, 0x2f, 0xfd, 0x00, 0x01, 0x02, 0x03]).unwrap();
let mut reader = PlainGzipExtractor.open_reader(zstd_file);
let Err(err) = reader.read_at(0, 8192).await else {
panic!("a zstd file must not decode as gzip");
};
let msg = format!("{err:#}");
assert!(
msg.contains("zstd"),
"decode error should name the detected format, got: {msg}"
);
assert!(
msg.contains("gzip"),
"decode error should state the expected format, got: {msg}"
);
}
#[tokio::test]
async fn test_plain_gzip_producer_does_not_mislabel_corrupt_gzip() {
// A genuinely corrupt gzip (valid header, truncated body) is corruption, not a
// format mismatch - the error must stay generic and never claim another format.
let temp_dir = TempDir::new().unwrap();
let gzip_file = temp_dir.path().join("corrupt.gz");
create_test_gzip_file(&"line\n".repeat(10000), &gzip_file).unwrap();
let full = std::fs::read(&gzip_file).unwrap();
std::fs::write(&gzip_file, &full[..20]).unwrap();
let mut reader = PlainGzipExtractor.open_reader(gzip_file);
let Err(err) = reader.read_at(0, 8192).await else {
panic!("truncated gzip must error");
};
let msg = format!("{err:#}");
assert!(
!msg.contains("zstd") && !msg.contains("compression setting"),
"corrupt gzip must not be mislabeled as a format mismatch, got: {msg}"
);
}
#[tokio::test]
async fn test_plain_gzip_extractor_newline_normalization() -> Result<()> {
// The extractor guarantees exactly one trailing newline on non-empty
// content and leaves empty content untouched — for gzip parts and for
// plaintext parts alike. Export APIs that compress via content
// negotiation skip gzip below a size threshold (Mixpanel sends bodies
// <= 1400 bytes uncompressed even when Accept-Encoding: gzip was
// requested), so a plaintext part must decode identically to its
// gzipped equivalent instead of failing with "invalid gzip header".
// (case name, raw input, gzip the input, expected output)
let cases: [(&str, &str, bool, &str); 6] = [
(
"gzip already terminated",
"line1\nline2\nline3\n",
true,
"line1\nline2\nline3\n",
),
(
"gzip missing trailing newline",
"line1\nline2\nline3",
true,
"line1\nline2\nline3\n",
),
("gzip empty file", "", true, ""),
(
"plaintext already terminated",
"line1\nline2\nline3\n",
false,
"line1\nline2\nline3\n",
),
(
"plaintext missing trailing newline",
"line1\nline2\nline3",
false,
"line1\nline2\nline3\n",
),
("plaintext empty file", "", false, ""),
];
for (name, input, gzip, expected) in cases {
let temp_dir = TempDir::new()?;
let part_file = temp_dir.path().join("test.raw");
if gzip {
create_test_gzip_file(input, &part_file)?;
} else {
std::fs::write(&part_file, input)?;
}
let mut reader = PlainGzipExtractor.open_reader(part_file);
let (data, size) = reader.read_to_end_for_test(8192).await;
assert_eq!(
size as usize,
expected.len(),
"size mismatch for case: {name}"
);
assert_eq!(data, expected.as_bytes(), "data mismatch for case: {name}");
}
Ok(())
}
#[tokio::test]
async fn test_plain_gzip_extractor_concatenated_members() -> Result<()> {
// The S3 batch export shape: each record is its own gzip member,
// concatenated into one file (see run_plain_gzip_producer). Every member
// must decode, and newline normalization must apply to the reassembled
// stream, not per member.
let temp_dir = TempDir::new()?;
let gzip_file = temp_dir.path().join("multi.gz");
create_multi_member_gzip_file(
&[
"{\"event\":\"a\"}\n",
"{\"event\":\"b\"}\n",
"{\"event\":\"c\"}", // no trailing newline on the last member
],
&gzip_file,
)?;
let mut reader = PlainGzipExtractor.open_reader(gzip_file);
let (data, size) = reader.read_to_end_for_test(8192).await;
let expected = "{\"event\":\"a\"}\n{\"event\":\"b\"}\n{\"event\":\"c\"}\n";
assert_eq!(size as usize, expected.len());
assert_eq!(data, expected.as_bytes());
Ok(())
}
#[tokio::test]
async fn test_plain_gzip_extractor_trailing_garbage_errors() {
// A valid member followed by non-gzip bytes must error instead of
// reporting a clean EOF: the trailing bytes may be real data, and a
// clean EOF would record the part as complete without them.
let temp_dir = TempDir::new().unwrap();
let gzip_file = temp_dir.path().join("trailing.gz");
create_multi_member_gzip_file(&["{\"event\":\"a\"}\n"], &gzip_file).unwrap();
let mut file = StdFile::options().append(true).open(&gzip_file).unwrap();
file.write_all(b"not gzip data").unwrap();
let mut reader = PlainGzipExtractor.open_reader(gzip_file);
let mut offset = 0u64;
loop {
match reader.read_at(offset, 8192).await {
Err(_) => return,
Ok(chunk) => {
assert!(
chunk.total.is_none(),
"trailing garbage must not decode to a clean EOF"
);
assert!(!chunk.bytes.is_empty(), "no error and no progress");
offset += chunk.bytes.len() as u64;
}
}
}
}
#[tokio::test]
async fn test_plain_gzip_extractor_corrupt_midstream_errors() {
let temp_dir = TempDir::new().unwrap();
let gzip_file = temp_dir.path().join("corrupt.gz");
// Write a valid gzip, then truncate it so the header is intact but the
// deflate body (and trailer) are incomplete.
create_test_gzip_file(&"line\n".repeat(10000), &gzip_file).unwrap();
let full = std::fs::read(&gzip_file).unwrap();
assert!(full.len() > 50, "compressed fixture should be non-trivial");
std::fs::write(&gzip_file, &full[..20]).unwrap();
// Decoding the truncated stream must surface an error rather than
// silently truncating output or panicking.
let mut reader = PlainGzipExtractor.open_reader(gzip_file);
let result = reader.read_at(0, 8192).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_decode_error_is_latched_across_reads() {
// Once a decode error is delivered, every subsequent read must keep
// returning it. The producer thread exits after sending the error, so a
// naive re-read would see the closed channel as a clean EOF and report the
// partially decoded length as the stream total - callers that retry reads
// (the sources' get_chunk retry loops) would then record a truncated part
// as complete: silent data loss.
let temp_dir = TempDir::new().unwrap();
let gzip_file = temp_dir.path().join("corrupt.gz");
// Large content so plenty of blocks decode successfully before the
// truncation point - the dangerous shape, since decoded_end is well past
// zero when the error arrives.
create_test_gzip_file(&"line\n".repeat(100_000), &gzip_file).unwrap();
let full = std::fs::read(&gzip_file).unwrap();
std::fs::write(&gzip_file, &full[..full.len() - 5]).unwrap();
let mut reader = PlainGzipExtractor.open_reader(gzip_file);
// Drive forward until the decode error surfaces (some reads succeed first).
let mut offset = 0u64;
while let Ok(chunk) = reader.read_at(offset, 8192).await {
assert!(
chunk.total.is_none(),
"truncated stream must never report a total"
);
assert!(!chunk.bytes.is_empty(), "no progress and no error");
offset += chunk.bytes.len() as u64;
}
// The error must be sticky: re-reads (at the same or an earlier retained
// offset) return the error again, never a clean EOF with a bogus total.
for _ in 0..2 {
let retry = reader.read_at(offset, 8192).await;
assert!(
retry.is_err(),
"retried read after a decode error must keep erroring, not report EOF"
);
}
}
#[tokio::test]
async fn test_zip_gzip_json_extractor_single_file() -> Result<()> {
let temp_dir = TempDir::new()?;
let zip_file = temp_dir.path().join("test.zip");
let json_content = r#"{"event": "test", "timestamp": 123}
{"event": "test2", "timestamp": 456}"#;
create_test_zip_with_gzip_json(vec![("data.json.gz", json_content)], &zip_file)?;
let mut reader = ZipGzipJsonExtractor.open_reader(zip_file);
let (data, size) = reader.read_to_end_for_test(8192).await;
assert!(size > 0);
let extracted = String::from_utf8(data).unwrap();
assert!(extracted.contains("test"));
assert!(extracted.contains("test2"));
Ok(())
}
#[tokio::test]
async fn test_zip_gzip_json_extractor_multiple_files() -> Result<()> {
let temp_dir = TempDir::new()?;
let zip_file = temp_dir.path().join("multi.zip");
let json_files = vec![
("001.json.gz", r#"{"id": 1}"#),
("002.json.gz", r#"{"id": 2}"#),
("003.json.gz", r#"{"id": 3}"#),
];
create_test_zip_with_gzip_json(json_files, &zip_file)?;
let mut reader = ZipGzipJsonExtractor.open_reader(zip_file);
let (data, _size) = reader.read_to_end_for_test(8192).await;
let extracted = String::from_utf8(data).unwrap();
// Natural sort keeps members ordered; newline appended after each.
assert_eq!(extracted, "{\"id\": 1}\n{\"id\": 2}\n{\"id\": 3}\n");
Ok(())
}
#[tokio::test]
async fn test_zip_gzip_json_extractor_concatenated_members_in_entry() -> Result<()> {
// An entry whose bytes are concatenated gzip members must decode past
// the first member, like the plain-gzip path.
let temp_dir = TempDir::new()?;
let zip_file = temp_dir.path().join("multi_member.zip");
let file = StdFile::create(&zip_file)?;
let mut zip = ZipWriter::new(file);
zip.start_file("data.json.gz", SimpleFileOptions::default())?;
for content in ["{\"id\":1}\n", "{\"id\":2}"] {
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
encoder.write_all(content.as_bytes())?;
zip.write_all(&encoder.finish()?)?;
}
zip.finish()?;
let mut reader = ZipGzipJsonExtractor.open_reader(zip_file);
let (data, _size) = reader.read_to_end_for_test(8192).await;
assert_eq!(data, b"{\"id\":1}\n{\"id\":2}\n");
Ok(())
}
#[tokio::test]
async fn test_zip_gzip_json_extractor_ignores_non_json_gz_files() -> Result<()> {
let temp_dir = TempDir::new()?;
let zip_file = temp_dir.path().join("mixed.zip");
let file = StdFile::create(&zip_file)?;
let mut zip = ZipWriter::new(file);
zip.start_file("data.json.gz", SimpleFileOptions::default())?;
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
encoder.write_all(b"valid json")?;
let compressed = encoder.finish()?;
zip.write_all(&compressed)?;
zip.start_file("readme.txt", SimpleFileOptions::default())?;
zip.write_all(b"ignore me")?;
zip.start_file("data.json", SimpleFileOptions::default())?;
zip.write_all(b"also ignore")?;
zip.finish()?;
let mut reader = ZipGzipJsonExtractor.open_reader(zip_file);
let (data, _size) = reader.read_to_end_for_test(8192).await;
let extracted = String::from_utf8(data).unwrap();
assert!(extracted.contains("valid json"));
assert!(!extracted.contains("ignore me"));
assert!(!extracted.contains("also ignore"));
Ok(())
}
#[tokio::test]
async fn test_plain_gzip_extractor_nonexistent_file() {
let temp_dir = TempDir::new().unwrap();
let nonexistent_file = temp_dir.path().join("nonexistent.gz");
let mut reader = PlainGzipExtractor.open_reader(nonexistent_file);
// The open error surfaces on the first read.
let result = reader.read_at(0, 8192).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_zip_gzip_extractor_invalid_zip() {
let temp_dir = TempDir::new().unwrap();
let invalid_zip = temp_dir.path().join("invalid.zip");
std::fs::write(&invalid_zip, b"not a zip file").unwrap();
let mut reader = ZipGzipJsonExtractor.open_reader(invalid_zip);
let result = reader.read_at(0, 8192).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_zip_gzip_extractor_empty_zip() -> Result<()> {
let temp_dir = TempDir::new()?;
let zip_file = temp_dir.path().join("empty.zip");
let file = StdFile::create(&zip_file)?;
let zip = ZipWriter::new(file);
zip.finish()?;
let mut reader = ZipGzipJsonExtractor.open_reader(zip_file);
let (data, size) = reader.read_to_end_for_test(8192).await;
assert_eq!(size, 0);
assert!(data.is_empty());
Ok(())
}
#[tokio::test]
async fn test_large_file_handling() -> Result<()> {
let temp_dir = TempDir::new()?;
let large_content = "line\n".repeat(10000);
let gzip_file = temp_dir.path().join("large.gz");
create_test_gzip_file(&large_content, &gzip_file)?;
// Small read chunk + large content exercises the carry buffer / front-trim.
let mut reader = PlainGzipExtractor.open_reader(gzip_file);
let (data, size) = reader.read_to_end_for_test(777).await;
assert_eq!(size as usize, large_content.len());
assert_eq!(data, large_content.as_bytes());
Ok(())
}
#[tokio::test]
async fn test_trait_object_usage() -> Result<()> {
let temp_dir = TempDir::new()?;
let gzip_file = temp_dir.path().join("trait_test.gz");
create_test_gzip_file("trait test content\n", &gzip_file)?;
let extractor: Arc<dyn PartExtractor> = Arc::new(PlainGzipExtractor);
let mut reader = extractor.open_reader(gzip_file);
let (data, size) = reader.read_to_end_for_test(8192).await;
assert_eq!(size as usize, "trait test content\n".len());
assert_eq!(data, b"trait test content\n");
Ok(())
}
#[tokio::test]
async fn test_read_at_clamps_and_reports_total_past_eof() {
let temp_dir = TempDir::new().unwrap();
let gzip_file = temp_dir.path().join("clamp.gz");
create_test_gzip_file("abc\n", &gzip_file).unwrap();
let mut reader = PlainGzipExtractor.open_reader(gzip_file);
// Reading well past the end returns only available bytes + the total.
let rc = reader.read_at(0, 1024).await.unwrap();
assert_eq!(rc.bytes, b"abc\n");
assert_eq!(rc.total, Some(4));
// A subsequent read at/after EOF returns empty with the known total.
let rc = reader.read_at(4, 1024).await.unwrap();
assert!(rc.bytes.is_empty());
assert_eq!(rc.total, Some(4));
}
#[tokio::test]
async fn test_read_at_rejects_backwards_seek() {
let temp_dir = TempDir::new().unwrap();
let gzip_file = temp_dir.path().join("seek.gz");
// Large enough that advancing far forward trims the front of the carry
// window, so the retained start moves well past 0.
create_test_gzip_file(&"x".repeat(1_000_000), &gzip_file).unwrap();
let mut reader = PlainGzipExtractor.open_reader(gzip_file);
reader.read_at(0, 1024).await.unwrap();
reader.read_at(900_000, 1024).await.unwrap();
// Seeking back before the (now-trimmed) retained window is rejected.
let result = reader.read_at(0, 1024).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_producer_panic_surfaces_as_error_not_truncated_eof() {
// A producer panic must not be mistaken for a clean end-of-stream, which would
// record a truncated part as complete (silent data loss). Drive the reader past
// the one block the producer emits before panicking, and require an error.
let mut reader = StreamingReader::open_panicking();
let mut offset = 0u64;
loop {
match reader.read_at(offset, 4).await {
// Expected: the panic is surfaced as a read error.
Err(_) => return,
Ok(chunk) => {
assert!(
chunk.total.is_none(),
"panic was swallowed as a clean EOF (total={:?})",
chunk.total
);
assert!(
!chunk.bytes.is_empty(),
"no error and no progress: panic was not surfaced"
);
offset += chunk.bytes.len() as u64;
}
}
}
}
}