Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

16.2 KB
Raw Blame
//! Container decoding with avsc's results, including its errors and its silent cases.

use std::cell::RefCell;

use super::schema::Schema;
use super::value::{AvroMap, AvroValue};
use super::{AvroError, Container, MAGIC};

/// A read past the end moves the position past the end and does not fail. The caller checks `is_valid`, as avsc does.
pub(crate) struct Tap<'a> {
    pub buf: &'a [u8],
    pub pos: usize,
    /// Values that consumed no bytes. A few bytes can declare any number of them, so they share one budget.
    pub empty_values: u64,
}

/// avsc's `isSafeLong` bound: one less than 2^53 - 1 on each side, for its rounding.
const SAFE_LONG: f64 = 9007199254740990.0;

const MAX_ARRAY_LENGTH: f64 = 4_294_967_295.0;

/// Node runs out of memory on a large count of empty values, and this port stops instead.
const MAX_EMPTY_VALUES: u64 = 1 << 20;

fn precision_loss() -> AvroError {
    AvroError::Decode("potential precision loss".to_string())
}

impl<'a> Tap<'a> {
    pub fn new(buf: &'a [u8]) -> Self {
        Self {
            buf,
            pos: 0,
            empty_values: 0,
        }
    }

    fn count_empty_value(&mut self) -> Result<(), AvroError> {
        self.empty_values += 1;
        if self.empty_values > MAX_EMPTY_VALUES {
            return Err(AvroError::Decode("too many empty records".to_string()));
        }
        Ok(())
    }

    pub fn is_valid(&self) -> bool {
        self.pos <= self.buf.len()
    }

    fn invalidate(&mut self) {
        self.pos = self.buf.len() + 1;
    }

    /// A byte, or 0 past the end as `buf[pos]` gives `undefined` in JavaScript.
    fn byte(&mut self) -> u8 {
        let b = self.buf.get(self.pos).copied().unwrap_or(0);
        self.pos = self.pos.saturating_add(1);
        b
    }

    /// The Node consumer patches avsc 5.7.9 to stop at 10 bytes, so an 11th byte invalidates the tap and every long is finite. See PORTING.md.
    pub fn read_long(&mut self) -> f64 {
        let mut n: i32 = 0;
        let mut k = 0;
        let mut b;
        let mut h;
        loop {
            b = self.byte();
            h = b & 0x80;
            n |= ((b & 0x7f) as i32) << k;
            k += 7;
            if h == 0 || k >= 28 {
                break;
            }
        }
        if h != 0 {
            let mut f = n as f64;
            let mut fk = 268_435_456.0;
            let mut read = 4;
            loop {
                if read == 10 {
                    self.invalidate();
                    return 0.0;
                }
                read += 1;
                b = self.byte();
                f += (b & 0x7f) as f64 * fk;
                fk *= 128.0;
                if b & 0x80 == 0 {
                    break;
                }
            }
            return (if f % 2.0 != 0.0 { -(f + 1.0) } else { f }) / 2.0;
        }
        ((n >> 1) ^ -(n & 1)) as f64
    }

    fn skip_long(&mut self) {
        while self.byte() & 0x80 != 0 {}
    }

    /// avsc moves `pos` by any length, but every later read past the end gives zeros, so the exact position does not matter.
    pub fn read_fixed(&mut self, len: usize) -> Option<&'a [u8]> {
        let start = self.pos;
        match start.checked_add(len) {
            Some(end) if end <= self.buf.len() => {
                self.pos = end;
                Some(&self.buf[start..end])
            }
            _ => {
                self.invalidate();
                None
            }
        }
    }

    /// A negative or oversized length leaves avsc's `pos` past the end, which is invalid too.
    pub fn read_bytes(&mut self) -> Option<&'a [u8]> {
        let len = self.read_long();
        let remaining = self.buf.len().saturating_sub(self.pos) as f64;
        if !(0.0..=remaining).contains(&len) {
            self.invalidate();
            return None;
        }
        self.read_fixed(len as usize)
    }

    /// Invalid UTF-8 decodes with U+FFFD for each maximal invalid subpart, as Node does.
    pub fn read_string(&mut self) -> Option<String> {
        self.read_bytes().map(|b| match std::str::from_utf8(b) {
            Ok(text) => text.to_owned(),
            Err(_) => String::from_utf8_lossy(b).into_owned(),
        })
    }

    fn remaining(&self) -> usize {
        self.buf.len().saturating_sub(self.pos)
    }

    /// A negative count carries a byte size, which avsc skips.
    fn read_array_size(&mut self) -> f64 {
        let n = self.read_long();
        if n < 0.0 {
            self.skip_long();
            return -n;
        }
        n
    }
}

/// Running out of data is not an error here, because the caller sees it through `is_valid`.
pub(crate) fn read_value(tap: &mut Tap<'_>, schema: &Schema) -> Result<AvroValue, AvroError> {
    let start = tap.pos;
    let value = read_schema_value(tap, schema)?;
    // Any value can expand into many empty children, so every value that reads no bytes is charged, at every depth.
    if tap.pos == start {
        tap.count_empty_value()?;
    }
    Ok(value)
}

fn read_schema_value(tap: &mut Tap<'_>, schema: &Schema) -> Result<AvroValue, AvroError> {
    Ok(match schema {
        Schema::Null => AvroValue::Null,
        Schema::Boolean => AvroValue::Boolean(tap.byte() != 0),
        Schema::Int => AvroValue::Int(tap.read_long() as i64),
        Schema::Long => {
            let n = tap.read_long();
            if !(-SAFE_LONG..=SAFE_LONG).contains(&n) {
                return Err(precision_loss());
            }
            AvroValue::Long(n as i64)
        }
        Schema::Float => {
            let b = tap.read_fixed(4);
            AvroValue::Float(b.map_or(0.0, |b| f32::from_le_bytes(b.try_into().unwrap()) as f64))
        }
        Schema::Double => {
            let b = tap.read_fixed(8);
            AvroValue::Double(b.map_or(0.0, |b| f64::from_le_bytes(b.try_into().unwrap())))
        }
        Schema::Bytes => AvroValue::Bytes(tap.read_bytes().map(<[u8]>::to_vec).unwrap_or_default()),
        Schema::String => AvroValue::String(tap.read_string().unwrap_or_default()),
        Schema::Fixed(f) => AvroValue::Fixed(
            tap.read_fixed(f.size)
                .map(<[u8]>::to_vec)
                .unwrap_or_default(),
        ),
        Schema::Enum(e) => {
            let index = tap.read_long();
            match e.symbols.get(index as usize).filter(|_| index >= 0.0) {
                Some(symbol) => AvroValue::Enum(symbol.clone()),
                None => {
                    return Err(AvroError::Decode(format!(
                        "invalid {} enum index: {}",
                        e.name,
                        crate::js::number_to_string(index)
                    )))
                }
            }
        }
        Schema::Union(branches) => {
            let index = tap.read_long();
            match branches
                .get(index as usize)
                .filter(|_| index >= 0.0 && index.fract() == 0.0)
            {
                Some(branch) => read_value(tap, branch)?,
                None => {
                    return Err(AvroError::Decode(format!(
                        "invalid union index: {}",
                        crate::js::number_to_string(index)
                    )))
                }
            }
        }
        Schema::Array(items) => {
            let mut out = Vec::new();
            let mut first_batch = true;
            loop {
                let mut n = tap.read_long();
                if n == 0.0 || n.is_nan() {
                    break;
                }
                if n < 0.0 {
                    n = -n;
                    tap.skip_long();
                }
                // avsc sizes the array from the first count with `new Array(n)`.
                if first_batch && n > MAX_ARRAY_LENGTH {
                    return Err(AvroError::Decode("Invalid array length".to_string()));
                }
                first_batch = false;
                if !tap.is_valid() {
                    break;
                }
                // A capacity hint only: a count can claim more items than the data holds.
                out.reserve((n as usize).min(tap.remaining()).min(4096));
                for _ in 0..n as u64 {
                    out.push(read_value(tap, items)?);
                    if !tap.is_valid() {
                        return Ok(AvroValue::Array(out));
                    }
                }
            }
            AvroValue::Array(out)
        }
        Schema::Map(values) => {
            let mut out = AvroMap::new();
            loop {
                let n = tap.read_array_size();
                if n == 0.0 || n.is_nan() || !tap.is_valid() {
                    break;
                }
                for _ in 0..n as u64 {
                    // avsc names an unreadable key `undefined`, and the record fails as a truncated block either way.
                    let key = tap.read_string().unwrap_or_else(|| "undefined".to_string());
                    let value = read_value(tap, values)?;
                    out.assign(key, value);
                    if !tap.is_valid() {
                        return Ok(AvroValue::Map(out));
                    }
                }
            }
            AvroValue::Map(out)
        }
        Schema::Record(record) => {
            let mut fields = Vec::with_capacity(record.fields.len());
            for field in &record.fields {
                fields.push((field.name.clone(), read_value(tap, &field.schema)?));
            }
            AvroValue::Record(fields)
        }
    })
}

struct Header {
    meta: Vec<(String, Vec<u8>)>,
    sync: [u8; 16],
}

/// `None` means it ran out of data, which avsc treats as "wait for more" and ends with no metadata.
fn read_header(tap: &mut Tap<'_>) -> Option<Header> {
    tap.read_fixed(4)?;
    let mut meta = Vec::new();
    loop {
        let n = tap.read_array_size();
        if n == 0.0 || n.is_nan() || !tap.is_valid() {
            break;
        }
        for _ in 0..n as u64 {
            let key = tap.read_string();
            let value = tap.read_bytes().map(<[u8]>::to_vec);
            if !tap.is_valid() {
                return None;
            }
            let key = key.unwrap_or_default();
            // A plain-object assignment: a later duplicate wins.
            match meta
                .iter_mut()
                .find(|(k, _): &&mut (String, Vec<u8>)| *k == key)
            {
                Some(entry) => entry.1 = value.unwrap_or_default(),
                None => meta.push((key, value.unwrap_or_default())),
            }
        }
    }
    let sync = tap.read_fixed(16)?;
    if !tap.is_valid() {
        return None;
    }
    Some(Header {
        meta,
        sync: sync.try_into().expect("16 bytes"),
    })
}

/// A larger stated size goes through the streaming decoder, which grows its buffer instead of trusting the header.
const MAX_PRESIZED_FRAME: u64 = 64 << 20;

const DEFAULT_WINDOW_LOG_MAX: u32 = 27;

/// The one-shot decoder skips zstd's default window limit, which Node enforces, so such frames take the streaming path.
fn declared_window_exceeds_default(data: &[u8]) -> bool {
    let Some(&descriptor) = data.get(4) else {
        return true;
    };
    if descriptor & 0x20 != 0 {
        return false;
    }
    let Some(&window) = data.get(5) else {
        return true;
    };
    let log = u32::from(window >> 3) + 10;
    let base = 1u64 << log;
    base + (base / 8) * u64::from(window & 7) > 1u64 << DEFAULT_WINDOW_LOG_MAX
}

thread_local! {
    /// A zstd context is large to set up, so each thread keeps one.
    static DECOMPRESSOR: RefCell<Option<zstd::bulk::Decompressor<'static>>> = const { RefCell::new(None) };
}

/// Any input other than one sized frame, or a failure there, takes the streaming path, whose result counts.
fn inflate(data: &[u8]) -> std::io::Result<Vec<u8>> {
    use zstd::zstd_safe::{find_frame_compressed_size, get_frame_content_size};
    if let (Ok(len), Ok(Some(size))) = (
        find_frame_compressed_size(data),
        get_frame_content_size(data),
    ) {
        if len == data.len() && size <= MAX_PRESIZED_FRAME && !declared_window_exceeds_default(data)
        {
            let fast = DECOMPRESSOR.with(|cell| {
                let mut slot = cell.borrow_mut();
                let decompressor = match slot.as_mut() {
                    Some(d) => d,
                    None => slot.insert(zstd::bulk::Decompressor::new()?),
                };
                let inflated = decompressor.decompress(data, size as usize);
                // A failed frame can leave state in the context, so the next frame gets a new one.
                if !matches!(&inflated, Ok(out) if out.len() as u64 == size) {
                    *slot = None;
                }
                inflated
            });
            if let Ok(inflated) = fast {
                if inflated.len() as u64 == size {
                    return Ok(inflated);
                }
            }
        }
    }
    zstd::stream::decode_all(data)
}

/// Only the zstandard codec is registered, as in `decodeLogRecords`.
pub fn decode_container(buf: &[u8]) -> Result<Container, AvroError> {
    if buf.len() < MAGIC.len() {
        return Err(AvroError::NoMetadata);
    }
    if buf[..4] != MAGIC {
        return Err(AvroError::InvalidMagic);
    }
    let mut tap = Tap::new(buf);
    let header = read_header(&mut tap).ok_or(AvroError::NoMetadata)?;
    let meta = |key: &str| header.meta.iter().find(|(k, _)| k == key).map(|(_, v)| v);
    let text = |v: &Vec<u8>| match std::str::from_utf8(v) {
        Ok(text) => text.to_owned(),
        Err(_) => String::from_utf8_lossy(v).into_owned(),
    };
    let codec = meta("avro.codec")
        .map(text)
        .unwrap_or_else(|| "null".to_string());
    if codec != "zstandard" {
        return Err(AvroError::UnknownCodec(codec));
    }
    let schema_json = meta("avro.schema")
        .map(text)
        .ok_or_else(|| AvroError::Schema("missing avro.schema".to_string()))?;
    let schema = Schema::parse_json_cached(&schema_json)?;

    // avsc checks every complete block's sync marker before any block decompresses, and it never reads an incomplete trailing block.
    let mut blocks = Vec::new();
    loop {
        // avsc reads the count as a `long`, which throws before the rest of the block is read.
        let count = tap.read_long();
        if !(-SAFE_LONG..=SAFE_LONG).contains(&count) {
            return Err(precision_loss());
        }
        let data = tap.read_bytes();
        let sync = tap.read_fixed(16);
        if !tap.is_valid() {
            break;
        }
        if sync != Some(&header.sync[..]) {
            return Err(AvroError::InvalidSyncMarker);
        }
        blocks.push((count, data.unwrap_or_default()));
        if tap.pos >= buf.len() {
            break;
        }
    }

    let mut records = Vec::new();
    let mut empty_values = 0u64;
    for (count, data) in blocks {
        let inflated =
            inflate(data).map_err(|_| AvroError::Decompression("zstandard".to_string()))?;
        // The sender controls `count`, so the reservation is bounded by bytes actually inflated.
        records.reserve(((count as i64).clamp(0, 4096) as usize).min(inflated.len()));
        // avsc keeps `count | 0`, and a negative count reads records until one fails.
        let mut remaining = i64::from((count as i64) as i32);
        let mut block = Tap::new(&inflated);
        block.empty_values = empty_values;
        while remaining != 0 {
            remaining -= 1;
            let start = block.pos;
            let value = read_value(&mut block, &schema)?;
            if !block.is_valid() {
                return Err(AvroError::Decode("truncated block".to_string()));
            }
            if block.pos == start && remaining < 0 {
                return Err(AvroError::Decode("too many empty records".to_string()));
            }
            records.push(value);
        }
        empty_values = block.empty_values;
    }

    Ok(Container {
        schema_json,
        schema,
        codec,
        records,
    })
}

#[cfg(test)]
mod tests {
    use super::Tap;

    #[test]
    fn read_long_stops_at_ten_bytes_like_the_patched_avsc() {
        let ten = [0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01];
        let mut tap = Tap::new(&ten);
        assert_eq!(tap.read_long(), 9_223_372_036_854_775_807.0);
        assert!(tap.is_valid());

        let eleven = [0xff; 11];
        let mut tap = Tap::new(&eleven);
        assert_eq!(tap.read_long(), 0.0);
        assert!(!tap.is_valid());
    }
}