:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! Container encoding laid out as avsc's `BlockEncoder` lays it out.
use std::cell::RefCell;
use rand::RngCore;
use super::schema::Schema;
use super::value::AvroValue;
use super::{AvroError, MAGIC};
/// avsc's default block size, in bytes of serialized records before compression.
const BLOCK_SIZE: usize = 65_536;
pub(crate) fn write_long(out: &mut Vec<u8>, n: i64) {
let mut z = ((n << 1) ^ (n >> 63)) as u64;
loop {
if z & !0x7f == 0 {
out.push(z as u8);
return;
}
out.push(((z & 0x7f) | 0x80) as u8);
z >>= 7;
}
}
fn write_bytes(out: &mut Vec<u8>, b: &[u8]) {
write_long(out, b.len() as i64);
out.extend_from_slice(b);
}
fn invalid(schema: &Schema, value: &AvroValue) -> AvroError {
AvroError::Encode(format!("invalid {}: {:?}", schema.short_name(), value))
}
fn is_safe_long(n: i64) -> bool {
(-9_007_199_254_740_990..=9_007_199_254_740_990).contains(&n)
}
/// avsc picks the branch by the value's JavaScript type.
fn union_branch<'s>(branches: &'s [Schema], value: &AvroValue) -> Option<(usize, &'s Schema)> {
branches.iter().enumerate().find(|(_, b)| {
matches!(
(b, value),
(Schema::Null, AvroValue::Null)
| (Schema::Boolean, AvroValue::Boolean(_))
| (
Schema::Int | Schema::Long | Schema::Float | Schema::Double,
AvroValue::Int(_)
| AvroValue::Long(_)
| AvroValue::Float(_)
| AvroValue::Double(_)
)
| (
Schema::Bytes | Schema::Fixed(_),
AvroValue::Bytes(_) | AvroValue::Fixed(_)
)
| (
Schema::String | Schema::Enum(_),
AvroValue::String(_) | AvroValue::Enum(_)
)
| (Schema::Array(_), AvroValue::Array(_))
| (Schema::Map(_), AvroValue::Map(_))
| (Schema::Record(_), AvroValue::Record(_))
)
})
}
fn as_number(value: &AvroValue) -> Option<f64> {
match value {
AvroValue::Int(n) | AvroValue::Long(n) => Some(*n as f64),
AvroValue::Float(f) | AvroValue::Double(f) => Some(*f),
_ => None,
}
}
pub(crate) fn write_value(
out: &mut Vec<u8>,
schema: &Schema,
value: &AvroValue,
) -> Result<(), AvroError> {
match schema {
Schema::Null => {
if !value.is_null() {
return Err(invalid(schema, value));
}
}
Schema::Boolean => match value {
AvroValue::Boolean(b) => out.push(*b as u8),
_ => return Err(invalid(schema, value)),
},
Schema::Int => match as_number(value) {
Some(n) if n.fract() == 0.0 && (i32::MIN as f64..=i32::MAX as f64).contains(&n) => {
write_long(out, n as i64)
}
_ => return Err(invalid(schema, value)),
},
Schema::Long => match as_number(value) {
Some(n) if n.fract() == 0.0 && is_safe_long(n as i64) => write_long(out, n as i64),
_ => return Err(invalid(schema, value)),
},
Schema::Float => match as_number(value) {
Some(n) => out.extend_from_slice(&(n as f32).to_le_bytes()),
None => return Err(invalid(schema, value)),
},
Schema::Double => match as_number(value) {
Some(n) => out.extend_from_slice(&n.to_le_bytes()),
None => return Err(invalid(schema, value)),
},
Schema::Bytes => match value.as_bytes() {
Some(b) => write_bytes(out, b),
None => return Err(invalid(schema, value)),
},
Schema::String => match value {
AvroValue::String(s) => write_bytes(out, s.as_bytes()),
_ => return Err(invalid(schema, value)),
},
Schema::Fixed(f) => match value.as_bytes() {
Some(b) if b.len() == f.size => out.extend_from_slice(b),
_ => return Err(invalid(schema, value)),
},
Schema::Enum(e) => {
let symbol = match value {
AvroValue::Enum(s) | AvroValue::String(s) => s,
_ => return Err(invalid(schema, value)),
};
match e.symbols.iter().position(|s| s == symbol) {
Some(i) => write_long(out, i as i64),
None => return Err(invalid(schema, value)),
}
}
Schema::Union(branches) => match union_branch(branches, value) {
Some((index, branch)) => {
write_long(out, index as i64);
write_value(out, branch, value)?;
}
None => return Err(invalid(schema, value)),
},
Schema::Array(items) => match value {
AvroValue::Array(values) => {
if !values.is_empty() {
write_long(out, values.len() as i64);
for v in values {
write_value(out, items, v)?;
}
}
out.push(0);
}
_ => return Err(invalid(schema, value)),
},
Schema::Map(values) => match value {
AvroValue::Map(map) => {
if !map.is_empty() {
write_long(out, map.len() as i64);
for (k, v) in map.iter() {
write_bytes(out, k.as_bytes());
write_value(out, values, v)?;
}
}
out.push(0);
}
_ => return Err(invalid(schema, value)),
},
Schema::Record(record) => match value {
AvroValue::Record(fields) => {
// Fields are in schema order, so each lookup tries the same position first. With unique names that match is the first field of the name.
let mut aligned = record.field_index.len() == record.fields.len();
for (i, field) in record.fields.iter().enumerate() {
let positional = match fields.get(i) {
Some((name, v)) if aligned && name == &field.name => Some(v),
_ => None,
};
// A missing field encodes as `undefined`, which a nullable union writes as null, as avsc does.
let v = match positional {
Some(v) => v,
None => {
aligned = false;
fields
.iter()
.find(|(name, _)| *name == field.name)
.map(|(_, v)| v)
.unwrap_or(&AvroValue::Null)
}
};
write_value(out, &field.schema, v)?;
}
}
_ => return Err(invalid(schema, value)),
},
}
Ok(())
}
thread_local! {
/// A zstd context is large to set up, so each thread keeps one at level 1.
static COMPRESSOR: RefCell<Option<zstd::bulk::Compressor<'static>>> = const { RefCell::new(None) };
}
fn zstd_compress(data: &[u8]) -> std::io::Result<Vec<u8>> {
COMPRESSOR.with(|cell| {
let mut slot = cell.borrow_mut();
let compressor = match slot.as_mut() {
Some(c) => c,
None => slot.insert(zstd::bulk::Compressor::new(1)?),
};
let compressed = compressor.compress(data);
if compressed.is_err() {
*slot = None;
}
compressed
})
}
fn compress(codec: &str, data: &[u8]) -> Result<Vec<u8>, AvroError> {
match codec {
"zstandard" => zstd_compress(data).map_err(|_| AvroError::Decompression(codec.to_string())),
"null" => Ok(data.to_vec()),
other => Err(AvroError::UnknownCodec(other.to_string())),
}
}
/// A record larger than the block size gets a block of its own and doubles the block size, as in avsc.
pub fn encode_container(
schema: &Schema,
codec: &str,
records: &[AvroValue],
) -> Result<Vec<u8>, AvroError> {
let mut sync = [0u8; 16];
rand::thread_rng().fill_bytes(&mut sync);
let mut out = Vec::new();
out.extend_from_slice(&MAGIC);
write_long(&mut out, 2);
write_bytes(&mut out, b"avro.schema");
write_bytes(&mut out, schema.avsc_json_str().as_bytes());
write_bytes(&mut out, b"avro.codec");
write_bytes(&mut out, codec.as_bytes());
out.push(0);
out.extend_from_slice(&sync);
let mut block_size = BLOCK_SIZE;
let mut block = Vec::new();
let mut count = 0i64;
let mut record = Vec::new();
let flush =
|out: &mut Vec<u8>, block: &mut Vec<u8>, count: &mut i64| -> Result<(), AvroError> {
let compressed = compress(codec, block)?;
write_long(out, *count);
write_bytes(out, &compressed);
out.extend_from_slice(&sync);
block.clear();
*count = 0;
Ok(())
};
for value in records {
record.clear();
write_value(&mut record, schema, value)?;
if block.len() + record.len() > block_size {
if !block.is_empty() {
flush(&mut out, &mut block, &mut count)?;
}
if record.len() > block_size {
block_size = record.len() * 2;
}
}
block.extend_from_slice(&record);
count += 1;
}
if !block.is_empty() {
flush(&mut out, &mut block, &mut count)?;
}
Ok(out)
}