:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! Avro schemas, read from a container header and written back as avsc writes them.
use std::collections::{HashMap, HashSet};
use std::sync::{Arc, LazyLock, Mutex, OnceLock};
use super::AvroError;
use crate::js::{json_parse, json_stringify, JsObject, JsString, JsValue};
#[derive(Debug, Clone, PartialEq)]
pub enum Schema {
Null,
Boolean,
Int,
Long,
Float,
Double,
Bytes,
String,
Record(Arc<RecordSchema>),
Enum(Arc<EnumSchema>),
Fixed(Arc<FixedSchema>),
Array(Box<Schema>),
Map(Box<Schema>),
/// avsc's default unwrapped union: the value carries its own branch.
Union(Vec<Schema>),
}
#[derive(Debug, Clone, PartialEq)]
pub struct RecordSchema {
pub name: String,
/// Fixed when the record is built, so a later definition with the same name cannot change it.
pub(crate) depth: usize,
pub fields: Vec<Field>,
pub aliases: Vec<String>,
pub doc: Option<String>,
pub field_index: HashMap<Arc<str>, usize>,
pub(crate) avsc_json: AvscJsonMemo,
}
/// Computed once. It takes no part in equality.
#[derive(Debug, Clone, Default)]
pub(crate) struct AvscJsonMemo(OnceLock<String>);
impl PartialEq for AvscJsonMemo {
fn eq(&self, _: &Self) -> bool {
true
}
}
/// Producers send the same few schemas, so parsed schemas are kept by their exact text.
const SCHEMA_CACHE_CAPACITY: usize = 64;
static SCHEMA_CACHE: LazyLock<Mutex<HashMap<String, Schema, ahash::RandomState>>> =
LazyLock::new(Default::default);
#[derive(Debug, Clone, PartialEq)]
pub struct Field {
pub name: Arc<str>,
pub schema: Schema,
pub default: Option<JsValue>,
pub order: Option<String>,
pub aliases: Vec<String>,
pub doc: Option<String>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct EnumSchema {
pub name: String,
pub symbols: Vec<String>,
pub default: Option<String>,
pub aliases: Vec<String>,
pub doc: Option<String>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct FixedSchema {
pub name: String,
pub size: usize,
pub aliases: Vec<String>,
}
fn schema_error(message: impl Into<String>) -> AvroError {
AvroError::Schema(message.into())
}
fn get_str<'a>(object: &'a JsObject, key: &str) -> Option<&'a str> {
object
.get(key)
.and_then(|v| v.as_js_string())
.and_then(JsString::as_str)
}
fn get_strings(object: &JsObject, key: &str) -> Vec<String> {
match object.get(key) {
Some(JsValue::Array(items)) => items
.iter()
.filter_map(|v| v.as_js_string().map(|s| s.to_string()))
.collect(),
_ => Vec::new(),
}
}
/// A name reference reaches past the JSON nesting limit, so resolved depth needs its own bound before any recursive walk.
const MAX_SCHEMA_DEPTH: usize = crate::js::MAX_BUILT_DEPTH;
/// Anonymous nesting follows the JSON nesting limit, and a record carries its own depth.
fn schema_depth(schema: &Schema) -> usize {
match schema {
Schema::Record(record) => record.depth,
Schema::Union(branches) => 1 + branches.iter().map(schema_depth).max().unwrap_or(0),
Schema::Array(inner) | Schema::Map(inner) => 1 + schema_depth(inner),
_ => 1,
}
}
struct Parser {
named: HashMap<String, Schema>,
}
fn qualify(name: &str, namespace: Option<&str>) -> String {
if name.contains('.') {
return name.to_string();
}
match namespace {
Some(ns) if !ns.is_empty() => format!("{ns}.{name}"),
_ => name.to_string(),
}
}
impl Parser {
/// avsc rejects a second definition of a name. Without this check, a redefinition that refers to the previous one can grow the schema exponentially.
fn check_unique(&self, name: &str) -> Result<(), AvroError> {
if self.named.contains_key(name) {
return Err(schema_error(format!("duplicate type name: {name}")));
}
Ok(())
}
fn parse(&mut self, value: &JsValue, namespace: Option<&str>) -> Result<Schema, AvroError> {
match value {
JsValue::String(name) => self.parse_name(&name.to_string(), namespace),
JsValue::Array(branches) => Ok(Schema::Union(
branches
.iter()
.map(|b| self.parse(b, namespace))
.collect::<Result<_, _>>()?,
)),
JsValue::Object(object) => self.parse_object(object, namespace),
_ => Err(schema_error("invalid schema")),
}
}
fn parse_name(&mut self, name: &str, namespace: Option<&str>) -> Result<Schema, AvroError> {
Ok(match name {
"null" => Schema::Null,
"boolean" => Schema::Boolean,
"int" => Schema::Int,
"long" => Schema::Long,
"float" => Schema::Float,
"double" => Schema::Double,
"bytes" => Schema::Bytes,
"string" => Schema::String,
other => self
.named
.get(&qualify(other, namespace))
.or_else(|| self.named.get(other))
.cloned()
.ok_or_else(|| schema_error(format!("undefined type name: {other}")))?,
})
}
fn parse_object(
&mut self,
object: &JsObject,
namespace: Option<&str>,
) -> Result<Schema, AvroError> {
let type_name = get_str(object, "type").ok_or_else(|| schema_error("missing type"))?;
match type_name {
"record" | "error" => {
let name = qualify(
get_str(object, "name").ok_or_else(|| schema_error("missing record name"))?,
get_str(object, "namespace").or(namespace),
);
self.check_unique(&name)?;
let ns = name.rsplit_once('.').map(|(ns, _)| ns.to_string());
let mut fields = Vec::new();
let Some(JsValue::Array(raw_fields)) = object.get("fields") else {
return Err(schema_error("missing fields"));
};
for raw in raw_fields {
let JsValue::Object(f) = raw else {
return Err(schema_error("invalid field"));
};
let field_type = f
.get("type")
.ok_or_else(|| schema_error("missing field type"))?;
fields.push(Field {
name: Arc::from(
get_str(f, "name").ok_or_else(|| schema_error("missing field name"))?,
),
schema: self.parse(field_type, ns.as_deref())?,
default: f.get("default").cloned(),
order: get_str(f, "order")
.filter(|o| *o != "ascending")
.map(str::to_string),
aliases: get_strings(f, "aliases"),
doc: get_str(f, "doc").map(str::to_string),
});
}
let field_index = fields
.iter()
.enumerate()
.map(|(i, f)| (f.name.clone(), i))
.collect();
let depth = 1 + fields
.iter()
.map(|f| schema_depth(&f.schema))
.max()
.unwrap_or(0);
if depth > MAX_SCHEMA_DEPTH {
return Err(schema_error("schema nesting too deep"));
}
let schema = Schema::Record(Arc::new(RecordSchema {
name: name.clone(),
depth,
fields,
aliases: get_strings(object, "aliases"),
doc: get_str(object, "doc").map(str::to_string),
field_index,
avsc_json: AvscJsonMemo::default(),
}));
self.named.insert(name, schema.clone());
Ok(schema)
}
"enum" => {
let name = qualify(
get_str(object, "name").ok_or_else(|| schema_error("missing enum name"))?,
get_str(object, "namespace").or(namespace),
);
self.check_unique(&name)?;
let schema = Schema::Enum(Arc::new(EnumSchema {
name: name.clone(),
symbols: get_strings(object, "symbols"),
default: get_str(object, "default").map(str::to_string),
aliases: get_strings(object, "aliases"),
doc: get_str(object, "doc").map(str::to_string),
}));
self.named.insert(name, schema.clone());
Ok(schema)
}
"fixed" => {
let name = qualify(
get_str(object, "name").ok_or_else(|| schema_error("missing fixed name"))?,
get_str(object, "namespace").or(namespace),
);
self.check_unique(&name)?;
let size = match object.get("size") {
Some(JsValue::Number(n)) if *n >= 0.0 => *n as usize,
_ => return Err(schema_error("invalid fixed size")),
};
let schema = Schema::Fixed(Arc::new(FixedSchema {
name: name.clone(),
size,
aliases: get_strings(object, "aliases"),
}));
self.named.insert(name, schema.clone());
Ok(schema)
}
"array" => Ok(Schema::Array(Box::new(
self.parse(
object
.get("items")
.ok_or_else(|| schema_error("missing items"))?,
namespace,
)?,
))),
"map" => Ok(Schema::Map(Box::new(
self.parse(
object
.get("values")
.ok_or_else(|| schema_error("missing values"))?,
namespace,
)?,
))),
// avsc reads a primitive written as an object as the plain primitive and drops an unregistered logical type.
primitive => self.parse_name(primitive, namespace),
}
}
}
impl Schema {
pub fn parse_json(text: &str) -> Result<Schema, AvroError> {
let value = json_parse(text).map_err(|e| schema_error(e.to_string()))?;
let mut parser = Parser {
named: HashMap::new(),
};
let schema = parser.parse(&value, None)?;
if schema_depth(&schema) > MAX_SCHEMA_DEPTH {
return Err(schema_error("schema nesting too deep"));
}
Ok(schema)
}
/// A parse is a pure function of the text, so a cached schema is the same value. Errors are not cached.
pub fn parse_json_cached(text: &str) -> Result<Schema, AvroError> {
if let Some(schema) = SCHEMA_CACHE.lock().unwrap().get(text) {
return Ok(schema.clone());
}
let schema = Schema::parse_json(text)?;
let mut cache = SCHEMA_CACHE.lock().unwrap();
if cache.len() >= SCHEMA_CACHE_CAPACITY {
cache.clear();
}
cache.insert(text.to_string(), schema.clone());
Ok(schema)
}
/// As avsc's `getSchema({ exportAttrs: true })`: it drops unregistered logical types and custom attributes, and names a type after its first definition.
pub fn to_avsc_json(&self) -> String {
self.avsc_json_str().to_string()
}
pub(crate) fn avsc_json_str(&self) -> std::borrow::Cow<'_, str> {
match self {
Schema::Record(record) => std::borrow::Cow::Borrowed(
record
.avsc_json
.0
.get_or_init(|| json_stringify(&self.export(&mut HashSet::new()))),
),
_ => std::borrow::Cow::Owned(json_stringify(&self.export(&mut HashSet::new()))),
}
}
fn export(&self, seen: &mut HashSet<String>) -> JsValue {
let s = |v: &str| JsValue::string(v);
let strings = |items: &[String]| JsValue::Array(items.iter().map(|i| s(i)).collect());
match self {
Schema::Null => s("null"),
Schema::Boolean => s("boolean"),
Schema::Int => s("int"),
Schema::Long => s("long"),
Schema::Float => s("float"),
Schema::Double => s("double"),
Schema::Bytes => s("bytes"),
Schema::String => s("string"),
Schema::Union(branches) => {
JsValue::Array(branches.iter().map(|b| b.export(seen)).collect())
}
Schema::Array(items) => {
let mut o = JsObject::new();
o.insert_str("type", s("array"));
o.insert_str("items", items.export(seen));
JsValue::Object(o)
}
Schema::Map(values) => {
let mut o = JsObject::new();
o.insert_str("type", s("map"));
o.insert_str("values", values.export(seen));
JsValue::Object(o)
}
Schema::Record(record) => {
if !seen.insert(record.name.clone()) {
return s(&record.name);
}
let mut o = JsObject::new();
o.insert_str("name", s(&record.name));
o.insert_str("type", s("record"));
let fields = record
.fields
.iter()
.map(|f| {
let mut fo = JsObject::new();
fo.insert_str("name", s(&f.name));
fo.insert_str("type", f.schema.export(seen));
if let Some(default) = &f.default {
fo.insert_str("default", default.clone());
}
if let Some(order) = &f.order {
fo.insert_str("order", s(order));
}
if !f.aliases.is_empty() {
fo.insert_str("aliases", strings(&f.aliases));
}
if let Some(doc) = &f.doc {
fo.insert_str("doc", s(doc));
}
JsValue::Object(fo)
})
.collect();
o.insert_str("fields", JsValue::Array(fields));
if !record.aliases.is_empty() {
o.insert_str("aliases", strings(&record.aliases));
}
if let Some(doc) = &record.doc {
o.insert_str("doc", s(doc));
}
JsValue::Object(o)
}
Schema::Enum(e) => {
if !seen.insert(e.name.clone()) {
return s(&e.name);
}
let mut o = JsObject::new();
o.insert_str("name", s(&e.name));
o.insert_str("type", s("enum"));
o.insert_str("symbols", strings(&e.symbols));
if let Some(default) = &e.default {
o.insert_str("default", s(default));
}
if !e.aliases.is_empty() {
o.insert_str("aliases", strings(&e.aliases));
}
if let Some(doc) = &e.doc {
o.insert_str("doc", s(doc));
}
JsValue::Object(o)
}
Schema::Fixed(f) => {
if !seen.insert(f.name.clone()) {
return s(&f.name);
}
let mut o = JsObject::new();
o.insert_str("name", s(&f.name));
o.insert_str("type", s("fixed"));
o.insert_str("size", JsValue::Number(f.size as f64));
if !f.aliases.is_empty() {
o.insert_str("aliases", strings(&f.aliases));
}
JsValue::Object(o)
}
}
}
/// The schema as avsc prints it in an "invalid value" error, for primitives.
pub(crate) fn short_name(&self) -> String {
match self {
Schema::Record(r) => r.name.clone(),
Schema::Enum(e) => e.name.clone(),
Schema::Fixed(f) => f.name.clone(),
other => {
let exported = other.export(&mut HashSet::new());
json_stringify(&exported)
}
}
}
}