Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

11.3 KB
Raw Blame
//! Messages decoded in place, for reading a file's node changes.
//!
//! [`Decoder`](super::Decoder) builds an owned tree (boxed messages, copied
//! strings) that saving can patch. Opening a file only reads each node change
//! once, so it decodes into a [`Flat`] instead: scalars are read, strings and
//! byte arrays stay in the message they arrived in, and nested messages and
//! lists go to flat tables that are cleared and reused for the next node
//! change. Nothing is allocated per value once the tables have grown.
//!
//! Every field also remembers the bytes it was read from, so what is built
//! from a field can be shared between node changes whose field is encoded
//! the same ([`super::MsgRef::shared`]): instances of one component carry
//! identical overrides and derived layout, and most paints repeat.

use super::{Decoder, Field, Kind, MAX_DEPTH, Reader, Schema, Wire};
use crate::error::{Result, corrupt};
use std::any::{Any, TypeId};
use std::cell::RefCell;
use std::hash::Hasher;
use std::sync::Arc;

/// A value of a [`Flat`] message: scalars, borrowed strings and bytes, and
/// indices into the flat tables.
#[derive(Clone, Copy, Debug)]
pub(crate) enum FlatVal<'d> {
    Bool(bool),
    Uint(u32),
    Int(i32),
    Float(f32),
    Uint64(u64),
    Int64(i64),
    Str(&'d str),
    Bytes(&'d [u8]),
    Enum(u32, u32),
    /// A message, by index into [`Flat::msgs`].
    Msg(u32),
    /// `(start, len)` in [`Flat::items`].
    List(u32, u32),
    /// `(start, len)` in [`Flat::floats`].
    Floats(u32, u32),
    /// `(start, len)` in [`Flat::uints`].
    Uints(u32, u32),
}

#[derive(Clone, Copy, Debug)]
pub(crate) struct FlatField<'d> {
    /// Index into the definition's fields.
    pub index: u16,
    /// Where the value was read: `(start, end)` in the message bytes.
    pub span: (u32, u32),
    pub value: FlatVal<'d>,
}

#[derive(Clone, Copy, Debug)]
pub(crate) struct FlatMsg {
    pub def: u32,
    /// `(start, len)` in [`Flat::fields`], sorted by field index.
    pub start: u32,
    pub len: u32,
    /// Bit `index % 128` is set for every field present.
    pub present: u128,
}

/// Node changes decoded in place (see the module docs).
pub struct Flat<'d> {
    pub(crate) data: &'d [u8],
    pub(crate) msgs: Vec<FlatMsg>,
    pub(crate) fields: Vec<FlatField<'d>>,
    pub(crate) items: Vec<FlatVal<'d>>,
    pub(crate) floats: Vec<f32>,
    pub(crate) uints: Vec<u32>,
    /// Fields and list items of the messages being decoded; each message
    /// moves its own to `fields` and `items` when it is complete, so those
    /// stay contiguous per message and per list.
    field_stack: Vec<FlatField<'d>>,
    item_stack: Vec<FlatVal<'d>>,
    pub(crate) shared: RefCell<Shared>,
    /// Strings handed out by [`super::MsgRef::intern`].
    pub(super) strings: RefCell<super::FxSet<Arc<str>>>,
}

impl<'d> Flat<'d> {
    /// Empty tables for messages read from `data`.
    pub fn new(data: &'d [u8]) -> Self {
        Self {
            data,
            msgs: Vec::new(),
            fields: Vec::new(),
            items: Vec::new(),
            floats: Vec::new(),
            uints: Vec::new(),
            field_stack: Vec::new(),
            item_stack: Vec::new(),
            shared: RefCell::default(),
            strings: RefCell::default(),
        }
    }

    /// Empty tables for messages read from `data`, going on sharing what
    /// an earlier table over the same data shared ([`Flat::into_shared`]).
    pub fn with_shared(data: &'d [u8], shared: FlatShared) -> Self {
        Self {
            shared: RefCell::new(shared.shared),
            strings: RefCell::new(shared.strings),
            ..Self::new(data)
        }
    }

    /// What was shared between the messages decoded here, to go on sharing
    /// it in another table over the same data.
    pub fn into_shared(self) -> FlatShared {
        FlatShared {
            shared: self.shared.into_inner(),
            strings: self.strings.into_inner(),
        }
    }

    /// Forgets the decoded messages (keeping the tables' memory, and what
    /// was shared).
    pub fn clear(&mut self) {
        self.msgs.clear();
        self.fields.clear();
        self.items.clear();
        self.floats.clear();
        self.uints.clear();
    }

    /// Decodes a struct or message of type `def` from `r` (a reader over
    /// this table's data); returns its index for [`super::MsgRef::flat`].
    pub fn decode(&mut self, schema: &Schema, r: &mut Reader<'d>, def: u32) -> Result<u32> {
        debug_assert!(std::ptr::eq(r.bytes, self.data));
        self.decode_at(schema, r, def, 0)
    }

    fn decode_at(
        &mut self,
        schema: &Schema,
        r: &mut Reader<'d>,
        def_index: u32,
        depth: u32,
    ) -> Result<u32> {
        if depth > MAX_DEPTH {
            return Err(corrupt("kiwi: nesting too deep"));
        }
        let def = schema.def(def_index);
        let mark = self.field_stack.len();
        match def.kind {
            Kind::Struct => {
                for (i, field) in def.fields.iter().enumerate() {
                    self.read_field(schema, r, i as u16, field, depth)?;
                }
            }
            Kind::Message => loop {
                let id = r.var_uint()?;
                if id == 0 {
                    break;
                }
                let index = def
                    .index_of_id(id)
                    .ok_or_else(|| corrupt(format!("kiwi: {} has no field {id}", def.name)))?;
                self.read_field(schema, r, index, &def.fields[index as usize], depth)?;
            },
            Kind::Enum => return Err(corrupt("kiwi: an enum is not a message")),
        }
        let own = &mut self.field_stack[mark..];
        // Lookups binary-search by field index. Writers emit fields in
        // declaration order, so this is almost always already true.
        if !own.is_sorted_by_key(|f| f.index) {
            own.sort_by_key(|f| f.index);
        }
        let present = own
            .iter()
            .fold(0u128, |bits, f| bits | (1u128 << (f.index & 127)));
        let start = self.fields.len() as u32;
        let len = own.len() as u32;
        self.fields.extend_from_slice(&self.field_stack[mark..]);
        self.field_stack.truncate(mark);
        self.msgs.push(FlatMsg {
            def: def_index,
            start,
            len,
            present,
        });
        Ok(self.msgs.len() as u32 - 1)
    }

    fn read_field(
        &mut self,
        schema: &Schema,
        r: &mut Reader<'d>,
        index: u16,
        field: &Field,
        depth: u32,
    ) -> Result<()> {
        if !field.keep {
            return Decoder::new(schema).skip_field(r, field, depth);
        }
        let start = r.at as u32;
        let value = self.field(schema, r, field, depth)?;
        self.field_stack.push(FlatField {
            index,
            span: (start, r.at as u32),
            value,
        });
        Ok(())
    }

    fn field(
        &mut self,
        schema: &Schema,
        r: &mut Reader<'d>,
        field: &Field,
        depth: u32,
    ) -> Result<FlatVal<'d>> {
        if !field.array {
            return self.value(schema, r, field.wire, depth);
        }
        let len = r.var_uint()? as usize;
        Ok(match field.wire {
            Wire::Byte => FlatVal::Bytes(r.take(len)?),
            Wire::Float => {
                let start = self.floats.len() as u32;
                for _ in 0..len {
                    self.floats.push(r.float()?);
                }
                FlatVal::Floats(start, len as u32)
            }
            Wire::Uint => {
                let start = self.uints.len() as u32;
                for _ in 0..len {
                    self.uints.push(r.var_uint()?);
                }
                FlatVal::Uints(start, len as u32)
            }
            wire => {
                let mark = self.item_stack.len();
                for _ in 0..len {
                    let v = self.value(schema, r, wire, depth)?;
                    self.item_stack.push(v);
                }
                let start = self.items.len() as u32;
                self.items.extend_from_slice(&self.item_stack[mark..]);
                self.item_stack.truncate(mark);
                FlatVal::List(start, len as u32)
            }
        })
    }

    fn value(
        &mut self,
        schema: &Schema,
        r: &mut Reader<'d>,
        wire: Wire,
        depth: u32,
    ) -> Result<FlatVal<'d>> {
        Ok(match wire {
            Wire::Bool => FlatVal::Bool(r.byte()? != 0),
            Wire::Byte => FlatVal::Uint(u32::from(r.byte()?)),
            Wire::Int => FlatVal::Int(r.var_int()?),
            Wire::Uint => FlatVal::Uint(r.var_uint()?),
            Wire::Float => FlatVal::Float(r.float()?),
            Wire::String => FlatVal::Str(r.string()?),
            Wire::Int64 => {
                let v = r.var_uint64()?;
                FlatVal::Int64(((v >> 1) as i64) ^ -((v & 1) as i64))
            }
            Wire::Uint64 => FlatVal::Uint64(r.var_uint64()?),
            Wire::Enum(i) => FlatVal::Enum(i, r.var_uint()?),
            Wire::Struct(i) | Wire::Message(i) => {
                FlatVal::Msg(self.decode_at(schema, r, i, depth + 1)?)
            }
        })
    }

    /// The field at `index` of message `slot`, if present.
    #[inline]
    pub(crate) fn field_of(&self, slot: u32, index: u16) -> Option<&FlatField<'d>> {
        let m = &self.msgs[slot as usize];
        if m.present & (1u128 << (index & 127)) == 0 {
            return None;
        }
        let fields = &self.fields[m.start as usize..(m.start + m.len) as usize];
        fields
            .binary_search_by_key(&index, |f| f.index)
            .ok()
            .map(|at| &fields[at])
    }
}

/// What a [`Flat`] shared between its messages, kept apart from the
/// tables: the values refer to the data by position only.
#[derive(Default)]
pub struct FlatShared {
    shared: Shared,
    strings: super::FxSet<Arc<str>>,
}

/// What was built from fields, by the bytes they were read from.
#[derive(Default)]
pub(crate) struct Shared {
    entries: super::FxMap<u64, Vec<SharedEntry>>,
}

struct SharedEntry {
    def: u32,
    index: u16,
    span: (u32, u32),
    value: Box<dyn Any>,
}

impl Shared {
    pub(crate) fn key<R: 'static>(def: u32, index: u16, bytes: &[u8]) -> u64 {
        let mut h = super::FxHasher::default();
        h.write(bytes);
        h.write_u32(def);
        h.write_u16(index);
        std::hash::Hash::hash(&TypeId::of::<R>(), &mut h);
        h.finish()
    }

    /// The value built from the field `index` of a `def` whose bytes are
    /// `data[span]`, if one was.
    pub(crate) fn get<R: Clone + 'static>(
        &self,
        key: u64,
        def: u32,
        index: u16,
        data: &[u8],
        span: (u32, u32),
    ) -> Option<R> {
        let bytes = &data[span.0 as usize..span.1 as usize];
        self.entries.get(&key)?.iter().find_map(|e| {
            (e.def == def
                && e.index == index
                && data[e.span.0 as usize..e.span.1 as usize] == *bytes)
                .then(|| e.value.downcast_ref::<R>().cloned())
                .flatten()
        })
    }

    pub(crate) fn insert<R: 'static>(
        &mut self,
        key: u64,
        def: u32,
        index: u16,
        span: (u32, u32),
        value: R,
    ) {
        self.entries.entry(key).or_default().push(SharedEntry {
            def,
            index,
            span,
            value: Box::new(value),
        });
    }
}