Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

12.8 KB
Raw Blame
//! Tokens (ISO 32000-1 §7.2, §7.3): whitespace and comments, numbers,
//! literal and hexadecimal strings, names, delimiters, and keywords.

#[cfg(test)]
mod test;

use super::Name;

/// Byte classes: regular characters, whitespace, and delimiters.
const REGULAR: u8 = 0;
const WHITE: u8 = 1;
const DELIMITER: u8 = 2;

const CLASS: [u8; 256] = {
    let mut class = [REGULAR; 256];
    let white = [0u8, 9, 10, 12, 13, 32];
    let mut i = 0;
    while i < white.len() {
        class[white[i] as usize] = WHITE;
        i += 1;
    }
    let delimiters = *b"()<>[]{}/%";
    let mut i = 0;
    while i < delimiters.len() {
        class[delimiters[i] as usize] = DELIMITER;
        i += 1;
    }
    class
};

/// Whether `b` is PDF whitespace (NUL, tab, LF, FF, CR, space).
pub(crate) fn is_white(b: u8) -> bool {
    CLASS[usize::from(b)] == WHITE
}

/// Whether `b` is a regular character (part of a name, number, or keyword).
pub(crate) fn is_regular(b: u8) -> bool {
    CLASS[usize::from(b)] == REGULAR
}

/// A hexadecimal digit's value.
pub(crate) fn hex_value(b: u8) -> Option<u8> {
    match b {
        b'0'..=b'9' => Some(b - b'0'),
        b'a'..=b'f' => Some(b - b'a' + 10),
        b'A'..=b'F' => Some(b - b'A' + 10),
        _ => None,
    }
}

/// A token.
#[derive(Clone, Debug, PartialEq)]
pub(crate) enum Token<'a> {
    Int(i64),
    Real(f64),
    /// A literal or hexadecimal string, escapes decoded.
    String(Vec<u8>),
    Name(Name),
    ArrayOpen,
    ArrayClose,
    DictOpen,
    DictClose,
    /// `{` or `}`: PostScript calculator braces, junk elsewhere.
    Brace(u8),
    /// A run of regular characters that is not a number: `obj`, `R`,
    /// `true`, content operators.
    Keyword(&'a [u8]),
    /// A delimiter that starts nothing (`)`, a lone `>`).
    Junk(u8),
}

/// A token and the bytes it came from.
#[derive(Clone, Debug, PartialEq)]
pub(crate) struct Lexed<'a> {
    pub token: Token<'a>,
    pub start: usize,
    pub end: usize,
}

/// Reads tokens from bytes, from a position.
pub(crate) struct Lexer<'a> {
    data: &'a [u8],
    pos: usize,
}

/// Exact powers of ten for the fast path of number conversion.
const POW10: [f64; 23] = [
    1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16,
    1e17, 1e18, 1e19, 1e20, 1e21, 1e22,
];

/// Past this, further digits no longer fit the mantissa.
const MANTISSA_LIMIT: u64 = 1_000_000_000_000_000_000;

impl<'a> Lexer<'a> {
    /// A lexer at `pos`.
    pub fn new(data: &'a [u8], pos: usize) -> Lexer<'a> {
        Lexer { data, pos }
    }

    /// Moves to `pos`.
    pub fn set_pos(&mut self, pos: usize) {
        self.pos = pos;
    }

    /// Skips whitespace and comments.
    pub fn skip_white(&mut self) {
        while let Some(&b) = self.data.get(self.pos) {
            if is_white(b) {
                self.pos += 1;
            } else if b == b'%' {
                while let Some(&c) = self.data.get(self.pos) {
                    if c == b'\r' || c == b'\n' {
                        break;
                    }
                    self.pos += 1;
                }
            } else {
                break;
            }
        }
    }

    /// The next token, `None` at the end.
    pub fn next(&mut self) -> Option<Lexed<'a>> {
        self.skip_white();
        let start = self.pos;
        let &b = self.data.get(start)?;
        let token = match b {
            b'0'..=b'9' | b'+' | b'-' | b'.' => self.number(),
            b'/' => {
                self.pos += 1;
                Token::Name(self.name())
            }
            b'(' => {
                self.pos += 1;
                Token::String(self.literal())
            }
            b'<' => {
                if self.data.get(start + 1) == Some(&b'<') {
                    self.pos += 2;
                    Token::DictOpen
                } else {
                    self.pos += 1;
                    Token::String(self.hex())
                }
            }
            b'>' => {
                if self.data.get(start + 1) == Some(&b'>') {
                    self.pos += 2;
                    Token::DictClose
                } else {
                    self.pos += 1;
                    Token::Junk(b)
                }
            }
            b'[' => {
                self.pos += 1;
                Token::ArrayOpen
            }
            b']' => {
                self.pos += 1;
                Token::ArrayClose
            }
            b'{' | b'}' => {
                self.pos += 1;
                Token::Brace(b)
            }
            b')' => {
                self.pos += 1;
                Token::Junk(b)
            }
            _ => {
                let end = self.regular_end(start);
                self.pos = end;
                Token::Keyword(&self.data[start..end])
            }
        };
        Some(Lexed {
            token,
            start,
            end: self.pos,
        })
    }

    /// Where the run of regular characters from `from` ends.
    fn regular_end(&self, from: usize) -> usize {
        self.data
            .get(from..)
            .and_then(|rest| rest.iter().position(|&c| !is_regular(c)))
            .map_or(self.data.len(), |n| from + n)
    }

    /// A number: `[+-]digits[.digits]`, as Acrobat reads them (`4.`, `.5`,
    /// `--5` as `-5`, a minus inside a number ignored, a lone sign as 0, an
    /// exponent when digits follow the `e`).
    fn number(&mut self) -> Token<'a> {
        let d = self.data;
        let mut i = self.pos;
        let mut negative = false;
        match d.get(i) {
            Some(b'-') => {
                negative = true;
                while d.get(i) == Some(&b'-') {
                    i += 1;
                }
            }
            Some(b'+') => i += 1,
            _ => {}
        }
        let mut mantissa: u64 = 0;
        let mut exp10: i32 = 0;
        let mut dot = false;
        let mut digits = false;
        let mut exponent = false;
        while let Some(&c) = d.get(i) {
            match c {
                b'0'..=b'9' => {
                    digits = true;
                    if mantissa < MANTISSA_LIMIT {
                        mantissa = mantissa * 10 + u64::from(c - b'0');
                        if dot {
                            exp10 = exp10.saturating_sub(1);
                        }
                    } else if !dot {
                        exp10 = exp10.saturating_add(1);
                    }
                }
                b'.' if !dot => dot = true,
                b'-' if digits => {}
                b'e' | b'E' if digits => {
                    let (sign, at) = match d.get(i + 1) {
                        Some(b'-') => (-1, i + 2),
                        Some(b'+') => (1, i + 2),
                        _ => (1, i + 1),
                    };
                    if !d.get(at).is_some_and(u8::is_ascii_digit) {
                        break;
                    }
                    let mut e: i32 = 0;
                    i = at;
                    while let Some(&c) = d.get(i)
                        && c.is_ascii_digit()
                    {
                        e = e.saturating_mul(10).saturating_add(i32::from(c - b'0'));
                        i += 1;
                    }
                    exp10 = exp10.saturating_add(sign * e);
                    exponent = true;
                    break;
                }
                _ => break,
            }
            i += 1;
        }
        self.pos = i;
        if !digits {
            return Token::Int(0);
        }
        if !dot
            && !exponent
            && exp10 == 0
            && let Ok(v) = i64::try_from(mantissa)
        {
            return Token::Int(if negative { -v } else { v });
        }
        let v = compose(mantissa, exp10);
        Token::Real(if negative { -v } else { v })
    }

    /// A name after its `/`, `#xx` escapes decoded.
    fn name(&mut self) -> Name {
        let start = self.pos;
        let end = self.regular_end(start);
        self.pos = end;
        let raw = &self.data[start..end];
        if !raw.contains(&b'#') {
            return Name(raw.to_vec());
        }
        let mut out = Vec::with_capacity(raw.len());
        let mut i = 0;
        while let Some(&b) = raw.get(i) {
            if b == b'#'
                && let Some(h) = raw.get(i + 1).copied().and_then(hex_value)
                && let Some(l) = raw.get(i + 2).copied().and_then(hex_value)
            {
                out.push((h << 4) | l);
                i += 3;
                continue;
            }
            out.push(b);
            i += 1;
        }
        Name(out)
    }

    /// A literal string after its `(`: balanced parentheses, escapes, line
    /// continuations, and end-of-line markers read as LF.
    fn literal(&mut self) -> Vec<u8> {
        let d = self.data;
        let mut out = Vec::new();
        let mut depth = 1usize;
        while let Some(&b) = d.get(self.pos) {
            self.pos += 1;
            match b {
                b'(' => {
                    depth += 1;
                    out.push(b);
                }
                b')' => {
                    depth -= 1;
                    if depth == 0 {
                        return out;
                    }
                    out.push(b);
                }
                b'\\' => {
                    let Some(&e) = d.get(self.pos) else {
                        break;
                    };
                    self.pos += 1;
                    match e {
                        b'n' => out.push(b'\n'),
                        b'r' => out.push(b'\r'),
                        b't' => out.push(b'\t'),
                        b'b' => out.push(0x08),
                        b'f' => out.push(0x0c),
                        b'0'..=b'7' => {
                            let mut v = u32::from(e - b'0');
                            for _ in 0..2 {
                                match d.get(self.pos) {
                                    Some(&c @ b'0'..=b'7') => {
                                        v = v * 8 + u32::from(c - b'0');
                                        self.pos += 1;
                                    }
                                    _ => break,
                                }
                            }
                            out.push((v & 0xff) as u8);
                        }
                        b'\r' => {
                            if d.get(self.pos) == Some(&b'\n') {
                                self.pos += 1;
                            }
                        }
                        b'\n' => {}
                        // `\(`, `\)`, `\\`, and unknown escapes: the character.
                        _ => out.push(e),
                    }
                }
                b'\r' => {
                    if d.get(self.pos) == Some(&b'\n') {
                        self.pos += 1;
                    }
                    out.push(b'\n');
                }
                _ => out.push(b),
            }
        }
        out
    }

    /// A hexadecimal string after its `<`: whitespace and junk skipped, an
    /// odd final digit read as if followed by 0.
    fn hex(&mut self) -> Vec<u8> {
        let mut out = Vec::new();
        let mut high: Option<u8> = None;
        while let Some(&b) = self.data.get(self.pos) {
            self.pos += 1;
            if b == b'>' {
                break;
            }
            let Some(v) = hex_value(b) else {
                continue;
            };
            match high.take() {
                Some(h) => out.push((h << 4) | v),
                None => high = Some(v),
            }
        }
        if let Some(h) = high {
            out.push(h << 4);
        }
        out
    }
}

/// `mantissa × 10^exp10`, correctly rounded.
fn compose(mantissa: u64, exp10: i32) -> f64 {
    let v = if mantissa < 1 << 53 && (-22..=22).contains(&exp10) {
        let m = mantissa as f64;
        let p = POW10[exp10.unsigned_abs() as usize];
        if exp10 >= 0 { m * p } else { m / p }
    } else {
        format!("{mantissa}e{exp10}").parse::<f64>().unwrap_or(0.0)
    };
    if v.is_finite() { v } else { 0.0 }
}

/// The first occurrence of `needle` in `hay` at or after `from`.
pub(crate) fn find(hay: &[u8], needle: &[u8], from: usize) -> Option<usize> {
    let &first = needle.first()?;
    let last_start = hay.len().checked_sub(needle.len())?;
    let mut i = from;
    while i <= last_start {
        let at = i + hay[i..=last_start].iter().position(|&b| b == first)?;
        if hay[at..].starts_with(needle) {
            return Some(at);
        }
        i = at + 1;
    }
    None
}

/// The last occurrence of `needle` in `hay` that starts before `before`.
pub(crate) fn rfind(hay: &[u8], needle: &[u8], before: usize) -> Option<usize> {
    let last_start = hay.len().checked_sub(needle.len())?;
    let end = before.min(last_start + 1);
    (0..end).rev().find(|&i| hay[i..].starts_with(needle))
}