Skip to content

The design language that makes your AI harness better at design.

JavaScript
30.0 KB
Raw Blame
//! Port of css-tree 3.2.1 `lib/tokenizer/` (CSS Syntax Level 3 tokenizer,
//! the char-code predicates, and `TokenStream` with its balance table).
//!
//! The JS walks UTF-16 code units; this port walks `char`s. Every predicate
//! treats any code point >= 0x80 the same way (name-start), so token
//! boundaries and the sliced text come out identical for both units.

// ─── Token types (lib/tokenizer/types.js) ───────────────────────────────────

pub type TokenType = u8;

pub const EOF: TokenType = 0;
pub const IDENT: TokenType = 1;
pub const FUNCTION: TokenType = 2;
pub const AT_KEYWORD: TokenType = 3;
pub const HASH: TokenType = 4;
pub const STRING: TokenType = 5;
pub const BAD_STRING: TokenType = 6;
pub const URL: TokenType = 7;
pub const BAD_URL: TokenType = 8;
pub const DELIM: TokenType = 9;
pub const NUMBER: TokenType = 10;
pub const PERCENTAGE: TokenType = 11;
pub const DIMENSION: TokenType = 12;
pub const WHITESPACE: TokenType = 13;
pub const CDO: TokenType = 14;
pub const CDC: TokenType = 15;
pub const COLON: TokenType = 16;
pub const SEMICOLON: TokenType = 17;
pub const COMMA: TokenType = 18;
pub const LEFT_SQUARE_BRACKET: TokenType = 19;
pub const RIGHT_SQUARE_BRACKET: TokenType = 20;
pub const LEFT_PARENTHESIS: TokenType = 21;
pub const RIGHT_PARENTHESIS: TokenType = 22;
pub const LEFT_CURLY_BRACKET: TokenType = 23;
pub const RIGHT_CURLY_BRACKET: TokenType = 24;
pub const COMMENT: TokenType = 25;

// ─── Char code definitions (lib/tokenizer/char-code-definitions.js) ────────

pub fn is_digit(code: u32) -> bool {
    (0x30..=0x39).contains(&code)
}

pub fn is_hex_digit(code: u32) -> bool {
    is_digit(code) || (0x41..=0x46).contains(&code) || (0x61..=0x66).contains(&code)
}

pub fn is_uppercase_letter(code: u32) -> bool {
    (0x41..=0x5A).contains(&code)
}

pub fn is_lowercase_letter(code: u32) -> bool {
    (0x61..=0x7A).contains(&code)
}

pub fn is_letter(code: u32) -> bool {
    is_uppercase_letter(code) || is_lowercase_letter(code)
}

pub fn is_non_ascii(code: u32) -> bool {
    code >= 0x80
}

pub fn is_name_start(code: u32) -> bool {
    is_letter(code) || is_non_ascii(code) || code == 0x5F
}

pub fn is_name(code: u32) -> bool {
    is_name_start(code) || is_digit(code) || code == 0x2D
}

pub fn is_non_printable(code: u32) -> bool {
    code <= 0x08 || code == 0x0B || (0x0E..=0x1F).contains(&code) || code == 0x7F
}

pub fn is_newline(code: u32) -> bool {
    code == 0x0A || code == 0x0D || code == 0x0C
}

pub fn is_white_space(code: u32) -> bool {
    is_newline(code) || code == 0x20 || code == 0x09
}

pub fn is_valid_escape(first: u32, second: u32) -> bool {
    if first != 0x5C {
        return false;
    }
    if is_newline(second) || second == 0 {
        return false;
    }
    true
}

pub fn is_identifier_start(first: u32, second: u32, third: u32) -> bool {
    if first == 0x2D {
        return is_name_start(second) || second == 0x2D || is_valid_escape(second, third);
    }
    if is_name_start(first) {
        return true;
    }
    if first == 0x5C {
        return is_valid_escape(first, second);
    }
    false
}

pub fn is_number_start(first: u32, second: u32, third: u32) -> u32 {
    if first == 0x2B || first == 0x2D {
        if is_digit(second) {
            return 2;
        }
        return if second == 0x2E && is_digit(third) {
            3
        } else {
            0
        };
    }
    if first == 0x2E {
        return if is_digit(second) { 2 } else { 0 };
    }
    if is_digit(first) {
        return 1;
    }
    0
}

pub fn is_bom(code: u32) -> usize {
    if code == 0xFEFF || code == 0xFFFE {
        1
    } else {
        0
    }
}

pub const EOF_CATEGORY: u32 = 0x80;
pub const WHITESPACE_CATEGORY: u32 = 0x82;
pub const DIGIT_CATEGORY: u32 = 0x83;
pub const NAME_START_CATEGORY: u32 = 0x84;
pub const NON_PRINTABLE_CATEGORY: u32 = 0x85;

pub fn char_code_category(code: u32) -> u32 {
    if code >= 0x80 {
        return NAME_START_CATEGORY;
    }
    if is_white_space(code) {
        WHITESPACE_CATEGORY
    } else if is_digit(code) {
        DIGIT_CATEGORY
    } else if is_name_start(code) {
        NAME_START_CATEGORY
    } else if is_non_printable(code) {
        NON_PRINTABLE_CATEGORY
    } else if code != 0 {
        code
    } else {
        EOF_CATEGORY
    }
}

// ─── Utils (lib/tokenizer/utils.js) ─────────────────────────────────────────

/// `source.charCodeAt(offset)` with 0 past the end (and for negative offsets).
pub fn char_at(source: &[char], offset: usize) -> u32 {
    if offset < source.len() {
        source[offset] as u32
    } else {
        0
    }
}

pub fn get_newline_length(source: &[char], offset: usize, code: u32) -> usize {
    if code == 13 && char_at(source, offset + 1) == 10 {
        2
    } else {
        1
    }
}

pub fn cmp_char(source: &[char], offset: usize, reference_code: u32) -> bool {
    // JS `testStr.charCodeAt(offset)` yields NaN past the end, which never
    // equals the reference; 0 does the same job here.
    let mut code = char_at(source, offset);
    if is_uppercase_letter(code) {
        code |= 32;
    }
    code == reference_code
}

pub fn cmp_str(source: &[char], start: usize, end: usize, reference: &str) -> bool {
    let reference: Vec<char> = reference.chars().collect();
    if end < start || end - start != reference.len() {
        return false;
    }
    if end > source.len() {
        return false;
    }
    for i in start..end {
        let reference_code = reference[i - start] as u32;
        let mut test_code = source[i] as u32;
        if is_uppercase_letter(test_code) {
            test_code |= 32;
        }
        if test_code != reference_code {
            return false;
        }
    }
    true
}

pub fn find_white_space_end(source: &[char], mut offset: usize) -> usize {
    while offset < source.len() {
        if !is_white_space(source[offset] as u32) {
            break;
        }
        offset += 1;
    }
    offset
}

pub fn find_decimal_number_end(source: &[char], mut offset: usize) -> usize {
    while offset < source.len() {
        if !is_digit(source[offset] as u32) {
            break;
        }
        offset += 1;
    }
    offset
}

/// § 4.3.7. Consume an escaped code point (offset is at the `\`).
pub fn consume_escaped(source: &[char], mut offset: usize) -> usize {
    offset += 2;
    if is_hex_digit(char_at(source, offset - 1)) {
        let max_offset = std::cmp::min(source.len(), offset + 5);
        while offset < max_offset {
            if !is_hex_digit(char_at(source, offset)) {
                break;
            }
            offset += 1;
        }
        let code = char_at(source, offset);
        if is_white_space(code) {
            offset += get_newline_length(source, offset, code);
        }
    }
    offset
}

/// § 4.3.11. Consume a name.
pub fn consume_name(source: &[char], mut offset: usize) -> usize {
    while offset < source.len() {
        let code = source[offset] as u32;
        if is_name(code) {
            offset += 1;
            continue;
        }
        if is_valid_escape(code, char_at(source, offset + 1)) {
            offset = consume_escaped(source, offset);
            continue;
        }
        break;
    }
    offset
}

/// § 4.3.12. Consume a number.
pub fn consume_number(source: &[char], mut offset: usize) -> usize {
    let mut code = char_at(source, offset);
    if code == 0x2B || code == 0x2D {
        offset += 1;
        code = char_at(source, offset);
    }
    if is_digit(code) {
        offset = find_decimal_number_end(source, offset + 1);
        code = char_at(source, offset);
    }
    if code == 0x2E && is_digit(char_at(source, offset + 1)) {
        offset += 2;
        offset = find_decimal_number_end(source, offset);
    }
    if cmp_char(source, offset, 101) {
        let mut sign = 0;
        code = char_at(source, offset + 1);
        if code == 0x2D || code == 0x2B {
            sign = 1;
            code = char_at(source, offset + 2);
        }
        if is_digit(code) {
            offset = find_decimal_number_end(source, offset + 1 + sign + 1);
        }
    }
    offset
}

/// § 4.3.14. Consume the remnants of a bad url.
pub fn consume_bad_url_remnants(source: &[char], mut offset: usize) -> usize {
    while offset < source.len() {
        let code = source[offset] as u32;
        if code == 0x29 {
            offset += 1;
            break;
        }
        if is_valid_escape(code, char_at(source, offset + 1)) {
            offset = consume_escaped(source, offset);
        }
        offset += 1;
    }
    offset
}

/// § 4.3.7. Decode an escape body (without the leading `\`).
pub fn decode_escaped(escaped: &[char]) -> String {
    if escaped.len() == 1 && !is_hex_digit(escaped[0] as u32) {
        return escaped[0].to_string();
    }
    let hex: String = escaped.iter().collect();
    // JS parseInt(escaped, 16): parses the leading hex digits.
    let digits: String = hex.chars().take_while(|c| c.is_ascii_hexdigit()).collect();
    let mut code: u32 = if digits.is_empty() {
        // NaN: JS then compares NaN === 0 (false), NaN in ranges (false)
        // and String.fromCodePoint(NaN) throws. Unreachable for valid escapes.
        0xFFFD
    } else {
        u32::from_str_radix(&digits, 16).unwrap_or(0xFFFD)
    };
    if code == 0 || (0xD800..=0xDFFF).contains(&code) || code > 0x10FFFF {
        code = 0xFFFD;
    }
    char::from_u32(code).unwrap_or('\u{FFFD}').to_string()
}

// ─── tokenize (lib/tokenizer/index.js) ──────────────────────────────────────

/// Runs the CSS tokenizer over `source`, calling `on_token(type, start, end)`
/// for each token in order.
pub fn tokenize<F: FnMut(TokenType, usize, usize)>(source: &[char], mut on_token: F) {
    let source_length = source.len();
    let get = |offset: usize| char_at(source, offset);
    let mut start = is_bom(get(0));
    let mut offset = start;
    let mut ty: TokenType;

    // § 4.3.3. Consume a numeric token
    let consume_numeric_token = |offset: &mut usize, ty: &mut TokenType| {
        *offset = consume_number(source, *offset);
        if is_identifier_start(get(*offset), get(*offset + 1), get(*offset + 2)) {
            *ty = DIMENSION;
            *offset = consume_name(source, *offset);
            return;
        }
        if get(*offset) == 0x25 {
            *ty = PERCENTAGE;
            *offset += 1;
            return;
        }
        *ty = NUMBER;
    };

    // § 4.3.6. Consume a url token
    let consume_url_token = |offset: &mut usize, ty: &mut TokenType| {
        *ty = URL;
        *offset = find_white_space_end(source, *offset);
        while *offset < source.len() {
            let code = source[*offset] as u32;
            match char_code_category(code) {
                0x29 => {
                    *offset += 1;
                    return;
                }
                WHITESPACE_CATEGORY => {
                    *offset = find_white_space_end(source, *offset);
                    if get(*offset) == 0x29 || *offset >= source.len() {
                        if *offset < source.len() {
                            *offset += 1;
                        }
                        return;
                    }
                    *offset = consume_bad_url_remnants(source, *offset);
                    *ty = BAD_URL;
                    return;
                }
                0x22 | 0x27 | 0x28 | NON_PRINTABLE_CATEGORY => {
                    *offset = consume_bad_url_remnants(source, *offset);
                    *ty = BAD_URL;
                    return;
                }
                0x5C => {
                    if is_valid_escape(code, get(*offset + 1)) {
                        *offset = consume_escaped(source, *offset) - 1;
                    } else {
                        *offset = consume_bad_url_remnants(source, *offset);
                        *ty = BAD_URL;
                        return;
                    }
                }
                _ => {}
            }
            *offset += 1;
        }
    };

    // § 4.3.4. Consume an ident-like token
    let consume_ident_like_token = |offset: &mut usize, ty: &mut TokenType| {
        let name_start_offset = *offset;
        *offset = consume_name(source, *offset);
        if cmp_str(source, name_start_offset, *offset, "url") && get(*offset) == 0x28 {
            *offset = find_white_space_end(source, *offset + 1);
            if get(*offset) == 0x22 || get(*offset) == 0x27 {
                *ty = FUNCTION;
                *offset = name_start_offset + 4;
                return;
            }
            consume_url_token(offset, ty);
            return;
        }
        if get(*offset) == 0x28 {
            *ty = FUNCTION;
            *offset += 1;
            return;
        }
        *ty = IDENT;
    };

    // § 4.3.5. Consume a string token
    let consume_string_token = |offset: &mut usize, ty: &mut TokenType| {
        let ending_code_point = get(*offset);
        *offset += 1;
        *ty = STRING;
        while *offset < source.len() {
            let code = source[*offset] as u32;
            let cat = char_code_category(code);
            if cat == ending_code_point {
                *offset += 1;
                return;
            }
            match cat {
                WHITESPACE_CATEGORY => {
                    if is_newline(code) {
                        *offset += get_newline_length(source, *offset, code);
                        *ty = BAD_STRING;
                        return;
                    }
                }
                0x5C => {
                    if *offset == source.len() - 1 {
                        // If the next input code point is EOF, do nothing.
                    } else {
                        let next_code = get(*offset + 1);
                        if is_newline(next_code) {
                            *offset += get_newline_length(source, *offset + 1, next_code);
                        } else if is_valid_escape(code, next_code) {
                            *offset = consume_escaped(source, *offset) - 1;
                        }
                    }
                }
                _ => {}
            }
            *offset += 1;
        }
    };

    while offset < source_length {
        let code = source[offset] as u32;
        match char_code_category(code) {
            WHITESPACE_CATEGORY => {
                ty = WHITESPACE;
                offset = find_white_space_end(source, offset + 1);
            }
            0x22 => {
                ty = STRING;
                consume_string_token(&mut offset, &mut ty);
            }
            0x23 => {
                if is_name(get(offset + 1)) || is_valid_escape(get(offset + 1), get(offset + 2)) {
                    ty = HASH;
                    offset = consume_name(source, offset + 1);
                } else {
                    ty = DELIM;
                    offset += 1;
                }
            }
            0x27 => {
                ty = STRING;
                consume_string_token(&mut offset, &mut ty);
            }
            0x28 => {
                ty = LEFT_PARENTHESIS;
                offset += 1;
            }
            0x29 => {
                ty = RIGHT_PARENTHESIS;
                offset += 1;
            }
            0x2B => {
                if is_number_start(code, get(offset + 1), get(offset + 2)) != 0 {
                    ty = NUMBER;
                    consume_numeric_token(&mut offset, &mut ty);
                } else {
                    ty = DELIM;
                    offset += 1;
                }
            }
            0x2C => {
                ty = COMMA;
                offset += 1;
            }
            0x2D => {
                if is_number_start(code, get(offset + 1), get(offset + 2)) != 0 {
                    ty = NUMBER;
                    consume_numeric_token(&mut offset, &mut ty);
                } else if get(offset + 1) == 0x2D && get(offset + 2) == 0x3E {
                    ty = CDC;
                    offset += 3;
                } else if is_identifier_start(code, get(offset + 1), get(offset + 2)) {
                    ty = IDENT;
                    consume_ident_like_token(&mut offset, &mut ty);
                } else {
                    ty = DELIM;
                    offset += 1;
                }
            }
            0x2E => {
                if is_number_start(code, get(offset + 1), get(offset + 2)) != 0 {
                    ty = NUMBER;
                    consume_numeric_token(&mut offset, &mut ty);
                } else {
                    ty = DELIM;
                    offset += 1;
                }
            }
            0x2F => {
                if get(offset + 1) == 0x2A {
                    ty = COMMENT;
                    // source.indexOf('*/', offset + 2)
                    let mut found = None;
                    let mut i = offset + 2;
                    while i + 1 < source_length {
                        if source[i] == '*' && source[i + 1] == '/' {
                            found = Some(i);
                            break;
                        }
                        i += 1;
                    }
                    offset = match found {
                        Some(i) => i + 2,
                        None => source_length,
                    };
                } else {
                    ty = DELIM;
                    offset += 1;
                }
            }
            0x3A => {
                ty = COLON;
                offset += 1;
            }
            0x3B => {
                ty = SEMICOLON;
                offset += 1;
            }
            0x3C => {
                if get(offset + 1) == 0x21 && get(offset + 2) == 0x2D && get(offset + 3) == 0x2D {
                    ty = CDO;
                    offset += 4;
                } else {
                    ty = DELIM;
                    offset += 1;
                }
            }
            0x40 => {
                if is_identifier_start(get(offset + 1), get(offset + 2), get(offset + 3)) {
                    ty = AT_KEYWORD;
                    offset = consume_name(source, offset + 1);
                } else {
                    ty = DELIM;
                    offset += 1;
                }
            }
            0x5B => {
                ty = LEFT_SQUARE_BRACKET;
                offset += 1;
            }
            0x5C => {
                if is_valid_escape(code, get(offset + 1)) {
                    ty = IDENT;
                    consume_ident_like_token(&mut offset, &mut ty);
                } else {
                    ty = DELIM;
                    offset += 1;
                }
            }
            0x5D => {
                ty = RIGHT_SQUARE_BRACKET;
                offset += 1;
            }
            0x7B => {
                ty = LEFT_CURLY_BRACKET;
                offset += 1;
            }
            0x7D => {
                ty = RIGHT_CURLY_BRACKET;
                offset += 1;
            }
            DIGIT_CATEGORY => {
                ty = NUMBER;
                consume_numeric_token(&mut offset, &mut ty);
            }
            NAME_START_CATEGORY => {
                ty = IDENT;
                consume_ident_like_token(&mut offset, &mut ty);
            }
            _ => {
                ty = DELIM;
                offset += 1;
            }
        }
        on_token(ty, start, offset);
        start = offset;
    }
}

// ─── TokenStream (lib/tokenizer/TokenStream.js) ─────────────────────────────

const BLOCK_OPEN_TOKEN: u8 = 1;
const BLOCK_CLOSE_TOKEN: u8 = 2;

fn balance_pair(ty: TokenType) -> TokenType {
    match ty {
        FUNCTION | LEFT_PARENTHESIS => RIGHT_PARENTHESIS,
        LEFT_SQUARE_BRACKET => RIGHT_SQUARE_BRACKET,
        LEFT_CURLY_BRACKET => RIGHT_CURLY_BRACKET,
        _ => 0,
    }
}

fn block_token(ty: TokenType) -> u8 {
    match ty {
        FUNCTION | LEFT_PARENTHESIS | LEFT_SQUARE_BRACKET | LEFT_CURLY_BRACKET => BLOCK_OPEN_TOKEN,
        RIGHT_PARENTHESIS | RIGHT_SQUARE_BRACKET | RIGHT_CURLY_BRACKET => BLOCK_CLOSE_TOKEN,
        _ => 0,
    }
}

pub fn is_block_opener_token_type(ty: TokenType) -> bool {
    block_token(ty) == BLOCK_OPEN_TOKEN
}

pub fn is_block_closer_token_type(ty: TokenType) -> bool {
    block_token(ty) == BLOCK_CLOSE_TOKEN
}

/// The tokenized source with css-tree's cursor and balance table.
pub struct TokenStream {
    pub source: Vec<char>,
    pub first_char_offset: usize,
    pub token_count: usize,
    types: Vec<TokenType>,
    ends: Vec<usize>,
    pub balance: Vec<usize>,
    pub eof: bool,
    pub token_index: usize,
    pub token_type: TokenType,
    pub token_start: usize,
    pub token_end: usize,
}

impl TokenStream {
    pub fn new(source: &str) -> TokenStream {
        let source: Vec<char> = source.chars().collect();
        let source_length = source.len();
        let mut types: Vec<TokenType> = Vec::with_capacity(source_length + 1);
        let mut ends: Vec<usize> = Vec::with_capacity(source_length + 1);
        // JS: Uint32Array(source.length + 1), zero-filled; indexed by token
        // index, and by `balanceStart` which starts at source.length.
        let mut balance: Vec<usize> = vec![0; source_length + 1];
        let mut token_count = 0usize;
        let mut first_char_offset: Option<usize> = None;
        let mut balance_close_type: TokenType = 0;
        let mut balance_start = source_length;

        tokenize(&source, |ty, start, end| {
            let index = token_count;
            token_count += 1;
            types.push(ty);
            ends.push(end);
            if first_char_offset.is_none() {
                first_char_offset = Some(start);
            }
            balance[index] = balance_start;
            if ty == balance_close_type {
                let prev_balance_start = balance[balance_start];
                balance[balance_start] = index;
                balance_start = prev_balance_start;
                balance_close_type = if prev_balance_start < types.len() {
                    balance_pair(types[prev_balance_start])
                } else {
                    // JS reads offsetAndType[prevBalanceStart] >> 24 which is
                    // 0 for an unwritten slot: no close type.
                    0
                };
            } else if is_block_opener_token_type(ty) {
                balance_start = index;
                balance_close_type = balance_pair(ty);
            }
        });

        types.push(EOF);
        ends.push(source_length);
        balance[token_count] = token_count;

        for i in 0..token_count {
            let bs = balance[i];
            if bs <= i {
                let balance_end = balance[bs];
                if balance_end != i {
                    balance[i] = balance_end;
                }
            } else if bs > token_count {
                balance[i] = token_count;
            }
        }

        let mut ts = TokenStream {
            source,
            first_char_offset: first_char_offset.unwrap_or(0),
            token_count,
            types,
            ends,
            balance,
            eof: false,
            token_index: 0,
            token_type: 0,
            token_start: 0,
            token_end: 0,
        };
        ts.reset();
        ts.next();
        ts
    }

    fn reset(&mut self) {
        self.eof = false;
        // JS tokenIndex = -1; we keep it as usize and special-case in next().
        self.token_index = usize::MAX;
        self.token_type = 0;
        self.token_start = self.first_char_offset;
        self.token_end = self.first_char_offset;
    }

    /// JS `offsetAndType[i] & OFFSET_MASK` for i in 0..=tokenCount, i.e. the
    /// end offset of token i (source length for the EOF slot).
    fn raw_end(&self, index: usize) -> usize {
        if index <= self.token_count {
            self.ends[index]
        } else {
            0
        }
    }

    fn raw_type(&self, index: usize) -> TokenType {
        if index <= self.token_count {
            self.types[index]
        } else {
            0
        }
    }

    pub fn lookup_type(&self, offset: isize) -> TokenType {
        let idx = self.token_index as isize + offset;
        if idx >= 0 && (idx as usize) < self.token_count {
            self.types[idx as usize]
        } else {
            EOF
        }
    }

    pub fn lookup_type_non_sc(&self, mut idx: usize) -> TokenType {
        let mut offset = self.token_index;
        while offset < self.token_count {
            let ty = self.types[offset];
            if ty != WHITESPACE && ty != COMMENT {
                if idx == 0 {
                    return ty;
                }
                idx -= 1;
            }
            offset += 1;
        }
        EOF
    }

    pub fn lookup_offset(&self, offset: isize) -> usize {
        let idx = self.token_index as isize + offset;
        if idx >= 0 && (idx as usize) < self.token_count {
            let idx = idx as usize;
            if idx == 0 {
                // JS: offsetAndType[-1] is undefined -> & mask -> 0
                0
            } else {
                self.raw_end(idx - 1)
            }
        } else {
            self.source.len()
        }
    }

    pub fn lookup_value(&self, offset: isize, reference: &str) -> bool {
        let idx = self.token_index as isize + offset;
        if idx >= 0 && (idx as usize) < self.token_count {
            let idx = idx as usize;
            let start = if idx == 0 { 0 } else { self.raw_end(idx - 1) };
            cmp_str(&self.source, start, self.raw_end(idx), reference)
        } else {
            false
        }
    }

    pub fn get_token_start(&self, token_index: usize) -> usize {
        if token_index == self.token_index {
            return self.token_start;
        }
        if token_index > 0 {
            return if token_index < self.token_count {
                self.raw_end(token_index - 1)
            } else {
                self.raw_end(self.token_count)
            };
        }
        self.first_char_offset
    }

    pub fn get_token_end(&self, token_index: usize) -> usize {
        if token_index == self.token_index {
            return self.token_end;
        }
        self.raw_end(std::cmp::min(token_index, self.token_count))
    }

    pub fn get_token_type(&self, token_index: usize) -> TokenType {
        if token_index == self.token_index {
            return self.token_type;
        }
        self.raw_type(std::cmp::min(token_index, self.token_count))
    }

    pub fn substr_to_cursor(&self, start: usize) -> String {
        self.substring(start, self.token_start)
    }

    pub fn substring(&self, start: usize, end: usize) -> String {
        // JS String.prototype.substring: swaps and clamps.
        let len = self.source.len();
        let (mut a, mut b) = (start.min(len), end.min(len));
        if a > b {
            std::mem::swap(&mut a, &mut b);
        }
        self.source[a..b].iter().collect()
    }

    pub fn char_code_at(&self, offset: usize) -> u32 {
        char_at(&self.source, offset)
    }

    pub fn is_balance_edge(&self, token_index: usize) -> bool {
        self.balance[self.token_index.min(self.token_count)] < token_index
    }

    pub fn is_delim(&self, code: u32) -> bool {
        self.token_type == DELIM && self.char_code_at(self.token_start) == code
    }

    pub fn is_delim_at(&self, code: u32, offset: isize) -> bool {
        if offset != 0 {
            return self.lookup_type(offset) == DELIM
                && self.char_code_at(self.lookup_offset(offset)) == code;
        }
        self.is_delim(code)
    }

    /// JS `skip(tokenCount)` (may be negative).
    pub fn skip(&mut self, count: isize) {
        let next = self.token_index as isize + count;
        if next >= 0 && (next as usize) < self.token_count {
            let next = next as usize;
            self.token_index = next;
            self.token_start = if next == 0 { 0 } else { self.raw_end(next - 1) };
            self.token_type = self.types[next];
            self.token_end = self.ends[next];
        } else {
            self.token_index = self.token_count;
            self.next();
        }
    }

    /// Move the cursor to an absolute token index.
    pub fn seek(&mut self, index: usize) {
        let delta = index as isize - self.token_index as isize;
        self.skip(delta);
    }

    pub fn next(&mut self) {
        let next = self.token_index.wrapping_add(1);
        if next < self.token_count {
            self.token_index = next;
            self.token_start = self.token_end;
            self.token_type = self.types[next];
            self.token_end = self.ends[next];
        } else {
            self.eof = true;
            self.token_index = self.token_count;
            self.token_type = EOF;
            self.token_start = self.source.len();
            self.token_end = self.source.len();
        }
    }

    pub fn skip_sc(&mut self) {
        while self.token_type == WHITESPACE || self.token_type == COMMENT {
            self.next();
        }
    }

    /// JS `skipUntilBalanced(startToken, stopConsume)`; `stop_consume` gets
    /// the char code at each token start and returns 0 (continue), 1 (stop
    /// before) or 2 (stop after).
    pub fn skip_until_balanced<F: Fn(u32) -> u8>(&mut self, start_token: usize, stop_consume: F) {
        let mut cursor = start_token;
        while cursor < self.token_count {
            let balance_end = self.balance[cursor];
            if balance_end < start_token {
                break;
            }
            let offset = if cursor > 0 {
                self.raw_end(cursor - 1)
            } else {
                self.first_char_offset
            };
            match stop_consume(self.char_code_at(offset)) {
                1 => break,
                2 => {
                    cursor += 1;
                    break;
                }
                _ => {
                    if is_block_opener_token_type(self.types[cursor]) {
                        cursor = balance_end;
                    }
                }
            }
            cursor += 1;
        }
        let delta = cursor as isize - self.token_index as isize;
        self.skip(delta);
    }
}