Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

13.2 KB
Raw Blame
//! Template-string body splitter for `f'…'` literals and the standalone
//! `parse_full_template_string` entry point.
//!
//! The body is a sequence of literal chunks separated by `{ expr }`
//! blocks. We sub-parse each block by spawning a fresh [`Parser`] seeked
//! to the byte after `{`, then advance past the closing `}` it stopped
//! on.

use super::Parser;
use crate::emit::Emitter;
use crate::error::ParseError;
use crate::lex::{Lexer, TokenKind};

/// Split a template-string body into literal chunks and embedded
/// expression blocks, then wrap as either a bare `Constant` (no
/// `{ … }` blocks) or a `Call("concat", chunks)`.
///
/// `full_src` is the source the outer parser sees; `body_offset` is the
/// byte offset where the template body begins (just past `f'`) and
/// `body_end` is the byte offset of the closing `'`. Positions emitted
/// on literal Constants are absolute in `full_src`, matching cpp's
/// `STRING_TEXT` ctx spans. Sub-parsed `{ … }` expression chunks already
/// get absolute positions because their sub-parser sees the full source
/// up to the closing brace.
///
/// Escapes follow cpp's f-string `STRING_TEXT` lexer rule and the
/// `parse_string_text_ctx` visitor: `\{`, `\'`, `\\`, `\n`, `\t`, `\r`,
/// `\b`, `\f`, `\a`, `\v` decode in place, `\0` contributes nothing,
/// and `\xHH` is kept verbatim. Any other `\X` is one the lexer rule
/// cannot span — cpp ends the `STRING_TEXT` token there, drops the
/// `\X`, and starts a fresh token, so the body splitter does the same:
/// it closes the current literal chunk and opens a new one (an extra
/// `concat` argument).
pub(super) fn parse_template_body<E: Emitter + Clone>(
    emit: &E,
    full_src: &str,
    body_offset: usize,
    body_end: usize,
) -> Result<E::Value, ParseError> {
    let body = &full_src[body_offset..body_end];
    let bytes = body.as_bytes();
    // (start_in_body, end_in_body, value) — absolute positions wrap on
    // each push so callers receive cpp-shaped `start` / `end` spans.
    let mut chunks: Vec<E::Value> = Vec::new();
    let mut literal = String::new();
    let mut literal_start = 0; // byte offset within `body` where the current literal began
    let mut i = 0;
    while i < bytes.len() {
        let c = bytes[i];
        // `\` always introduces an escape: cpp's `STRING_TEXT` lexer
        // sweeps over `ESCAPE_CHAR_COMMON`, then `parse_string_text_ctx`
        // decodes recognised ones and drops the byte for unrecognised
        // ones. We do both in one pass.
        if c == b'\\' && i + 1 < bytes.len() {
            let next = bytes[i + 1];
            match next {
                b'{' => {
                    literal.push('{');
                    i += 2;
                    continue;
                }
                b'}' => {
                    literal.push('}');
                    i += 2;
                    continue;
                }
                b'\'' => {
                    literal.push('\'');
                    i += 2;
                    continue;
                }
                b'\\' => {
                    literal.push('\\');
                    i += 2;
                    continue;
                }
                b'n' => {
                    literal.push('\n');
                    i += 2;
                    continue;
                }
                b't' => {
                    literal.push('\t');
                    i += 2;
                    continue;
                }
                b'r' => {
                    literal.push('\r');
                    i += 2;
                    continue;
                }
                b'b' => {
                    literal.push('\u{08}');
                    i += 2;
                    continue;
                }
                b'f' => {
                    literal.push('\u{0C}');
                    i += 2;
                    continue;
                }
                b'a' => {
                    literal.push('\u{07}');
                    i += 2;
                    continue;
                }
                b'v' => {
                    literal.push('\u{0B}');
                    i += 2;
                    continue;
                }
                b'0' => {
                    // cpp's `parse_string_text_ctx` drops `\0` (NUL is
                    // not emitted, contributing nothing to the output).
                    i += 2;
                    continue;
                }
                b'x' => {
                    // cpp keeps `\xHH` verbatim as `\xHH` — the lexer
                    // doesn't decode it, and `parse_string_text_ctx`
                    // copies the backslash + the 'x' + the two hex
                    // digits literally.
                    literal.push('\\');
                    i += 1;
                    continue;
                }
                _ => {
                    // Unknown `\X` — cpp ends the current `STRING_TEXT`
                    // token at the `\`, then starts a fresh token on
                    // the next valid character. The body splitter
                    // models that by closing the current literal chunk
                    // and dropping the `\X` sequence (both the backslash
                    // and the escaped char — cpp drops `\q`, `\é`, and
                    // `\😀` alike, all contributing nothing).
                    if !literal.is_empty() {
                        chunks.push(wrap_literal_chunk(
                            emit,
                            full_src,
                            std::mem::take(&mut literal),
                            body_offset + literal_start,
                            body_offset + i,
                        ));
                    }
                    // `X` may be a multibyte codepoint (`\é`, `\😀`), so step
                    // past the whole escaped char, not a fixed 2 bytes — a
                    // fixed step lands mid-char and panics the `&body[i..]`
                    // slice at the bottom of the loop.
                    let escaped_char = body[i + 1..]
                        .chars()
                        .next()
                        .expect("guarded by `i + 1 < bytes.len()`");
                    i += 1 + escaped_char.len_utf8();
                    literal_start = i;
                    continue;
                }
            }
        }
        if c == b'{' {
            if !literal.is_empty() {
                chunks.push(wrap_literal_chunk(
                    emit,
                    full_src,
                    std::mem::take(&mut literal),
                    body_offset + literal_start,
                    body_offset + i,
                ));
            }
            // The body inside `{ … }` is parsed as a `columnExpr`. cpp's
            // `parse_string_template` lifts the `{ … }` lexer mode into
            // a sub-`columnExpr` parse — paren-balance via the
            // sub-lexer so nested `{}` (Hog blocks, dicts) within the
            // expression don't terminate the substitution. We mirror
            // that here by lexing through `{`/`}` until the matching
            // close-brace lands at depth zero.
            //
            // Positions stay absolute in `full_src`. cpp's grammar
            // emits ctx spans relative to the *outer* source — the
            // visitor doesn't reset them per template chunk — so the
            // sub-parser's nodes share an `offset` space with the
            // whole template.
            //
            // Scan via the FULL source so the lexer's positions are
            // absolute (`scan.next_token()?.start` is a `full_src`
            // offset), then sub-parse from the same full source so the
            // emitted positions stay absolute too.
            let abs_brace_start = body_offset + i;
            let mut scan = Lexer::with_pos(full_src, abs_brace_start + 1);
            let mut depth: i32 = 1;
            let close_brace_start = loop {
                let t = scan.next_token()?;
                match t.kind {
                    TokenKind::LBrace => depth += 1,
                    TokenKind::RBrace => {
                        depth -= 1;
                        if depth == 0 {
                            break t.start;
                        }
                    }
                    TokenKind::Eof => {
                        return Err(ParseError::syntax(
                            "expected `}` to close template expression",
                            abs_brace_start,
                            body_end,
                        ));
                    }
                    _ => {}
                }
            };
            // Sub-parse the brace-internal expression. The slice stops
            // at the `}`, so the sub-parser sees EOF where the block
            // ends instead of the surrounding literal text. Slicing
            // `full_src` keeps the absolute offsets up to that point —
            // a Parser::with_pos started at `abs_brace_start + 1`
            // emits absolute positions in `full_src`.
            let mut sub = Parser::<'_, E>::with_pos_emit(
                &full_src[..close_brace_start],
                abs_brace_start + 1,
                emit.clone(),
            )?;
            let expr = sub.parse_expr_bp(0)?;
            if !matches!(sub.peek(), TokenKind::Eof) {
                return Err(ParseError::syntax(
                    format!(
                        "expected `}}` to close template expression, got {:?}",
                        sub.peek()
                    ),
                    sub.peek0.start,
                    sub.peek0.end,
                ));
            }
            // Convert absolute close-brace offset back to body-relative
            // for the byte-level scan loop.
            i = close_brace_start - body_offset + 1;
            literal_start = i;
            chunks.push(expr);
            continue;
        }
        // Generic char: decode the next UTF-8 scalar so we copy whole
        // codepoints, not stray bytes.
        let rest = &body[i..];
        let ch = rest.chars().next().expect("non-empty rest");
        literal.push(ch);
        i += ch.len_utf8();
    }
    if !literal.is_empty() {
        chunks.push(wrap_literal_chunk(
            emit,
            full_src,
            literal,
            body_offset + literal_start,
            body_offset + i,
        ));
    }
    if chunks.is_empty() {
        // Empty body — cpp spans the empty-string Constant over the WHOLE `f'…'`
        // token (no interior text), from `body_offset - 2` (`f'`) through the
        // closing `'`. Inline `f'…'` has that quote at `body_end` (span ends
        // `body_end + 1`); the standalone `parse_full_template_string` entry has
        // no trailing quote (`body_end == full_src.len()`), so it ends at `body_end`.
        let token_end = if body_end < full_src.len() {
            body_end + 1
        } else {
            body_end
        };
        return Ok(wrap_literal_chunk(
            emit,
            full_src,
            String::new(),
            body_offset.saturating_sub(2),
            token_end,
        ));
    }
    // A single-chunk template IS that chunk — whether a literal
    // `Constant` or one `{ … }` substitution (`f'{x}'` is just `x`).
    // cpp only wraps in `concat` when there are two or more pieces.
    if chunks.len() == 1 {
        return Ok(chunks.remove(0));
    }
    // The outer concat Call wraps the chunks. Its position spans the
    // whole template body — the caller adds the outer wrap based on
    // the `f'…'` token bounds via the standard pratt-loop wrap.
    Ok(emit.call("concat", chunks))
}

/// Wrap a literal-chunk Constant with the cpp `STRING_TEXT` ctx span.
/// Positions are absolute byte offsets in the full source. The
/// helper directly assembles the cpp-shape `{line, column, offset}`
/// envelope without spinning up a fresh `Parser` (which would try to
/// lex the rest of the source starting from `start`, and inside an
/// `f'…'` body that lex would fail on the trailing unclosed `'`).
fn wrap_literal_chunk<E: Emitter>(
    emit: &E,
    full_src: &str,
    value: String,
    start: usize,
    end: usize,
) -> E::Value {
    let constant = emit.constant(emit.string(&value));
    let start_pos = pos_in_source(emit, full_src, start);
    let end_pos = pos_in_source(emit, full_src, end);
    emit.with_pos(constant, start_pos, end_pos)
}

/// Compute the cpp-shape `{line, column, offset}` envelope for an
/// absolute byte offset in `src`. `offset` is the character index
/// (Unicode code points), matching cpp's ANTLR `getStartIndex()`
/// semantics; `column` is character-position-in-line.
pub(crate) fn pos_in_source<E: Emitter>(emit: &E, src: &str, byte_offset: usize) -> E::Value {
    // Line: count `\n` bytes before `byte_offset`.
    let preceding = &src[..byte_offset.min(src.len())];
    let line_breaks = preceding.bytes().filter(|&b| b == b'\n').count();
    let line = (line_breaks + 1) as u32;
    // Column + char offset: count chars in the preceding-source slice.
    let line_start = preceding.rfind('\n').map(|i| i + 1).unwrap_or(0);
    let column = src[line_start..byte_offset.min(src.len())].chars().count() as u32;
    let char_offset = if src.is_ascii() {
        byte_offset
    } else {
        preceding.chars().count()
    };
    emit.position(line, column, char_offset)
}