:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! HogQL lexer (default mode).
//!
//! Produces tokens matching the ANTLR grammar at
//! [`posthog/hogql/grammar/HogQLLexer.common.g4`] for everything outside the
//! template-string and HogQLX modes. Those modes maintain a stack and switch
//! lexer state mid-stream; they're deferred until the relevant tests are
//! reached.
//!
//! Keywords are case-insensitive (the ANTLR fragments expand to `[aA]`-style
//! character classes). The keyword table is a static slice; lookup is a
//! linear scan after lowering bytes — fine at this size and well below the
//! cost of the parser's per-decision work.
use crate::error::ParseError;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Token {
pub kind: TokenKind,
pub start: usize,
pub end: usize,
}
impl Token {
pub fn eof(at: usize) -> Self {
Self {
kind: TokenKind::Eof,
start: at,
end: at,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TokenKind {
// ---- Literals / names ----------------------------------------------
Ident,
QuotedIdent,
/// Decimal/octal/hex/float literal. The exact subform is decoded at
/// parse time from the source slice.
Number,
/// Single-quoted string literal. Source slice still contains the
/// surrounding quotes and any embedded escapes — `parse_string_literal`
/// in the parser strips and unescapes.
String,
/// `f'...'` / `F'...'` template-string literal. Source slice spans
/// the entire `f'…'`, including the leading `f`/`F`, both quotes,
/// and any embedded `{expr}` blocks. Body interpretation and
/// chunk-splitting happen in `parse_full_template_string` so the
/// lexer only has to find the matching closing quote (skipping
/// nested `{ ... }` expression blocks that may themselves contain
/// strings or further template strings).
TemplateString,
// ---- Punctuation / brackets ---------------------------------------
LParen,
RParen,
LBracket,
RBracket,
LBrace,
RBrace,
Comma,
Dot,
Colon,
Semicolon,
QMark, // `?`
Hash, // `#`
LtSlash, // `</` for HogQLX closing tags (recognised even outside
// tag mode so the parser can route on it)
SlashGt, // `/>` for HogQLX self-closing tags
// ---- Arithmetic / bitwise / concat --------------------------------
Plus,
Dash,
Asterisk,
Slash,
Percent,
Concat, // `||`
// ---- Comparison / equality ----------------------------------------
EqDouble, // `=` (the SQL "single equals" assignment-style equality)
EqSingle, // `==` (the SQL "double equals")
NotEq, // `!=` or `<>`
Lt,
LtEq,
Gt,
GtEq,
NullSafeEq, // `<=>` (MySQL null-safe equality, sugar for IS NOT DISTINCT FROM)
// ---- Regex --------------------------------------------------------
RegexSingle, // `~`
RegexDouble, // `=~`
NotRegex, // `!~`
IRegexSingle, // `~*`
IRegexDouble, // `=~*`
NotIRegex, // `!~*`
// ---- Composite operators -----------------------------------------
Arrow, // `->`
DoubleColon, // `::`
ColonEquals, // `:=`
Nullish, // `??`
NullProperty, // `?.`
// ---- Keywords -----------------------------------------------------
Keyword(Kw),
Eof,
}
/// One variant per keyword recognised by the HogQL lexer. Kept in
/// sync with [`KEYWORDS`] below; if you add a variant, add the entry.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Kw {
All,
And,
Anti,
Any,
Array,
As,
Ascending,
Asof,
Between,
Both,
By,
Case,
Cast,
Catch,
Cohort,
Collate,
Columns,
Cross,
Cube,
Current,
Date,
Day,
Desc,
Descending,
Distinct,
Else,
End,
Except,
Exclude,
Extract,
False,
Fill,
Filter,
Final,
Finally,
First,
Fn,
Following,
For,
From,
Full,
Fun,
Group,
Grouping,
Having,
Hour,
Id,
If,
Ignore,
Ilike,
In,
Include,
Inf,
Inner,
Interpolate,
Intersect,
Interval,
Is,
Join,
Key,
Lambda,
Last,
Leading,
Left,
Let,
Like,
Limit,
Local,
Materialized,
Minute,
Month,
Name,
Nan,
Natural,
Not,
Null,
Nulls,
Offset,
On,
Or,
Order,
Outer,
Over,
Partition,
Pivot,
Positional,
Preceding,
Prewhere,
Qualify,
Quarter,
Range,
Recursive,
Replace,
Return,
Right,
Rollup,
Row,
Rows,
Sample,
Second,
Select,
Semi,
Sets,
Settings,
Step,
Substring,
Then,
Throw,
Ties,
Time,
Timestamp,
To,
Top,
Totals,
Trailing,
Trim,
True,
Truncate,
Try,
TryCast,
Unbounded,
Union,
Unpivot,
Using,
Values,
Week,
When,
Where,
While,
Window,
With,
Within,
Year,
Zone,
}
/// Static keyword table. Entries are matched against the lowercased
/// identifier text. Order doesn't matter functionally; grouped roughly by
/// the grammar's `keyword` rule for readability.
const KEYWORDS: &[(&str, Kw)] = &[
("all", Kw::All),
("and", Kw::And),
("anti", Kw::Anti),
("any", Kw::Any),
("array", Kw::Array),
("as", Kw::As),
("ascending", Kw::Ascending),
("asc", Kw::Ascending),
("asof", Kw::Asof),
("between", Kw::Between),
("both", Kw::Both),
("by", Kw::By),
("case", Kw::Case),
("cast", Kw::Cast),
("catch", Kw::Catch),
("cohort", Kw::Cohort),
("collate", Kw::Collate),
("columns", Kw::Columns),
("cross", Kw::Cross),
("cube", Kw::Cube),
("current", Kw::Current),
("date", Kw::Date),
("day", Kw::Day),
("desc", Kw::Desc),
("descending", Kw::Descending),
("distinct", Kw::Distinct),
("else", Kw::Else),
("end", Kw::End),
("except", Kw::Except),
("exclude", Kw::Exclude),
("extract", Kw::Extract),
("false", Kw::False),
("fill", Kw::Fill),
("filter", Kw::Filter),
("final", Kw::Final),
("finally", Kw::Finally),
("first", Kw::First),
("fn", Kw::Fn),
("following", Kw::Following),
("for", Kw::For),
("from", Kw::From),
("full", Kw::Full),
("fun", Kw::Fun),
("group", Kw::Group),
("grouping", Kw::Grouping),
("having", Kw::Having),
("hour", Kw::Hour),
("id", Kw::Id),
("if", Kw::If),
("ignore", Kw::Ignore),
("ilike", Kw::Ilike),
("in", Kw::In),
("include", Kw::Include),
("inf", Kw::Inf),
("infinity", Kw::Inf), // cpp grammar: `INF: I N F | I N F I N I T Y` — same keyword
("inner", Kw::Inner),
("interpolate", Kw::Interpolate),
("intersect", Kw::Intersect),
("interval", Kw::Interval),
("is", Kw::Is),
("join", Kw::Join),
("key", Kw::Key),
("lambda", Kw::Lambda),
("last", Kw::Last),
("leading", Kw::Leading),
("left", Kw::Left),
("let", Kw::Let),
("like", Kw::Like),
("limit", Kw::Limit),
("local", Kw::Local),
("materialized", Kw::Materialized),
("minute", Kw::Minute),
("month", Kw::Month),
("name", Kw::Name),
("nan", Kw::Nan),
("natural", Kw::Natural),
("not", Kw::Not),
("null", Kw::Null),
("nulls", Kw::Nulls),
("offset", Kw::Offset),
("on", Kw::On),
("or", Kw::Or),
("order", Kw::Order),
("outer", Kw::Outer),
("over", Kw::Over),
("partition", Kw::Partition),
("pivot", Kw::Pivot),
("positional", Kw::Positional),
("preceding", Kw::Preceding),
("prewhere", Kw::Prewhere),
("qualify", Kw::Qualify),
("quarter", Kw::Quarter),
("range", Kw::Range),
("recursive", Kw::Recursive),
("replace", Kw::Replace),
("return", Kw::Return),
("right", Kw::Right),
("rollup", Kw::Rollup),
("row", Kw::Row),
("rows", Kw::Rows),
("sample", Kw::Sample),
("second", Kw::Second),
("select", Kw::Select),
("semi", Kw::Semi),
("sets", Kw::Sets),
("settings", Kw::Settings),
("step", Kw::Step),
("substring", Kw::Substring),
("then", Kw::Then),
("throw", Kw::Throw),
("ties", Kw::Ties),
("time", Kw::Time),
("timestamp", Kw::Timestamp),
("to", Kw::To),
("top", Kw::Top),
("totals", Kw::Totals),
("trailing", Kw::Trailing),
("trim", Kw::Trim),
("true", Kw::True),
("truncate", Kw::Truncate),
("try", Kw::Try),
("try_cast", Kw::TryCast),
("unbounded", Kw::Unbounded),
("union", Kw::Union),
("unpivot", Kw::Unpivot),
("using", Kw::Using),
("values", Kw::Values),
("week", Kw::Week),
("when", Kw::When),
("where", Kw::Where),
("while", Kw::While),
("window", Kw::Window),
("with", Kw::With),
("within", Kw::Within),
// YEAR has a `YYYY` alias per HogQLLexer.common.g4 line 140
// (`YEAR: Y E A R | Y Y Y Y`). The interval-unit keywords are
// singular only — cpp's lexer has no plural forms, so `hours`,
// `days`, … lex as plain identifiers and only the in-string
// `INTERVAL '5 days'` form accepts a plural (handled by the
// visitor, not the lexer).
("year", Kw::Year),
("yyyy", Kw::Year),
("zone", Kw::Zone),
];
pub struct Lexer<'a> {
src: &'a [u8],
pos: usize,
/// Mirrors cpp's `HOGQLX_TAG_OPEN` / `HOGQLX_TAG_CLOSE` lexer modes
/// for the one trivia rule that differs there: `HASH_COMMENT` is
/// default-mode-only, so a `#` between tag attributes must stay a
/// `Hash` token (which the tag parser rejects, matching the
/// grammar's `TAG_UNEXPECTED` catch-all) instead of starting a
/// comment. Toggled by the HogQLX tag parser; see
/// `parse_hogqlx_tag_element`.
in_hogqlx_tag: bool,
}
impl<'a> Lexer<'a> {
/// Construct a lexer at the given byte offset of `src`. The parser
/// passes 0 for normal top-level lexing and a forward position when
/// it needs to spawn a shadow lexer for bounded lookahead (lambda
/// heads, CTE shape probes) or to resume after a template-string
/// expression block.
pub fn with_pos(src: &'a str, pos: usize) -> Self {
Self {
src: src.as_bytes(),
pos,
in_hogqlx_tag: false,
}
}
pub fn pos(&self) -> usize {
self.pos
}
pub fn in_hogqlx_tag(&self) -> bool {
self.in_hogqlx_tag
}
pub fn set_in_hogqlx_tag(&mut self, on: bool) {
self.in_hogqlx_tag = on;
}
pub fn next_token(&mut self) -> Result<Token, ParseError> {
self.skip_trivia()?;
let start = self.pos;
let Some(b) = self.peek_byte(0) else {
return Ok(Token::eof(start));
};
let kind = match b {
b'(' => {
self.pos += 1;
TokenKind::LParen
}
b')' => {
self.pos += 1;
TokenKind::RParen
}
b'[' => {
self.pos += 1;
TokenKind::LBracket
}
b']' => {
self.pos += 1;
TokenKind::RBracket
}
b'{' => {
self.pos += 1;
TokenKind::LBrace
}
b'}' => {
self.pos += 1;
TokenKind::RBrace
}
b',' => {
self.pos += 1;
TokenKind::Comma
}
b';' => {
self.pos += 1;
TokenKind::Semicolon
}
b'#' => match self.peek_byte(1) {
// `#` immediately followed by a digit is the positional-ref
// HASH token (`#1`); the grammar's HASH_COMMENT rule
// explicitly excludes a leading digit so those keep lexing.
Some(b'0'..=b'9') => {
self.pos += 1;
TokenKind::Hash
}
// Anything else (including EOL / EOF right after the `#`)
// is a MySQL-style `#` line comment, skipped like `--`.
// skip_trivia caught most of these already; this catches a
// `#` that follows a non-trivia token mid-expression.
_ if !self.in_hogqlx_tag => {
self.skip_line_comment();
return self.next_token();
}
// Inside a HogQLX tag there is no HASH_COMMENT rule; emit
// Hash and let the tag parser reject it (TAG_UNEXPECTED).
_ => {
self.pos += 1;
TokenKind::Hash
}
},
// Always emit a single dot. `.5` style literals are assembled
// at parse time via the grammar's `floatingLiteral` (DOT then
// DECIMAL_LITERAL) — folding here would misread `t.1` (tuple
// access) as `t` followed by the float `.1`.
b'.' => {
self.pos += 1;
TokenKind::Dot
}
b':' => match self.peek_byte(1) {
Some(b':') => {
self.pos += 2;
TokenKind::DoubleColon
}
Some(b'=') => {
self.pos += 2;
TokenKind::ColonEquals
}
_ => {
self.pos += 1;
TokenKind::Colon
}
},
b'?' => match self.peek_byte(1) {
Some(b'.') => {
self.pos += 2;
TokenKind::NullProperty
}
Some(b'?') => {
self.pos += 2;
TokenKind::Nullish
}
_ => {
self.pos += 1;
TokenKind::QMark
}
},
b'+' => {
self.pos += 1;
TokenKind::Plus
}
b'-' => match self.peek_byte(1) {
Some(b'>') => {
self.pos += 2;
TokenKind::Arrow
}
Some(b'-') => {
// `--` line comment. skip_trivia caught most of these
// already; this catches a `--` that follows a non-trivia
// token mid-expression. Re-route through trivia and
// restart.
self.skip_line_comment();
return self.next_token();
}
_ => {
self.pos += 1;
TokenKind::Dash
}
},
b'*' => {
self.pos += 1;
TokenKind::Asterisk
}
b'/' => match self.peek_byte(1) {
Some(b'>') => {
self.pos += 2;
TokenKind::SlashGt
}
Some(b'*') if self.block_comment_has_closer() => {
self.skip_block_comment(start)?;
return self.next_token();
}
Some(b'/') => {
// `//` line comment (C-style alias for `--`).
// `skip_trivia` catches most occurrences; this
// mid-stream branch covers `//` that follows a
// non-trivia token, same way the mid-stream `--`
// line below the dash arm does.
self.skip_line_comment();
return self.next_token();
}
_ => {
self.pos += 1;
TokenKind::Slash
}
},
b'%' => {
self.pos += 1;
TokenKind::Percent
}
b'|' => match self.peek_byte(1) {
Some(b'|') => {
self.pos += 2;
TokenKind::Concat
}
_ => return Err(self.err(start, "unexpected '|' (expected '||')")),
},
b'=' => match self.peek_byte(1) {
Some(b'=') => {
self.pos += 2;
TokenKind::EqSingle
}
Some(b'~') => match self.peek_byte(2) {
Some(b'*') => {
self.pos += 3;
TokenKind::IRegexDouble
}
_ => {
self.pos += 2;
TokenKind::RegexDouble
}
},
_ => {
self.pos += 1;
TokenKind::EqDouble
}
},
b'!' => match self.peek_byte(1) {
Some(b'=') => {
self.pos += 2;
TokenKind::NotEq
}
Some(b'~') => match self.peek_byte(2) {
Some(b'*') => {
self.pos += 3;
TokenKind::NotIRegex
}
_ => {
self.pos += 2;
TokenKind::NotRegex
}
},
// A bare `!` (not `!=` / `!~` / `!~*`) matches no token
// — the grammar's catch-all `UNEXPECTED_CHARACTER`.
// Report it by code point, same as the generic arm.
_ => return Err(self.err(start, "unexpected character '!' (U+0021)")),
},
b'<' => match self.peek_byte(1) {
// `<=>` must win over `<=` (longest match, mirroring the
// grammar's NULL_SAFE_EQ-before-LT_EQ declaration order).
Some(b'=') if self.peek_byte(2) == Some(b'>') => {
self.pos += 3;
TokenKind::NullSafeEq
}
Some(b'=') => {
self.pos += 2;
TokenKind::LtEq
}
Some(b'>') => {
self.pos += 2;
TokenKind::NotEq
}
Some(b'/') => {
self.pos += 2;
TokenKind::LtSlash
}
_ => {
self.pos += 1;
TokenKind::Lt
}
},
b'>' => match self.peek_byte(1) {
Some(b'=') => {
self.pos += 2;
TokenKind::GtEq
}
_ => {
self.pos += 1;
TokenKind::Gt
}
},
b'~' => match self.peek_byte(1) {
Some(b'*') => {
self.pos += 2;
TokenKind::IRegexSingle
}
_ => {
self.pos += 1;
TokenKind::RegexSingle
}
},
b'\'' => self.lex_string(start)?,
b'`' | b'"' => self.lex_quoted_ident(start)?,
b'0'..=b'9' => self.lex_number(start),
// `f'...'` / `F'...'` template string. Must intercept BEFORE
// the generic ident path, since `f` is a valid ident-start.
// Requires zero gap between the `f` and the `'` — `f 'x'` is
// still `Ident("f")` followed by `String`.
b'f' | b'F' if self.peek_byte(1) == Some(b'\'') => self.lex_template_string(start)?,
b'a'..=b'z' | b'A'..=b'Z' | b'_' | b'$' => self.lex_ident_or_keyword(start),
other => {
// No lexer rule matched — the grammar's catch-all
// `UNEXPECTED_CHARACTER` token. Decode the whole UTF-8
// scalar (not just the lead byte) and name it by code
// point: the only actionable signal when the character
// is invisible (a zero-width space / joiner).
let ch = self.peek_char().map(|(c, _)| c).unwrap_or(other as char);
return Err(self.err(
start,
format!("unexpected character {:?} (U+{:04X})", ch, ch as u32),
));
}
};
Ok(Token {
kind,
start,
end: self.pos,
})
}
// ---- internals ---------------------------------------------------------
fn peek_byte(&self, ahead: usize) -> Option<u8> {
self.src.get(self.pos + ahead).copied()
}
/// Decode the UTF-8 scalar starting at `self.pos`, returning it
/// alongside its byte length. `None` at EOF or when `self.pos`
/// sits on a continuation / invalid lead byte — the lexer input is
/// always a valid `&str`, so the latter only happens mid-scalar.
fn peek_char(&self) -> Option<(char, usize)> {
let lead = self.peek_byte(0)?;
let len = match lead {
0x00..=0x7F => 1,
0xC0..=0xDF => 2,
0xE0..=0xEF => 3,
0xF0..=0xF7 => 4,
_ => return None,
};
let slice = self.src.get(self.pos..self.pos + len)?;
let c = std::str::from_utf8(slice).ok()?.chars().next()?;
Some((c, len))
}
fn err(&self, start: usize, msg: impl Into<String>) -> ParseError {
ParseError::syntax(msg, start, self.pos.max(start + 1))
}
fn skip_trivia(&mut self) -> Result<(), ParseError> {
loop {
// Whitespace. The ANTLR `WHITESPACE` rule is
// `[ \t\r\n]`; `u8::is_ascii_whitespace`
// covers all of those bar the vertical tab (``),
// which is added explicitly. Non-ASCII whitespace
// (NO-BREAK SPACE, the Unicode line/paragraph separators,
// etc.) is also skipped: cpp's ANTLR lexer error-recovers
// past such characters, leaving the parser a clean token
// stream, and treating them as trivia reaches the same
// result without a hard reject.
while let Some(b) = self.peek_byte(0) {
if b < 0x80 {
if !(b.is_ascii_whitespace() || b == 0x0B) {
break;
}
self.pos += 1;
} else {
// `char::is_whitespace` is the Unicode `White_Space`
// set; the grammar's `WHITESPACE` rule also admits
// U+FEFF (BOM), so a file saved with a byte-order
// mark still parses.
match self.peek_char() {
Some((c, len)) if c.is_whitespace() || c == '\u{FEFF}' => self.pos += len,
_ => break,
}
}
}
// `--` or `//` line comment (HogQL accepts both forms).
if (self.peek_byte(0) == Some(b'-') && self.peek_byte(1) == Some(b'-'))
|| (self.peek_byte(0) == Some(b'/') && self.peek_byte(1) == Some(b'/'))
{
self.skip_line_comment();
continue;
}
// MySQL-style `#` line comment (the grammar's HASH_COMMENT;
// default mode only — HogQLX tag modes reject `#` instead).
// `#` immediately followed by a digit is NOT a comment — it
// stays a HASH token so positional references (`#1`) keep
// lexing. Everything else after `#` (including EOL / EOF) is
// comment: the ANTLR rule's tail matches '\n' | '\r' | EOF, so
// a bare `#` at end of line is skipped too.
if !self.in_hogqlx_tag
&& self.peek_byte(0) == Some(b'#')
&& !matches!(self.peek_byte(1), Some(b'0'..=b'9'))
{
self.skip_line_comment();
continue;
}
// `/* ... */` block comment. cpp's ANTLR lexer only matches
// the comment rule when a closing `*/` is found; an
// unterminated `/*` falls back to `/` and `*` tokens (which
// the parser then evaluates per the normal expression
// grammar). Probe for `*/` first; if missing, leave the
// characters in place for the regular lex path.
if self.peek_byte(0) == Some(b'/')
&& self.peek_byte(1) == Some(b'*')
&& self.block_comment_has_closer()
{
self.skip_block_comment(self.pos)?;
continue;
}
break;
}
Ok(())
}
/// Lookahead from a `/*` at `self.pos`: is there a matching `*/`
/// before EOF? Used by `skip_trivia` to commit to the comment-skip
/// path only when the comment is well-formed (matching cpp's ANTLR
/// `/* ... */` rule, which fails to match on unterminated comments).
fn block_comment_has_closer(&self) -> bool {
// We're positioned at `/`; the `*` is at +1, content at +2.
let mut i = self.pos + 2;
let bytes = self.src;
while i + 1 < bytes.len() {
if bytes[i] == b'*' && bytes[i + 1] == b'/' {
return true;
}
i += 1;
}
false
}
fn skip_line_comment(&mut self) {
// Caller already established the leading `--`.
while let Some(b) = self.peek_byte(0) {
if b == b'\n' {
break;
}
self.pos += 1;
}
}
fn skip_block_comment(&mut self, start: usize) -> Result<(), ParseError> {
// Caller has confirmed a closing `*/` exists ahead via
// `block_comment_has_closer` (matching cpp's ANTLR `/*...*/`
// rule, which fails on unterminated). Just walk to the close.
self.pos += 2;
loop {
match (self.peek_byte(0), self.peek_byte(1)) {
(Some(b'*'), Some(b'/')) => {
self.pos += 2;
return Ok(());
}
(Some(_), _) => self.pos += 1,
(None, _) => {
// Shouldn't be reachable when callers gate on
// `block_comment_has_closer`, but surface as an
// error rather than silently advancing.
return Err(ParseError::syntax(
"unterminated block comment",
start,
self.pos,
));
}
}
}
}
fn lex_number(&mut self, start: usize) -> TokenKind {
let _ = start;
// Hex / octal prefix.
if self.peek_byte(0) == Some(b'0') {
match self.peek_byte(1) {
Some(b'x') | Some(b'X')
if self.peek_byte(2).is_some_and(|b| b.is_ascii_hexdigit()) =>
{
// Require at least one hex digit after `0x` — the
// grammar token is `'0' X HEX_DIGIT+`. A bare `0x`
// lexes as `0` + ident `x` (matches the `0b` / `0o`
// arms below); cpp's `HEXADECIMAL_LITERAL` rejects
// an empty body, so `SELECT 0x AS y` is a parse
// error, not a `0x` literal aliased to `y`.
self.pos += 2;
let prefix_end = self.pos;
while self.peek_byte(0).is_some_and(|b| b.is_ascii_hexdigit()) {
self.pos += 1;
}
let int_end = self.pos;
// Try to extend to a FLOATING_LITERAL hex-float —
// `HEX (DOT HEX*)? P [+-]? DEC+`. `p`/`P` is the
// only marker per grammar; `e`/`E` stays a hex
// digit. Requires at least one hex digit in the
// mantissa (per `HEXADECIMAL_LITERAL: '0' X HEX+`),
// so `0x.8p3` is *not* a hex-float — it stays a
// bare `0x` token the parser then rejects. If the
// suffix doesn't match cleanly we leave self.pos
// at int_end (never commit the `.` / `p` probes).
if int_end > prefix_end {
let mut probe = 0usize;
if self.peek_byte(probe) == Some(b'.') {
probe += 1;
while self.peek_byte(probe).is_some_and(|b| b.is_ascii_hexdigit()) {
probe += 1;
}
}
if matches!(self.peek_byte(probe), Some(b'p') | Some(b'P')) {
let mut exp_probe = probe + 1;
if matches!(self.peek_byte(exp_probe), Some(b'+') | Some(b'-')) {
exp_probe += 1;
}
if self
.peek_byte(exp_probe)
.is_some_and(|b| b.is_ascii_digit())
{
self.pos += exp_probe;
while self.peek_byte(0).is_some_and(|b| b.is_ascii_digit()) {
self.pos += 1;
}
}
}
}
return TokenKind::Number;
}
// `0b…` — BINARY_LITERAL (`0b<bin>+`) and the grammar's
// MALFORMED_BINARY_LITERAL (`0b<dec>+`, e.g. `0b22`).
// Scan the full decimal-digit run as one Number token;
// `parse_number_literal` decodes the value and rejects
// a non-binary digit. The `peek_byte(2)` digit guard
// keeps a bare `0b` lexing as `0` + ident `b`.
Some(b'b') | Some(b'B')
if self.peek_byte(2).is_some_and(|b| b.is_ascii_digit()) =>
{
self.pos += 2;
while self.peek_byte(0).is_some_and(|b| b.is_ascii_digit()) {
self.pos += 1;
}
return TokenKind::Number;
}
// `0o…` — OCTAL_PREFIX_LITERAL (`0o<dec>+`). cpp's
// `VISIT(NumberLiteral)` rejects it; scan the decimal
// run as one Number token so `parse_number_literal`
// surfaces that rejection. (`OCT_DIGIT` would stop at
// `0o9` and strand the `9`; the grammar token spans
// DEC_DIGIT.) The digit guard keeps `0o` lexing as
// `0` + ident `o`.
Some(b'o') | Some(b'O')
if self.peek_byte(2).is_some_and(|b| b.is_ascii_digit()) =>
{
self.pos += 2;
while self.peek_byte(0).is_some_and(|b| b.is_ascii_digit()) {
self.pos += 1;
}
return TokenKind::Number;
}
_ => {}
}
}
// Integer part only — the grammar's `floatingLiteral` rule is
// `DECIMAL_LITERAL DOT (DECIMAL_LITERAL | OCTAL_LITERAL)?`, i.e.
// the fractional `.<digits>` tail is assembled at parse time, NOT
// at lex time. This is deliberate: lexing it eagerly would
// misread `t.1.2` (tuple access) as `t.<float 1.2>`.
while self.peek_byte(0).is_some_and(|b| b.is_ascii_digit()) {
self.pos += 1;
}
// FLOATING_LITERAL with an exponent but no fractional digits:
// `1.e5` / `1.E+5`. The grammar's FLOATING_LITERAL token is
// `DECIMAL_LITERAL DOT DEC_DIGIT* E (PLUS|DASH)? DEC_DIGIT+`, so
// cpp's lexer munches the whole thing as one token. Consume the
// `.` here only when an exponent unambiguously follows; `1.2` /
// `t.1.2` (dot then digit, no `e`) is left for parse-time
// fractional assembly. The exponent block below then consumes
// the `e<digits>` tail.
if self.peek_byte(0) == Some(b'.') && matches!(self.peek_byte(1), Some(b'e') | Some(b'E')) {
let digit_at = if matches!(self.peek_byte(2), Some(b'+') | Some(b'-')) {
3
} else {
2
};
if self.peek_byte(digit_at).is_some_and(|b| b.is_ascii_digit()) {
self.pos += 1;
}
}
// Exponent
// Exponent: only consume `e`/`E` when followed by an optional
// sign and at least one digit (else it's a following identifier
// like `1eq`).
if matches!(self.peek_byte(0), Some(b'e') | Some(b'E')) {
let mut probe = 1;
if matches!(self.peek_byte(probe), Some(b'+') | Some(b'-')) {
probe += 1;
}
if self.peek_byte(probe).is_some_and(|b| b.is_ascii_digit()) {
self.pos += probe;
while self.peek_byte(0).is_some_and(|b| b.is_ascii_digit()) {
self.pos += 1;
}
}
}
TokenKind::Number
}
fn lex_ident_or_keyword(&mut self, start: usize) -> TokenKind {
while let Some(b) = self.peek_byte(0) {
if b.is_ascii_alphanumeric() || b == b'_' || b == b'$' {
self.pos += 1;
} else {
break;
}
}
let text = &self.src[start..self.pos];
if let Some(kw) = lookup_keyword(text) {
TokenKind::Keyword(kw)
} else {
TokenKind::Ident
}
}
fn lex_string(&mut self, start: usize) -> Result<TokenKind, ParseError> {
self.pos += 1; // opening quote
loop {
match self.peek_byte(0) {
None => {
return Err(ParseError::syntax(
"unterminated string literal",
start,
self.pos,
))
}
Some(b'\'') => {
// `''` inside a single-quoted string is an escaped quote.
if self.peek_byte(1) == Some(b'\'') {
self.pos += 2;
continue;
}
self.pos += 1;
return Ok(TokenKind::String);
}
Some(b'\\') => {
// `ESCAPE_CHAR_COMMON` (`HogQLLexer.common.g4:145`)
// is a closed set: `\b \f \r \n \t \0 \a \v \\ \xNN`,
// plus the string-literal-only `\'`. cpp rejects
// anything else; rust was silently keeping the
// backslash + char as literal text.
let escape_pos = self.pos;
self.pos += 1;
match self.peek_byte(0) {
None => {} // unterminated — caught by the outer loop's None arm
Some(c) => match c {
b'b' | b'B' | b'f' | b'F' | b'r' | b'R' | b'n' | b'N' | b't' | b'T'
| b'0' | b'a' | b'A' | b'v' | b'V' | b'\\' | b'\'' => {
self.pos += 1;
}
b'x' | b'X' => {
// `\xNN` — exactly two hex digits required.
self.pos += 1;
if self.peek_byte(0).is_some_and(|b| b.is_ascii_hexdigit())
&& self.peek_byte(1).is_some_and(|b| b.is_ascii_hexdigit())
{
self.pos += 2;
} else {
return Err(ParseError::syntax(
r"\x escape requires two hex digits",
escape_pos,
self.pos,
));
}
}
_ => {
return Err(ParseError::syntax(
format!("unrecognised escape '\\{}'", c as char),
escape_pos,
self.pos + 1,
));
}
},
}
}
Some(_) => self.pos += 1,
}
}
}
/// `f'…'` template string. Caller positioned at the leading `f`/`F`,
/// with byte 1 confirmed to be `'`. Scans the body until the
/// matching unescaped `'` at the outermost level, descending into
/// `{ … }` expression blocks via recursive token-level lexing so
/// nested strings and nested template strings inside `{…}` don't
/// confuse the boundary search.
fn lex_template_string(&mut self, start: usize) -> Result<TokenKind, ParseError> {
self.pos += 2; // consume `f'`
loop {
let Some(b) = self.peek_byte(0) else {
return Err(ParseError::syntax(
"unterminated template string",
start,
self.pos,
));
};
match b {
b'\\' => {
// Generic escape: consume backslash + the escaped byte.
// The body parser re-interprets these.
self.pos += 1;
if self.peek_byte(0).is_some() {
self.pos += 1;
}
}
b'\'' => {
self.pos += 1;
return Ok(TokenKind::TemplateString);
}
b'{' => {
// Switch into expression-token mode for the body of
// the `{ … }` block. Recursively call next_token so
// nested strings, parens, and nested template
// strings are all parsed correctly. Stop when the
// matching `}` arrives at depth 0.
self.pos += 1; // consume `{`
let mut depth: i32 = 1;
while depth > 0 {
let t = self.next_token()?;
match t.kind {
TokenKind::LBrace => depth += 1,
TokenKind::RBrace => depth -= 1,
TokenKind::Eof => {
return Err(ParseError::syntax(
"unterminated template-string expression block",
start,
self.pos,
));
}
_ => {}
}
}
}
_ => self.pos += 1,
}
}
}
fn lex_quoted_ident(&mut self, start: usize) -> Result<TokenKind, ParseError> {
let quote = self.src[self.pos];
self.pos += 1;
loop {
match self.peek_byte(0) {
None => {
return Err(ParseError::syntax(
"unterminated quoted identifier",
start,
self.pos,
))
}
Some(b) if b == quote => {
// Doubled-quote inside the ident is an escaped quote
// (grammar: `BACKQUOTE BACKQUOTE` / `QUOTE_DOUBLE
// QUOTE_DOUBLE`).
if self.peek_byte(1) == Some(quote) {
self.pos += 2;
continue;
}
self.pos += 1;
return Ok(TokenKind::QuotedIdent);
}
Some(b'\\') => {
// The grammar's quoted-identifier rule admits:
// `ESCAPE_CHAR_COMMON` — `\b \f \r \n \t \0 \a \v \\ \xHH`
// `BACKSLASH QUOTE_DOUBLE` — only inside `"..."`
// `BACKSLASH QUOTE_SINGLE` — only inside `` `...` ``
// Otherwise `\` is a stray char and the grammar
// rejects (`~([\\<quote>])` excludes a bare
// backslash). Mirror the closed set so `"\"abc"`
// is a single quoted ident containing `"abc`.
let escape_pos = self.pos;
self.pos += 1;
match self.peek_byte(0) {
None => {}
Some(c) => match c {
b'b' | b'B' | b'f' | b'F' | b'r' | b'R' | b'n' | b'N' | b't' | b'T'
| b'0' | b'a' | b'A' | b'v' | b'V' | b'\\' => {
self.pos += 1;
}
b'"' if quote == b'"' => self.pos += 1,
b'\'' if quote == b'`' => self.pos += 1,
b'x' | b'X' => {
self.pos += 1;
if self.peek_byte(0).is_some_and(|b| b.is_ascii_hexdigit())
&& self.peek_byte(1).is_some_and(|b| b.is_ascii_hexdigit())
{
self.pos += 2;
} else {
return Err(ParseError::syntax(
r"\x escape in quoted identifier requires two hex digits",
escape_pos,
self.pos,
));
}
}
_ => {
return Err(ParseError::syntax(
format!(
"unrecognised escape '\\{}' in quoted identifier",
c as char
),
escape_pos,
self.pos + 1,
));
}
},
}
}
Some(_) => self.pos += 1,
}
}
}
}
/// Case-insensitive keyword lookup against [`KEYWORDS`]. Linear scan over
/// ~120 entries, ASCII-lowered byte compare; well under the cost of any
/// per-token decision. Returns `None` for plain identifiers.
fn lookup_keyword(bytes: &[u8]) -> Option<Kw> {
for (text, kw) in KEYWORDS {
if eq_ignore_ascii_case(bytes, text.as_bytes()) {
return Some(*kw);
}
}
None
}
fn eq_ignore_ascii_case(a: &[u8], b: &[u8]) -> bool {
if a.len() != b.len() {
return false;
}
a.iter()
.zip(b.iter())
.all(|(x, y)| x.eq_ignore_ascii_case(y))
}
#[cfg(test)]
mod tests {
use super::*;
fn lex_all(src: &str) -> Vec<TokenKind> {
let mut lx = Lexer::with_pos(src, 0);
let mut kinds = Vec::new();
loop {
let t = lx.next_token().expect("lex error in test corpus");
if t.kind == TokenKind::Eof {
break;
}
kinds.push(t.kind);
}
kinds
}
#[test]
fn arithmetic() {
assert_eq!(
lex_all("1 + 2 * 3"),
vec![
TokenKind::Number,
TokenKind::Plus,
TokenKind::Number,
TokenKind::Asterisk,
TokenKind::Number
],
);
}
#[test]
fn keywords_case_insensitive() {
assert_eq!(lex_all("SELECT")[0], TokenKind::Keyword(Kw::Select));
assert_eq!(lex_all("select")[0], TokenKind::Keyword(Kw::Select));
assert_eq!(lex_all("SeLeCt")[0], TokenKind::Keyword(Kw::Select));
}
#[test]
fn tuple_access_is_dot_then_number() {
assert_eq!(
lex_all("t.1"),
vec![TokenKind::Ident, TokenKind::Dot, TokenKind::Number],
);
}
#[test]
fn strings_with_doubled_quote() {
let kinds = lex_all("'it''s'");
assert_eq!(kinds, vec![TokenKind::String]);
}
#[test]
fn line_comment_skipped() {
assert_eq!(
lex_all("1 -- the answer\n+ 2"),
vec![TokenKind::Number, TokenKind::Plus, TokenKind::Number]
);
}
#[test]
fn block_comment_skipped() {
assert_eq!(
lex_all("1 /* nope */ + 2"),
vec![TokenKind::Number, TokenKind::Plus, TokenKind::Number]
);
}
#[test]
fn hash_comment_skipped() {
assert_eq!(
lex_all("1 # the answer\n+ 2"),
vec![TokenKind::Number, TokenKind::Plus, TokenKind::Number]
);
// A bare `#` at EOF / EOL is a comment too (the ANTLR rule's
// tail admits '\n' | '\r' | EOF directly after the `#`).
assert_eq!(lex_all("1 #"), vec![TokenKind::Number]);
assert_eq!(lex_all("#\n1"), vec![TokenKind::Number]);
assert_eq!(lex_all("# only a comment"), vec![]);
}
#[test]
fn hash_before_digit_stays_positional_ref() {
assert_eq!(lex_all("#1"), vec![TokenKind::Hash, TokenKind::Number]);
assert_eq!(
lex_all("select #2"),
vec![
TokenKind::Keyword(Kw::Select),
TokenKind::Hash,
TokenKind::Number
],
);
}
#[test]
fn null_safe_eq_wins_over_lt_eq() {
assert_eq!(
lex_all("1 <=> 2"),
vec![TokenKind::Number, TokenKind::NullSafeEq, TokenKind::Number]
);
assert_eq!(
lex_all("1 <= 2"),
vec![TokenKind::Number, TokenKind::LtEq, TokenKind::Number]
);
assert_eq!(
lex_all("1 <= > 2"),
vec![
TokenKind::Number,
TokenKind::LtEq,
TokenKind::Gt,
TokenKind::Number
],
);
}
#[test]
fn regex_family() {
assert_eq!(
lex_all("a ~ b =~ c !~ d ~* e =~* f !~* g"),
vec![
TokenKind::Ident,
TokenKind::RegexSingle,
TokenKind::Ident,
TokenKind::RegexDouble,
TokenKind::Ident,
TokenKind::NotRegex,
TokenKind::Ident,
TokenKind::IRegexSingle,
TokenKind::Ident,
TokenKind::IRegexDouble,
TokenKind::Ident,
TokenKind::NotIRegex,
TokenKind::Ident,
],
);
}
}