VectorCraft: An open-source, vector art tool written by scratch in pure Rust
//! The scanner of the editing data: PostScript-like tokens, read line by line.
//!
//! - A line starting with `%_` holds tokens like any other line (they are marked [`Token::hidden`]:
//! readers of the printed format skip them as comments).
//! - Any other line starting with `%` is a [`Tok::Comment`] (the section markers).
//! - `%_` later in a line starts tokens too, hidden (a pseudo-comment operator after its
//! operands, as a gradient stop's `… 50 100 %_Bs`: AI7 spec §1.4.3); any other `%` later in a
//! line starts a comment that is skipped.
//! - The binary image data after `%%BeginData:` is one [`Tok::Data`]; the binary preview of a placed
//! file is one [`Tok::Preview`].
//! - A document included whole (`%%BeginDocument:` … `%%EndDocument` lines, DSC 3.0 p. 68, nested
//! ones counted, the counted payloads of `%%BeginData:` and `%%BeginBinary:` inside skipped) is
//! one [`Tok::Document`].
use super::super::lex::{find, hex_decode};
/// A token.
#[derive(Clone, Debug, PartialEq)]
pub(super) enum Tok<'a> {
Num(f64),
/// A literal `(…)` string, escapes undone.
Str(Vec<u8>),
/// A hexadecimal `<…>` string.
Hex(Vec<u8>),
/// A literal `/name`.
Name(&'a [u8]),
/// An operator (any other word, `[` and `]` included).
Word(&'a [u8]),
/// A comment line, without its first `%`.
Comment(&'a [u8]),
/// An image's samples: what follows `XI` in a `%%BeginData:` section.
Data(&'a [u8]),
/// A placed file's preview: what is between `%AI26_BeginPlacedObjectPreview` and its end.
Preview(&'a [u8]),
/// A document included whole: what is between `%%BeginDocument:`'s line and the line of the
/// `%%EndDocument` that ends it, or `None` when none does (the rest of the data is skipped).
Document(Option<&'a [u8]>),
}
#[derive(Clone, Debug, PartialEq)]
pub(super) struct Token<'a> {
pub tok: Tok<'a>,
/// Read from a `%_` line.
pub hidden: bool,
}
/// A placed file's preview (binary): (begin, end) markers.
const PREVIEW: (&[u8], &[u8]) = (b"%AI26_BeginPlacedObjectPreview", b"%AI26_EndPlacedObjectPreview");
/// A placed file's markers, which a hidden one has on `%_` lines.
const PLACE: [&[u8]; 2] = [b"%AI5_BeginPlace", b"%AI5_EndPlace"];
/// A path's note comments, also read on `%_` lines.
const NOTES: [&[u8]; 3] = [b"%AI3_Note:", b"%%Note:", b"%AI5_NoteMore:"];
/// The markers around a placed file's relative folder, which a hidden one has on `%_` lines too.
const RELATIVE: [&[u8]; 2] = [b"%AI28_BeginPlacedRelativePath", b"%AI28_EndPlacedRelativePath"];
/// A document included whole: (begin, end) markers.
const DOCUMENT: (&[u8], &[u8]) = (b"%%BeginDocument", b"%%EndDocument");
fn is_space(b: u8) -> bool {
matches!(b, b' ' | b'\t' | b'\r' | b'\n' | b'\x0c' | b'\0')
}
fn is_delim(b: u8) -> bool {
matches!(b, b'(' | b')' | b'<' | b'>' | b'[' | b']' | b'{' | b'}' | b'/' | b'%')
}
fn is_eol(b: u8) -> bool {
matches!(b, b'\r' | b'\n')
}
pub(super) struct Lexer<'a> {
src: &'a [u8],
pos: usize,
/// The current line started with `%_`.
hidden: bool,
}
impl<'a> Lexer<'a> {
pub fn new(src: &'a [u8]) -> Self {
Self { src, pos: 0, hidden: false }
}
fn peek(&self) -> Option<u8> {
self.src.get(self.pos).copied()
}
fn rest(&self) -> &'a [u8] {
self.src.get(self.pos..).unwrap_or_default()
}
fn at_line_start(&self) -> bool {
self.pos == 0 || self.src.get(self.pos - 1).is_some_and(|b| is_eol(*b))
}
/// The rest of the current line (without its end), the position moved past it.
fn line(&mut self) -> &'a [u8] {
let rest = self.rest();
let n = rest.iter().position(|b| is_eol(*b)).unwrap_or(rest.len());
self.pos += n;
rest.get(..n).unwrap_or_default()
}
/// [`Self::line`], the position moved past its end too (`\r`, `\n` or `\r\n`).
fn whole_line(&mut self) -> &'a [u8] {
let line = self.line();
if self.peek() == Some(b'\r') {
self.pos += 1;
}
if self.peek() == Some(b'\n') {
self.pos += 1;
}
line
}
/// Where the next token is looked for.
pub fn offset(&self) -> usize {
self.pos
}
/// Look for the next token at `pos` again (the end of the data at most).
pub fn seek(&mut self, pos: usize) {
self.pos = pos.min(self.src.len());
}
/// The data from `from` to `to` (empty where that isn't in it).
pub fn slice(&self, from: usize, to: usize) -> &'a [u8] {
self.src.get(from..to).unwrap_or_default()
}
/// Skip past the next `needle` (the end of data that isn't tokens) → what was skipped.
pub fn skip_past(&mut self, needle: &[u8]) -> &'a [u8] {
let rest = self.rest();
let n = find(rest, needle).map_or(rest.len(), |i| i + needle.len());
self.pos += n;
rest.get(..n).unwrap_or_default()
}
/// The next token; `None` at the end.
pub fn next_token(&mut self) -> Option<Token<'a>> {
loop {
let b = self.peek()?;
if is_eol(b) {
self.pos += 1;
self.hidden = false;
continue;
}
if is_space(b) {
self.pos += 1;
continue;
}
if b == b'%' {
let start = self.at_line_start();
// A pseudo-comment: `%_` at the line's start, or after operands.
if self.rest().starts_with(b"%_") && (start || self.rest().get(2).is_some_and(|b| b.is_ascii_alphabetic())) {
self.pos += 2;
self.hidden = true;
continue;
}
// Right after `%_` a section still starts (a hidden image's data, #1264).
let prefixed = self.hidden && self.pos >= 2 && self.src.get(self.pos - 2..self.pos) == Some(b"%_");
let line = self.line();
if (start || prefixed)
&& let Some(t) = self.special(line)
{
return Some(Token { hidden: prefixed, ..t });
}
if start {
return Some(Token { tok: Tok::Comment(line.get(1..).unwrap_or_default()), hidden: false });
}
// So does a hidden placed file: its `~` ends what `%_%AI5_BeginPlace` begins. And a
// hidden path's note.
if prefixed && PLACE.iter().chain(&NOTES).chain(&RELATIVE).any(|m| line.starts_with(m)) {
return Some(Token { tok: Tok::Comment(line.get(1..).unwrap_or_default()), hidden: true });
}
// A comment later in a line.
continue;
}
let hidden = self.hidden;
self.pos += 1;
let tok = match b {
b'(' => Tok::Str(self.string()),
b'<' => {
let rest = self.rest();
let end = rest.iter().position(|b| *b == b'>').unwrap_or(rest.len());
self.pos += (end + 1).min(rest.len());
// Malformed digits read as an empty string.
Tok::Hex(rest.get(..end).and_then(hex_decode).unwrap_or_default())
}
b'[' | b']' | b'{' | b'}' | b')' | b'>' => Tok::Word(self.src.get(self.pos - 1..self.pos).unwrap_or_default()),
b'/' => Tok::Name(self.word()),
_ => {
self.pos -= 1;
let w = self.word();
match number(w) {
Some(v) => Tok::Num(v),
None => Tok::Word(w),
}
}
};
return Some(Token { tok, hidden });
}
}
/// The token a comment line stands for when it starts a section read differently: image data
/// (`%%BeginData:` followed by `XI`), a placed file's preview and skipped binary sections.
fn special(&mut self, line: &'a [u8]) -> Option<Token<'a>> {
if line.starts_with(PREVIEW.0) {
let from = self.pos;
let after = find(self.rest(), PREVIEW.1).map_or(self.src.len(), |i| self.pos + i);
self.pos = after;
self.line();
return Some(Token { tok: Tok::Preview(self.slice(from, after)), hidden: false });
}
if line.starts_with(DOCUMENT.0) {
return Some(Token { tok: Tok::Document(self.document()), hidden: false });
}
let n = line.strip_prefix(b"%%BeginData:")?;
let n: usize = std::str::from_utf8(n).ok()?.split_whitespace().next()?.parse().ok()?;
// The count starts after the comment's line end character (a following `\n` counts).
let from = self.pos + 1;
let region = self.src.get(from..)?;
let mut skip = region.iter().take_while(|b| is_space(**b)).count();
// A hidden object's section is on `%_` lines up to its samples: `%_XI`, then `%_%%EndData`.
let hidden = region.get(skip..)?.starts_with(b"%_XI");
if hidden {
skip += 2;
} else if !region.get(skip..)?.starts_with(b"XI") {
return None;
}
let mut data = from + skip + 2;
match (self.src.get(data), self.src.get(data + 1)) {
(Some(b'\r'), Some(b'\n')) => data += 2,
(Some(b'\r' | b'\n'), _) => data += 1,
_ => {}
}
// The count leaves out an alpha channel: the data ends where `%%EndData` starts.
let declared = from.saturating_add(n).min(self.src.len()).max(data);
let window =
self.src.get(declared.saturating_sub(4).max(data)..declared.saturating_add(n.saturating_mul(2)).saturating_add(64).min(self.src.len()));
let end = match window.and_then(|w| find(w, b"%%EndData")) {
Some(i) => declared.saturating_sub(4).max(data) + i,
None => declared,
};
self.pos = end;
let samples = self.src.get(data..end).unwrap_or_default();
let samples = if hidden { samples.strip_suffix(b"%_").unwrap_or(samples) } else { samples };
Some(Token { tok: Tok::Data(samples), hidden: false })
}
/// After a `%%BeginDocument:` line: the document up to the start of the `%%EndDocument` line
/// that ends it, the position moved past that line. A `%%BeginDocument:` line inside starts a
/// nested document, and the bytes a `%%BeginData:` or `%%BeginBinary:` line inside counts are
/// skipped (DSC 3.0 pp. 68 and 72): a marker in them ends nothing. `None` when no line ends it:
/// the position moves to the end of the data.
fn document(&mut self) -> Option<&'a [u8]> {
// The rest of the `%%BeginDocument:` line is its end; the line after the document starts
// tokens afresh (not hidden unless it starts with `%_`).
self.whole_line();
self.hidden = false;
let from = self.pos;
let mut depth = 0usize;
while self.pos < self.src.len() {
let start = self.pos;
// A hidden object's document is on `%_` lines.
let line = self.whole_line();
let line = line.strip_prefix(b"%_").unwrap_or(line);
if line.starts_with(DOCUMENT.1) {
if depth == 0 {
return Some(self.slice(from, start));
}
depth -= 1;
} else if line.starts_with(DOCUMENT.0) {
depth = depth.saturating_add(1);
} else if let Some(n) = line.strip_prefix(b"%%BeginData:").or_else(|| line.strip_prefix(b"%%BeginBinary:")) {
self.skip_counted(n);
}
}
None
}
/// Skip what a `%%BeginData:` or `%%BeginBinary:` comment counts (`n`, its text after the
/// colon): `count [type [Bytes|Lines]]`, bytes unless it says `Lines`.
fn skip_counted(&mut self, n: &[u8]) {
let mut words = std::str::from_utf8(n).unwrap_or_default().split_whitespace();
let Some(count) = words.next().and_then(|c| c.parse::<usize>().ok()) else { return };
if words.nth(1).is_some_and(|unit| unit.eq_ignore_ascii_case("lines")) {
for _ in 0..count {
if self.pos >= self.src.len() {
break;
}
self.whole_line();
}
} else {
// The count starts after the comment's line end, which `document` has passed.
self.pos = self.pos.saturating_add(count).min(self.src.len());
}
}
/// A regular token's characters.
fn word(&mut self) -> &'a [u8] {
let start = self.pos;
while let Some(b) = self.peek() {
if is_space(b) || is_delim(b) {
break;
}
self.pos += 1;
}
self.src.get(start..self.pos).unwrap_or_default()
}
/// A literal string after its `(` (an unterminated one ends with the data).
fn string(&mut self) -> Vec<u8> {
let mut out = vec![];
let mut depth = 0usize;
while let Some(b) = self.peek() {
self.pos += 1;
match b {
b'(' => {
depth += 1;
out.push(b);
}
b')' if depth == 0 => break,
b')' => {
depth -= 1;
out.push(b);
}
b'\\' => {
let Some(e) = self.peek() else { break };
self.pos += 1;
match e {
b'n' => out.push(b'\n'),
b'r' => out.push(b'\r'),
b't' => out.push(b'\t'),
b'b' => out.push(8),
b'f' => out.push(12),
b'\r' => {
if self.peek() == Some(b'\n') {
self.pos += 1;
}
}
b'\n' => {}
b'0'..=b'7' => {
let mut v = u32::from(e - b'0');
for _ in 0..2 {
match self.peek() {
Some(d @ b'0'..=b'7') => {
v = v * 8 + u32::from(d - b'0');
self.pos += 1;
}
_ => break,
}
}
out.push((v & 0xff) as u8);
}
other => out.push(other),
}
}
_ => out.push(b),
}
}
out
}
}
/// A finite number token (`12`, `-3.5`, `.5`, `1e-3`), or `None`.
pub(super) fn number(w: &[u8]) -> Option<f64> {
let first = *w.first()?;
if !(first.is_ascii_digit() || matches!(first, b'+' | b'-' | b'.')) {
return None;
}
if w.iter().any(|b| b.is_ascii_alphabetic() && !matches!(b, b'e' | b'E')) {
return None;
}
let v: f64 = std::str::from_utf8(w).ok()?.parse().ok()?;
v.is_finite().then_some(v)
}
#[cfg(test)]
mod tests {
use super::*;
fn all(src: &[u8]) -> Vec<Token<'_>> {
let mut l = Lexer::new(src);
std::iter::from_fn(|| l.next_token()).collect()
}
fn t(tok: Tok<'_>, hidden: bool) -> Token<'_> {
Token { tok, hidden }
}
#[test]
fn hidden_lines_and_comments() {
let toks = all(b"%AI5_BeginLayer\r\n1 (a\\)b) Ln\r%_/X : ;\n0 %_BS\n<4142>\n");
assert_eq!(
toks,
vec![
t(Tok::Comment(b"AI5_BeginLayer"), false),
t(Tok::Num(1.0), false),
t(Tok::Str(b"a)b".to_vec()), false),
t(Tok::Word(b"Ln"), false),
t(Tok::Name(b"X"), true),
t(Tok::Word(b":"), true),
t(Tok::Word(b";"), true),
t(Tok::Num(0.0), false),
// A pseudo-comment operator after its operands.
t(Tok::Word(b"BS"), true),
t(Tok::Hex(b"AB".to_vec()), false),
]
);
}
#[test]
fn a_percent_later_in_a_line_is_a_comment_unless_an_operator_follows_it() {
let toks = all(b"1 % a note (\n2 %_Bs\n3 %_ 4\n");
let words: Vec<_> = toks.iter().map(|t| (t.tok.clone(), t.hidden)).collect();
assert_eq!(words, [(Tok::Num(1.0), false), (Tok::Num(2.0), false), (Tok::Word(b"Bs"), true), (Tok::Num(3.0), false)]);
}
#[test]
fn image_data_is_one_token() {
// The count starts after the comment's `\r`: `\nXI\n` and 3 samples; an alpha channel
// after them isn't counted.
let src = b"[1 0 0 1 0 0] XI\r\n%%BeginData: 7\r\nXI\n(%)A%%EndData\r\nXH\r\n";
let toks = all(src);
assert!(toks.contains(&t(Tok::Data(b"(%)A"), false)), "{toks:?}");
assert_eq!(toks.last(), Some(&t(Tok::Word(b"XH"), false)));
// #1264: on `%_` lines too, where the samples would otherwise be read as tokens.
let toks = all(b"%_[1 0 0 1 0 0] XI\r%_%%BeginData: 7\rXI\n(U)A%%EndData\r%_XH\ru\r");
assert!(toks.contains(&t(Tok::Data(b"(U)A"), true)), "{toks:?}");
assert_eq!(toks.last(), Some(&t(Tok::Word(b"u"), false)), "{toks:?}");
// A section of hex lines is comments.
let toks = all(b"%%BeginData: 9 Hex Bytes\r\n%00FF\r\n%%EndData\r\n1\r\n");
assert_eq!(toks.last(), Some(&t(Tok::Num(1.0), false)));
// On `%_` lines (an opacity mask's image, say) too: a `(` in the samples starts no string.
let toks = all(b"%_XI\r%_%%BeginData: 4\r%_XI\r(\x00\x00\x00%_%%EndData\r%_XH\r1\r");
assert!(toks.contains(&t(Tok::Data(b"(\x00\x00\x00"), true)), "{toks:?}");
assert_eq!(toks.last(), Some(&t(Tok::Num(1.0), false)));
}
#[test]
fn binary_sections_are_one_token() {
let toks = all(b"%AI26_BeginPlacedObjectPreview\r\n2 1 (\x00\xff\r\n%AI26_EndPlacedObjectPreview\r\nN\r\n");
assert_eq!(toks, vec![t(Tok::Preview(b"\r\n2 1 (\x00\xff\r\n"), false), t(Tok::Word(b"N"), false)]);
let toks = all(b"%%BeginDocument: a.eps\n(\x00 frobnicate\n%%EndDocument\nN\n");
assert_eq!(toks, vec![t(Tok::Document(Some(b"(\x00 frobnicate\n")), false), t(Tok::Word(b"N"), false)]);
// One that doesn't end takes the rest.
assert_eq!(all(b"%AI26_BeginPlacedObjectPreview\r1 (").len(), 1);
}
#[test]
fn an_included_document_ends_at_its_own_end_line() {
// A nested document, `%%EndDocument` later in a line and in counted data end nothing; `\r\n`,
// `\r` and `\n` lines alike.
let body = b"1 0 0 setrgbcolor\r\n%%BeginDocument: inner.eps\r(%%EndDocument) pop\r%%EndDocument\r\n\
%%BeginData: 14 Binary Bytes\n%%EndDocument\n%%BeginBinary: 14\n%%EndDocument\n\
%%BeginData: 1 Hex Lines\n%%EndDocument\n";
let mut src = b"%%BeginDocument: outer.eps\r\n".to_vec();
src.extend_from_slice(body);
src.extend_from_slice(b"%%EndDocument\r\n~\r\n");
assert_eq!(all(&src), vec![t(Tok::Document(Some(body)), false), t(Tok::Word(b"~"), false)]);
// On a hidden object's `%_` lines.
let toks = all(b"%_%%BeginDocument: a.eps\r%_1 0 0 setrgbcolor\r%_%%EndDocument\r%_~\r");
assert_eq!(toks, vec![t(Tok::Document(Some(b"%_1 0 0 setrgbcolor\r")), true), t(Tok::Word(b"~"), true)]);
// The line after it is hidden only when it starts with `%_`.
let toks = all(b"%_%%BeginDocument: a.eps\r%_%%EndDocument\rN\r");
assert_eq!(toks, vec![t(Tok::Document(Some(b"")), true), t(Tok::Word(b"N"), false)]);
}
#[test]
fn an_included_document_that_doesnt_end_takes_the_rest() {
for src in [&b"%%BeginDocument: a.eps\n%%BeginDocument: b.eps\n%%EndDocument\n~\n"[..], b"%%BeginDocument: a.eps\n%%BeginData: 99999999999\n"]
{
assert_eq!(all(src), vec![t(Tok::Document(None), false)]);
}
}
#[test]
fn skip_past_data() {
let mut l = Lexer::new(b"/Binary : /ASCII85Decode ,\r\n%_(abc[%\r\n%x~>\r\n%_; 1");
for _ in 0..4 {
l.next_token();
}
assert_eq!(l.skip_past(b"~>"), b"\r\n%_(abc[%\r\n%x~>");
assert_eq!(l.next_token().map(|t| t.tok), Some(Tok::Word(b";")));
assert_eq!(l.next_token().map(|t| t.tok), Some(Tok::Num(1.0)));
}
#[test]
fn hostile_input_ends() {
for src in [&b"(unterminated \\"[..], b"<12", b"%%BeginData: 99999999999\r\nXI", b"%%BeginData: 5\rXI\r", b"/", b"1e999 nan"] {
let toks = all(src);
assert!(toks.len() < 10);
}
}
}