Skip to content

armgit / E2B

Public Imported from GitHub
0

Open-source, secure environment with real-world tools for enterprise-grade agents.

15.0 KB
Raw Blame
/**
 * Shell-style word lexer for Dockerfile instruction arguments.
 *
 * This is a port of BuildKit's `frontend/dockerfile/shell` lexer
 * (https://github.com/moby/buildkit). Variable references (`$VAR`,
 * `${VAR:-default}`, ...) are expanded from the `env` option; without it
 * they are kept verbatim, so they can be evaluated later inside the sandbox.
 */

const EOF = null

/** Equivalent of Go's `unicode.IsSpace`. */
export function isSpace(ch: string): boolean {
  return /^[\t\n\v\f\r \u0085\u00a0\p{Z}]$/u.test(ch)
}

function isLetter(ch: string): boolean {
  return /^\p{L}$/u.test(ch)
}

function isDigit(ch: string): boolean {
  return /^\p{Nd}$/u.test(ch)
}

function isSpecialParam(ch: string): boolean {
  return '@*#?-$!0'.includes(ch)
}

class Scanner {
  private readonly chars: string[]
  private pos = 0

  constructor(input: string) {
    this.chars = Array.from(input)
  }

  peek(): string | null {
    return this.pos < this.chars.length ? this.chars[this.pos] : EOF
  }

  next(): string | null {
    return this.pos < this.chars.length ? this.chars[this.pos++] : EOF
  }
}

class Words {
  private words: string[] = []
  private buf = ''
  private inWord = false

  addChar(ch: string) {
    if (isSpace(ch) && this.inWord) {
      if (this.buf.length !== 0) {
        this.words.push(this.buf)
        this.buf = ''
        this.inWord = false
      }
    } else if (!isSpace(ch)) {
      this.addRawChar(ch)
    }
  }

  addRawChar(ch: string) {
    this.buf += ch
    this.inWord = true
  }

  addString(str: string) {
    for (const ch of str) {
      this.addChar(ch)
    }
  }

  addRawString(str: string) {
    this.buf += str
    this.inWord = true
  }

  getWords(): string[] {
    if (this.buf.length > 0) {
      this.words.push(this.buf)
      this.buf = ''
      this.inWord = false
    }
    return this.words
  }
}

export interface ShellLexOptions {
  /** Keep quote characters in the output instead of removing them. */
  rawQuotes?: boolean
  /** Keep escape characters in the output instead of removing them. */
  rawEscapes?: boolean
  /** Do not treat quote characters specially at all. */
  skipProcessQuotes?: boolean
  /** Variables to expand. Defaults to no variables. */
  env?: ReadonlyMap<string, string>
  /**
   * Keep references to variables missing from `env` verbatim instead of
   * expanding them to an empty string. Defaults to `true`.
   */
  skipUnsetEnv?: boolean
}

export class ShellLex {
  private readonly escapeToken: string
  private readonly rawQuotes: boolean
  private readonly rawEscapes: boolean
  private readonly skipProcessQuotes: boolean
  private readonly env: ReadonlyMap<string, string>
  private readonly skipUnsetEnv: boolean

  constructor(escapeToken: string, options: ShellLexOptions = {}) {
    this.escapeToken = escapeToken
    this.rawQuotes = options.rawQuotes ?? false
    this.rawEscapes = options.rawEscapes ?? false
    this.skipProcessQuotes = options.skipProcessQuotes ?? false
    this.env = options.env ?? new Map()
    this.skipUnsetEnv = options.skipUnsetEnv ?? true
  }

  /** Process a single word: remove quotes/escapes, keep whitespace. */
  processWord(word: string): string {
    return this.process(word).result
  }

  /** Process a line into whitespace separated words, honoring quotes. */
  processWords(word: string): string[] {
    return this.process(word).words
  }

  private process(word: string): { result: string; words: string[] } {
    const sw = new ShellWord(
      new Scanner(word),
      this.escapeToken,
      this.rawQuotes,
      this.rawEscapes,
      this.skipProcessQuotes,
      this.env,
      this.skipUnsetEnv
    )
    try {
      return sw.processStopOn(EOF, this.rawEscapes)
    } catch (err) {
      const message = err instanceof Error ? err.message : String(err)
      throw new Error(`failed to process ${JSON.stringify(word)}: ${message}`)
    }
  }
}

class ShellWord {
  constructor(
    private readonly scanner: Scanner,
    private readonly escapeToken: string,
    private readonly rawQuotes: boolean,
    private rawEscapes: boolean,
    private readonly skipProcessQuotes: boolean,
    private readonly env: ReadonlyMap<string, string>,
    private readonly skipUnsetEnv: boolean
  ) {}

  processStopOn(
    stopChar: string | null,
    rawEscapes: boolean
  ): { result: string; words: string[] } {
    let result = ''
    const words = new Words()

    const previousRawEscapes = this.rawEscapes
    this.rawEscapes = rawEscapes
    try {
      while (this.scanner.peek() !== EOF) {
        let ch = this.scanner.peek() as string

        if (stopChar !== EOF && ch === stopChar) {
          this.scanner.next()
          return { result, words: words.getWords() }
        }

        const fn = this.specialHandler(ch)
        if (fn) {
          const tmp = fn()
          result += tmp
          if (ch === '$') {
            words.addString(tmp)
          } else {
            words.addRawString(tmp)
          }
          continue
        }

        ch = this.scanner.next() as string
        if (ch === this.escapeToken) {
          if (this.rawEscapes) {
            words.addRawChar(ch)
            result += ch
          }

          // the escape token escapes the next character, except at end of line
          const next = this.scanner.next()
          if (next === EOF) {
            break
          }
          ch = next
          words.addRawChar(ch)
        } else {
          words.addChar(ch)
        }
        result += ch
      }
    } finally {
      this.rawEscapes = previousRawEscapes
    }

    if (stopChar !== EOF) {
      throw new Error(
        `unexpected end of statement while looking for matching ${stopChar}`
      )
    }
    return { result, words: words.getWords() }
  }

  private specialHandler(ch: string): (() => string) | undefined {
    switch (ch) {
      case '$':
        return () => this.processDollar()
      case '<':
        return () => this.processPossibleHeredoc()
      case "'":
        return this.skipProcessQuotes
          ? undefined
          : () => this.processSingleQuote()
      case '"':
        return this.skipProcessQuotes
          ? undefined
          : () => this.processDoubleQuote()
      default:
        return undefined
    }
  }

  private processSingleQuote(): string {
    // All chars between single quotes are taken as-is; a single quote
    // cannot be escaped inside single quotes.
    let result = ''
    let ch = this.scanner.next() as string
    if (this.rawQuotes) {
      result += ch
    }

    for (;;) {
      const next = this.scanner.next()
      if (next === EOF) {
        throw new Error(
          'unexpected end of statement while looking for matching single-quote'
        )
      }
      ch = next
      if (ch === "'") {
        if (this.rawQuotes) {
          result += ch
        }
        return result
      }
      result += ch
    }
  }

  private processDoubleQuote(): string {
    // All chars up to the next " are taken as-is, even ', except `$`.
    // The escape token only escapes `"`, `$` and itself.
    let result = ''
    const open = this.scanner.next() as string
    if (this.rawQuotes) {
      result += open
    }

    for (;;) {
      const peeked = this.scanner.peek()
      if (peeked === EOF) {
        throw new Error(
          'unexpected end of statement while looking for matching double-quote'
        )
      }
      if (peeked === '"') {
        const ch = this.scanner.next() as string
        if (this.rawQuotes) {
          result += ch
        }
        return result
      }
      if (peeked === '$') {
        result += this.processDollar()
        continue
      }

      let ch = this.scanner.next() as string
      if (ch === this.escapeToken) {
        if (this.rawEscapes) {
          result += ch
        }
        const after = this.scanner.peek()
        if (after === EOF) {
          // ignore escape token at end of word
          continue
        }
        if (after === '"' || after === '$' || after === this.escapeToken) {
          ch = this.scanner.next() as string
        }
      }
      result += ch
    }
  }

  private processDollar(): string {
    this.scanner.next() // '$'

    if (this.scanner.peek() !== '{') {
      const name = this.processName()
      if (name === '') {
        return '$'
      }
      const value = this.env.get(name)
      if (value === undefined && this.skipUnsetEnv) {
        return '$' + name
      }
      return value ?? ''
    }

    this.scanner.next() // '{'
    const first = this.scanner.peek()
    if (first === EOF) {
      throw new Error("syntax error: missing '}'")
    }
    if (first === '{' || first === '}' || first === ':') {
      throw new Error('syntax error: bad substitution')
    }

    const name = this.processName()
    let ch = this.scanner.next()
    if (ch === EOF) {
      throw new Error("syntax error: missing '}'")
    }
    let chs = ch
    const value = this.env.get(name)

    if (ch === '}') {
      if (value === undefined && this.skipUnsetEnv) {
        return '${' + name + '}'
      }
      return value ?? ''
    }

    let nullIsUnset = false
    if (ch === ':') {
      nullIsUnset = true
      const modifier = this.scanner.next()
      if (modifier === EOF) {
        throw new Error("syntax error: missing '}'")
      }
      ch = modifier
      chs += ch
    }

    if (!nullIsUnset) {
      if (ch === '/') {
        return this.processDollarReplace(name)
      }
      if (!'+-?#%'.includes(ch)) {
        throw new Error(`unsupported modifier (${chs}) in substitution`)
      }
    }

    const rawEscapes = ch === '#' || ch === '%'
    if (nullIsUnset && rawEscapes) {
      throw new Error(`unsupported modifier (${chs}) in substitution`)
    }
    const word = this.processStopOnOrMissingBrace('}', rawEscapes)
    if (value === undefined && this.skipUnsetEnv) {
      return '${' + name + chs + word + '}'
    }

    const unset = value === undefined || (nullIsUnset && value === '')
    switch (ch) {
      case '-':
        return unset ? word : (value as string)
      case '+':
        return unset ? '' : word
      case '?':
        if (value === undefined) {
          throw new Error(`${name}: ${word || 'is not allowed to be unset'}`)
        }
        if (unset) {
          throw new Error(`${name}: ${word || 'is not allowed to be empty'}`)
        }
        return value
      case '#':
      case '%': {
        // `#`/`%` remove the shortest match, `##`/`%%` the longest
        const greedy = word.startsWith(ch)
        const pattern = greedy ? word.slice(1) : word
        return ch === '%'
          ? trimSuffix(pattern, value ?? '', greedy)
          : trimPrefix(pattern, value ?? '', greedy)
      }
      default:
        throw new Error(`unsupported modifier (${chs}) in substitution`)
    }
  }

  private processDollarReplace(name: string): string {
    let op = '/'
    if (this.scanner.peek() === '/') {
      this.scanner.next()
      op = '//'
    }
    let pattern: string
    try {
      pattern = this.processStopOn('/', true).result
    } catch (err) {
      if (this.scanner.peek() === EOF) {
        throw new Error("syntax error: missing '/' in ${}")
      }
      throw err
    }
    const replacement = this.processStopOnOrMissingBrace('}', true)

    const value = this.env.get(name)
    if (value === undefined && this.skipUnsetEnv) {
      return '${' + name + op + pattern + '/' + replacement + '}'
    }
    const re = shellPatternToRegex(pattern, true, false, op === '//')
    return (value ?? '').replace(re, () => replacement)
  }

  private processStopOnOrMissingBrace(
    stopChar: string,
    rawEscapes: boolean
  ): string {
    try {
      return this.processStopOn(stopChar, rawEscapes).result
    } catch (err) {
      if (this.scanner.peek() === EOF) {
        throw new Error("syntax error: missing '}'")
      }
      throw err
    }
  }

  private processName(): string {
    // Read in a name (alphanumeric or _). A leading digit sequence or a
    // special parameter character is a complete name on its own.
    let name = ''
    for (;;) {
      const ch = this.scanner.peek()
      if (ch === EOF) {
        break
      }
      if (name.length === 0 && isDigit(ch)) {
        let digits = ''
        for (;;) {
          const d = this.scanner.peek()
          if (d === EOF || !isDigit(d)) {
            break
          }
          digits += this.scanner.next() as string
        }
        return digits
      }
      if (name.length === 0 && isSpecialParam(ch)) {
        return this.scanner.next() as string
      }
      if (!isLetter(ch) && !isDigit(ch) && ch !== '_') {
        break
      }
      name += this.scanner.next() as string
    }
    return name
  }

  private processPossibleHeredoc(): string {
    this.scanner.next() // '<'
    if (this.scanner.peek() !== '<') {
      return '<'
    }
    this.scanner.next()

    // a heredoc may have whitespace between `<<` and the terminator word
    let space = ''
    for (;;) {
      const ch = this.scanner.peek()
      if (ch !== '\t' && ch !== '\r' && ch !== ' ') {
        break
      }
      space += this.scanner.next() as string
    }
    return '<<' + space
  }
}

/**
 * Convert a shell wildcard pattern (`?` one character, `*` the shortest or,
 * when greedy, the longest run) to a regular expression. Bracket
 * expressions are not supported, like in BuildKit.
 */
function shellPatternToRegex(
  pattern: string,
  greedy: boolean,
  anchored: boolean,
  global = false
): RegExp {
  const chars = Array.from(pattern)
  let out = anchored ? '^' : ''
  const star = greedy ? '.*' : '.*?'
  for (let i = 0; i < chars.length; i++) {
    let ch = chars[i]
    if (ch === '*') {
      out += star
      continue
    }
    if (ch === '?') {
      out += '.'
      continue
    }
    if (ch === '\\') {
      // `}` and `/` are escaped to be part of the `${}` word, the escape
      // itself is not part of the pattern
      const next = chars[i + 1]
      if (next === '}' || next === '/') {
        continue
      }
      i++
      if (next !== '*' && next !== '?' && next !== '\\') {
        throw new Error(
          `invalid pattern (${pattern}) in substitution: invalid escape '\\${next ?? ''}'`
        )
      }
      ch = next
    }
    if ('[]{}.+()|^$*?\\'.includes(ch)) {
      out += '\\'
    }
    out += ch
  }
  return new RegExp(out, global ? 'gu' : 'u')
}

function trimPrefix(pattern: string, value: string, greedy: boolean): string {
  return value.replace(shellPatternToRegex(pattern, greedy, true), '')
}

/** Reverse a pattern without turning `a\*c` into `c*\a`. */
function reversePattern(pattern: string): string {
  const chars = Array.from(pattern)
  const out: string[] = []
  for (let i = 0; i < chars.length; i++) {
    if (chars[i] === '\\' && i + 1 < chars.length) {
      out.unshift(chars[i], chars[i + 1])
      i++
    } else {
      out.unshift(chars[i])
    }
  }
  return out.join('')
}

function reverseString(str: string): string {
  return Array.from(str).reverse().join('')
}

function trimSuffix(pattern: string, value: string, greedy: boolean): string {
  // regular expressions cannot find the shortest rightmost match, so both
  // the pattern and the value are reversed to turn it into a leftmost search
  return reverseString(
    trimPrefix(reversePattern(pattern), reverseString(value), greedy)
  )
}