Open-source, secure environment with real-world tools for enterprise-grade agents.
Python agentaiai-agentai-agentscode-interpretercopilotdevelopmentdevtoolsgptgpt-4javascriptllmnextjsopenaipythonreactsoftwaretypescript
/**
* Dockerfile syntax parser.
*
* Port of BuildKit's `frontend/dockerfile/parser` (https://github.com/moby/buildkit):
* handles parser directives (`# escape=`), comments, line continuations,
* builder flags (`--chown=...`), JSON/exec forms and heredocs.
*/
import { ShellLex, isSpace } from './lexer'
export interface DockerfileHeredoc {
name: string
/** Raw content including trailing newlines of every line. */
content: string
/** `<<-`: leading tabs are stripped from content lines. */
chomp: boolean
/** Unquoted terminator: content is subject to variable expansion. */
expand: boolean
fileDescriptor: number
}
export interface DockerfileInstruction {
/** Upper-cased instruction keyword, e.g. `RUN`. */
name: string
/** The full (continuation-joined) instruction line. */
original: string
/** Builder flags, e.g. `--chown=user:group`, with quotes removed. */
flags: string[]
/**
* Instruction arguments. For `ENV` and `LABEL` these are
* `[key, value, delimiter, ...]` triples where delimiter is `=` or ``.
*/
args: string[]
/** Arguments were given in JSON (exec) form. */
json: boolean
heredocs: DockerfileHeredoc[]
startLine: number
endLine: number
}
export interface DockerfileWarning {
message: string
line: number
}
export interface DockerfileAst {
instructions: DockerfileInstruction[]
escapeToken: string
warnings: DockerfileWarning[]
}
export class DockerfileSyntaxError extends Error {
constructor(
message: string,
public readonly line?: number
) {
super(
line === undefined
? `Dockerfile parse error: ${message}`
: `Dockerfile parse error on line ${line}: ${message}`
)
this.name = 'DockerfileSyntaxError'
}
}
const DEFAULT_ESCAPE_TOKEN = '\\'
function withLine<T>(line: number, fn: () => T): T {
try {
return fn()
} catch (err) {
if (err instanceof DockerfileSyntaxError) {
throw err
}
throw new DockerfileSyntaxError(
err instanceof Error ? err.message : String(err),
line
)
}
}
const WHITESPACE = /[\t\v\f\r ]+/
const DIRECTIVE = /^#\s*([a-zA-Z][a-zA-Z0-9]*)\s*=\s*(.+?)\s*$/
const VALID_DIRECTIVES = new Set(['syntax', 'escape', 'check'])
const HEREDOC_INSTRUCTIONS = new Set(['ADD', 'COPY', 'RUN'])
type LineParser = (
rest: string,
escapeToken: string
) => { args: string[]; json: boolean }
function trimNewline(line: string): string {
let end = line.length
while (end > 0 && (line[end - 1] === '\n' || line[end - 1] === '\r')) {
end--
}
return line.slice(0, end)
}
function trimLeadingWhitespace(line: string): string {
return line.replace(/^[\t\n\v\f\r \u0085\u00a0\p{Z}]+/u, '')
}
function isComment(line: string): boolean {
return trimLeadingWhitespace(line).startsWith('#')
}
function escapeRegExp(s: string): string {
return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
}
/** Split into lines the way a `bufio.Scanner` with newline-preserving split does. */
function splitLines(content: string): string[] {
return content.match(/[^\n]*\n|[^\n]+$/g) ?? []
}
class Directives {
escapeToken = DEFAULT_ESCAPE_TOKEN
private continuation = this.buildContinuationRegex(DEFAULT_ESCAPE_TOKEN)
private done = false
private seen = new Set<string>()
/** Returns `true` when the line was a parser directive. */
possibleParserDirective(line: string): boolean {
if (this.done) {
return false
}
const match = DIRECTIVE.exec(line)
if (!match) {
this.done = true
return false
}
const key = match[1].toLowerCase()
if (!VALID_DIRECTIVES.has(key)) {
this.done = true
return false
}
if (this.seen.has(key)) {
throw new Error(`only one ${key} parser directive can be used`)
}
this.seen.add(key)
if (key === 'escape') {
this.setEscapeToken(match[2])
}
return true
}
trimContinuation(line: string): { line: string; isEndOfLine: boolean } {
if (this.continuation.test(line)) {
return { line: line.replace(this.continuation, '$1'), isEndOfLine: false }
}
return { line, isEndOfLine: true }
}
private setEscapeToken(token: string) {
if (token !== '`' && token !== '\\') {
throw new Error(`invalid escape token '${token}' does not match \` or \\`)
}
this.escapeToken = token
this.continuation = this.buildContinuationRegex(token)
}
private buildContinuationRegex(token: string): RegExp {
const t = escapeRegExp(token)
// The escape token is a line continuation when it is the last
// non-whitespace character and not itself escaped.
return new RegExp(`([^${t}])${t}[ \\t]*$|^${t}[ \\t]*$`)
}
}
function parseString(rest: string) {
return { args: rest === '' ? [] : [rest], json: false }
}
function parseIgnore() {
return { args: [], json: false }
}
function parseStringsWhitespaceDelimited(rest: string) {
if (rest === '') {
return { args: [], json: false }
}
return { args: rest.split(WHITESPACE), json: false }
}
class NotJsonArrayError extends Error {}
class NotStringArrayError extends Error {}
function parseJSON(rest: string): string[] {
rest = trimLeadingWhitespace(rest)
if (!rest.startsWith('[')) {
throw new NotJsonArrayError()
}
let parsed: unknown
try {
parsed = JSON.parse(rest)
} catch (err) {
throw new NotJsonArrayError()
}
if (!Array.isArray(parsed)) {
throw new NotJsonArrayError()
}
for (const item of parsed) {
if (typeof item !== 'string') {
throw new NotStringArrayError('Only strings are supported in JSON arrays')
}
}
return parsed as string[]
}
function parseMaybeJSON(rest: string) {
if (rest === '') {
return { args: [], json: false }
}
try {
return { args: parseJSON(rest), json: true }
} catch (err) {
if (err instanceof NotStringArrayError) {
throw err
}
return { args: [rest], json: false }
}
}
// `HEALTHCHECK [flags] <type> <command>`: the type (`CMD`/`NONE`) is the first
// argument, the command follows in JSON or shell form
function parseHealthConfig(rest: string) {
const chars = Array.from(rest)
let sep = 0
while (sep < chars.length && !isSpace(chars[sep])) {
sep++
}
let next = sep
while (next < chars.length && isSpace(chars[next])) {
next++
}
if (sep === 0) {
return { args: [], json: false }
}
const { args, json } = parseMaybeJSON(chars.slice(next).join(''))
return { args: [chars.slice(0, sep).join(''), ...args], json }
}
function parseMaybeJSONToList(rest: string) {
try {
return { args: parseJSON(rest), json: true }
} catch (err) {
if (err instanceof NotStringArrayError) {
throw err
}
return parseStringsWhitespaceDelimited(rest)
}
}
/**
* Split a line into whitespace separated words, keeping quotes and escape
* characters in place (they are resolved later by the shell lexer).
*/
export function parseWords(rest: string, escapeToken: string): string[] {
const chars = Array.from(rest)
const words: string[] = []
let phase: 'spaces' | 'word' | 'quote' = 'spaces'
let quote = ''
let blankOK = false
let word = ''
for (let pos = 0; pos <= chars.length; pos++) {
const ch = chars[pos]
if (phase === 'spaces') {
if (pos === chars.length) {
break
}
if (isSpace(ch)) {
continue
}
phase = 'word'
}
if (pos === chars.length) {
if (blankOK || word.length > 0) {
words.push(word)
}
break
}
if (phase === 'word') {
if (isSpace(ch)) {
phase = 'spaces'
if (blankOK || word.length > 0) {
words.push(word)
}
word = ''
blankOK = false
continue
}
if (ch === "'" || ch === '"') {
quote = ch
blankOK = true
phase = 'quote'
}
if (ch === escapeToken) {
if (pos + 1 === chars.length) {
continue // skip an escape token at end of line
}
// outside quotes an escape token always keeps itself and the
// following character, even if that character is a quote
word += ch
pos++
word += chars[pos]
continue
}
word += ch
continue
}
// phase === 'quote'
if (ch === quote) {
phase = 'word'
}
if (ch === escapeToken && quote !== "'") {
if (pos + 1 === chars.length) {
phase = 'word'
continue // skip the escape token at end
}
word += ch
pos++
word += chars[pos]
continue
}
word += ch
}
return words
}
/**
* Like `parseMaybeJSONToList`, but the shell form is split with quote
* awareness so that `COPY "my file.txt" /dest/` works (Docker only
* supports such paths in JSON form).
*/
function parseMaybeJSONToWords(rest: string, escapeToken: string) {
try {
return { args: parseJSON(rest), json: true }
} catch (err) {
if (err instanceof NotStringArrayError) {
throw err
}
return { args: parseWords(rest, escapeToken), json: false }
}
}
function parseNameOrNameVal(rest: string, escapeToken: string) {
return { args: parseWords(rest, escapeToken), json: false }
}
function parseNameVal(key: string): LineParser {
return (rest, escapeToken) => {
const words = parseWords(rest, escapeToken)
if (words.length === 0) {
return { args: [], json: false }
}
// Old format: KEY name value
if (!words[0].includes('=')) {
const match = WHITESPACE.exec(rest)
if (!match) {
throw new Error(`${key} must have two arguments`)
}
const name = rest.slice(0, match.index)
const value = rest.slice(match.index + match[0].length)
return { args: [name, value, ''], json: false }
}
const args: string[] = []
for (const word of words) {
if (!word.includes('=')) {
throw new Error(
`Syntax error - can't find = in "${word}". Must be of the form: name=value`
)
}
const index = word.indexOf('=')
args.push(word.slice(0, index), word.slice(index + 1), '=')
}
return { args, json: false }
}
}
const LINE_PARSERS: Record<string, LineParser> = {
ADD: parseMaybeJSONToWords,
ARG: parseNameOrNameVal,
CMD: parseMaybeJSON,
COPY: parseMaybeJSONToWords,
ENTRYPOINT: parseMaybeJSON,
ENV: parseNameVal('ENV'),
EXPOSE: parseStringsWhitespaceDelimited,
FROM: parseStringsWhitespaceDelimited,
HEALTHCHECK: parseHealthConfig,
LABEL: parseNameVal('LABEL'),
MAINTAINER: parseString,
ONBUILD: parseIgnore,
RUN: parseMaybeJSON,
SHELL: parseMaybeJSON,
STOPSIGNAL: parseString,
USER: parseString,
VOLUME: parseMaybeJSONToList,
WORKDIR: parseString,
}
/** Parses leading `--flag[=value]` words. Returns the remaining line and flags. */
function extractBuilderFlags(
line: string,
escapeToken: string
): { rest: string; flags: string[] } {
const chars = Array.from(line)
const flags: string[] = []
let phase: 'spaces' | 'word' | 'quote' = 'spaces'
let quote = ''
let blankOK = false
let word = ''
for (let pos = 0; pos <= chars.length; pos++) {
const ch = chars[pos]
if (phase === 'spaces') {
if (pos === chars.length) {
break
}
if (isSpace(ch)) {
continue
}
// only keep going if the next word starts with --
if (ch !== '-' || pos + 1 === chars.length || chars[pos + 1] !== '-') {
return { rest: chars.slice(pos).join(''), flags }
}
phase = 'word'
}
if (pos === chars.length) {
if (word !== '--' && (blankOK || word.length > 0)) {
flags.push(word)
}
break
}
if (phase === 'word') {
if (isSpace(ch)) {
phase = 'spaces'
if (word === '--') {
return { rest: chars.slice(pos).join(''), flags }
}
if (blankOK || word.length > 0) {
flags.push(word)
}
word = ''
blankOK = false
continue
}
if (ch === "'" || ch === '"') {
quote = ch
blankOK = true
phase = 'quote'
continue
}
if (ch === escapeToken) {
if (pos + 1 === chars.length) {
continue
}
pos++
word += chars[pos]
continue
}
word += ch
continue
}
// phase === 'quote'
if (ch === quote) {
phase = 'word'
continue
}
if (ch === escapeToken) {
if (pos + 1 === chars.length) {
phase = 'word'
continue
}
pos++
word += chars[pos]
continue
}
word += ch
}
return { rest: '', flags }
}
function splitCommand(
line: string,
escapeToken: string
): { command: string; flags: string[]; rest: string } {
const trimmed = line.trim()
const match = WHITESPACE.exec(trimmed)
if (!match) {
return { command: trimmed, flags: [], rest: '' }
}
const command = trimmed.slice(0, match.index)
const { rest, flags } = extractBuilderFlags(
trimmed.slice(match.index + match[0].length),
escapeToken
)
return { command, flags, rest: rest.trim() }
}
// Matches `^(\d*)<<(-?)\s*([^<]*)$` without a regex, whose adjacent `\s*`
// and `[^<]*` are ambiguous on whitespace runs (polynomial backtracking).
const HEREDOC_SPACES = '\t\n\f\r '
function matchHeredocMarker(
word: string
): { fileDescriptor: string; chomp: boolean; rest: string } | undefined {
let i = 0
while (i < word.length && word[i] >= '0' && word[i] <= '9') {
i++
}
if (!word.startsWith('<<', i)) {
return undefined
}
const fileDescriptor = word.slice(0, i)
i += 2
const chomp = word[i] === '-'
if (chomp) {
i++
}
while (i < word.length && HEREDOC_SPACES.includes(word[i])) {
i++
}
const rest = word.slice(i)
if (rest.includes('<')) {
return undefined
}
return { fileDescriptor, chomp, rest }
}
export function parseHeredoc(word: string): DockerfileHeredoc | undefined {
const match = matchHeredocMarker(word)
if (!match) {
return undefined
}
const fileDescriptor =
match.fileDescriptor === '' ? 0 : parseInt(match.fileDescriptor, 10)
const { chomp, rest } = match
if (rest.length === 0) {
return undefined
}
// Lex the terminator both with and without quotes. If the results
// differ, part of the word was quoted and the content must not expand.
const words = new ShellLex('\\').processWords(rest)
if (words.length !== 1) {
return undefined
}
const wordsRaw = new ShellLex('\\', { rawQuotes: true }).processWords(rest)
if (wordsRaw.length !== words.length) {
throw new Error(
`internal lexing of heredoc produced inconsistent results: ${rest}`
)
}
const countQuotes = (s: string) =>
(s.match(/'/g) ?? []).length + (s.match(/"/g) ?? []).length
const expand = countQuotes(words[0]) === countQuotes(wordsRaw[0])
return { name: words[0], content: '', chomp, expand, fileDescriptor }
}
function heredocsFromLine(line: string): DockerfileHeredoc[] {
let words: string[]
try {
words = new ShellLex('\\', {
rawQuotes: true,
rawEscapes: true,
}).processWords(line)
} catch {
return []
}
const heredocs: DockerfileHeredoc[] = []
for (const word of words) {
const heredoc = parseHeredoc(word)
if (heredoc) {
heredocs.push(heredoc)
}
}
return heredocs
}
/** Strips leading tabs from every line (`<<-` heredocs). */
export function chompHeredocContent(content: string): string {
return content.replace(/^\t+/gm, '')
}
export function parseDockerfileAst(content: string): DockerfileAst {
const directives = new Directives()
const lines = splitLines(content.replace(/^\uFEFF/, ''))
const instructions: DockerfileInstruction[] = []
const warnings: DockerfileWarning[] = []
let currentLine = 0
let index = 0
const processLine = (raw: string, stripLeftWhitespace: boolean): string => {
let line = trimNewline(raw)
if (stripLeftWhitespace) {
line = trimLeadingWhitespace(line)
}
withLine(currentLine + 1, () => directives.possibleParserDirective(line))
return isComment(line) ? '' : line
}
while (index < lines.length) {
const raw = lines[index++]
let line = processLine(raw, true)
currentLine++
const startLine = currentLine
const first = directives.trimContinuation(line)
line = first.line
let isEndOfLine = first.isEndOfLine
if (isEndOfLine && line.length === 0) {
continue
}
let hasEmptyContinuationLine = false
while (!isEndOfLine && index < lines.length) {
const continuationRaw = lines[index++]
const continuation = processLine(continuationRaw, false)
currentLine++
if (isComment(continuationRaw)) {
continue
}
if (trimLeadingWhitespace(trimNewline(continuationRaw)).length === 0) {
hasEmptyContinuationLine = true
continue
}
const trimmed = directives.trimContinuation(continuation)
isEndOfLine = trimmed.isEndOfLine
line += trimmed.line
}
if (hasEmptyContinuationLine) {
warnings.push({
message: `Empty continuation line found in: ${line}`,
line: currentLine,
})
}
const { command, flags, rest } = splitCommand(line, directives.escapeToken)
const name = command.toUpperCase()
const lineParser = LINE_PARSERS[name]
if (!lineParser) {
throw new DockerfileSyntaxError(
`unknown instruction: ${command}`,
startLine
)
}
const parsed = withLine(startLine, () =>
lineParser(rest, directives.escapeToken)
)
const instruction: DockerfileInstruction = {
name,
original: line,
flags,
args: parsed.args,
json: parsed.json,
heredocs: [],
startLine,
endLine: currentLine,
}
if (HEREDOC_INSTRUCTIONS.has(name) && !parsed.json && line.includes('<<')) {
const heredocs = withLine(startLine, () => heredocsFromLine(line))
for (const heredoc of heredocs) {
let terminated = false
while (index < lines.length) {
const heredocLine = lines[index++]
currentLine++
let possibleTerminator = trimNewline(heredocLine)
if (heredoc.chomp) {
possibleTerminator = possibleTerminator.replace(/^\t+/, '')
}
if (possibleTerminator === heredoc.name) {
terminated = true
break
}
heredoc.content += heredocLine
}
if (!terminated) {
throw new DockerfileSyntaxError('unterminated heredoc', startLine)
}
instruction.heredocs.push(heredoc)
}
instruction.endLine = currentLine
}
instructions.push(instruction)
}
return { instructions, escapeToken: directives.escapeToken, warnings }
}