Skip to content

armgit / E2B

Public Imported from GitHub
0

Open-source, secure environment with real-world tools for enterprise-grade agents.

fix(sdk): match BuildKit in .dockerignore pattern matching (#1935)

devin-ai-integration[bot]cc296178f208
diff --git a/.changeset/dockerignore-buildkit-parity.md b/.changeset/dockerignore-buildkit-parity.md
new file mode 100644
index 000000000..fad7b05a5
--- /dev/null
+++ b/.changeset/dockerignore-buildkit-parity.md
@@ -0,0 +1,6 @@
+---
+'e2b': patch
+'@e2b/python-sdk': patch
+---
+
+Match BuildKit when filtering the template build context with `.dockerignore`: a leading `**` followed by literal characters (e.g. `**.txt`) now matches at any depth, a negated pattern matching only a parent directory (e.g. `src/app.ts` then `!src`) no longer re-includes the path, and in the JS SDK `?` and bracket expressions match characters outside the BMP (e.g. emoji) as a single character.
diff --git a/packages/js-sdk/src/template/dockerignore.ts b/packages/js-sdk/src/template/dockerignore.ts
index 7bb99f014..7ddb3da96 100644
--- a/packages/js-sdk/src/template/dockerignore.ts
+++ b/packages/js-sdk/src/template/dockerignore.ts
@@ -25,7 +25,64 @@ const WILDCARD_CHARS = /[*?[\\]/
const backslashIsSeparator = () => path.sep === '\\'
function escapeRegex(ch: string): string {
- return ch.replace(/[.*+?^${}()|[\]\\/-]/g, '\\$&')
+ return ch.replace(/[.*+?^${}()|[\]\\/]/g, '\\$&')
+}
+
+// Escapes that keep their regex meaning in a bracket expression, like in Python
+const CLASS_ESCAPES = new Set('dDsSwWfnrtv')
+const CLASS_CONTROL_ESCAPES: Record<string, string> = { a: '\\x07', b: '\\x08' }
+const CLASS_HEX_ESCAPE_LENGTHS: Record<string, number> = { x: 2, u: 4, U: 8 }
+// Characters that must stay escaped in a bracket expression with the `u` flag
+const CLASS_SYNTAX_CHARS = new Set('^$\\.*+?()[]{}|/-')
+
+// Make a bracket expression body valid with the regex `u` flag
+function translateClass(body: string, backslashIsEscape: boolean): string {
+ let cls = ''
+ let i = 0
+ while (i < body.length) {
+ const ch = body[i++]
+ if (ch !== '\\') {
+ cls += ch
+ } else if (!backslashIsEscape || i >= body.length) {
+ cls += '\\\\'
+ } else {
+ const next = body[i++]
+ if (CLASS_ESCAPES.has(next)) {
+ cls += '\\' + next
+ } else if (next in CLASS_CONTROL_ESCAPES) {
+ cls += CLASS_CONTROL_ESCAPES[next]
+ } else if (next in CLASS_HEX_ESCAPE_LENGTHS) {
+ const length = CLASS_HEX_ESCAPE_LENGTHS[next]
+ const hex = body.slice(i, i + length)
+ if (hex.length < length || !/^[0-9a-fA-F]+$/.test(hex)) {
+ throw new Error(`incomplete escape \\${next}${hex}`)
+ }
+ const code = parseInt(hex, 16)
+ if (code > 0x10ffff) {
+ throw new Error(`bad escape \\${next}${hex}`)
+ }
+ cls += `\\u{${code.toString(16)}}`
+ i += length
+ } else if (/[0-7]/.test(next)) {
+ let octal = next
+ while (octal.length < 3 && /[0-7]/.test(body[i] ?? '')) {
+ octal += body[i++]
+ }
+ const code = parseInt(octal, 8)
+ if (code > 0o377) {
+ throw new Error(
+ `octal escape value \\${octal} outside of range 0-0o377`
+ )
+ }
+ cls += `\\u{${code.toString(16)}}`
+ } else if (/[A-Za-z0-9]/.test(next)) {
+ throw new Error(`bad escape \\${next}`)
+ } else {
+ cls += CLASS_SYNTAX_CHARS.has(next) ? '\\' + next : next
+ }
+ }
+ }
+ return cls
}
// Equivalent of Go's filepath.Clean followed by filepath.ToSlash
@@ -36,12 +93,25 @@ function clean(pattern: string): string {
return path.posix.normalize(pattern).replace(/(.)\/$/, '$1')
}
-function compile(pattern: string): RegExp {
+type Matcher = (p: string) => boolean
+
+enum MatchType {
+ Exact = 'exact',
+ Prefix = 'prefix',
+ Suffix = 'suffix',
+ Regex = 'regex',
+}
+
+// Like moby, use plain string checks for patterns without wildcards and only
+// fall back to a regex otherwise
+function compile(pattern: string): Matcher {
let regex = '^'
+ let matchType = MatchType.Exact
const n = pattern.length
const backslashIsEscape = !backslashIsSeparator()
let i = 0
while (i < n) {
+ const first = i === 0
const ch = pattern[i++]
if (ch === '*') {
if (pattern[i] === '*') {
@@ -50,14 +120,26 @@ function compile(pattern: string): RegExp {
if (pattern[i] === '/') {
i++
}
- regex += i >= n ? '.*' : '(.*/)?'
+ if (i >= n) {
+ regex += '.*'
+ matchType =
+ matchType === MatchType.Exact ? MatchType.Prefix : MatchType.Regex
+ } else {
+ regex += '(.*/)?'
+ matchType = MatchType.Regex
+ }
+ if (first) {
+ matchType = MatchType.Suffix
+ }
} else {
regex += '[^/]*'
+ matchType = MatchType.Regex
}
} else if (ch === '?') {
regex += '[^/]'
+ matchType = MatchType.Regex
} else if (ch === '[') {
- // Copy a bracket expression as is, a leading "^" negates it
+ // Copy a bracket expression, a leading "^" negates it
let j = i
if (pattern[j] === '^') {
j++
@@ -68,14 +150,22 @@ function compile(pattern: string): RegExp {
}
j++
}
- regex += pattern.slice(i - 1, j + 1)
+ regex +=
+ '[' +
+ translateClass(pattern.slice(i, Math.min(j, n)), backslashIsEscape) +
+ (j < n ? ']' : '')
i = j + 1
+ matchType = MatchType.Regex
+ } else if (ch === ']') {
+ regex += '\\]'
+ matchType = MatchType.Regex
} else if (LITERAL_REGEX_CHARS.has(ch)) {
regex += '\\' + ch
} else if (ch === '\\' && backslashIsEscape) {
// Escape the next character
if (i < n) {
regex += escapeRegex(pattern[i++])
+ matchType = MatchType.Regex
} else {
regex += '\\\\'
}
@@ -83,13 +173,31 @@ function compile(pattern: string): RegExp {
regex += ch
}
}
- return new RegExp(regex + '$')
+
+ switch (matchType) {
+ case MatchType.Exact:
+ return (p) => p === pattern
+ case MatchType.Prefix: {
+ const prefix = pattern.slice(0, -2)
+ return (p) => p.startsWith(prefix)
+ }
+ case MatchType.Suffix: {
+ const suffix = pattern.slice(2)
+ // "**/foo" also matches "foo"
+ return (p) =>
+ p.endsWith(suffix) || (suffix[0] === '/' && p === suffix.slice(1))
+ }
+ case MatchType.Regex: {
+ const re = new RegExp(regex + '$', 'u')
+ return (p) => re.test(p)
+ }
+ }
}
interface Pattern {
exclusion: boolean
dirs: string[]
- regex: RegExp
+ match: Matcher
}
/**
@@ -119,43 +227,48 @@ export class PatternMatcher {
if (pattern.length > 1 && pattern.startsWith('/')) {
pattern = pattern.slice(1)
}
- let regex: RegExp
+ let match: Matcher
try {
- regex = compile(pattern)
+ match = compile(pattern)
} catch (err) {
throw new TemplateError(
`Invalid ignore pattern '${original}': ${(err as Error).message}`
)
}
- this.patterns.push({ exclusion, dirs: pattern.split('/'), regex })
+ this.patterns.push({ exclusion, dirs: pattern.split('/'), match })
}
}
/**
- * Whether the path or one of its parent directories is excluded.
+ * Whether the path is excluded. Like BuildKit, the patterns are evaluated
+ * on each parent directory first, and a pattern that matched a parent
+ * directory also matches the paths under it.
*
* @param p Slash-separated path relative to the context root
* @returns True if the path is excluded
*/
matches(p: string): boolean {
- const parentPath = path.posix.dirname(p)
- const parentDirs = parentPath === '.' ? [] : parentPath.split('/')
-
+ const segments = p.split('/')
+ let parentMatched: boolean[] = []
let matched = false
- for (const pattern of this.patterns) {
- // An inclusion can't change an already matched path, and an
- // exclusion can't change a path that hasn't matched yet
- if (pattern.exclusion !== matched) {
- continue
- }
- const match =
- pattern.regex.test(p) ||
- parentDirs.some((_, i) =>
- pattern.regex.test(parentDirs.slice(0, i + 1).join('/'))
- )
- if (match) {
- matched = !pattern.exclusion
- }
+ for (let depth = 1; depth <= segments.length; depth++) {
+ const current = segments.slice(0, depth).join('/')
+ matched = false
+ parentMatched = this.patterns.map((pattern, i) => {
+ let match = parentMatched[i] ?? false
+ if (!match) {
+ // An inclusion can't change an already matched path, and an
+ // exclusion can't change a path that hasn't matched yet
+ if (pattern.exclusion !== matched) {
+ return false
+ }
+ match = pattern.match(current)
+ }
+ if (match) {
+ matched = !pattern.exclusion
+ }
+ return match
+ })
}
return matched
}
diff --git a/packages/js-sdk/tests/template/utils/getAllFilesInPath.test.ts b/packages/js-sdk/tests/template/utils/getAllFilesInPath.test.ts
index b5b8c6e54..dd2ebdbe4 100644
--- a/packages/js-sdk/tests/template/utils/getAllFilesInPath.test.ts
+++ b/packages/js-sdk/tests/template/utils/getAllFilesInPath.test.ts
@@ -413,6 +413,34 @@ describe('getAllFilesInPath', () => {
])
})
+ test('should not re-include a path through a negated parent directory', async () => {
+ const files = await relativePaths('.', ['src/app.ts', '**/*.js', '!src'])
+ expect(files).not.toContain('src/app.ts')
+ expect(files).not.toContain('src/node_modules/lib.js')
+ expect(files).toContain('src/app.spec.ts')
+ })
+
+ test('should match a leading globstar with a literal suffix', async () => {
+ for (const name of ['root.txt', 'src/nested.txt', 'keep.txt.bak']) {
+ await writeFile(join(testDir, name), 'x')
+ }
+ const files = await relativePaths('.', ['**.txt', '**/generated'])
+ expect(files).not.toContain('root.txt')
+ expect(files).not.toContain('src/nested.txt')
+ expect(files).not.toContain('src/generated/api.ts')
+ expect(files).toContain('keep.txt.bak')
+ })
+
+ test('should match characters outside the BMP as a single character', async () => {
+ for (const name of ['πŸ˜€.txt', 'πŸ˜€πŸ˜€.txt']) {
+ await writeFile(join(testDir, name), 'x')
+ }
+ const files = await relativePaths('*', ['?.txt'])
+ expect(files).not.toContain('πŸ˜€.txt')
+ expect(files).toContain('πŸ˜€πŸ˜€.txt')
+ expect(await relativePaths('*', ['[πŸ˜€]?.txt'])).not.toContain('πŸ˜€πŸ˜€.txt')
+ })
+
test('should make absolute patterns inside the context relative', async () => {
const files = await relativePaths('.', [
join(testDir, 'src'),
@@ -438,6 +466,23 @@ describe('getAllFilesInPath', () => {
)
})
+ test.skipIf(process.platform === 'win32')(
+ 'should handle escapes in a bracket expression like Python',
+ async () => {
+ await expect(
+ getAllFilesInPath('.', testDir, ['[\\q]'])
+ ).rejects.toThrow("Invalid ignore pattern '[\\q]'")
+ await writeFile(join(testDir, 'a'), 'x')
+ expect(await relativePaths('*', ['[\\a]'])).toContain('a')
+ await writeFile(join(testDir, 'b'), 'x')
+ await writeFile(join(testDir, 'c'), 'x')
+ const files = await relativePaths('*', ['[\\x61]', '[\\142]'])
+ expect(files).not.toContain('a')
+ expect(files).not.toContain('b')
+ expect(files).toContain('c')
+ }
+ )
+
test('should keep the files hash stable when ignored files change', async () => {
const hash = () =>
calculateFilesHash('.', '/app', testDir, ['.git'], false, undefined)
diff --git a/packages/python-sdk/e2b/template/dockerignore.py b/packages/python-sdk/e2b/template/dockerignore.py
index d6909a15d..5a02be1d9 100644
--- a/packages/python-sdk/e2b/template/dockerignore.py
+++ b/packages/python-sdk/e2b/template/dockerignore.py
@@ -9,7 +9,8 @@ https://github.com/moby/patternmatcher
import os
import posixpath
import re
-from typing import List
+from enum import Enum
+from typing import Callable, List
from e2b.exceptions import TemplateException
@@ -19,6 +20,13 @@ _WILDCARD_CHARS = re.compile(r"[*?\[\\]")
_BACKSLASH_IS_SEPARATOR = os.sep == "\\"
+class _MatchType(Enum):
+ EXACT = "exact"
+ PREFIX = "prefix"
+ SUFFIX = "suffix"
+ REGEX = "regex"
+
+
def _clean(pattern: str) -> str:
# Equivalent of Go's filepath.Clean followed by filepath.ToSlash
if _BACKSLASH_IS_SEPARATOR:
@@ -26,10 +34,14 @@ def _clean(pattern: str) -> str:
return posixpath.normpath(re.sub("/+", "/", pattern))
-def _compile(pattern: str) -> "re.Pattern[str]":
+def _compile(pattern: str) -> Callable[[str], bool]:
+ # Like moby, use plain string checks for patterns without wildcards and
+ # only fall back to a regex otherwise
regex = "^"
+ match_type = _MatchType.EXACT
i, n = 0, len(pattern)
while i < n:
+ first = i == 0
ch = pattern[i]
i += 1
if ch == "*":
@@ -38,11 +50,24 @@ def _compile(pattern: str) -> "re.Pattern[str]":
# Treat "**/" as "**"
if i < n and pattern[i] == "/":
i += 1
- regex += ".*" if i >= n else "(.*/)?"
+ if i >= n:
+ regex += ".*"
+ match_type = (
+ _MatchType.PREFIX
+ if match_type is _MatchType.EXACT
+ else _MatchType.REGEX
+ )
+ else:
+ regex += "(.*/)?"
+ match_type = _MatchType.REGEX
+ if first:
+ match_type = _MatchType.SUFFIX
else:
regex += "[^/]*"
+ match_type = _MatchType.REGEX
elif ch == "?":
regex += "[^/]"
+ match_type = _MatchType.REGEX
elif ch == "[":
# Copy a bracket expression as is, a leading "^" negates it
j = i
@@ -54,6 +79,10 @@ def _compile(pattern: str) -> "re.Pattern[str]":
j += 1
regex += pattern[i - 1 : j + 1]
i = j + 1
+ match_type = _MatchType.REGEX
+ elif ch == "]":
+ regex += ch
+ match_type = _MatchType.REGEX
elif ch in _LITERAL_REGEX_CHARS:
regex += "\\" + ch
elif ch == "\\" and not _BACKSLASH_IS_SEPARATOR:
@@ -61,21 +90,32 @@ def _compile(pattern: str) -> "re.Pattern[str]":
if i < n:
regex += re.escape(pattern[i])
i += 1
+ match_type = _MatchType.REGEX
else:
regex += "\\\\"
else:
regex += ch
- return re.compile(regex + "$")
+
+ if match_type is _MatchType.EXACT:
+ return lambda path: path == pattern
+ if match_type is _MatchType.PREFIX:
+ prefix = pattern[:-2]
+ return lambda path: path.startswith(prefix)
+ if match_type is _MatchType.SUFFIX:
+ suffix = pattern[2:]
+ # "**/foo" also matches "foo"
+ return lambda path: path.endswith(suffix) or (
+ suffix.startswith("/") and path == suffix[1:]
+ )
+ compiled = re.compile(regex + r"\Z")
+ return lambda path: compiled.match(path) is not None
class _Pattern:
def __init__(self, cleaned_pattern: str, exclusion: bool):
self.exclusion = exclusion
self.dirs = cleaned_pattern.split("/")
- self._regex = _compile(cleaned_pattern)
-
- def match(self, path: str) -> bool:
- return self._regex.match(path) is not None
+ self.match = _compile(cleaned_pattern)
class PatternMatcher:
@@ -110,26 +150,33 @@ class PatternMatcher:
def matches(self, path: str) -> bool:
"""
- Whether the path or one of its parent directories is excluded.
+ Whether the path is excluded. Like BuildKit, the patterns are evaluated
+ on each parent directory first, and a pattern that matched a parent
+ directory also matches the paths under it.
:param path: Slash-separated path relative to the context root
:return: True if the path is excluded
"""
- parent_path = posixpath.dirname(path)
- parent_dirs = parent_path.split("/") if parent_path else []
-
+ segments = path.split("/")
+ parent_matched: List[bool] = []
matched = False
- for pattern in self._patterns:
- # An inclusion can't change an already matched path, and an
- # exclusion can't change a path that hasn't matched yet
- if pattern.exclusion != matched:
- continue
- match = pattern.match(path) or any(
- pattern.match("/".join(parent_dirs[: i + 1]))
- for i in range(len(parent_dirs))
- )
- if match:
- matched = not pattern.exclusion
+ for depth in range(1, len(segments) + 1):
+ current = "/".join(segments[:depth])
+ current_matched: List[bool] = []
+ matched = False
+ for i, pattern in enumerate(self._patterns):
+ match = bool(parent_matched) and parent_matched[i]
+ if not match:
+ # An inclusion can't change an already matched path, and
+ # an exclusion can't change a path that hasn't matched yet
+ if pattern.exclusion != matched:
+ current_matched.append(False)
+ continue
+ match = pattern.match(current)
+ current_matched.append(match)
+ if match:
+ matched = not pattern.exclusion
+ parent_matched = current_matched
return matched
def may_match_under(self, dir_path: str) -> bool:
diff --git a/packages/python-sdk/tests/shared/template/utils/test_get_all_files_in_path.py b/packages/python-sdk/tests/shared/template/utils/test_get_all_files_in_path.py
index 88a9bf0d3..fbf194d98 100644
--- a/packages/python-sdk/tests/shared/template/utils/test_get_all_files_in_path.py
+++ b/packages/python-sdk/tests/shared/template/utils/test_get_all_files_in_path.py
@@ -407,6 +407,35 @@ class TestDockerignoreSemantics:
"src/node_modules/lib.js",
]
+ def test_should_not_reinclude_a_path_through_a_negated_parent_directory(
+ self, test_dir
+ ):
+ files = self.relative_paths(".", test_dir, ["src/app.ts", "**/*.js", "!src"])
+ assert "src/app.ts" not in files
+ assert "src/node_modules/lib.js" not in files
+ assert "src/app.spec.ts" in files
+
+ def test_should_match_a_leading_globstar_with_a_literal_suffix(self, test_dir):
+ for name in ["root.txt", "src/nested.txt", "keep.txt.bak"]:
+ with open(os.path.join(test_dir, name), "w") as f:
+ f.write("x")
+ files = self.relative_paths(".", test_dir, ["**.txt", "**/generated"])
+ assert "root.txt" not in files
+ assert "src/nested.txt" not in files
+ assert "src/generated/api.ts" not in files
+ assert "keep.txt.bak" in files
+
+ def test_should_match_characters_outside_the_bmp_as_a_single_character(
+ self, test_dir
+ ):
+ for name in ["πŸ˜€.txt", "πŸ˜€πŸ˜€.txt"]:
+ with open(os.path.join(test_dir, name), "w") as f:
+ f.write("x")
+ files = self.relative_paths("*", test_dir, ["?.txt"])
+ assert "πŸ˜€.txt" not in files
+ assert "πŸ˜€πŸ˜€.txt" in files
+ assert "πŸ˜€πŸ˜€.txt" not in self.relative_paths("*", test_dir, ["[πŸ˜€]?.txt"])
+
def test_should_make_absolute_patterns_inside_the_context_relative(self, test_dir):
files = self.relative_paths(
".",
@@ -432,6 +461,23 @@ class TestDockerignoreSemantics:
with pytest.raises(TemplateException, match="Invalid ignore pattern '\\[abc'"):
get_all_files_in_path(".", test_dir, ["[abc"])
+ @pytest.mark.skipif(sys.platform == "win32", reason="Backslash is a separator")
+ def test_should_handle_escapes_in_a_bracket_expression_like_python(self, test_dir):
+ with pytest.raises(
+ TemplateException, match=r"Invalid ignore pattern '\[\\q\]'"
+ ):
+ get_all_files_in_path(".", test_dir, ["[\\q]"])
+ with open(os.path.join(test_dir, "a"), "w") as f:
+ f.write("x")
+ assert "a" in self.relative_paths("*", test_dir, ["[\\a]"])
+ for name in ["b", "c"]:
+ with open(os.path.join(test_dir, name), "w") as f:
+ f.write("x")
+ files = self.relative_paths("*", test_dir, ["[\\x61]", "[\\142]"])
+ assert "a" not in files
+ assert "b" not in files
+ assert "c" in files
+
def test_should_keep_the_files_hash_stable_when_ignored_files_change(
self, test_dir
):