Open-source, secure environment with real-world tools for enterprise-grade agents.
Python agentaiai-agentai-agentscode-interpretercopilotdevelopmentdevtoolsgptgpt-4javascriptllmnextjsopenaipythonreactsoftwaretypescript
fix(sdk): match BuildKit in .dockerignore pattern matching (#1935)
devin-ai-integration[bot]cc296178f208
diff --git a/.changeset/dockerignore-buildkit-parity.md b/.changeset/dockerignore-buildkit-parity.mdnew file mode 100644index 000000000..fad7b05a5--- /dev/null+++ b/.changeset/dockerignore-buildkit-parity.md@@ -0,0 +1,6 @@+---+'e2b': patch+'@e2b/python-sdk': patch+---++Match BuildKit when filtering the template build context with `.dockerignore`: a leading `**` followed by literal characters (e.g. `**.txt`) now matches at any depth, a negated pattern matching only a parent directory (e.g. `src/app.ts` then `!src`) no longer re-includes the path, and in the JS SDK `?` and bracket expressions match characters outside the BMP (e.g. emoji) as a single character.diff --git a/packages/js-sdk/src/template/dockerignore.ts b/packages/js-sdk/src/template/dockerignore.tsindex 7bb99f014..7ddb3da96 100644--- a/packages/js-sdk/src/template/dockerignore.ts+++ b/packages/js-sdk/src/template/dockerignore.ts@@ -25,7 +25,64 @@ const WILDCARD_CHARS = /[*?[\\]/const backslashIsSeparator = () => path.sep === '\\'function escapeRegex(ch: string): string {- return ch.replace(/[.*+?^${}()|[\]\\/-]/g, '\\$&')+ return ch.replace(/[.*+?^${}()|[\]\\/]/g, '\\$&')+}++// Escapes that keep their regex meaning in a bracket expression, like in Python+const CLASS_ESCAPES = new Set('dDsSwWfnrtv')+const CLASS_CONTROL_ESCAPES: Record<string, string> = { a: '\\x07', b: '\\x08' }+const CLASS_HEX_ESCAPE_LENGTHS: Record<string, number> = { x: 2, u: 4, U: 8 }+// Characters that must stay escaped in a bracket expression with the `u` flag+const CLASS_SYNTAX_CHARS = new Set('^$\\.*+?()[]{}|/-')++// Make a bracket expression body valid with the regex `u` flag+function translateClass(body: string, backslashIsEscape: boolean): string {+ let cls = ''+ let i = 0+ while (i < body.length) {+ const ch = body[i++]+ if (ch !== '\\') {+ cls += ch+ } else if (!backslashIsEscape || i >= body.length) {+ cls += '\\\\'+ } else {+ const next = body[i++]+ if (CLASS_ESCAPES.has(next)) {+ cls += '\\' + next+ } else if (next in CLASS_CONTROL_ESCAPES) {+ cls += CLASS_CONTROL_ESCAPES[next]+ } else if (next in CLASS_HEX_ESCAPE_LENGTHS) {+ const length = CLASS_HEX_ESCAPE_LENGTHS[next]+ const hex = body.slice(i, i + length)+ if (hex.length < length || !/^[0-9a-fA-F]+$/.test(hex)) {+ throw new Error(`incomplete escape \\${next}${hex}`)+ }+ const code = parseInt(hex, 16)+ if (code > 0x10ffff) {+ throw new Error(`bad escape \\${next}${hex}`)+ }+ cls += `\\u{${code.toString(16)}}`+ i += length+ } else if (/[0-7]/.test(next)) {+ let octal = next+ while (octal.length < 3 && /[0-7]/.test(body[i] ?? '')) {+ octal += body[i++]+ }+ const code = parseInt(octal, 8)+ if (code > 0o377) {+ throw new Error(+ `octal escape value \\${octal} outside of range 0-0o377`+ )+ }+ cls += `\\u{${code.toString(16)}}`+ } else if (/[A-Za-z0-9]/.test(next)) {+ throw new Error(`bad escape \\${next}`)+ } else {+ cls += CLASS_SYNTAX_CHARS.has(next) ? '\\' + next : next+ }+ }+ }+ return cls}// Equivalent of Go's filepath.Clean followed by filepath.ToSlash@@ -36,12 +93,25 @@ function clean(pattern: string): string {return path.posix.normalize(pattern).replace(/(.)\/$/, '$1')}-function compile(pattern: string): RegExp {+type Matcher = (p: string) => boolean++enum MatchType {+ Exact = 'exact',+ Prefix = 'prefix',+ Suffix = 'suffix',+ Regex = 'regex',+}++// Like moby, use plain string checks for patterns without wildcards and only+// fall back to a regex otherwise+function compile(pattern: string): Matcher {let regex = '^'+ let matchType = MatchType.Exactconst n = pattern.lengthconst backslashIsEscape = !backslashIsSeparator()let i = 0while (i < n) {+ const first = i === 0const ch = pattern[i++]if (ch === '*') {if (pattern[i] === '*') {@@ -50,14 +120,26 @@ function compile(pattern: string): RegExp {if (pattern[i] === '/') {i++}- regex += i >= n ? '.*' : '(.*/)?'+ if (i >= n) {+ regex += '.*'+ matchType =+ matchType === MatchType.Exact ? MatchType.Prefix : MatchType.Regex+ } else {+ regex += '(.*/)?'+ matchType = MatchType.Regex+ }+ if (first) {+ matchType = MatchType.Suffix+ }} else {regex += '[^/]*'+ matchType = MatchType.Regex}} else if (ch === '?') {regex += '[^/]'+ matchType = MatchType.Regex} else if (ch === '[') {- // Copy a bracket expression as is, a leading "^" negates it+ // Copy a bracket expression, a leading "^" negates itlet j = iif (pattern[j] === '^') {j++@@ -68,14 +150,22 @@ function compile(pattern: string): RegExp {}j++}- regex += pattern.slice(i - 1, j + 1)+ regex +=+ '[' ++ translateClass(pattern.slice(i, Math.min(j, n)), backslashIsEscape) ++ (j < n ? ']' : '')i = j + 1+ matchType = MatchType.Regex+ } else if (ch === ']') {+ regex += '\\]'+ matchType = MatchType.Regex} else if (LITERAL_REGEX_CHARS.has(ch)) {regex += '\\' + ch} else if (ch === '\\' && backslashIsEscape) {// Escape the next characterif (i < n) {regex += escapeRegex(pattern[i++])+ matchType = MatchType.Regex} else {regex += '\\\\'}@@ -83,13 +173,31 @@ function compile(pattern: string): RegExp {regex += ch}}- return new RegExp(regex + '$')++ switch (matchType) {+ case MatchType.Exact:+ return (p) => p === pattern+ case MatchType.Prefix: {+ const prefix = pattern.slice(0, -2)+ return (p) => p.startsWith(prefix)+ }+ case MatchType.Suffix: {+ const suffix = pattern.slice(2)+ // "**/foo" also matches "foo"+ return (p) =>+ p.endsWith(suffix) || (suffix[0] === '/' && p === suffix.slice(1))+ }+ case MatchType.Regex: {+ const re = new RegExp(regex + '$', 'u')+ return (p) => re.test(p)+ }+ }}interface Pattern {exclusion: booleandirs: string[]- regex: RegExp+ match: Matcher}/**@@ -119,43 +227,48 @@ export class PatternMatcher {if (pattern.length > 1 && pattern.startsWith('/')) {pattern = pattern.slice(1)}- let regex: RegExp+ let match: Matchertry {- regex = compile(pattern)+ match = compile(pattern)} catch (err) {throw new TemplateError(`Invalid ignore pattern '${original}': ${(err as Error).message}`)}- this.patterns.push({ exclusion, dirs: pattern.split('/'), regex })+ this.patterns.push({ exclusion, dirs: pattern.split('/'), match })}}/**- * Whether the path or one of its parent directories is excluded.+ * Whether the path is excluded. Like BuildKit, the patterns are evaluated+ * on each parent directory first, and a pattern that matched a parent+ * directory also matches the paths under it.** @param p Slash-separated path relative to the context root* @returns True if the path is excluded*/matches(p: string): boolean {- const parentPath = path.posix.dirname(p)- const parentDirs = parentPath === '.' ? [] : parentPath.split('/')-+ const segments = p.split('/')+ let parentMatched: boolean[] = []let matched = false- for (const pattern of this.patterns) {- // An inclusion can't change an already matched path, and an- // exclusion can't change a path that hasn't matched yet- if (pattern.exclusion !== matched) {- continue- }- const match =- pattern.regex.test(p) ||- parentDirs.some((_, i) =>- pattern.regex.test(parentDirs.slice(0, i + 1).join('/'))- )- if (match) {- matched = !pattern.exclusion- }+ for (let depth = 1; depth <= segments.length; depth++) {+ const current = segments.slice(0, depth).join('/')+ matched = false+ parentMatched = this.patterns.map((pattern, i) => {+ let match = parentMatched[i] ?? false+ if (!match) {+ // An inclusion can't change an already matched path, and an+ // exclusion can't change a path that hasn't matched yet+ if (pattern.exclusion !== matched) {+ return false+ }+ match = pattern.match(current)+ }+ if (match) {+ matched = !pattern.exclusion+ }+ return match+ })}return matched}diff --git a/packages/js-sdk/tests/template/utils/getAllFilesInPath.test.ts b/packages/js-sdk/tests/template/utils/getAllFilesInPath.test.tsindex b5b8c6e54..dd2ebdbe4 100644--- a/packages/js-sdk/tests/template/utils/getAllFilesInPath.test.ts+++ b/packages/js-sdk/tests/template/utils/getAllFilesInPath.test.ts@@ -413,6 +413,34 @@ describe('getAllFilesInPath', () => {])})+ test('should not re-include a path through a negated parent directory', async () => {+ const files = await relativePaths('.', ['src/app.ts', '**/*.js', '!src'])+ expect(files).not.toContain('src/app.ts')+ expect(files).not.toContain('src/node_modules/lib.js')+ expect(files).toContain('src/app.spec.ts')+ })++ test('should match a leading globstar with a literal suffix', async () => {+ for (const name of ['root.txt', 'src/nested.txt', 'keep.txt.bak']) {+ await writeFile(join(testDir, name), 'x')+ }+ const files = await relativePaths('.', ['**.txt', '**/generated'])+ expect(files).not.toContain('root.txt')+ expect(files).not.toContain('src/nested.txt')+ expect(files).not.toContain('src/generated/api.ts')+ expect(files).toContain('keep.txt.bak')+ })++ test('should match characters outside the BMP as a single character', async () => {+ for (const name of ['π.txt', 'ππ.txt']) {+ await writeFile(join(testDir, name), 'x')+ }+ const files = await relativePaths('*', ['?.txt'])+ expect(files).not.toContain('π.txt')+ expect(files).toContain('ππ.txt')+ expect(await relativePaths('*', ['[π]?.txt'])).not.toContain('ππ.txt')+ })+test('should make absolute patterns inside the context relative', async () => {const files = await relativePaths('.', [join(testDir, 'src'),@@ -438,6 +466,23 @@ describe('getAllFilesInPath', () => {)})+ test.skipIf(process.platform === 'win32')(+ 'should handle escapes in a bracket expression like Python',+ async () => {+ await expect(+ getAllFilesInPath('.', testDir, ['[\\q]'])+ ).rejects.toThrow("Invalid ignore pattern '[\\q]'")+ await writeFile(join(testDir, 'a'), 'x')+ expect(await relativePaths('*', ['[\\a]'])).toContain('a')+ await writeFile(join(testDir, 'b'), 'x')+ await writeFile(join(testDir, 'c'), 'x')+ const files = await relativePaths('*', ['[\\x61]', '[\\142]'])+ expect(files).not.toContain('a')+ expect(files).not.toContain('b')+ expect(files).toContain('c')+ }+ )+test('should keep the files hash stable when ignored files change', async () => {const hash = () =>calculateFilesHash('.', '/app', testDir, ['.git'], false, undefined)diff --git a/packages/python-sdk/e2b/template/dockerignore.py b/packages/python-sdk/e2b/template/dockerignore.pyindex d6909a15d..5a02be1d9 100644--- a/packages/python-sdk/e2b/template/dockerignore.py+++ b/packages/python-sdk/e2b/template/dockerignore.py@@ -9,7 +9,8 @@ https://github.com/moby/patternmatcherimport osimport posixpathimport re-from typing import List+from enum import Enum+from typing import Callable, Listfrom e2b.exceptions import TemplateException@@ -19,6 +20,13 @@ _WILDCARD_CHARS = re.compile(r"[*?\[\\]")_BACKSLASH_IS_SEPARATOR = os.sep == "\\"+class _MatchType(Enum):+ EXACT = "exact"+ PREFIX = "prefix"+ SUFFIX = "suffix"+ REGEX = "regex"++def _clean(pattern: str) -> str:# Equivalent of Go's filepath.Clean followed by filepath.ToSlashif _BACKSLASH_IS_SEPARATOR:@@ -26,10 +34,14 @@ def _clean(pattern: str) -> str:return posixpath.normpath(re.sub("/+", "/", pattern))-def _compile(pattern: str) -> "re.Pattern[str]":+def _compile(pattern: str) -> Callable[[str], bool]:+ # Like moby, use plain string checks for patterns without wildcards and+ # only fall back to a regex otherwiseregex = "^"+ match_type = _MatchType.EXACTi, n = 0, len(pattern)while i < n:+ first = i == 0ch = pattern[i]i += 1if ch == "*":@@ -38,11 +50,24 @@ def _compile(pattern: str) -> "re.Pattern[str]":# Treat "**/" as "**"if i < n and pattern[i] == "/":i += 1- regex += ".*" if i >= n else "(.*/)?"+ if i >= n:+ regex += ".*"+ match_type = (+ _MatchType.PREFIX+ if match_type is _MatchType.EXACT+ else _MatchType.REGEX+ )+ else:+ regex += "(.*/)?"+ match_type = _MatchType.REGEX+ if first:+ match_type = _MatchType.SUFFIXelse:regex += "[^/]*"+ match_type = _MatchType.REGEXelif ch == "?":regex += "[^/]"+ match_type = _MatchType.REGEXelif ch == "[":# Copy a bracket expression as is, a leading "^" negates itj = i@@ -54,6 +79,10 @@ def _compile(pattern: str) -> "re.Pattern[str]":j += 1regex += pattern[i - 1 : j + 1]i = j + 1+ match_type = _MatchType.REGEX+ elif ch == "]":+ regex += ch+ match_type = _MatchType.REGEXelif ch in _LITERAL_REGEX_CHARS:regex += "\\" + chelif ch == "\\" and not _BACKSLASH_IS_SEPARATOR:@@ -61,21 +90,32 @@ def _compile(pattern: str) -> "re.Pattern[str]":if i < n:regex += re.escape(pattern[i])i += 1+ match_type = _MatchType.REGEXelse:regex += "\\\\"else:regex += ch- return re.compile(regex + "$")++ if match_type is _MatchType.EXACT:+ return lambda path: path == pattern+ if match_type is _MatchType.PREFIX:+ prefix = pattern[:-2]+ return lambda path: path.startswith(prefix)+ if match_type is _MatchType.SUFFIX:+ suffix = pattern[2:]+ # "**/foo" also matches "foo"+ return lambda path: path.endswith(suffix) or (+ suffix.startswith("/") and path == suffix[1:]+ )+ compiled = re.compile(regex + r"\Z")+ return lambda path: compiled.match(path) is not Noneclass _Pattern:def __init__(self, cleaned_pattern: str, exclusion: bool):self.exclusion = exclusionself.dirs = cleaned_pattern.split("/")- self._regex = _compile(cleaned_pattern)-- def match(self, path: str) -> bool:- return self._regex.match(path) is not None+ self.match = _compile(cleaned_pattern)class PatternMatcher:@@ -110,26 +150,33 @@ class PatternMatcher:def matches(self, path: str) -> bool:"""- Whether the path or one of its parent directories is excluded.+ Whether the path is excluded. Like BuildKit, the patterns are evaluated+ on each parent directory first, and a pattern that matched a parent+ directory also matches the paths under it.:param path: Slash-separated path relative to the context root:return: True if the path is excluded"""- parent_path = posixpath.dirname(path)- parent_dirs = parent_path.split("/") if parent_path else []-+ segments = path.split("/")+ parent_matched: List[bool] = []matched = False- for pattern in self._patterns:- # An inclusion can't change an already matched path, and an- # exclusion can't change a path that hasn't matched yet- if pattern.exclusion != matched:- continue- match = pattern.match(path) or any(- pattern.match("/".join(parent_dirs[: i + 1]))- for i in range(len(parent_dirs))- )- if match:- matched = not pattern.exclusion+ for depth in range(1, len(segments) + 1):+ current = "/".join(segments[:depth])+ current_matched: List[bool] = []+ matched = False+ for i, pattern in enumerate(self._patterns):+ match = bool(parent_matched) and parent_matched[i]+ if not match:+ # An inclusion can't change an already matched path, and+ # an exclusion can't change a path that hasn't matched yet+ if pattern.exclusion != matched:+ current_matched.append(False)+ continue+ match = pattern.match(current)+ current_matched.append(match)+ if match:+ matched = not pattern.exclusion+ parent_matched = current_matchedreturn matcheddef may_match_under(self, dir_path: str) -> bool:diff --git a/packages/python-sdk/tests/shared/template/utils/test_get_all_files_in_path.py b/packages/python-sdk/tests/shared/template/utils/test_get_all_files_in_path.pyindex 88a9bf0d3..fbf194d98 100644--- a/packages/python-sdk/tests/shared/template/utils/test_get_all_files_in_path.py+++ b/packages/python-sdk/tests/shared/template/utils/test_get_all_files_in_path.py@@ -407,6 +407,35 @@ class TestDockerignoreSemantics:"src/node_modules/lib.js",]+ def test_should_not_reinclude_a_path_through_a_negated_parent_directory(+ self, test_dir+ ):+ files = self.relative_paths(".", test_dir, ["src/app.ts", "**/*.js", "!src"])+ assert "src/app.ts" not in files+ assert "src/node_modules/lib.js" not in files+ assert "src/app.spec.ts" in files++ def test_should_match_a_leading_globstar_with_a_literal_suffix(self, test_dir):+ for name in ["root.txt", "src/nested.txt", "keep.txt.bak"]:+ with open(os.path.join(test_dir, name), "w") as f:+ f.write("x")+ files = self.relative_paths(".", test_dir, ["**.txt", "**/generated"])+ assert "root.txt" not in files+ assert "src/nested.txt" not in files+ assert "src/generated/api.ts" not in files+ assert "keep.txt.bak" in files++ def test_should_match_characters_outside_the_bmp_as_a_single_character(+ self, test_dir+ ):+ for name in ["π.txt", "ππ.txt"]:+ with open(os.path.join(test_dir, name), "w") as f:+ f.write("x")+ files = self.relative_paths("*", test_dir, ["?.txt"])+ assert "π.txt" not in files+ assert "ππ.txt" in files+ assert "ππ.txt" not in self.relative_paths("*", test_dir, ["[π]?.txt"])+def test_should_make_absolute_patterns_inside_the_context_relative(self, test_dir):files = self.relative_paths(".",@@ -432,6 +461,23 @@ class TestDockerignoreSemantics:with pytest.raises(TemplateException, match="Invalid ignore pattern '\\[abc'"):get_all_files_in_path(".", test_dir, ["[abc"])+ @pytest.mark.skipif(sys.platform == "win32", reason="Backslash is a separator")+ def test_should_handle_escapes_in_a_bracket_expression_like_python(self, test_dir):+ with pytest.raises(+ TemplateException, match=r"Invalid ignore pattern '\[\\q\]'"+ ):+ get_all_files_in_path(".", test_dir, ["[\\q]"])+ with open(os.path.join(test_dir, "a"), "w") as f:+ f.write("x")+ assert "a" in self.relative_paths("*", test_dir, ["[\\a]"])+ for name in ["b", "c"]:+ with open(os.path.join(test_dir, name), "w") as f:+ f.write("x")+ files = self.relative_paths("*", test_dir, ["[\\x61]", "[\\142]"])+ assert "a" not in files+ assert "b" not in files+ assert "c" in files+def test_should_keep_the_files_hash_stable_when_ignored_files_change(self, test_dir):