Skip to content

armgit / E2B

Public Imported from GitHub
0

Open-source, secure environment with real-world tools for enterprise-grade agents.

Add custom registries support for pulling base images (#897)

Mish Ushakov3d6418184ff6
diff --git a/.changeset/tough-squids-wash.md b/.changeset/tough-squids-wash.md
new file mode 100644
index 000000000..3218e210b
--- /dev/null
+++ b/.changeset/tough-squids-wash.md
@@ -0,0 +1,6 @@
+---
+'@e2b/python-sdk': patch
+'e2b': patch
+---
+
+added registry helpers to Template
diff --git a/.vscode/settings.json b/.vscode/settings.json
index 7180ff610..e151f437a 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -73,6 +73,12 @@
// Disable Pylint and use just Pylance
"python.linting.enabled": false,
"python.analysis.typeCheckingMode": "basic",
+ "cursorpyright.analysis.extraPaths": [
+ "./**",
+ "./packages/python-sdk/**",
+ "./packages/python-sdk/.venv/bin/python"
+ ],
+ "cursorpyright.analysis.typeCheckingMode": "basic",
// "python.analysis.pylanceLspClientEnabled": true,
// "python.analysis.inlayHints.functionReturnTypes": true,
// "python.analysis.inlayHints.pytestParameters": true,
diff --git a/packages/js-sdk/src/template/index.ts b/packages/js-sdk/src/template/index.ts
index 7af6ccc98..6b0f9ad3c 100644
--- a/packages/js-sdk/src/template/index.ts
+++ b/packages/js-sdk/src/template/index.ts
@@ -9,7 +9,6 @@ import {
waitForBuildFinish,
} from './buildApi'
import { parseDockerfile } from './dockerfileParser'
-import { BuildError } from './errors'
import {
Instruction,
TemplateFromImage,
@@ -17,12 +16,14 @@ import {
TemplateFinal,
CopyItem,
LogEntry,
+ RegistryConfig,
} from './types'
import {
calculateFilesHash,
getCallerDirectory,
padOctal,
readDockerignore,
+ readGCPServiceAccountJSON,
} from './utils'
import { ConnectionConfig } from '../connectionConfig'
import { ReadyCmd } from './readycmd'
@@ -51,6 +52,7 @@ export class TemplateBase
private defaultBaseImage: string = 'e2bdev/base'
private baseImage: string | undefined = this.defaultBaseImage
private baseTemplate: string | undefined = undefined
+ private registryConfig: RegistryConfig | undefined = undefined
private startCmd: string | undefined = undefined
private readyCmd: string | undefined = undefined
// Force the whole template to be rebuilt
@@ -101,10 +103,18 @@ export class TemplateBase
return this.fromImage(this.defaultBaseImage)
}
- fromImage(baseImage: string): TemplateBuilder {
+ fromImage(
+ baseImage: string,
+ options?: { registryConfig?: RegistryConfig }
+ ): TemplateBuilder {
this.baseImage = baseImage
this.baseTemplate = undefined
+ // Set the registry config if provided
+ if (options?.registryConfig) {
+ this.registryConfig = options.registryConfig
+ }
+
// If we should force the next layer and it's a FROM command, invalidate whole template
if (this.forceNextLayer) {
this.force = true
@@ -145,6 +155,57 @@ export class TemplateBase
return this
}
+ fromRegistry(
+ image: string,
+ options: {
+ username: string
+ password: string
+ }
+ ): TemplateBuilder {
+ return this.fromImage(image, {
+ registryConfig: {
+ type: 'registry',
+ username: options.username,
+ password: options.password,
+ },
+ })
+ }
+
+ fromAWSRegistry(
+ image: string,
+ options: {
+ accessKeyId: string
+ secretAccessKey: string
+ region: string
+ }
+ ): TemplateBuilder {
+ return this.fromImage(image, {
+ registryConfig: {
+ type: 'aws',
+ awsAccessKeyId: options.accessKeyId,
+ awsSecretAccessKey: options.secretAccessKey,
+ awsRegion: options.region,
+ },
+ })
+ }
+
+ fromGCPRegistry(
+ image: string,
+ options: {
+ serviceAccountJSON: string | object
+ }
+ ): TemplateBuilder {
+ return this.fromImage(image, {
+ registryConfig: {
+ type: 'gcp',
+ serviceAccountJson: readGCPServiceAccountJSON(
+ this.fileContextPath,
+ options.serviceAccountJSON
+ ),
+ },
+ })
+ }
+
copy(
src: string,
dest: string,
@@ -548,7 +609,7 @@ export class TemplateBase
}
private serialize(steps: Instruction[]): TriggerBuildTemplate {
- const baseData = {
+ const templateData: TriggerBuildTemplate = {
startCmd: this.startCmd,
readyCmd: this.readyCmd,
steps,
@@ -556,20 +617,18 @@ export class TemplateBase
}
if (this.baseImage !== undefined) {
- return {
- ...baseData,
- fromImage: this.baseImage,
- }
- } else if (this.baseTemplate !== undefined) {
- return {
- ...baseData,
- fromTemplate: this.baseTemplate,
- }
- } else {
- throw new BuildError(
- 'Template must specify either fromImage or fromTemplate'
- )
+ templateData.fromImage = this.baseImage
+ }
+
+ if (this.baseTemplate !== undefined) {
+ templateData.fromTemplate = this.baseTemplate
}
+
+ if (this.registryConfig !== undefined) {
+ templateData.fromImageRegistry = this.registryConfig
+ }
+
+ return templateData
}
}
diff --git a/packages/js-sdk/src/template/types.ts b/packages/js-sdk/src/template/types.ts
index f8971bf9b..dbcbedd75 100644
--- a/packages/js-sdk/src/template/types.ts
+++ b/packages/js-sdk/src/template/types.ts
@@ -35,6 +35,30 @@ export interface TemplateFromImage {
fromDockerfile(dockerfileContent: string): TemplateBuilder
+ fromRegistry(
+ image: string,
+ options: {
+ username: string
+ password: string
+ }
+ ): TemplateBuilder
+
+ fromAWSRegistry(
+ image: string,
+ options: {
+ accessKeyId: string
+ secretAccessKey: string
+ region: string
+ }
+ ): TemplateBuilder
+
+ fromGCPRegistry(
+ image: string,
+ options: {
+ serviceAccountJSON: object | string
+ }
+ ): TemplateBuilder
+
skipCache(): TemplateBuilder
}
@@ -118,3 +142,23 @@ export class LogEntry {
)}`
}
}
+
+export type GenericDockerRegistry = {
+ type: 'registry'
+ username: string
+ password: string
+}
+
+export type AWSRegistry = {
+ type: 'aws'
+ awsAccessKeyId: string
+ awsSecretAccessKey: string
+ awsRegion: string
+}
+
+export type GCPRegistry = {
+ type: 'gcp'
+ serviceAccountJson: string
+}
+
+export type RegistryConfig = GenericDockerRegistry | AWSRegistry | GCPRegistry
diff --git a/packages/js-sdk/src/template/utils.ts b/packages/js-sdk/src/template/utils.ts
index 7f35a89b9..6c6be85e2 100644
--- a/packages/js-sdk/src/template/utils.ts
+++ b/packages/js-sdk/src/template/utils.ts
@@ -97,3 +97,13 @@ export async function tarFileStreamUpload(
uploadStream: await tarFileStream(fileName, fileContextPath),
}
}
+
+export function readGCPServiceAccountJSON(
+ contextPath: string,
+ pathOrContent: string | object
+): string {
+ if (typeof pathOrContent === 'string') {
+ return fs.readFileSync(path.join(contextPath, pathOrContent), 'utf-8')
+ }
+ return JSON.stringify(pathOrContent)
+}
diff --git a/packages/python-sdk/e2b/template/main.py b/packages/python-sdk/e2b/template/main.py
index 13825078c..03bbbe003 100644
--- a/packages/python-sdk/e2b/template/main.py
+++ b/packages/python-sdk/e2b/template/main.py
@@ -3,12 +3,18 @@ from typing import Dict, List, Optional, Union
from httpx import Limits
from e2b.template.dockerfile_parser import parse_dockerfile
-from e2b.template.types import CopyItem, Instruction, TemplateType
+from e2b.template.types import (
+ CopyItem,
+ Instruction,
+ TemplateType,
+ RegistryConfig,
+)
from e2b.template.utils import (
calculate_files_hash,
get_caller_directory,
pad_octal,
read_dockerignore,
+ read_gcp_service_account_json,
)
from e2b.template.readycmd import ReadyCmd
@@ -249,6 +255,7 @@ class TemplateBase:
self._default_base_image: str = "e2bdev/base"
self._base_image: Optional[str] = self._default_base_image
self._base_template: Optional[str] = None
+ self._registry_config: Optional[RegistryConfig] = None
self._start_cmd: Optional[str] = None
self._ready_cmd: Optional[str] = None
# Force the whole template to be rebuilt
@@ -283,10 +290,16 @@ class TemplateBase:
def from_base_image(self) -> TemplateBuilder:
return self.from_image(self._default_base_image)
- def from_image(self, base_image: str) -> TemplateBuilder:
+ def from_image(
+ self, base_image: str, registry_config: Optional[RegistryConfig] = None
+ ) -> TemplateBuilder:
self._base_image = base_image
self._base_template = None
+ # Set the registry config if provided
+ if registry_config is not None:
+ self._registry_config = registry_config
+
# If we should force the next layer and it's a FROM command, invalidate whole template
if self._force_next_layer:
self._force = True
@@ -323,6 +336,48 @@ class TemplateBase:
return builder
+ def from_registry(
+ self, image: str, username: str, password: str
+ ) -> TemplateBuilder:
+ return self.from_image(
+ image,
+ registry_config={
+ "type": "registry",
+ "username": username,
+ "password": password,
+ },
+ )
+
+ def from_aws_registry(
+ self,
+ image: str,
+ access_key_id: str,
+ secret_access_key: str,
+ region: str,
+ ) -> TemplateBuilder:
+ return self.from_image(
+ image,
+ registry_config={
+ "type": "aws",
+ "awsAccessKeyId": access_key_id,
+ "awsSecretAccessKey": secret_access_key,
+ "awsRegion": region,
+ },
+ )
+
+ def from_gcp_registry(
+ self, image: str, service_account_json: Union[str, dict]
+ ) -> TemplateBuilder:
+ return self.from_image(
+ image,
+ registry_config={
+ "type": "gcp",
+ "serviceAccountJson": read_gcp_service_account_json(
+ self._file_context_path, service_account_json
+ ),
+ },
+ )
+
@staticmethod
def to_json(template: "TemplateClass") -> str:
return json.dumps(
@@ -391,6 +446,9 @@ class TemplateBase:
if self._base_template is not None:
template_data["fromTemplate"] = self._base_template
+ if self._registry_config is not None:
+ template_data["fromImageRegistry"] = self._registry_config
+
if self._start_cmd is not None:
template_data["startCmd"] = self._start_cmd
diff --git a/packages/python-sdk/e2b/template/types.py b/packages/python-sdk/e2b/template/types.py
index 6ae2cf809..9cbbb9c64 100644
--- a/packages/python-sdk/e2b/template/types.py
+++ b/packages/python-sdk/e2b/template/types.py
@@ -1,4 +1,4 @@
-from typing import List, Optional, TypedDict
+from typing import List, Optional, TypedDict, Union
from typing_extensions import NotRequired
from dataclasses import dataclass
from datetime import datetime
@@ -22,16 +22,6 @@ class Instruction(TypedDict):
filesHash: NotRequired[str]
-class TemplateType(TypedDict):
- fromImage: NotRequired[str]
- fromTemplate: NotRequired[str]
- startCmd: NotRequired[str]
- readyCmd: NotRequired[str]
- readyCmdTimeoutMs: NotRequired[int]
- steps: List[Instruction]
- force: bool
-
-
@dataclass
class LogEntry:
timestamp: datetime
@@ -43,3 +33,34 @@ class LogEntry:
def __str__(self) -> str:
return f"[{self.timestamp.isoformat()}] [{self.level}] {self.message}"
+
+
+class GenericDockerRegistry(TypedDict):
+ type: Literal["registry"]
+ username: str
+ password: str
+
+
+class AWSRegistry(TypedDict):
+ type: Literal["aws"]
+ awsAccessKeyId: str
+ awsSecretAccessKey: str
+ awsRegion: str
+
+
+class GCPRegistry(TypedDict):
+ type: Literal["gcp"]
+ serviceAccountJson: str
+
+
+RegistryConfig = Union[GenericDockerRegistry, AWSRegistry, GCPRegistry]
+
+
+class TemplateType(TypedDict):
+ fromImage: NotRequired[str]
+ fromTemplate: NotRequired[str]
+ fromImageRegistry: NotRequired[RegistryConfig]
+ startCmd: NotRequired[str]
+ readyCmd: NotRequired[str]
+ steps: List[Instruction]
+ force: bool
diff --git a/packages/python-sdk/e2b/template/utils.py b/packages/python-sdk/e2b/template/utils.py
index e8c331dde..8ba2d1de7 100644
--- a/packages/python-sdk/e2b/template/utils.py
+++ b/packages/python-sdk/e2b/template/utils.py
@@ -1,10 +1,11 @@
import hashlib
import os
+import json
from glob import glob
import fnmatch
import re
import inspect
-from typing import List, Optional
+from typing import List, Optional, Union
def read_dockerignore(context_path: str) -> List[str]:
@@ -86,3 +87,15 @@ def get_caller_directory() -> Optional[str]:
def pad_octal(mode: int) -> str:
return f"{mode:04o}"
+
+
+def read_gcp_service_account_json(
+ context_path: str, path_or_content: Union[str, dict]
+) -> str:
+ if isinstance(path_or_content, str):
+ with open(
+ os.path.join(context_path, path_or_content), "r", encoding="utf-8"
+ ) as f:
+ return f.read()
+ else:
+ return json.dumps(path_or_content)
diff --git a/packages/python-sdk/e2b/template_async/build_api.py b/packages/python-sdk/e2b/template_async/build_api.py
index c122b60f5..a9930fa18 100644
--- a/packages/python-sdk/e2b/template_async/build_api.py
+++ b/packages/python-sdk/e2b/template_async/build_api.py
@@ -7,7 +7,6 @@ from typing import Callable, Literal, Optional
import httpx
-from e2b.api.client.types import UNSET
from e2b.template.types import TemplateType, LogEntry
from e2b.api.client.client import AuthenticatedClient
from e2b.api.client.api.templates import (
@@ -21,7 +20,6 @@ from e2b.api.client.models import (
TemplateBuildStartV2,
TemplateBuildFileUpload,
TemplateBuild,
- TemplateStep,
Error,
)
from e2b.api import handle_api_exception
@@ -94,31 +92,8 @@ async def trigger_build(
build_id: str,
template: TemplateType,
) -> None:
- # Convert template dict to TemplateBuildStartV2 model
- template_steps = []
- for step in template.get("steps", []):
- template_step = TemplateStep(
- type_=step["type"],
- args=step.get("args", []),
- force=step.get("force", False),
- )
- if "filesHash" in step:
- template_step.files_hash = step["filesHash"]
- template_steps.append(template_step)
-
- # Create the appropriate template data type based on fromImage or fromTemplate
- template_data = TemplateBuildStartV2(
- from_image=template.get("fromImage", UNSET),
- from_template=template.get("fromTemplate", UNSET),
- force=template.get("force", False),
- steps=template_steps,
- start_cmd=template.get("startCmd", UNSET),
- ready_cmd=template.get("readyCmd", UNSET),
- )
-
- # Validate that either fromImage or fromTemplate is specified
- if template_data.from_image is UNSET and template_data.from_template is UNSET:
- raise BuildException("Template must specify either fromImage or fromTemplate")
+ # Convert template dict to TemplateBuildStartV2 model using from_dict
+ template_data = TemplateBuildStartV2.from_dict(template)
res = await post_v_2_templates_template_id_builds_build_id.asyncio_detailed(
template_id=template_id,
diff --git a/packages/python-sdk/e2b/template_sync/build_api.py b/packages/python-sdk/e2b/template_sync/build_api.py
index b3b0d4f4b..cec5c08f1 100644
--- a/packages/python-sdk/e2b/template_sync/build_api.py
+++ b/packages/python-sdk/e2b/template_sync/build_api.py
@@ -7,7 +7,6 @@ from typing import Callable, Literal, Optional
import httpx
-from e2b.api.client.types import UNSET
from e2b.template.types import TemplateType, LogEntry
from e2b.api.client.client import AuthenticatedClient
from e2b.api.client.api.templates import (
@@ -21,7 +20,6 @@ from e2b.api.client.models import (
TemplateBuildStartV2,
TemplateBuildFileUpload,
TemplateBuild,
- TemplateStep,
Error,
)
from e2b.api import handle_api_exception
@@ -94,31 +92,8 @@ def trigger_build(
build_id: str,
template: TemplateType,
) -> None:
- # Convert template dict to TemplateBuildStartV2 model
- template_steps = []
- for step in template.get("steps", []):
- template_step = TemplateStep(
- type_=step["type"],
- args=step.get("args", []),
- force=step.get("force", False),
- )
- if "filesHash" in step:
- template_step.files_hash = step["filesHash"]
- template_steps.append(template_step)
-
- # Create the appropriate template data type based on fromImage or fromTemplate
- template_data = TemplateBuildStartV2(
- from_image=template.get("fromImage", UNSET),
- from_template=template.get("fromTemplate", UNSET),
- force=template.get("force", False),
- steps=template_steps,
- start_cmd=template.get("startCmd", UNSET),
- ready_cmd=template.get("readyCmd", UNSET),
- )
-
- # Validate that either fromImage or fromTemplate is specified
- if template_data.from_image is UNSET and template_data.from_template is UNSET:
- raise BuildException("Template must specify either fromImage or fromTemplate")
+ # Convert template dict to TemplateBuildStartV2 model using from_dict
+ template_data = TemplateBuildStartV2.from_dict(template)
res = post_v_2_templates_template_id_builds_build_id.sync_detailed(
template_id=template_id,