AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
refactor(platform): put the single-container image's code layers last (#14969)
Nicholas Tindle6b97e95ffda3
diff --git a/.github/scripts/single_container_cache.py b/.github/scripts/single_container_cache.pyindex 8fc9508b25..f51aaa9004 100644--- a/.github/scripts/single_container_cache.py+++ b/.github/scripts/single_container_cache.py@@ -20,7 +20,7 @@ def cache_settings(event: str, ref: str, image: str, arch: str) -> list[str]:export = sharedsettings = [f"{target}.cache-from=type=registry,ref={cache_ref}"- for target in ("backend-server", "single-container")+ for target in ("backend-server-base", "backend-server", "single-container")for cache_ref in imports]if export:diff --git a/.github/scripts/verify_single_container_reports.py b/.github/scripts/verify_single_container_reports.pyindex 5102af7745..df6cde5590 100644--- a/.github/scripts/verify_single_container_reports.py+++ b/.github/scripts/verify_single_container_reports.py@@ -116,7 +116,11 @@ def verify_trivy(path: Path, expected_image: str) -> None:for result in results:if not isinstance(result, dict):raise ValueError(f"{path.name} has an invalid result entry")- for key in ("Target", "Class", "Type"):+ # Trivy writes secret results with a Target and Class but no Type.+ identity = ("Target", "Class", "Type")+ if result.get("Class") == "secret":+ identity = ("Target", "Class")+ for key in identity:if not isinstance(result.get(key), str) or not result[key]:raise ValueError(f"{path.name} result is missing {key}")for key in TRIVY_FINDING_KEYS:diff --git a/.github/workflows/platform-single-container-docker.yml b/.github/workflows/platform-single-container-docker.ymlindex 2b45700a29..1702e577c5 100644--- a/.github/workflows/platform-single-container-docker.yml+++ b/.github/workflows/platform-single-container-docker.yml@@ -206,6 +206,7 @@ jobs:severity: CRITICALexit-code: "1"timeout: 30m+ trivyignores: autogpt_platform/single-container/.trivyignore.yaml- name: Scan image filesystem for embedded secretsid: secret_scan@@ -219,6 +220,7 @@ jobs:severity: HIGH,CRITICALexit-code: "1"timeout: 30m+ trivyignores: autogpt_platform/single-container/.trivyignore.yaml- name: Signal concurrent scans completeif: ${{ !cancelled() }}@@ -376,6 +378,7 @@ jobs:severity: CRITICALexit-code: "1"timeout: 30m+ trivyignores: autogpt_platform/single-container/.trivyignore.yaml- name: Scan the exact pushed digest for embedded secretsuses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0@@ -388,6 +391,7 @@ jobs:severity: HIGH,CRITICALexit-code: "1"timeout: 30m+ trivyignores: autogpt_platform/single-container/.trivyignore.yaml- name: Export verified digestshell: bashdiff --git a/autogpt_platform/backend/Dockerfile b/autogpt_platform/backend/Dockerfileindex 030ecce4ad..279395b203 100644--- a/autogpt_platform/backend/Dockerfile+++ b/autogpt_platform/backend/Dockerfile@@ -99,7 +99,7 @@ COPY docs /docsRUN find /docs -type f ! -name '*.md' ! -name '*.mdx' -delete \&& find /docs -type d -empty -delete-FROM debian:13-slim AS server+FROM debian:13-slim AS server-baseWORKDIR /app@@ -159,6 +159,10 @@ WORKDIR /app/autogpt_platform/backendCOPY --from=builder /app/autogpt_platform/backend/.venv ./.venvENV PATH="/app/autogpt_platform/backend/.venv/bin:$PATH"+# Dependencies end here; AutoGPT code starts below. The single-container+# image builds on server-base so its own dependencies sit under the code.+FROM server-base AS server+# Copy dependency files + autogpt_libs (path dependency)COPY autogpt_platform/autogpt_libs /app/autogpt_platform/autogpt_libsCOPY autogpt_platform/backend/poetry.lock autogpt_platform/backend/pyproject.toml ./diff --git a/autogpt_platform/single-container/.trivyignore.yaml b/autogpt_platform/single-container/.trivyignore.yamlnew file mode 100644index 0000000000..b5161e7889--- /dev/null+++ b/autogpt_platform/single-container/.trivyignore.yaml@@ -0,0 +1,11 @@+# Findings Trivy reports in the single-container image that are not AutoGPT+# secrets. Keep every entry to one rule in one third-party file.+secrets:+ - id: aws-access-key-id+ paths:+ - "**/site-packages/yt_dlp/extractor/shahid.py"+ statement: >-+ yt-dlp's Shahid extractor embeds the site's public client key. Trivy only+ reports it since the appliance stopped building on the finished backend+ server image, whose CMD made Trivy treat every backend layer as a base+ image layer and skip it during secret scanning.diff --git a/autogpt_platform/single-container/Dockerfile b/autogpt_platform/single-container/Dockerfileindex ccd002cfd4..b1b9b58546 100644--- a/autogpt_platform/single-container/Dockerfile+++ b/autogpt_platform/single-container/Dockerfile@@ -48,12 +48,19 @@ FROM rabbitmq:4.1.8@sha256:b921426eabfb39e201c60f0623f6b45a3160cf97b2dc3af2f5340FROM falkordb/falkordb-server:v4.20.1@sha256:8b676c2e72271f5d06541136eb214b0db50180636cc86f7ee38d63287e9b8d13 AS falkordb# autogpt-backend is supplied by Docker Bake from the existing backend-# Dockerfile's server target. This keeps the backend image contract independent-# from the opt-in single-container distribution.-FROM autogpt-backend AS single-container+# Dockerfile's server target. The appliance takes only AutoGPT's code from it+# (what the server target adds on top of server-base), so drop the dependency+# venv that server-base already provides.+FROM autogpt-backend AS backend-code-ARG IMAGE_VERSION=dev-ARG VCS_REF=unknown+RUN rm -rf /app/autogpt_platform/backend/.venv++# autogpt-backend-base is supplied by Docker Bake from the backend Dockerfile's+# server-base target: the backend image before AutoGPT's code is added. The+# appliance stacks its own dependencies on it and adds the code last, so a new+# commit only rebuilds and re-stores the code layers. This keeps the backend+# image contract independent from the opt-in single-container distribution.+FROM autogpt-backend-base AS single-container# The normal backend image stores generated Prisma engines under root's cache.# Appliance services run unprivileged, so relocate those inherited engines to a@@ -73,13 +80,6 @@ RUN install -d -m 0755 /opt/prisma-python \&& ln -s "${prisma_node_modules}" /opt/prisma-python/binaries/node_modules \&& ln -s "${query_engine}" /opt/prisma-python/query-engine-LABEL org.opencontainers.image.title="AutoGPT Platform single-container" \- org.opencontainers.image.description="Experimental single-node AutoGPT Platform distribution" \- org.opencontainers.image.source="https://github.com/Significant-Gravitas/AutoGPT" \- org.opencontainers.image.licenses="LicenseRef-PolyForm-Shield-1.0.0 AND SSPL-1.0" \- org.opencontainers.image.version="${IMAGE_VERSION}" \- org.opencontainers.image.revision="${VCS_REF}"-ENV DEBIAN_FRONTEND=noninteractive# Block service auto-start during package installation; Supervisor owns every@@ -133,7 +133,7 @@ RUN printf '#!/bin/sh\nexit 101\n' > /usr/sbin/policy-rc.d \&& rm -rf /var/lib/apt/lists/* /var/lib/postgresql/15/main# Node 24 is required by the frontend. Backend Prisma and agent-browser also-# use this binary through PATH; their npm packages remain from the server stage.+# use this binary through PATH; their npm packages remain from server-base.COPY --from=frontend /usr/local/bin/node /usr/local/bin/nodeCOPY --from=frontend /usr/local/LICENSE /usr/share/licenses/nodejs/LICENSERUN rm -rf /usr/lib/node_modules/npm@@ -148,19 +148,6 @@ COPY --from=rabbitmq /opt/rabbitmq /opt/rabbitmqCOPY --from=falkordb /usr/local/bin/redis-server /opt/falkordb/redis-serverCOPY --from=falkordb /var/lib/falkordb/bin/falkordb.so /opt/falkordb/falkordb.so-# Next standalone server and immutable assets.-COPY --from=frontend /app/.next/standalone /app/frontend-COPY --from=frontend /app/.next/static /app/frontend/.next/static-COPY --from=frontend /app/public /app/frontend/public--# Next's standalone trace over-includes Orval's build-only esbuild dependency.-# It is never used by the production server and must not ship as runtime code.-RUN rm -rf \- /app/frontend/node_modules/.pnpm/esbuild@* \- /app/frontend/node_modules/.pnpm/@esbuild+linux-*@* \- && find /app/frontend/node_modules -type l \- \( -name esbuild -o -path '*/node_modules/@esbuild/linux-*' \) -delete-# Runtime orchestration, proxy, health tooling, and database compatibility shim.COPY autogpt_platform/single-container/bootstrap.sh \autogpt_platform/single-container/common.sh \@@ -207,10 +194,6 @@ RUN groupadd --system --gid 10001 autogpt \&& find /opt/autogpt/single-container -type f \( -name '*.sh' -o -name 'entrypoint' -o -name 'autogpt-*' \) -exec chmod 0755 {} + \&& ln -sf /opt/autogpt/single-container/healthcheck.sh /usr/local/bin/autogpt-healthcheck \&& ln -sf /opt/autogpt/single-container/promote-admin.sh /usr/local/bin/autogpt-admin \- && rm -f /app/autogpt_platform/backend/config.json \- && ln -s /data/config/backend.json /app/autogpt_platform/backend/config.json \- && rm -rf /app/frontend/.next/cache \- && ln -s /data/cache/next /app/frontend/.next/cache \&& site_packages="$(/app/autogpt_platform/backend/.venv/bin/python -c \'import sysconfig; print(sysconfig.get_paths()["purelib"])')" \&& rm -f \@@ -219,6 +202,35 @@ RUN groupadd --system --gid 10001 autogpt \&& test ! -e "${site_packages}/ldclient/testing/selfsigned.key" \&& test ! -e "${site_packages}/ldclient/testing/selfsigned.pem"+# Everything above changes only when dependencies do. Everything below changes+# with AutoGPT's code, so it goes last and a new commit reuses the layers above.++# Next standalone server and immutable assets.+COPY --from=frontend /app/.next/standalone /app/frontend+COPY --from=frontend /app/.next/static /app/frontend/.next/static+COPY --from=frontend /app/public /app/frontend/public++# Next's standalone trace over-includes Orval's build-only esbuild dependency.+# It is never used by the production server and must not ship as runtime code.+RUN rm -rf \+ /app/frontend/node_modules/.pnpm/esbuild@* \+ /app/frontend/node_modules/.pnpm/@esbuild+linux-*@* \+ && find /app/frontend/node_modules -type l \+ \( -name esbuild -o -path '*/node_modules/@esbuild/linux-*' \) -delete \+ && rm -rf /app/frontend/.next/cache \+ && ln -s /data/cache/next /app/frontend/.next/cache++# AutoGPT's backend code and docs, then its entry points (rest, executor, ...),+# installed the same way the backend server target does on top of server-base.+WORKDIR /app/autogpt_platform/backend+COPY --from=backend-code /app/autogpt_platform/autogpt_libs /app/autogpt_platform/autogpt_libs+COPY --from=backend-code /app/autogpt_platform/backend /app/autogpt_platform/backend+COPY --from=backend-code /app/docs /app/docs+RUN POETRY_VIRTUALENVS_CREATE=true POETRY_VIRTUALENVS_IN_PROJECT=true \+ poetry install --no-ansi --only-root \+ && rm -f /app/autogpt_platform/backend/config.json \+ && ln -s /data/config/backend.json /app/autogpt_platform/backend/config.json+ENV PATH="/opt/rabbitmq/sbin:/opt/erlang/bin:/opt/openssl/bin:/usr/lib/postgresql/15/bin:/app/autogpt_platform/backend/.venv/bin:${PATH}" \ERLANG_INSTALL_PATH_PREFIX=/opt/erlang \OPENSSL_INSTALL_PATH_PREFIX=/opt/openssl \@@ -240,6 +252,18 @@ ENV PATH="/opt/rabbitmq/sbin:/opt/erlang/bin:/opt/openssl/bin:/usr/lib/postgresqFORCE_FLAG_GRAPHITI_MEMORY=true \AUTOGPT_PUBLIC_URL=http://localhost:3000+# Build arguments change on every build, so they are declared only after the+# last layer-producing step to keep them out of the layer cache keys.+ARG IMAGE_VERSION=dev+ARG VCS_REF=unknown++LABEL org.opencontainers.image.title="AutoGPT Platform single-container" \+ org.opencontainers.image.description="Experimental single-node AutoGPT Platform distribution" \+ org.opencontainers.image.source="https://github.com/Significant-Gravitas/AutoGPT" \+ org.opencontainers.image.licenses="LicenseRef-PolyForm-Shield-1.0.0 AND SSPL-1.0" \+ org.opencontainers.image.version="${IMAGE_VERSION}" \+ org.opencontainers.image.revision="${VCS_REF}"+WORKDIR /app/autogpt_platform/backendVOLUME ["/data"]diff --git a/autogpt_platform/single-container/docker-bake.hcl b/autogpt_platform/single-container/docker-bake.hclindex f7b731fca8..e10b9baf38 100644--- a/autogpt_platform/single-container/docker-bake.hcl+++ b/autogpt_platform/single-container/docker-bake.hcl@@ -2,6 +2,12 @@ group "default" {targets = ["single-container"]}+target "backend-server-base" {+ context = "."+ dockerfile = "autogpt_platform/backend/Dockerfile"+ target = "server-base"+}+target "backend-server" {context = "."dockerfile = "autogpt_platform/backend/Dockerfile"@@ -13,7 +19,8 @@ target "single-container" {dockerfile = "autogpt_platform/single-container/Dockerfile"target = "single-container"contexts = {- autogpt-backend = "target:backend-server"+ autogpt-backend-base = "target:backend-server-base"+ autogpt-backend = "target:backend-server"}tags = ["autogpt-platform:single-container-dev"]}diff --git a/autogpt_platform/single-container/tests/test_ci_registry_cache.py b/autogpt_platform/single-container/tests/test_ci_registry_cache.pyindex 13be35c767..3f16733342 100644--- a/autogpt_platform/single-container/tests/test_ci_registry_cache.py+++ b/autogpt_platform/single-container/tests/test_ci_registry_cache.py@@ -1,4 +1,5 @@import importlib.util+import reimport unittestfrom pathlib import Path@@ -47,6 +48,18 @@ class RegistryCacheTests(unittest.TestCase):self.settings("workflow_dispatch", "refs/heads/two"),)+ def test_every_bake_target_reads_the_validation_cache(self):+ bake = (+ SCRIPT.parents[2] / "autogpt_platform/single-container/docker-bake.hcl"+ ).read_text(encoding="utf-8")+ targets = re.findall(r'^target "([^"]+)"', bake, re.MULTILINE)+ self.assertIn("backend-server-base", targets)+ settings = self.settings("push")+ for target in targets:+ self.assertIn(+ f"{target}.cache-from=type=registry,ref={IMAGE}:v4-amd64", settings+ )+def test_validation_cache_reads_are_architecture_scoped(self):settings = self.settings("release", "refs/tags/v1")self.assertTrue(all("cache-from" in s and ":v4-amd64" in s for s in settings))diff --git a/autogpt_platform/single-container/tests/test_ci_report_verifier.py b/autogpt_platform/single-container/tests/test_ci_report_verifier.pyindex 79e06bc966..3340058c2d 100644--- a/autogpt_platform/single-container/tests/test_ci_report_verifier.py+++ b/autogpt_platform/single-container/tests/test_ci_report_verifier.py@@ -198,6 +198,55 @@ class ReportVerifierTests(unittest.TestCase):self.assertNotEqual(result.returncode, 0)self.assertIn("Vulnerabilities=1", result.stdout)+ def add_result(self, name, result):+ path = self.report_dir / name+ report = json.loads(path.read_text(encoding="utf-8"))+ report["Results"].append(result)+ path.write_text(json.dumps(report), encoding="utf-8")++ def test_clean_secret_results_need_no_type(self):+ # Trivy keeps a secret result for a file whose findings were all+ # filtered out by severity or an ignore entry, and never gives it a Type.+ for name in ("trivy-critical.json", "trivy-secrets.json"):+ self.add_result(+ name,+ {+ "Target": "/app/vendor/extractor.py",+ "Class": "secret",+ "Secrets": [],+ },+ )++ result = self.run_verifier()++ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)++ def test_secret_findings_are_rejected(self):+ self.add_result(+ "trivy-secrets.json",+ {+ "Target": "/app/leak.py",+ "Class": "secret",+ "Secrets": [{"RuleID": "aws-access-key-id"}],+ },+ )++ result = self.run_verifier()++ self.assertNotEqual(result.returncode, 0)+ self.assertIn("Secrets=1", result.stdout)++ def test_package_result_without_type_is_rejected(self):+ self.add_result(+ "trivy-critical.json",+ {"Target": "Python", "Class": "lang-pkgs", "Vulnerabilities": []},+ )++ result = self.run_verifier()++ self.assertNotEqual(result.returncode, 0)+ self.assertIn("result is missing Type", result.stdout)+def test_non_trivy_json_is_rejected(self):(self.report_dir / "trivy-secrets.json").write_text("{}", encoding="utf-8")diff --git a/autogpt_platform/single-container/tests/test_ci_scan_ignores.py b/autogpt_platform/single-container/tests/test_ci_scan_ignores.pynew file mode 100644index 0000000000..cef3185e23--- /dev/null+++ b/autogpt_platform/single-container/tests/test_ci_scan_ignores.py@@ -0,0 +1,26 @@+from __future__ import annotations++import unittest+from pathlib import Path++REPO_ROOT = Path(__file__).resolve().parents[3]+WORKFLOW_PATH = (+ REPO_ROOT / ".github" / "workflows" / "platform-single-container-docker.yml"+)+IGNORE_FILE = "autogpt_platform/single-container/.trivyignore.yaml"+++class ScanIgnoreTest(unittest.TestCase):+ def test_every_trivy_scan_uses_the_shared_ignore_file(self) -> None:+ # The CI build and the release digest are scanned by separate steps; an+ # entry that only one of them honours fails the other on a false positive.+ steps = WORKFLOW_PATH.read_text(encoding="utf-8").split("- name: ")[1:]+ scans = [step for step in steps if "aquasecurity/trivy-action@" in step]+ self.assertGreaterEqual(len(scans), 4)+ for step in scans:+ self.assertIn(f"trivyignores: {IGNORE_FILE}", step, step.splitlines()[0])+ self.assertTrue((REPO_ROOT / IGNORE_FILE).is_file())+++if __name__ == "__main__":+ unittest.main()diff --git a/autogpt_platform/single-container/tests/test_dockerfile_layers.py b/autogpt_platform/single-container/tests/test_dockerfile_layers.pynew file mode 100644index 0000000000..56792f8427--- /dev/null+++ b/autogpt_platform/single-container/tests/test_dockerfile_layers.py@@ -0,0 +1,127 @@+from __future__ import annotations++import re+import unittest+from pathlib import Path, PurePosixPath++ASSET_DIR = Path(__file__).resolve().parents[1]+APPLIANCE_DOCKERFILE = ASSET_DIR / "Dockerfile"+BACKEND_DOCKERFILE = ASSET_DIR.parent / "backend" / "Dockerfile"+BAKE_PATH = ASSET_DIR / "docker-bake.hcl"++SERVER_BASE = "FROM debian:13-slim AS server-base"+SERVER = "FROM server-base AS server"+APPLIANCE = "FROM autogpt-backend-base AS single-container"+LAYER_STEPS = {"ADD", "COPY", "RUN"}+++def instructions(path: Path) -> list[str]:+ """Return a Dockerfile's instructions with continuation lines joined."""+ result: list[str] = []+ current = ""+ for line in path.read_text(encoding="utf-8").splitlines():+ stripped = line.strip()+ if not stripped or stripped.startswith("#"):+ continue+ current = f"{current} {stripped}" if current else stripped+ if current.endswith("\\"):+ current = current[:-1].rstrip()+ continue+ result.append(re.sub(r"\s+", " ", current))+ current = ""+ return result+++def stage(lines: list[str], header: str) -> list[str]:+ start = lines.index(header) + 1+ end = next(+ (i for i in range(start, len(lines)) if lines[i].startswith("FROM ")),+ len(lines),+ )+ return lines[start:end]+++def run_commands(lines: list[str]) -> set[str]:+ return {+ command.strip()+ for line in lines+ if line.startswith("RUN ")+ for command in line.removeprefix("RUN ").split("&&")+ }+++class ApplianceLayerOrderTest(unittest.TestCase):+ """The appliance stacks its dependencies under AutoGPT's code.++ It builds on the backend's server-base target instead of the finished+ server image, so the layers that only change with dependencies are reused+ by every build, and a new commit only re-stores the code layers on top.+ """++ def setUp(self) -> None:+ self.backend = instructions(BACKEND_DOCKERFILE)+ self.appliance = stage(instructions(APPLIANCE_DOCKERFILE), APPLIANCE)++ def test_server_base_holds_dependencies_only(self) -> None:+ for line in stage(self.backend, SERVER_BASE):+ if line.startswith(("COPY ", "ADD ")):+ self.assertTrue(+ line.startswith("COPY --from=builder "),+ f"server-base must not copy AutoGPT code: {line}",+ )++ def test_appliance_adds_everything_the_server_target_adds(self) -> None:+ copied_from_backend = {+ parts[-1]+ for parts in (line.split() for line in self.appliance)+ if parts[:2] == ["COPY", "--from=backend-code"] and parts[2] == parts[-1]+ }+ commands = run_commands(self.appliance)++ workdir = PurePosixPath("/")+ for line in stage(self.backend, SERVER_BASE):+ if line.startswith("WORKDIR "):+ workdir = workdir / line.split()[1]+ for line in stage(self.backend, SERVER):+ keyword, *arguments = line.split()+ if keyword == "WORKDIR":+ workdir = workdir / arguments[0]+ elif keyword in {"COPY", "ADD"}:+ destination = str(workdir / arguments[-1])+ self.assertTrue(+ any(+ destination == copied or destination.startswith(f"{copied}/")+ for copied in copied_from_backend+ ),+ f"the appliance never copies {destination} from backend-code",+ )+ elif keyword == "RUN":+ self.assertIn(line.removeprefix("RUN "), commands)++ def test_build_arguments_follow_every_layer(self) -> None:+ arguments = [+ i for i, line in enumerate(self.appliance) if line.startswith("ARG ")+ ]+ self.assertTrue(arguments, "the appliance declares its build arguments")+ self.assertEqual(+ [+ line+ for line in self.appliance[min(arguments) :]+ if line.split()[0] in LAYER_STEPS+ ],+ [],+ "build arguments change every build and would invalidate later layers",+ )++ def test_bake_supplies_server_base(self) -> None:+ bake = BAKE_PATH.read_text(encoding="utf-8")+ self.assertRegex(+ bake, r'target "backend-server-base" \{[^}]*target\s*=\s*"server-base"'+ )+ self.assertRegex(+ bake, r'autogpt-backend-base\s*=\s*"target:backend-server-base"'+ )+++if __name__ == "__main__":+ unittest.main()