Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(tasks): rebuild the sandbox images when node ships a release (#122319)

Tom Piccirello7a171d4c258b
diff --git a/.github/workflows/cd-sandbox-base-image.yml b/.github/workflows/cd-sandbox-base-image.yml
index 554c16e26620..a6b1a64a8e48 100644
--- a/.github/workflows/cd-sandbox-base-image.yml
+++ b/.github/workflows/cd-sandbox-base-image.yml
@@ -52,6 +52,7 @@ jobs:
sandbox_image: ${{ steps.filter.outputs.sandbox_image == 'true' || steps.drift.outputs.drift == 'true' }}
skills_source: ${{ steps.filter.outputs.skills_source }}
agent_version: ${{ steps.desired.outputs.agent_version }}
+ nodejs_version: ${{ steps.desired.outputs.nodejs_version }}
inputs_digest: ${{ steps.desired.outputs.inputs_digest }}
drift_reason: ${{ steps.filter.outputs.sandbox_image != 'true' && steps.drift.outputs.drift_reason || '' }}
steps:
@@ -107,6 +108,26 @@ jobs:
echo "::error::no ARG AGENT_VERSION line in $DOCKERFILE"
exit 1
fi
+ # A Node release changes no file in the repo, so the newest nodesource nodejs
+ # package is an input too. A new release then fails the drift check below, and the
+ # NODEJS_VERSION build-arg makes the build miss the cached Node layer. Only a version
+ # that nodesource serves for both architectures counts, because the images build for both.
+ node_major=$(sed -n 's|.*deb\.nodesource\.com/setup_\([0-9]*\)\.x.*|\1|p' "$DOCKERFILE" | head -n1)
+ if [ -z "$node_major" ]; then
+ echo "::error::no nodesource setup_<major>.x line in $DOCKERFILE"
+ exit 1
+ fi
+ nodejs_versions() {
+ curl -fsSL --retry 5 --retry-all-errors --retry-max-time 60 --connect-timeout 10 \
+ "https://deb.nodesource.com/node_${node_major}.x/dists/nodistro/main/binary-$1/Packages" \
+ | awk '$1 == "Package:" { pkg = $2 } $1 == "Version:" && pkg == "nodejs" { print $2 }' \
+ | LC_ALL=C sort -u
+ }
+ nodejs_version=$(LC_ALL=C comm -12 <(nodejs_versions amd64) <(nodejs_versions arm64) | sort -V | tail -n1)
+ if [[ ! "$nodejs_version" =~ ^${node_major}\.[0-9]+\.[0-9]+-[0-9]+nodesource[0-9]+$ ]]; then
+ echo "::error::nodesource serves no nodejs ${node_major}.x package for both amd64 and arm64 (got '${nodejs_version}')"
+ exit 1
+ fi
inputs=(
.github/workflows/cd-sandbox-base-image.yml
.github/actions/setup-uv
@@ -121,15 +142,19 @@ jobs:
inputs+=("$product/skills")
done < <(git ls-tree -d --name-only HEAD products/)
inputs_digest=$(
- for path in "${inputs[@]}"; do
- if object=$(git rev-parse --verify -q "HEAD:$path"); then
- printf '%s\t%s\n' "$path" "$object"
- fi
- done | LC_ALL=C sort | sha256sum | cut -d' ' -f1
+ {
+ for path in "${inputs[@]}"; do
+ if object=$(git rev-parse --verify -q "HEAD:$path"); then
+ printf '%s\t%s\n' "$path" "$object"
+ fi
+ done
+ printf 'nodesource nodejs\t%s\n' "$nodejs_version"
+ } | LC_ALL=C sort | sha256sum | cut -d' ' -f1
)
- echo "HEAD wants @posthog/agent $agent_version from inputs $inputs_digest"
+ echo "HEAD wants @posthog/agent $agent_version and nodejs $nodejs_version from inputs $inputs_digest"
{
echo "agent_version=$agent_version"
+ echo "nodejs_version=$nodejs_version"
echo "inputs_digest=$inputs_digest"
} >> "$GITHUB_OUTPUT"
@@ -578,6 +603,7 @@ jobs:
# pinned, reviewed version. Bump it there, not by resolving npm "latest".
build-args: |
COMMIT_HASH=${{ github.sha }}
+ NODEJS_VERSION=${{ needs.changes.outputs.nodejs_version }}
labels: |
org.opencontainers.image.revision=${{ github.sha }}
com.posthog.sandbox.agent-version=${{ needs.changes.outputs.agent_version }}
@@ -594,7 +620,9 @@ jobs:
push: ${{ github.event_name == 'pull_request' || vars.CD_DEPLOY_ENABLED == 'true' }}
tags: ${{ github.ref == 'refs/heads/master' && format('ghcr.io/posthog/posthog-sandbox-notebook:master,ghcr.io/posthog/posthog-sandbox-notebook:{0}', github.sha) || format('ghcr.io/posthog/posthog-sandbox-notebook:{0}', github.sha) }}
platforms: linux/arm64,linux/amd64
- build-args: COMMIT_HASH=${{ github.sha }}
+ build-args: |
+ COMMIT_HASH=${{ github.sha }}
+ NODEJS_VERSION=${{ needs.changes.outputs.nodejs_version }}
labels: |
org.opencontainers.image.revision=${{ github.sha }}
com.posthog.sandbox.agent-version=${{ needs.changes.outputs.agent_version }}
diff --git a/products/tasks/backend/logic/services/modal_sandbox.py b/products/tasks/backend/logic/services/modal_sandbox.py
index 8a22130fb982..4f7d37693855 100644
--- a/products/tasks/backend/logic/services/modal_sandbox.py
+++ b/products/tasks/backend/logic/services/modal_sandbox.py
@@ -14,6 +14,7 @@ import threading
import dataclasses
from collections.abc import Callable, Iterable
from dataclasses import dataclass
+from datetime import UTC, datetime
from functools import lru_cache
from http import HTTPStatus
from io import StringIO
@@ -617,8 +618,13 @@ def _build_slim_template_image() -> modal.Image:
changes pays a one-time Modal-side build; every create after that reuses the cached
image layers.
"""
+ # Modal caches each layer by its definition, so the apt packages below, nodejs included,
+ # keep the versions of their first build. The ISO week in the first layer changes the
+ # definition once a week, so the first create of each week rebuilds them from the newest packages.
+ year, week, _ = datetime.now(UTC).isocalendar()
return (
modal.Image.debian_slim()
+ .run_commands(f"echo 'Refresh the apt packages for {year}-W{week:02d}'")
# bash is required by the NodeSource setup script below; debian_slim doesn't guarantee it.
.apt_install("git", "ca-certificates", "curl", "bash")
.run_commands(
diff --git a/products/tasks/backend/sandbox/images/Dockerfile.sandbox-base b/products/tasks/backend/sandbox/images/Dockerfile.sandbox-base
index bc641da0fe0d..e735e70181c4 100644
--- a/products/tasks/backend/sandbox/images/Dockerfile.sandbox-base
+++ b/products/tasks/backend/sandbox/images/Dockerfile.sandbox-base
@@ -78,9 +78,12 @@ RUN set -eux; \
git --version; \
git help -a | grep -q '[[:space:]]backfill'
-# Install Node.js 24.x
+# Install Node.js 24.x. The layer cache keys on this RUN text, so cd-sandbox-base-image.yml
+# passes the newest nodesource package as NODEJS_VERSION to rebuild the layer when Node ships
+# a release. A build without the arg installs the newest package.
+ARG NODEJS_VERSION
RUN curl -fsSL https://deb.nodesource.com/setup_24.x | bash - && \
- apt-get install -y --no-install-recommends nodejs && \
+ apt-get install -y --no-install-recommends "nodejs${NODEJS_VERSION:+=$NODEJS_VERSION}" && \
rm -rf /var/lib/apt/lists/*
# Install additional language package managers
diff --git a/products/tasks/backend/sandbox/images/Dockerfile.sandbox-notebook b/products/tasks/backend/sandbox/images/Dockerfile.sandbox-notebook
index 9daa974934dd..e5384c2f880c 100644
--- a/products/tasks/backend/sandbox/images/Dockerfile.sandbox-notebook
+++ b/products/tasks/backend/sandbox/images/Dockerfile.sandbox-notebook
@@ -50,9 +50,10 @@ RUN apt-get update && \
sudo \
&& rm -rf /var/lib/apt/lists/*
-# Install Node.js 24.x
+# Install Node.js 24.x. NODEJS_VERSION works as in Dockerfile.sandbox-base (the reason is there).
+ARG NODEJS_VERSION
RUN curl -fsSL https://deb.nodesource.com/setup_24.x | bash - && \
- apt-get install -y nodejs && \
+ apt-get install -y "nodejs${NODEJS_VERSION:+=$NODEJS_VERSION}" && \
rm -rf /var/lib/apt/lists/*
# Install additional language package managers