Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(ci): make Pulumi Preview on PR see the changed files (#6823)

gbirman4e12c8651f57
diff --git a/.github/scripts/detect-affected-services.sh b/.github/scripts/detect-affected-services.sh
deleted file mode 100755
index e8d9881bcb..0000000000
--- a/.github/scripts/detect-affected-services.sh
+++ /dev/null
@@ -1,149 +0,0 @@
-#!/bin/bash
-set -e
-
-# This script detects which services are affected by changes in the root Rust workspace.
-# It uses cargo metadata to build a proper dependency graph
-
-# Get changed files from git
-CHANGED_FILES=$(git diff --name-only HEAD~1 HEAD | grep -E "^(Cargo\.(toml|lock)|rust-toolchain\.toml|Cross\.toml|clippy\.toml|deny\.toml|\.cargo/|\.config/|\.sqlx/|crates/|services/|tooling/xtask/|docker/|infra/|\.github/services-config\.json|\.github/workspace-dep-closures\.json|\.github/scripts/build-cloud-storage-lambdas(-nix)?\.sh)" || true)
-
-if [ -z "$CHANGED_FILES" ]; then
- echo "No Rust service or deployment files changed"
- echo "services=[]"
- echo "has_changes=false"
- exit 0
-fi
-
-echo "Changed files:" >&2
-echo "$CHANGED_FILES" >&2
-
-# Get cargo metadata for the entire workspace with dependencies
-METADATA=$(cargo metadata --format-version 1 --no-deps)
-WORKSPACE_ROOT=$(echo "$METADATA" | jq -r '.workspace_root')
-
-# Extract changed packages from the changed files
-CHANGED_PACKAGES=()
-while IFS= read -r file; do
- if [[ "$file" =~ ^(crates/[^/]+|services/[^/]+)/ ]]; then
- PKG_DIR="${BASH_REMATCH[1]}"
- PKG_NAME=$(echo "$METADATA" | jq -r \
- --arg manifest "$WORKSPACE_ROOT/$PKG_DIR/Cargo.toml" \
- '.packages[] | select(.manifest_path == $manifest) | .name' | head -n1)
- if [ -n "$PKG_NAME" ]; then
- # Avoid duplicates
- if [[ ! " ${CHANGED_PACKAGES[@]} " =~ " ${PKG_NAME} " ]]; then
- CHANGED_PACKAGES+=("$PKG_NAME")
- echo "Package $PKG_NAME has direct changes" >&2
- fi
- fi
- fi
-done <<< "$CHANGED_FILES"
-
-# Now find all services that are affected
-AFFECTED_SERVICES=()
-
-# Get services from the config file
-SERVICES=$(jq -r '.services | keys[]' .github/services-config.json)
-
-# Changes to shared build/deploy machinery can affect every deployable service.
-GLOBAL_CHANGE=false
-if echo "$CHANGED_FILES" | grep -qE '^(Cargo\.(toml|lock)|rust-toolchain\.toml|Cross\.toml|clippy\.toml|deny\.toml|\.cargo/|\.config/|\.sqlx/|tooling/xtask/|docker/|infra/[^/]+$|infra/packages/|\.github/services-config\.json|\.github/workspace-dep-closures\.json|\.github/scripts/build-cloud-storage-lambdas(-nix)?\.sh)'; then
- GLOBAL_CHANGE=true
-fi
-
-for service in $SERVICES; do
- SERVICE_AFFECTED=false
-
- # Get the source paths for this service from config
- SOURCE_PATHS=$(jq -r --arg svc "$service" '.services[$svc].source_paths[]? // empty' .github/services-config.json)
-
- # Check stack path changes
- STACK_PATH=$(jq -r --arg svc "$service" '.services[$svc].stack_path // empty' .github/services-config.json)
-
- if [ "$GLOBAL_CHANGE" = true ]; then
- SERVICE_AFFECTED=true
- echo "Service $service affected by shared workspace/deployment changes" >&2
- fi
- if [ -n "$STACK_PATH" ]; then
- STACK_PATH_PATTERN="${STACK_PATH%/**}"
- if echo "$CHANGED_FILES" | grep -q "^$STACK_PATH_PATTERN"; then
- SERVICE_AFFECTED=true
- echo "Service $service affected by stack changes" >&2
- fi
- fi
-
- # If no source paths, check if already affected by stack
- if [ -z "$SOURCE_PATHS" ]; then
- if [ "$SERVICE_AFFECTED" = true ]; then
- AFFECTED_SERVICES+=("$service")
- fi
- continue
- fi
-
- # Check if any source path has direct changes
- while IFS= read -r source_path; do
- if [ -n "$source_path" ]; then
- SOURCE_PATH_PATTERN="${source_path%/**}"
- if echo "$CHANGED_FILES" | grep -q "^$SOURCE_PATH_PATTERN"; then
- SERVICE_AFFECTED=true
- echo "Service $service has direct source changes in $source_path" >&2
- break
- fi
- fi
- done <<< "$SOURCE_PATHS"
-
- # Check if any of the service's packages depend on changed packages
- if [ "$SERVICE_AFFECTED" = false ] && [ ${#CHANGED_PACKAGES[@]} -gt 0 ]; then
- echo "Checking dependencies for service $service..." >&2
-
- # For each source path of the service, extract the package name and check its dependencies
- while IFS= read -r source_path; do
- if [ -n "$source_path" ]; then
- # Resolve the package from its manifest path instead of assuming
- # the directory name and Cargo package name are identical.
- SOURCE_PATH_CLEAN="${source_path%/**}"
- SOURCE_PKG_NAME=$(echo "$METADATA" | jq -r \
- --arg manifest "$WORKSPACE_ROOT/$SOURCE_PATH_CLEAN/Cargo.toml" \
- '.packages[] | select(.manifest_path == $manifest) | .name' | head -n1)
- if [ -z "$SOURCE_PKG_NAME" ]; then
- continue
- fi
- echo " Checking if $SOURCE_PKG_NAME depends on changed packages..." >&2
-
- # Get all dependencies of this package
- DEPS=$(echo "$METADATA" | jq -r --arg name "$SOURCE_PKG_NAME" '
- .packages[] |
- select(.name == $name) |
- .dependencies[] |
- .name
- ' | sort -u)
-
- # Check if any changed package is in the dependencies
- for changed_pkg in "${CHANGED_PACKAGES[@]}"; do
- if echo "$DEPS" | grep -q "^$changed_pkg$"; then
- SERVICE_AFFECTED=true
- echo " ✓ Service $service (via $SOURCE_PKG_NAME) depends on changed package: $changed_pkg" >&2
- break 2 # Break both loops
- fi
- done
- fi
- done <<< "$SOURCE_PATHS"
- fi
-
- if [ "$SERVICE_AFFECTED" = true ]; then
- AFFECTED_SERVICES+=("$service")
- fi
-done
-
-# Output results
-if [ ${#AFFECTED_SERVICES[@]} -gt 0 ]; then
- # Create JSON array
- SERVICES_JSON=$(printf '%s\n' "${AFFECTED_SERVICES[@]}" | jq -R . | jq -s . | jq -c .)
- echo "services=$SERVICES_JSON"
- echo "has_changes=true"
- echo "Affected services: ${AFFECTED_SERVICES[@]}" >&2
-else
- echo "services=[]"
- echo "has_changes=false"
- echo "No services affected by changes" >&2
-fi
diff --git a/.github/workflows/pulumi_preview_pr.yml b/.github/workflows/pulumi_preview_pr.yml
index 9ebad9544e..2c85502811 100644
--- a/.github/workflows/pulumi_preview_pr.yml
+++ b/.github/workflows/pulumi_preview_pr.yml
@@ -77,12 +77,21 @@ jobs:
.github/scripts/build-cloud-storage-lambdas-nix.sh
.github/services-config.json
.github/workspace-dep-closures.json
+ json: 'true'
+ escape_json: 'false'
write_output_files: 'true'
- id: detect
name: Detect affected services
run: |
services=()
+ # The action joins its plain-text output with spaces and no trailing
+ # newline, which `while read` never yields a line from, so read the
+ # JSON list and split it one path per line. `all_modified_files`,
+ # unlike `all_changed_files`, includes deletions.
+ changed_files=.github/outputs/changed_files.txt
+ jq -r '.[]' .github/outputs/all_modified_files.json > "$changed_files"
+
# Read service config
config=$(cat .github/services-config.json)
@@ -123,7 +132,7 @@ jobs:
service_changed=true
break
fi
- done < .github/outputs/all_changed_files.txt
+ done < "$changed_files"
# Get all source and stack globs for this service.
service_paths=$(echo "$config" | jq -r --arg s "$service" \
@@ -140,7 +149,7 @@ jobs:
break 2
fi
done <<< "$service_paths"
- done < .github/outputs/all_changed_files.txt
+ done < "$changed_files"
fi
# Match crate/service source against each deployable's workspace
@@ -167,7 +176,7 @@ jobs:
service_changed=true
break 3
fi
- done < .github/outputs/all_changed_files.txt
+ done < "$changed_files"
done < <(jq -r --arg c "$crate" '.closures[$c] // empty | .[]' .github/workspace-dep-closures.json)
done <<< "$crates"
fi
@@ -177,6 +186,18 @@ jobs:
fi
done
+ # A stack directory no service's `stack_path` claims gets no preview.
+ unmapped=$(sed -n 's|^infra/stacks/\([^/]*\)/.*|\1|p' "$changed_files" | sort -u |
+ while IFS= read -r stack; do
+ if ! jq -e --arg p "infra/stacks/$stack/**" \
+ 'any(.services[]; .stack_path == $p)' <<< "$config" > /dev/null; then
+ echo "$stack"
+ fi
+ done)
+ if [[ -n "$unmapped" ]]; then
+ echo "::warning::No preview for infra/stacks/{$(paste -sd, <<< "$unmapped")}: no service in .github/services-config.json has that stack_path"
+ fi
+
if [ ${#services[@]} -eq 0 ]; then
echo "has-changes=false" >> $GITHUB_OUTPUT
echo "services=[]" >> $GITHUB_OUTPUT
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/pulumi_preview_pr.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/pulumi_preview_pr.rs
index 9f365107de..369c71512c 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/pulumi_preview_pr.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/pulumi_preview_pr.rs
@@ -181,6 +181,8 @@ fn changed_files() -> Step<Use> {
"#}
.trim_end(),
))
+ .add_with(("json", true))
+ .add_with(("escape_json", false))
.add_with(("write_output_files", true))
}
@@ -189,6 +191,13 @@ fn detect_affected_services() -> Step<Run> {
.run(indoc::indoc! {r#"
services=()
+ # The action joins its plain-text output with spaces and no trailing
+ # newline, which `while read` never yields a line from, so read the
+ # JSON list and split it one path per line. `all_modified_files`,
+ # unlike `all_changed_files`, includes deletions.
+ changed_files=.github/outputs/changed_files.txt
+ jq -r '.[]' .github/outputs/all_modified_files.json > "$changed_files"
+
# Read service config
config=$(cat .github/services-config.json)
@@ -229,7 +238,7 @@ fn detect_affected_services() -> Step<Run> {
service_changed=true
break
fi
- done < .github/outputs/all_changed_files.txt
+ done < "$changed_files"
# Get all source and stack globs for this service.
service_paths=$(echo "$config" | jq -r --arg s "$service" \
@@ -246,7 +255,7 @@ fn detect_affected_services() -> Step<Run> {
break 2
fi
done <<< "$service_paths"
- done < .github/outputs/all_changed_files.txt
+ done < "$changed_files"
fi
# Match crate/service source against each deployable's workspace
@@ -273,7 +282,7 @@ fn detect_affected_services() -> Step<Run> {
service_changed=true
break 3
fi
- done < .github/outputs/all_changed_files.txt
+ done < "$changed_files"
done < <(jq -r --arg c "$crate" '.closures[$c] // empty | .[]' .github/workspace-dep-closures.json)
done <<< "$crates"
fi
@@ -283,6 +292,18 @@ fn detect_affected_services() -> Step<Run> {
fi
done
+ # A stack directory no service's `stack_path` claims gets no preview.
+ unmapped=$(sed -n 's|^infra/stacks/\([^/]*\)/.*|\1|p' "$changed_files" | sort -u |
+ while IFS= read -r stack; do
+ if ! jq -e --arg p "infra/stacks/$stack/**" \
+ 'any(.services[]; .stack_path == $p)' <<< "$config" > /dev/null; then
+ echo "$stack"
+ fi
+ done)
+ if [[ -n "$unmapped" ]]; then
+ echo "::warning::No preview for infra/stacks/{$(paste -sd, <<< "$unmapped")}: no service in .github/services-config.json has that stack_path"
+ fi
+
if [ ${#services[@]} -eq 0 ]; then
echo "has-changes=false" >> $GITHUB_OUTPUT
echo "services=[]" >> $GITHUB_OUTPUT
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/pulumi_preview_pr/test.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/pulumi_preview_pr/test.rs
index 19776628bd..2689250acb 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/pulumi_preview_pr/test.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/pulumi_preview_pr/test.rs
@@ -12,3 +12,25 @@ fn sqlx_changes_preview_every_service() {
"a root-only .sqlx change must fan out to every stack, not services=[]: {run}"
);
}
+
+#[test]
+fn detect_reads_the_json_changed_file_list() {
+ let run = detect_affected_services()
+ .value
+ .run
+ .expect("detect step should be a run script");
+ assert!(
+ !run.contains("all_changed_files.txt"),
+ "the space-joined .txt output has no newline, so `while read` sees no files: {run}"
+ );
+ assert!(
+ run.contains(r#"jq -r '.[]' .github/outputs/all_modified_files.json > "$changed_files""#)
+ && run.matches(r#"done < "$changed_files""#).count() == 3,
+ "every matching loop must read the JSON list, deletions included: {run}"
+ );
+ let with = serde_json::to_string(&changed_files().value.with).expect("with serializes");
+ assert!(
+ with.contains(r#""json":"true""#) && with.contains(r#""escape_json":"false""#),
+ "{with}"
+ );
+}