Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(ci): avoid full history in the rust test planner (#127007)

posthog[bot]950ca74c5dcd
diff --git a/.github/scripts/resolve-changed-files.sh b/.github/scripts/resolve-changed-files.sh
new file mode 100644
index 000000000000..cad3d9e9d507
--- /dev/null
+++ b/.github/scripts/resolve-changed-files.sh
@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# Resolve a complete change set against MERGE_TARGET without fetching history.
+set -euo pipefail
+
+# An API failure or a truncated file list has to widen to ALL
+# rather than upload a partial file list, which Trunk cannot
+# distinguish from a complete one. The computation step reads
+# `resolved` and reports ALL when it is anything but true.
+unresolved() {
+ echo "::warning::$1; reporting ALL."
+ echo 'resolved=false' >> "$GITHUB_OUTPUT"
+ exit 0
+}
+
+# Each attempt overwrites the output file, so a failed attempt
+# leaves no partial page behind for the next one to append to.
+# A 4xx, rate limits included, would fail again on the same
+# token bucket, so only network errors and 5xx are retried.
+gh_api_to() {
+ local out="$1" err="$RUNNER_TEMP/gh-api-error.txt" attempt
+ shift
+ for attempt in 1 2 3; do
+ if gh api "$@" > "$out" 2> "$err"; then
+ return 0
+ fi
+ cat "$err" >&2
+ grep -qE '\(HTTP 4[0-9]{2}\)' "$err" && return 1
+ [ "$attempt" -lt 3 ] && sleep $((attempt * 5))
+ done
+ return 1
+}
+
+# A renamed file is listed as its old path and its new one. The
+# old path is what the target script reads as deleted, and it
+# widens on that: the tach map has no importer edge for it.
+file_paths='.filename, (select(.previous_filename) | .previous_filename)'
+
+[[ -n "$MERGE_TARGET" && ! "$MERGE_TARGET" =~ ^0+$ ]] || unresolved "No valid comparison base"
+
+head_sha="$(git rev-parse HEAD)"
+compare="$RUNNER_TEMP/compare.json"
+gh_api_to "$compare" "repos/$REPOSITORY/compare/$MERGE_TARGET...$head_sha?per_page=1" \
+ || unresolved "The compare API failed for $MERGE_TARGET...$head_sha"
+
+merge_base="$(jq -r '.merge_base_commit.sha // empty' "$compare")"
+[[ "$merge_base" =~ ^[0-9a-f]{40}$ ]] || unresolved "The compare API returned no merge base"
+jq -e '.files | type == "array"' "$compare" > /dev/null || unresolved "The compare API returned no file list"
+
+# The compare API lists at most 300 files. Past that, the merge
+# base commit is fetched without blobs and diffed against the
+# checked-out merge commit, which needs only their trees. The pull
+# request files API is no substitute: it describes the PR head
+# against its own base, and reads whatever head the PR has now.
+if [ "$(jq '.files | length' "$compare")" -lt 300 ]; then
+ jq -r ".files[] | $file_paths" "$compare" > "$RUNNER_TEMP/listed-files.txt"
+else
+ git fetch --no-tags --depth=1 --filter=blob:none origin "$merge_base" \
+ || unresolved "Could not fetch the merge base $merge_base"
+ git diff --name-only --no-renames "$merge_base" HEAD > "$RUNNER_TEMP/listed-files.txt" \
+ || unresolved "Could not diff against the merge base"
+fi
+
+sort -u "$RUNNER_TEMP/listed-files.txt" > "$RUNNER_TEMP/changed-files.txt"
+sed 's/^/ /' "$RUNNER_TEMP/changed-files.txt"
+changed="$(cat "$RUNNER_TEMP/changed-files.txt")"
+
+echo 'resolved=true' >> "$GITHUB_OUTPUT"
+echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
+if printf '%s\n' "$changed" | grep -qE '^(rust|proto)/'; then
+ echo 'rust_affected=true' >> "$GITHUB_OUTPUT"
+fi
+if printf '%s\n' "$changed" | grep -qxF 'pnpm-lock.yaml'; then
+ echo 'js_lockfile_changed=true' >> "$GITHUB_OUTPUT"
+fi
diff --git a/.github/scripts/test_resolve_changed_files.py b/.github/scripts/test_resolve_changed_files.py
new file mode 100644
index 000000000000..d701cedf4778
--- /dev/null
+++ b/.github/scripts/test_resolve_changed_files.py
@@ -0,0 +1,156 @@
+import os
+import json
+import subprocess
+from pathlib import Path
+
+import pytest
+
+SCRIPT = Path(__file__).with_name("resolve-changed-files.sh")
+BASE = "a" * 40
+HEAD = "b" * 40
+
+
+def run_resolver(
+ tmp_path: Path,
+ response: dict[str, object],
+ target: str,
+ paths: list[str],
+ fail_fetch: bool = False,
+ fail_api: bool = False,
+) -> tuple[subprocess.CompletedProcess[str], str]:
+ (tmp_path / "compare-response.json").write_text(json.dumps(response))
+ (tmp_path / "diff-paths").write_text("\n".join(paths) + "\n")
+ for name, body in {
+ "gh": """echo "$*" > "$RUNNER_TEMP/api-call"
+if [ "$FAIL_API" = true ]; then
+ echo '(HTTP 403)' >&2
+ exit 1
+fi
+cat "$RUNNER_TEMP/compare-response.json"
+""",
+ "git": """case "$1" in
+ rev-parse) echo "$HEAD_SHA" ;;
+ fetch)
+ echo "$*" > "$RUNNER_TEMP/fetch-call"
+ [ "$FAIL_FETCH" != true ] ;;
+ diff)
+ [ "$*" = "diff --name-only --no-renames $BASE_SHA HEAD" ] || exit 1
+ cat "$RUNNER_TEMP/diff-paths" ;;
+ *) exit 1 ;;
+esac
+""",
+ }.items():
+ executable = tmp_path / name
+ executable.write_text("#!/usr/bin/env bash\n" + body)
+ executable.chmod(0o755)
+ output = tmp_path / "output"
+ result = subprocess.run(
+ ["bash", str(SCRIPT)],
+ env={
+ **os.environ,
+ "PATH": f"{tmp_path}:{os.environ['PATH']}",
+ "RUNNER_TEMP": str(tmp_path),
+ "GITHUB_OUTPUT": str(output),
+ "REPOSITORY": "example/repo",
+ "MERGE_TARGET": target,
+ "HEAD_SHA": HEAD,
+ "BASE_SHA": BASE,
+ "FAIL_FETCH": str(fail_fetch).lower(),
+ "FAIL_API": str(fail_api).lower(),
+ },
+ capture_output=True,
+ text=True,
+ )
+ return result, output.read_text()
+
+
[email protected]("target", ["stack-parent", "master", "c" * 40])
[email protected]("file_count", [2, 300])
+def test_complete_paths_and_event_base(tmp_path: Path, target: str, file_count: int) -> None:
+ paths = ["rust/old/src/lib.rs", "rust/new/src/lib.rs", "rust/deleted/Cargo.toml", "rust/Cargo.lock"]
+ response: dict[str, object] = {
+ "merge_base_commit": {"sha": BASE},
+ "files": [
+ {"filename": paths[1], "previous_filename": paths[0]},
+ {"filename": paths[2]},
+ ],
+ }
+ if file_count == 300:
+ response["files"] = [{"filename": f"rust/file-{i}"} for i in range(300)]
+ else:
+ paths = paths[:3]
+ result, outputs = run_resolver(tmp_path, response, target, paths)
+ assert result.returncode == 0
+ assert "resolved=true" in outputs
+ assert f"merge_base={BASE}" in outputs
+ assert (tmp_path / "changed-files.txt").read_text().splitlines() == sorted(paths)
+ assert f"compare/{target}...{HEAD}?per_page=1" in (tmp_path / "api-call").read_text()
+ assert (tmp_path / "fetch-call").exists() == (file_count == 300)
+
+
+ "response,fail_fetch,fail_api,target",
+ [
+ ({"merge_base_commit": {"sha": "invalid"}, "files": []}, False, False, "master"),
+ ({"merge_base_commit": {"sha": BASE}}, False, False, "master"),
+ ({"merge_base_commit": {"sha": BASE}, "files": [{}] * 300}, True, False, "master"),
+ ({}, False, True, "master"),
+ ({}, False, False, "0" * 40),
+ ({}, False, False, ""),
+ ],
+)
+def test_uncertain_diff_does_not_publish_a_partial_list(
+ tmp_path: Path, response: dict[str, object], fail_fetch: bool, fail_api: bool, target: str
+) -> None:
+ result, outputs = run_resolver(tmp_path, response, target, [], fail_fetch, fail_api)
+ assert result.returncode == 0
+ assert outputs.strip() == "resolved=false"
+ assert not (tmp_path / "changed-files.txt").exists()
+
+
+def test_truncated_compare_uses_only_endpoint_trees(tmp_path: Path) -> None:
+ source = tmp_path / "source"
+ source.mkdir()
+
+ def git(*args: str, cwd: Path = source) -> str:
+ return subprocess.check_output(["git", *args], cwd=cwd, text=True).strip()
+
+ git("init", "-q")
+ git("config", "user.email", "[email protected]")
+ git("config", "user.name", "Test")
+ (source / "old.rs").write_text("unchanged contents\n")
+ (source / "deleted.rs").write_text("deleted contents\n")
+ git("add", ".")
+ git("commit", "-qm", "base")
+ base = git("rev-parse", "HEAD")
+ git("mv", "old.rs", "new.rs")
+ git("rm", "deleted.rs")
+ (source / "Cargo.lock").write_text("lockfile\n")
+ git("add", ".")
+ git("commit", "-qm", "change")
+ expected = set(git("diff", "--name-only", "--no-renames", f"{base}...HEAD").splitlines())
+ shallow = tmp_path / "shallow"
+ git("clone", "-q", "--depth=1", source.as_uri(), str(shallow))
+ assert git("rev-parse", "--is-shallow-repository", cwd=shallow) == "true"
+ response = {"merge_base_commit": {"sha": base}, "files": [{}] * 300}
+ (tmp_path / "response.json").write_text(json.dumps(response))
+ executable = tmp_path / "gh"
+ executable.write_text('#!/usr/bin/env bash\ncat "$RUNNER_TEMP/response.json"\n')
+ executable.chmod(0o755)
+ result = subprocess.run(
+ ["bash", str(SCRIPT.resolve())],
+ cwd=shallow,
+ env={
+ **os.environ,
+ "PATH": f"{tmp_path}:{os.environ['PATH']}",
+ "RUNNER_TEMP": str(tmp_path),
+ "GITHUB_OUTPUT": str(tmp_path / "output"),
+ "REPOSITORY": "example/repo",
+ "MERGE_TARGET": "stack-parent",
+ },
+ capture_output=True,
+ text=True,
+ check=True,
+ )
+ assert "resolved=true" in (tmp_path / "output").read_text(), result.stdout
+ assert set((tmp_path / "changed-files.txt").read_text().splitlines()) == expected
diff --git a/.github/workflows/ci-rust.yml b/.github/workflows/ci-rust.yml
index 366950940b68..7326daa94771 100644
--- a/.github/workflows/ci-rust.yml
+++ b/.github/workflows/ci-rust.yml
@@ -77,6 +77,7 @@ jobs:
- 'proto/**'
- '.github/workflows/ci-rust.yml'
- '.github/scripts/rust_test_shards.py'
+ - '.github/scripts/resolve-changed-files.sh'
- '.github/actions/setup-uv/**'
- '.github/workflows/rust.yml'
- '.github/workflows/rust-docker-build.yml'
@@ -129,18 +130,33 @@ jobs:
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
- fetch-depth: 0
+ fetch-depth: 1
filter: blob:none
sparse-checkout: |
rust/
proto/
.github/rust-images.yml
.github/scripts/rust_test_shards.py
+ .github/scripts/resolve-changed-files.sh
.github/actions/rust-compute-affected/
.github/actions/setup-rust/
sparse-checkout-cone-mode: false
clean: false
+ - name: Resolve the changed files
+ id: changed-files
+ if: github.event_name != 'workflow_dispatch' && needs.changes.outputs.all_crates != 'true'
+ env:
+ GH_TOKEN: ${{ github.token }}
+ REPOSITORY: ${{ github.repository }}
+ MERGE_TARGET: ${{ github.event_name == 'push' && github.event.before || github.event.pull_request.base.ref }}
+ run: |
+ if [ -f .github/scripts/resolve-changed-files.sh ]; then
+ bash .github/scripts/resolve-changed-files.sh
+ else
+ echo 'resolved=false' >> "$GITHUB_OUTPUT"
+ fi
+
- name: Install rust
uses: ./.github/actions/setup-rust
with:
@@ -153,7 +169,9 @@ jobs:
event-name: ${{ github.event_name }}
pr-base-sha: ${{ github.event.pull_request.base.sha }}
push-before-sha: ${{ github.event.before }}
- rebuild-all: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.all_crates == 'true') && 'true' || 'false' }}
+ base-ref: ${{ github.event_name == 'push' && github.event.before || steps.changed-files.outputs.merge_base }}
+ changed-files: ${{ runner.temp }}/changed-files.txt
+ rebuild-all: ${{ steps.changed-files.outputs.resolved != 'true' && 'true' || 'false' }}
# Bin-pack affected packages into balanced test shards.
- name: Compute test shards
diff --git a/.github/workflows/trunk-impacted-targets.yml b/.github/workflows/trunk-impacted-targets.yml
index fb2182d1f2a9..5716fedeaa1e 100644
--- a/.github/workflows/trunk-impacted-targets.yml
+++ b/.github/workflows/trunk-impacted-targets.yml
@@ -97,75 +97,10 @@ jobs:
REPOSITORY: ${{ github.repository }}
MERGE_TARGET: ${{ github.event.repository.default_branch }}
run: |
- set -uo pipefail
-
- # An API failure or a truncated file list has to widen to ALL
- # rather than upload a partial file list, which Trunk cannot
- # distinguish from a complete one. The computation step reads
- # `resolved` and reports ALL when it is anything but true.
- unresolved() {
- echo "::warning::$1; reporting ALL."
- echo 'resolved=false' >> "$GITHUB_OUTPUT"
- exit 0
- }
-
- # Each attempt overwrites the output file, so a failed attempt
- # leaves no partial page behind for the next one to append to.
- # A 4xx, rate limits included, would fail again on the same
- # token bucket, so only network errors and 5xx are retried.
- gh_api_to() {
- local out="$1" err="$RUNNER_TEMP/gh-api-error.txt" attempt
- shift
- for attempt in 1 2 3; do
- if gh api "$@" > "$out" 2> "$err"; then
- return 0
- fi
- cat "$err" >&2
- grep -qE '\(HTTP 4[0-9]{2}\)' "$err" && return 1
- [ "$attempt" -lt 3 ] && sleep $((attempt * 5))
- done
- return 1
- }
-
- # A renamed file is listed as its old path and its new one. The
- # old path is what the target script reads as deleted, and it
- # widens on that: the tach map has no importer edge for it.
- file_paths='.filename, (select(.previous_filename) | .previous_filename)'
-
- head_sha="$(git rev-parse HEAD)"
- compare="$RUNNER_TEMP/compare.json"
- gh_api_to "$compare" "repos/$REPOSITORY/compare/$MERGE_TARGET...$head_sha?per_page=1" \
- || unresolved "The compare API failed for $MERGE_TARGET...$head_sha"
-
- merge_base="$(jq -r '.merge_base_commit.sha // empty' "$compare")"
- [[ "$merge_base" =~ ^[0-9a-f]{40}$ ]] || unresolved "The compare API returned no merge base"
- jq -e '.files | type == "array"' "$compare" > /dev/null || unresolved "The compare API returned no file list"
-
- # The compare API lists at most 300 files. Past that, the merge
- # base commit is fetched without blobs and diffed against the
- # checked-out merge commit, which needs only their trees. The pull
- # request files API is no substitute: it describes the PR head
- # against its own base, and reads whatever head the PR has now.
- if [ "$(jq '.files | length' "$compare")" -lt 300 ]; then
- jq -r ".files[] | $file_paths" "$compare" > "$RUNNER_TEMP/listed-files.txt"
+ if [ -f .github/scripts/resolve-changed-files.sh ]; then
+ bash .github/scripts/resolve-changed-files.sh
else
- git fetch --no-tags --depth=1 --filter=blob:none origin "$merge_base" \
- || unresolved "Could not fetch the merge base $merge_base"
- git diff --name-only --no-renames "$merge_base" HEAD > "$RUNNER_TEMP/listed-files.txt" \
- || unresolved "Could not diff against the merge base"
- fi
-
- sort -u "$RUNNER_TEMP/listed-files.txt" > "$RUNNER_TEMP/changed-files.txt"
- sed 's/^/ /' "$RUNNER_TEMP/changed-files.txt"
- changed="$(cat "$RUNNER_TEMP/changed-files.txt")"
-
- echo 'resolved=true' >> "$GITHUB_OUTPUT"
- echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
- if printf '%s\n' "$changed" | grep -qE '^(rust|proto)/'; then
- echo 'rust_affected=true' >> "$GITHUB_OUTPUT"
- fi
- if printf '%s\n' "$changed" | grep -qxF 'pnpm-lock.yaml'; then
- echo 'js_lockfile_changed=true' >> "$GITHUB_OUTPUT"
+ echo 'resolved=false' >> "$GITHUB_OUTPUT"
fi
# The target script reads the base lockfile with `git show` at the
diff --git a/tools/workflow-plan/tests/workflows.test.ts b/tools/workflow-plan/tests/workflows.test.ts
index 975b7b7b15b5..c2ef97e3cc7e 100644
--- a/tools/workflow-plan/tests/workflows.test.ts
+++ b/tools/workflow-plan/tests/workflows.test.ts
@@ -1033,6 +1033,26 @@ describe('CI workflow run plans', () => {
expect(plan.jobs['code-quality']?.steps.find((step) => step.name === 'Save mypy cache')?.runs).toBe(runs)
})
+ it.each([
+ ['PR', pullRequest(), false, true],
+ ['merge queue', mergeQueue(), false, true],
+ ['push', push(), false, true],
+ ['manual full rebuild', workflowDispatch(), false, false],
+ ['all-crate change', pullRequest(), true, false],
+ ] as const)('Rust change resolver on %s', (name, github, allCrates, runs) => {
+ const wf = workflow('ci-rust.yml')
+ const plan = planWorkflow(wf, {
+ name,
+ github,
+ steps: {
+ changes: { filter: pathsFilter({ rust: true, all_crates: allCrates }) },
+ affected: { shards: { outputs: { matrix: '{"include":[{"packages":"common-types"}]}' } } },
+ },
+ })
+ expect(plan.errors).toEqual([])
+ expect(plan.jobs.affected?.steps.find((step) => step.name === 'Resolve the changed files')?.runs).toBe(runs)
+ })
+
it.each(['success', 'failure'] as const)(
'sccache counters survive a %s build without changing its verdict',
(outcome) => {