:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(ci): avoid full history in the rust test planner (#127007)
posthog[bot]950ca74c5dcd
diff --git a/.github/scripts/resolve-changed-files.sh b/.github/scripts/resolve-changed-files.shnew file mode 100644index 000000000000..cad3d9e9d507--- /dev/null+++ b/.github/scripts/resolve-changed-files.sh@@ -0,0 +1,74 @@+#!/usr/bin/env bash+# Resolve a complete change set against MERGE_TARGET without fetching history.+set -euo pipefail++# An API failure or a truncated file list has to widen to ALL+# rather than upload a partial file list, which Trunk cannot+# distinguish from a complete one. The computation step reads+# `resolved` and reports ALL when it is anything but true.+unresolved() {+ echo "::warning::$1; reporting ALL."+ echo 'resolved=false' >> "$GITHUB_OUTPUT"+ exit 0+}++# Each attempt overwrites the output file, so a failed attempt+# leaves no partial page behind for the next one to append to.+# A 4xx, rate limits included, would fail again on the same+# token bucket, so only network errors and 5xx are retried.+gh_api_to() {+ local out="$1" err="$RUNNER_TEMP/gh-api-error.txt" attempt+ shift+ for attempt in 1 2 3; do+ if gh api "$@" > "$out" 2> "$err"; then+ return 0+ fi+ cat "$err" >&2+ grep -qE '\(HTTP 4[0-9]{2}\)' "$err" && return 1+ [ "$attempt" -lt 3 ] && sleep $((attempt * 5))+ done+ return 1+}++# A renamed file is listed as its old path and its new one. The+# old path is what the target script reads as deleted, and it+# widens on that: the tach map has no importer edge for it.+file_paths='.filename, (select(.previous_filename) | .previous_filename)'++[[ -n "$MERGE_TARGET" && ! "$MERGE_TARGET" =~ ^0+$ ]] || unresolved "No valid comparison base"++head_sha="$(git rev-parse HEAD)"+compare="$RUNNER_TEMP/compare.json"+gh_api_to "$compare" "repos/$REPOSITORY/compare/$MERGE_TARGET...$head_sha?per_page=1" \+ || unresolved "The compare API failed for $MERGE_TARGET...$head_sha"++merge_base="$(jq -r '.merge_base_commit.sha // empty' "$compare")"+[[ "$merge_base" =~ ^[0-9a-f]{40}$ ]] || unresolved "The compare API returned no merge base"+jq -e '.files | type == "array"' "$compare" > /dev/null || unresolved "The compare API returned no file list"++# The compare API lists at most 300 files. Past that, the merge+# base commit is fetched without blobs and diffed against the+# checked-out merge commit, which needs only their trees. The pull+# request files API is no substitute: it describes the PR head+# against its own base, and reads whatever head the PR has now.+if [ "$(jq '.files | length' "$compare")" -lt 300 ]; then+ jq -r ".files[] | $file_paths" "$compare" > "$RUNNER_TEMP/listed-files.txt"+else+ git fetch --no-tags --depth=1 --filter=blob:none origin "$merge_base" \+ || unresolved "Could not fetch the merge base $merge_base"+ git diff --name-only --no-renames "$merge_base" HEAD > "$RUNNER_TEMP/listed-files.txt" \+ || unresolved "Could not diff against the merge base"+fi++sort -u "$RUNNER_TEMP/listed-files.txt" > "$RUNNER_TEMP/changed-files.txt"+sed 's/^/ /' "$RUNNER_TEMP/changed-files.txt"+changed="$(cat "$RUNNER_TEMP/changed-files.txt")"++echo 'resolved=true' >> "$GITHUB_OUTPUT"+echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"+if printf '%s\n' "$changed" | grep -qE '^(rust|proto)/'; then+ echo 'rust_affected=true' >> "$GITHUB_OUTPUT"+fi+if printf '%s\n' "$changed" | grep -qxF 'pnpm-lock.yaml'; then+ echo 'js_lockfile_changed=true' >> "$GITHUB_OUTPUT"+fidiff --git a/.github/scripts/test_resolve_changed_files.py b/.github/scripts/test_resolve_changed_files.pynew file mode 100644index 000000000000..d701cedf4778--- /dev/null+++ b/.github/scripts/test_resolve_changed_files.py@@ -0,0 +1,156 @@+import os+import json+import subprocess+from pathlib import Path++import pytest++SCRIPT = Path(__file__).with_name("resolve-changed-files.sh")+BASE = "a" * 40+HEAD = "b" * 40+++def run_resolver(+ tmp_path: Path,+ response: dict[str, object],+ target: str,+ paths: list[str],+ fail_fetch: bool = False,+ fail_api: bool = False,+) -> tuple[subprocess.CompletedProcess[str], str]:+ (tmp_path / "compare-response.json").write_text(json.dumps(response))+ (tmp_path / "diff-paths").write_text("\n".join(paths) + "\n")+ for name, body in {+ "gh": """echo "$*" > "$RUNNER_TEMP/api-call"+if [ "$FAIL_API" = true ]; then+ echo '(HTTP 403)' >&2+ exit 1+fi+cat "$RUNNER_TEMP/compare-response.json"+""",+ "git": """case "$1" in+ rev-parse) echo "$HEAD_SHA" ;;+ fetch)+ echo "$*" > "$RUNNER_TEMP/fetch-call"+ [ "$FAIL_FETCH" != true ] ;;+ diff)+ [ "$*" = "diff --name-only --no-renames $BASE_SHA HEAD" ] || exit 1+ cat "$RUNNER_TEMP/diff-paths" ;;+ *) exit 1 ;;+esac+""",+ }.items():+ executable = tmp_path / name+ executable.write_text("#!/usr/bin/env bash\n" + body)+ executable.chmod(0o755)+ output = tmp_path / "output"+ result = subprocess.run(+ ["bash", str(SCRIPT)],+ env={+ **os.environ,+ "PATH": f"{tmp_path}:{os.environ['PATH']}",+ "RUNNER_TEMP": str(tmp_path),+ "GITHUB_OUTPUT": str(output),+ "REPOSITORY": "example/repo",+ "MERGE_TARGET": target,+ "HEAD_SHA": HEAD,+ "BASE_SHA": BASE,+ "FAIL_FETCH": str(fail_fetch).lower(),+ "FAIL_API": str(fail_api).lower(),+ },+ capture_output=True,+ text=True,+ )+ return result, output.read_text()++[email protected]("target", ["stack-parent", "master", "c" * 40])[email protected]("file_count", [2, 300])+def test_complete_paths_and_event_base(tmp_path: Path, target: str, file_count: int) -> None:+ paths = ["rust/old/src/lib.rs", "rust/new/src/lib.rs", "rust/deleted/Cargo.toml", "rust/Cargo.lock"]+ response: dict[str, object] = {+ "merge_base_commit": {"sha": BASE},+ "files": [+ {"filename": paths[1], "previous_filename": paths[0]},+ {"filename": paths[2]},+ ],+ }+ if file_count == 300:+ response["files"] = [{"filename": f"rust/file-{i}"} for i in range(300)]+ else:+ paths = paths[:3]+ result, outputs = run_resolver(tmp_path, response, target, paths)+ assert result.returncode == 0+ assert "resolved=true" in outputs+ assert f"merge_base={BASE}" in outputs+ assert (tmp_path / "changed-files.txt").read_text().splitlines() == sorted(paths)+ assert f"compare/{target}...{HEAD}?per_page=1" in (tmp_path / "api-call").read_text()+ assert (tmp_path / "fetch-call").exists() == (file_count == 300)+++ "response,fail_fetch,fail_api,target",+ [+ ({"merge_base_commit": {"sha": "invalid"}, "files": []}, False, False, "master"),+ ({"merge_base_commit": {"sha": BASE}}, False, False, "master"),+ ({"merge_base_commit": {"sha": BASE}, "files": [{}] * 300}, True, False, "master"),+ ({}, False, True, "master"),+ ({}, False, False, "0" * 40),+ ({}, False, False, ""),+ ],+)+def test_uncertain_diff_does_not_publish_a_partial_list(+ tmp_path: Path, response: dict[str, object], fail_fetch: bool, fail_api: bool, target: str+) -> None:+ result, outputs = run_resolver(tmp_path, response, target, [], fail_fetch, fail_api)+ assert result.returncode == 0+ assert outputs.strip() == "resolved=false"+ assert not (tmp_path / "changed-files.txt").exists()+++def test_truncated_compare_uses_only_endpoint_trees(tmp_path: Path) -> None:+ source = tmp_path / "source"+ source.mkdir()++ def git(*args: str, cwd: Path = source) -> str:+ return subprocess.check_output(["git", *args], cwd=cwd, text=True).strip()++ git("init", "-q")+ git("config", "user.email", "[email protected]")+ git("config", "user.name", "Test")+ (source / "old.rs").write_text("unchanged contents\n")+ (source / "deleted.rs").write_text("deleted contents\n")+ git("add", ".")+ git("commit", "-qm", "base")+ base = git("rev-parse", "HEAD")+ git("mv", "old.rs", "new.rs")+ git("rm", "deleted.rs")+ (source / "Cargo.lock").write_text("lockfile\n")+ git("add", ".")+ git("commit", "-qm", "change")+ expected = set(git("diff", "--name-only", "--no-renames", f"{base}...HEAD").splitlines())+ shallow = tmp_path / "shallow"+ git("clone", "-q", "--depth=1", source.as_uri(), str(shallow))+ assert git("rev-parse", "--is-shallow-repository", cwd=shallow) == "true"+ response = {"merge_base_commit": {"sha": base}, "files": [{}] * 300}+ (tmp_path / "response.json").write_text(json.dumps(response))+ executable = tmp_path / "gh"+ executable.write_text('#!/usr/bin/env bash\ncat "$RUNNER_TEMP/response.json"\n')+ executable.chmod(0o755)+ result = subprocess.run(+ ["bash", str(SCRIPT.resolve())],+ cwd=shallow,+ env={+ **os.environ,+ "PATH": f"{tmp_path}:{os.environ['PATH']}",+ "RUNNER_TEMP": str(tmp_path),+ "GITHUB_OUTPUT": str(tmp_path / "output"),+ "REPOSITORY": "example/repo",+ "MERGE_TARGET": "stack-parent",+ },+ capture_output=True,+ text=True,+ check=True,+ )+ assert "resolved=true" in (tmp_path / "output").read_text(), result.stdout+ assert set((tmp_path / "changed-files.txt").read_text().splitlines()) == expecteddiff --git a/.github/workflows/ci-rust.yml b/.github/workflows/ci-rust.ymlindex 366950940b68..7326daa94771 100644--- a/.github/workflows/ci-rust.yml+++ b/.github/workflows/ci-rust.yml@@ -77,6 +77,7 @@ jobs:- 'proto/**'- '.github/workflows/ci-rust.yml'- '.github/scripts/rust_test_shards.py'+ - '.github/scripts/resolve-changed-files.sh'- '.github/actions/setup-uv/**'- '.github/workflows/rust.yml'- '.github/workflows/rust-docker-build.yml'@@ -129,18 +130,33 @@ jobs:steps:- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2with:- fetch-depth: 0+ fetch-depth: 1filter: blob:nonesparse-checkout: |rust/proto/.github/rust-images.yml.github/scripts/rust_test_shards.py+ .github/scripts/resolve-changed-files.sh.github/actions/rust-compute-affected/.github/actions/setup-rust/sparse-checkout-cone-mode: falseclean: false+ - name: Resolve the changed files+ id: changed-files+ if: github.event_name != 'workflow_dispatch' && needs.changes.outputs.all_crates != 'true'+ env:+ GH_TOKEN: ${{ github.token }}+ REPOSITORY: ${{ github.repository }}+ MERGE_TARGET: ${{ github.event_name == 'push' && github.event.before || github.event.pull_request.base.ref }}+ run: |+ if [ -f .github/scripts/resolve-changed-files.sh ]; then+ bash .github/scripts/resolve-changed-files.sh+ else+ echo 'resolved=false' >> "$GITHUB_OUTPUT"+ fi+- name: Install rustuses: ./.github/actions/setup-rustwith:@@ -153,7 +169,9 @@ jobs:event-name: ${{ github.event_name }}pr-base-sha: ${{ github.event.pull_request.base.sha }}push-before-sha: ${{ github.event.before }}- rebuild-all: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.all_crates == 'true') && 'true' || 'false' }}+ base-ref: ${{ github.event_name == 'push' && github.event.before || steps.changed-files.outputs.merge_base }}+ changed-files: ${{ runner.temp }}/changed-files.txt+ rebuild-all: ${{ steps.changed-files.outputs.resolved != 'true' && 'true' || 'false' }}# Bin-pack affected packages into balanced test shards.- name: Compute test shardsdiff --git a/.github/workflows/trunk-impacted-targets.yml b/.github/workflows/trunk-impacted-targets.ymlindex fb2182d1f2a9..5716fedeaa1e 100644--- a/.github/workflows/trunk-impacted-targets.yml+++ b/.github/workflows/trunk-impacted-targets.yml@@ -97,75 +97,10 @@ jobs:REPOSITORY: ${{ github.repository }}MERGE_TARGET: ${{ github.event.repository.default_branch }}run: |- set -uo pipefail-- # An API failure or a truncated file list has to widen to ALL- # rather than upload a partial file list, which Trunk cannot- # distinguish from a complete one. The computation step reads- # `resolved` and reports ALL when it is anything but true.- unresolved() {- echo "::warning::$1; reporting ALL."- echo 'resolved=false' >> "$GITHUB_OUTPUT"- exit 0- }-- # Each attempt overwrites the output file, so a failed attempt- # leaves no partial page behind for the next one to append to.- # A 4xx, rate limits included, would fail again on the same- # token bucket, so only network errors and 5xx are retried.- gh_api_to() {- local out="$1" err="$RUNNER_TEMP/gh-api-error.txt" attempt- shift- for attempt in 1 2 3; do- if gh api "$@" > "$out" 2> "$err"; then- return 0- fi- cat "$err" >&2- grep -qE '\(HTTP 4[0-9]{2}\)' "$err" && return 1- [ "$attempt" -lt 3 ] && sleep $((attempt * 5))- done- return 1- }-- # A renamed file is listed as its old path and its new one. The- # old path is what the target script reads as deleted, and it- # widens on that: the tach map has no importer edge for it.- file_paths='.filename, (select(.previous_filename) | .previous_filename)'-- head_sha="$(git rev-parse HEAD)"- compare="$RUNNER_TEMP/compare.json"- gh_api_to "$compare" "repos/$REPOSITORY/compare/$MERGE_TARGET...$head_sha?per_page=1" \- || unresolved "The compare API failed for $MERGE_TARGET...$head_sha"-- merge_base="$(jq -r '.merge_base_commit.sha // empty' "$compare")"- [[ "$merge_base" =~ ^[0-9a-f]{40}$ ]] || unresolved "The compare API returned no merge base"- jq -e '.files | type == "array"' "$compare" > /dev/null || unresolved "The compare API returned no file list"-- # The compare API lists at most 300 files. Past that, the merge- # base commit is fetched without blobs and diffed against the- # checked-out merge commit, which needs only their trees. The pull- # request files API is no substitute: it describes the PR head- # against its own base, and reads whatever head the PR has now.- if [ "$(jq '.files | length' "$compare")" -lt 300 ]; then- jq -r ".files[] | $file_paths" "$compare" > "$RUNNER_TEMP/listed-files.txt"+ if [ -f .github/scripts/resolve-changed-files.sh ]; then+ bash .github/scripts/resolve-changed-files.shelse- git fetch --no-tags --depth=1 --filter=blob:none origin "$merge_base" \- || unresolved "Could not fetch the merge base $merge_base"- git diff --name-only --no-renames "$merge_base" HEAD > "$RUNNER_TEMP/listed-files.txt" \- || unresolved "Could not diff against the merge base"- fi-- sort -u "$RUNNER_TEMP/listed-files.txt" > "$RUNNER_TEMP/changed-files.txt"- sed 's/^/ /' "$RUNNER_TEMP/changed-files.txt"- changed="$(cat "$RUNNER_TEMP/changed-files.txt")"-- echo 'resolved=true' >> "$GITHUB_OUTPUT"- echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"- if printf '%s\n' "$changed" | grep -qE '^(rust|proto)/'; then- echo 'rust_affected=true' >> "$GITHUB_OUTPUT"- fi- if printf '%s\n' "$changed" | grep -qxF 'pnpm-lock.yaml'; then- echo 'js_lockfile_changed=true' >> "$GITHUB_OUTPUT"+ echo 'resolved=false' >> "$GITHUB_OUTPUT"fi# The target script reads the base lockfile with `git show` at thediff --git a/tools/workflow-plan/tests/workflows.test.ts b/tools/workflow-plan/tests/workflows.test.tsindex 975b7b7b15b5..c2ef97e3cc7e 100644--- a/tools/workflow-plan/tests/workflows.test.ts+++ b/tools/workflow-plan/tests/workflows.test.ts@@ -1033,6 +1033,26 @@ describe('CI workflow run plans', () => {expect(plan.jobs['code-quality']?.steps.find((step) => step.name === 'Save mypy cache')?.runs).toBe(runs)})+ it.each([+ ['PR', pullRequest(), false, true],+ ['merge queue', mergeQueue(), false, true],+ ['push', push(), false, true],+ ['manual full rebuild', workflowDispatch(), false, false],+ ['all-crate change', pullRequest(), true, false],+ ] as const)('Rust change resolver on %s', (name, github, allCrates, runs) => {+ const wf = workflow('ci-rust.yml')+ const plan = planWorkflow(wf, {+ name,+ github,+ steps: {+ changes: { filter: pathsFilter({ rust: true, all_crates: allCrates }) },+ affected: { shards: { outputs: { matrix: '{"include":[{"packages":"common-types"}]}' } } },+ },+ })+ expect(plan.errors).toEqual([])+ expect(plan.jobs.affected?.steps.find((step) => step.name === 'Resolve the changed files')?.runs).toBe(runs)+ })+it.each(['success', 'failure'] as const)('sccache counters survive a %s build without changing its verdict',(outcome) => {