Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(ci): isolate rust shard planning (#119759)

Raúl Negrón-Otero9fe8361f421a
diff --git a/.github/scripts/rust_test_shards.py b/.github/scripts/rust_test_shards.py
new file mode 100644
index 000000000000..b27dcd990e62
--- /dev/null
+++ b/.github/scripts/rust_test_shards.py
@@ -0,0 +1,131 @@
+# ruff: noqa: T201
+import os
+import json
+import math
+from pathlib import Path
+
+# Approximate marginal compile and test durations, fitted by non-negative least
+# squares over shard durations. Fixed setup overhead is excluded from the weights.
+PACKAGE_WEIGHTS: dict[str, int] = {
+ "affected-services": 37,
+ "assignment-coordination": 5,
+ "batch-import-worker": 69,
+ "capture": 102,
+ "capture-load-gen": 38,
+ "capture-logs": 87,
+ "capture-apm-metrics": 5,
+ "cohort-core": 22,
+ "cohort-event-shuffler": 25,
+ "cohort-seeder": 77,
+ "cohort-stream-processor": 285,
+ "common-alloc": 5,
+ "common-cache": 6,
+ "common-compression": 5,
+ "common-continuous-profiling": 9,
+ "common-cookieless": 103,
+ "common-database": 35,
+ "common-dns": 18,
+ "common-geoip": 5,
+ "common-hypercache": 36,
+ "common-ingestion-warnings": 59,
+ "common-kafka": 5,
+ "common-liveness": 5,
+ "common-metrics": 5,
+ "common-posthog": 18,
+ "common-profiler": 59,
+ "common-redis": 12,
+ "common-s3": 5,
+ "common-temporal": 12,
+ "common-types": 16,
+ "cymbal": 236,
+ "cymbal-proto": 5,
+ "deltalite-core": 5,
+ "deltalite-python": 5,
+ "embedding-worker": 90,
+ "feature-flags": 241,
+ "flags-consumer": 23,
+ "health": 5,
+ "hogql_parser_rs": 7,
+ "hogvm": 5,
+ "hypercache-server": 40,
+ "ingestion-consumer": 20,
+ "ingestion-control-plane": 41,
+ "pgcollector": 45,
+ "pgapi": 40,
+ "k8s-awareness": 116,
+ "kafka-assigner": 175,
+ "kafka-assigner-proto": 5,
+ "kafka-deduplicator": 129,
+ "keyed-stash": 5,
+ "lifecycle": 5,
+ "limiters": 5,
+ "personhog-common": 7,
+ "personhog-coordination": 230,
+ "personhog-identity": 80,
+ "personhog-leader": 113,
+ "personhog-proto": 9,
+ "personhog-replica": 37,
+ "personhog-router": 114,
+ "personhog-stateright": 173,
+ "personhog-test-harness": 33,
+ "personhog-writer": 63,
+ "posthog-replay-anonymizer": 147,
+ "posthog-symbol-data": 15,
+ "property-defs-rs": 74,
+ "property-vals-rs": 70,
+ "replay-anonymizer-node": 5,
+ "replay-core-v4": 10,
+ "replay-core-v4-node": 5,
+ "replay-recorder-v4": 15,
+ "serve-metrics": 5,
+ "usage-ingestion": 70,
+ "usage-ingestion-proto": 5,
+}
+
+DEFAULT_WEIGHT = 60
+TARGET_SECONDS = 8 * 60
+
+
+def plan_shards(affected_crates: set[str]) -> list[list[str]]:
+ # An empty affected set means a full build. Affected new crates still receive
+ # coverage before their measured weight has been added to the table.
+ packages = PACKAGE_WEIGHTS.copy()
+ if affected_crates:
+ for name in affected_crates - packages.keys():
+ packages[name] = DEFAULT_WEIGHT
+ packages = {name: weight for name, weight in packages.items() if name in affected_crates}
+ shard_count = max(1, math.ceil(sum(packages.values()) / TARGET_SECONDS))
+ buckets: list[list[str]] = [[] for _ in range(shard_count)]
+ times = [0] * shard_count
+ for name in sorted(packages, key=lambda name: -packages[name]):
+ lightest = min(range(shard_count), key=lambda index: times[index])
+ buckets[lightest].append(name)
+ times[lightest] += packages[name]
+ return [sorted(bucket) for bucket in buckets if bucket]
+
+
+def main() -> None:
+ affected: set[str] = set(json.loads(os.environ.get("AFFECTED_CRATES", "[]")))
+ unknown = affected - PACKAGE_WEIGHTS.keys()
+ if unknown:
+ print(
+ f"WARNING: affected crates missing from weight table (using {DEFAULT_WEIGHT}s default): {', '.join(sorted(unknown))}"
+ )
+ print("Add them to the packages dict in .github/scripts/rust_test_shards.py with measured weights.")
+ print(
+ f"Selective build: testing {len(affected)} of affected crates" if affected else "Full build: testing all crates"
+ )
+ shards = plan_shards(affected)
+ total = 0
+ for index, shard in enumerate(shards, start=1):
+ seconds = sum(PACKAGE_WEIGHTS.get(name, DEFAULT_WEIGHT) for name in shard)
+ total += seconds
+ print(f"Shard {index} ({seconds}s): {', '.join(shard)}")
+ print(f"\nTotal: {total}s across {len(shards)} shards (target ≤{TARGET_SECONDS}s/shard)")
+ matrix = {"include": [{"packages": " ".join(shard)} for shard in shards]}
+ with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output:
+ output.write(f"matrix={json.dumps(matrix)}\n")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/.github/scripts/test_rust_test_shards.py b/.github/scripts/test_rust_test_shards.py
new file mode 100644
index 000000000000..30464e4ec064
--- /dev/null
+++ b/.github/scripts/test_rust_test_shards.py
@@ -0,0 +1,53 @@
+import os
+import sys
+import json
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from rust_test_shards import PACKAGE_WEIGHTS, plan_shards
+
+
+ "affected,expected",
+ [
+ ({"capture"}, [["capture"]]),
+ (
+ {"capture", "feature-flags", "cohort-stream-processor"},
+ [["cohort-stream-processor"], ["capture", "feature-flags"]],
+ ),
+ ({"new-crate", "capture"}, [["capture", "new-crate"]]),
+ ],
+)
+def test_selective_shards(affected: set[str], expected: list[list[str]]) -> None:
+ assert plan_shards(affected) == expected
+
+
+def test_empty_selection_runs_every_known_crate_once() -> None:
+ shards = plan_shards(set())
+ packages = [name for shard in shards for name in shard]
+ assert sorted(packages) == sorted(PACKAGE_WEIGHTS)
+ assert all(shard for shard in shards)
+
+
+def test_new_crates_use_default_weights_without_losing_coverage() -> None:
+ affected = {f"new-crate-{index}" for index in range(9)}
+ shards = plan_shards(affected)
+ assert len(shards) == 2
+ assert sorted(name for shard in shards for name in shard) == sorted(affected)
+
+
+def test_cli_emits_the_build_matrix(tmp_path: Path) -> None:
+ output = tmp_path / "output"
+ result = subprocess.run(
+ [sys.executable, str(Path(__file__).with_name("rust_test_shards.py"))],
+ env={**os.environ, "AFFECTED_CRATES": '["capture", "new-crate"]', "GITHUB_OUTPUT": str(output)},
+ check=True,
+ capture_output=True,
+ text=True,
+ )
+ assert "using 60s default): new-crate" in result.stdout
+ assert "Selective build: testing 2 of affected crates" in result.stdout
+ assert "Total: 162s across 1 shards (target ≤480s/shard)" in result.stdout
+ assert json.loads(output.read_text().removeprefix("matrix=")) == {"include": [{"packages": "capture new-crate"}]}
diff --git a/.github/workflows/ci-rust.yml b/.github/workflows/ci-rust.yml
index 3cf0b6033051..0b20c8d57fd6 100644
--- a/.github/workflows/ci-rust.yml
+++ b/.github/workflows/ci-rust.yml
@@ -75,6 +75,7 @@ jobs:
- 'rust/**'
- 'proto/**'
- '.github/workflows/ci-rust.yml'
+ - '.github/scripts/rust_test_shards.py'
- '.github/actions/setup-uv/**'
- '.github/workflows/rust.yml'
- '.github/workflows/rust-docker-build.yml'
@@ -98,6 +99,7 @@ jobs:
- 'nodejs/src/ingestion/pipelines/sessionreplay/ml-mirror/keys/encryption-vector*.json'
- 'nodejs/src/ingestion/pipelines/sessionreplay/ai-research/v4-keys/encryption-vector*.json'
all_crates:
+ - '.github/scripts/rust_test_shards.py'
# Pins the service images that every crate's tests run against.
# A determinator rule would also rebuild and deploy every Rust image.
- 'docker-compose.base.yml'
@@ -124,6 +126,7 @@ jobs:
rust/
proto/
.github/rust-images.yml
+ .github/scripts/rust_test_shards.py
.github/actions/rust-compute-affected/
.github/actions/setup-rust/
sparse-checkout-cone-mode: false
@@ -148,143 +151,7 @@ jobs:
id: shards
env:
AFFECTED_CRATES: ${{ steps.affected.outputs.crates }}
- shell: python3 {0}
- run: |
- import json, math, os
-
- affected = set(json.loads(os.environ.get("AFFECTED_CRATES", "[]")))
-
- packages = {
- "affected-services": 37,
- "assignment-coordination": 5,
- "batch-import-worker": 69,
- "capture": 102,
- "capture-load-gen": 38,
- "capture-logs": 87,
- "capture-apm-metrics": 5,
- "cohort-core": 22,
- "cohort-event-shuffler": 25,
- "cohort-seeder": 77,
- "cohort-stream-processor": 285,
- "common-alloc": 5,
- "common-cache": 6,
- "common-compression": 5,
- "common-continuous-profiling": 9,
- "common-cookieless": 103,
- "common-database": 35,
- "common-dns": 18,
- "common-geoip": 5,
- "common-hypercache": 36,
- "common-ingestion-warnings": 59,
- "common-kafka": 5,
- "common-liveness": 5,
- "common-metrics": 5,
- "common-posthog": 18,
- "common-profiler": 59,
- "common-redis": 12,
- "common-s3": 5,
- "common-temporal": 12,
- "common-types": 16,
- "cymbal": 236,
- "cymbal-proto": 5,
- "deltalite-core": 5,
- "deltalite-python": 5,
- "embedding-worker": 90,
- "feature-flags": 241,
- "flags-consumer": 23,
- "health": 5,
- "hogql_parser_rs": 7,
- "hogvm": 5,
- "hypercache-server": 40,
- "ingestion-consumer": 20,
- "ingestion-control-plane": 41,
- "pgcollector": 45,
- "pgapi": 40,
- "k8s-awareness": 116,
- "kafka-assigner": 175,
- "kafka-assigner-proto": 5,
- "kafka-deduplicator": 129,
- "keyed-stash": 5,
- "lifecycle": 5,
- "limiters": 5,
- "personhog-common": 7,
- "personhog-coordination": 230,
- "personhog-identity": 80,
- "personhog-leader": 113,
- "personhog-proto": 9,
- "personhog-replica": 37,
- "personhog-router": 114,
- "personhog-stateright": 173,
- "personhog-test-harness": 33,
- "personhog-writer": 63,
- "posthog-replay-anonymizer": 147,
- "posthog-symbol-data": 15,
- "property-defs-rs": 74,
- "property-vals-rs": 70,
- "replay-anonymizer-node": 5,
- "replay-core-v4": 10,
- "replay-core-v4-node": 5,
- "replay-recorder-v4": 15,
- "serve-metrics": 5,
- "usage-ingestion": 70,
- "usage-ingestion-proto": 5,
- }
-
- # Filter to only affected crates when selective builds are active.
- # Unknown crates (not in packages dict) get a default weight so new
- # services are never silently dropped from CI.
- DEFAULT_WEIGHT = 60
- if affected:
- unknown = affected - packages.keys()
- if unknown:
- print(f"WARNING: affected crates missing from weight table (using {DEFAULT_WEIGHT}s default): {', '.join(sorted(unknown))}")
- print("Add them to the packages dict in ci-rust.yml with measured weights.")
- for crate in unknown:
- packages[crate] = DEFAULT_WEIGHT
- packages = {k: v for k, v in packages.items() if k in affected}
- print(f"Selective build: testing {len(packages)} of affected crates")
- else:
- print("Full build: testing all crates")
-
- if not packages:
- with open(os.environ["GITHUB_OUTPUT"], "a") as f:
- f.write('matrix={"include":[]}\n')
- raise SystemExit(0)
-
- # Weights are approximate marginal durations (compile + test) in seconds,
- # fitted by non-negative least squares over 2 weeks of shard job
- # durations (job time = fixed overhead + sum of member weights;
- # last updated: 2026-08-18).
- # TARGET_MINUTES controls max wall-clock per shard; shard count is derived.
- TARGET_MINUTES = 8
- target_seconds = TARGET_MINUTES * 60
- total = sum(packages.values())
- num_shards = max(1, math.ceil(total / target_seconds))
-
- # Greedy bin-packing: largest first into lightest bucket
- buckets = [[] for _ in range(num_shards)]
- times = [0] * num_shards
-
- for pkg, t in sorted(packages.items(), key=lambda x: -x[1]):
- lightest = min(range(num_shards), key=lambda i: times[i])
- buckets[lightest].append(pkg)
- times[lightest] += t
-
- matrix = {
- "include": [
- {"packages": " ".join(sorted(b))}
- for b in buckets
- if b
- ]
- }
-
- for i, b in enumerate(buckets):
- if b:
- print(f"Shard {i + 1} ({times[i]}s): {', '.join(sorted(b))}")
- print(f"\nTotal: {total}s across {num_shards} shards (target ≤{target_seconds}s/shard)")
-
- with open(os.environ["GITHUB_OUTPUT"], "a") as f:
- f.write(f"matrix={json.dumps(matrix)}\n")
+ run: python3 .github/scripts/rust_test_shards.py
build:
name: Build Rust (${{ matrix.packages }})