:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(ci): isolate rust shard planning (#119759)
Raúl Negrón-Otero9fe8361f421a
diff --git a/.github/scripts/rust_test_shards.py b/.github/scripts/rust_test_shards.pynew file mode 100644index 000000000000..b27dcd990e62--- /dev/null+++ b/.github/scripts/rust_test_shards.py@@ -0,0 +1,131 @@+# ruff: noqa: T201+import os+import json+import math+from pathlib import Path++# Approximate marginal compile and test durations, fitted by non-negative least+# squares over shard durations. Fixed setup overhead is excluded from the weights.+PACKAGE_WEIGHTS: dict[str, int] = {+ "affected-services": 37,+ "assignment-coordination": 5,+ "batch-import-worker": 69,+ "capture": 102,+ "capture-load-gen": 38,+ "capture-logs": 87,+ "capture-apm-metrics": 5,+ "cohort-core": 22,+ "cohort-event-shuffler": 25,+ "cohort-seeder": 77,+ "cohort-stream-processor": 285,+ "common-alloc": 5,+ "common-cache": 6,+ "common-compression": 5,+ "common-continuous-profiling": 9,+ "common-cookieless": 103,+ "common-database": 35,+ "common-dns": 18,+ "common-geoip": 5,+ "common-hypercache": 36,+ "common-ingestion-warnings": 59,+ "common-kafka": 5,+ "common-liveness": 5,+ "common-metrics": 5,+ "common-posthog": 18,+ "common-profiler": 59,+ "common-redis": 12,+ "common-s3": 5,+ "common-temporal": 12,+ "common-types": 16,+ "cymbal": 236,+ "cymbal-proto": 5,+ "deltalite-core": 5,+ "deltalite-python": 5,+ "embedding-worker": 90,+ "feature-flags": 241,+ "flags-consumer": 23,+ "health": 5,+ "hogql_parser_rs": 7,+ "hogvm": 5,+ "hypercache-server": 40,+ "ingestion-consumer": 20,+ "ingestion-control-plane": 41,+ "pgcollector": 45,+ "pgapi": 40,+ "k8s-awareness": 116,+ "kafka-assigner": 175,+ "kafka-assigner-proto": 5,+ "kafka-deduplicator": 129,+ "keyed-stash": 5,+ "lifecycle": 5,+ "limiters": 5,+ "personhog-common": 7,+ "personhog-coordination": 230,+ "personhog-identity": 80,+ "personhog-leader": 113,+ "personhog-proto": 9,+ "personhog-replica": 37,+ "personhog-router": 114,+ "personhog-stateright": 173,+ "personhog-test-harness": 33,+ "personhog-writer": 63,+ "posthog-replay-anonymizer": 147,+ "posthog-symbol-data": 15,+ "property-defs-rs": 74,+ "property-vals-rs": 70,+ "replay-anonymizer-node": 5,+ "replay-core-v4": 10,+ "replay-core-v4-node": 5,+ "replay-recorder-v4": 15,+ "serve-metrics": 5,+ "usage-ingestion": 70,+ "usage-ingestion-proto": 5,+}++DEFAULT_WEIGHT = 60+TARGET_SECONDS = 8 * 60+++def plan_shards(affected_crates: set[str]) -> list[list[str]]:+ # An empty affected set means a full build. Affected new crates still receive+ # coverage before their measured weight has been added to the table.+ packages = PACKAGE_WEIGHTS.copy()+ if affected_crates:+ for name in affected_crates - packages.keys():+ packages[name] = DEFAULT_WEIGHT+ packages = {name: weight for name, weight in packages.items() if name in affected_crates}+ shard_count = max(1, math.ceil(sum(packages.values()) / TARGET_SECONDS))+ buckets: list[list[str]] = [[] for _ in range(shard_count)]+ times = [0] * shard_count+ for name in sorted(packages, key=lambda name: -packages[name]):+ lightest = min(range(shard_count), key=lambda index: times[index])+ buckets[lightest].append(name)+ times[lightest] += packages[name]+ return [sorted(bucket) for bucket in buckets if bucket]+++def main() -> None:+ affected: set[str] = set(json.loads(os.environ.get("AFFECTED_CRATES", "[]")))+ unknown = affected - PACKAGE_WEIGHTS.keys()+ if unknown:+ print(+ f"WARNING: affected crates missing from weight table (using {DEFAULT_WEIGHT}s default): {', '.join(sorted(unknown))}"+ )+ print("Add them to the packages dict in .github/scripts/rust_test_shards.py with measured weights.")+ print(+ f"Selective build: testing {len(affected)} of affected crates" if affected else "Full build: testing all crates"+ )+ shards = plan_shards(affected)+ total = 0+ for index, shard in enumerate(shards, start=1):+ seconds = sum(PACKAGE_WEIGHTS.get(name, DEFAULT_WEIGHT) for name in shard)+ total += seconds+ print(f"Shard {index} ({seconds}s): {', '.join(shard)}")+ print(f"\nTotal: {total}s across {len(shards)} shards (target ≤{TARGET_SECONDS}s/shard)")+ matrix = {"include": [{"packages": " ".join(shard)} for shard in shards]}+ with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output:+ output.write(f"matrix={json.dumps(matrix)}\n")+++if __name__ == "__main__":+ main()diff --git a/.github/scripts/test_rust_test_shards.py b/.github/scripts/test_rust_test_shards.pynew file mode 100644index 000000000000..30464e4ec064--- /dev/null+++ b/.github/scripts/test_rust_test_shards.py@@ -0,0 +1,53 @@+import os+import sys+import json+import subprocess+from pathlib import Path++import pytest++from rust_test_shards import PACKAGE_WEIGHTS, plan_shards+++ "affected,expected",+ [+ ({"capture"}, [["capture"]]),+ (+ {"capture", "feature-flags", "cohort-stream-processor"},+ [["cohort-stream-processor"], ["capture", "feature-flags"]],+ ),+ ({"new-crate", "capture"}, [["capture", "new-crate"]]),+ ],+)+def test_selective_shards(affected: set[str], expected: list[list[str]]) -> None:+ assert plan_shards(affected) == expected+++def test_empty_selection_runs_every_known_crate_once() -> None:+ shards = plan_shards(set())+ packages = [name for shard in shards for name in shard]+ assert sorted(packages) == sorted(PACKAGE_WEIGHTS)+ assert all(shard for shard in shards)+++def test_new_crates_use_default_weights_without_losing_coverage() -> None:+ affected = {f"new-crate-{index}" for index in range(9)}+ shards = plan_shards(affected)+ assert len(shards) == 2+ assert sorted(name for shard in shards for name in shard) == sorted(affected)+++def test_cli_emits_the_build_matrix(tmp_path: Path) -> None:+ output = tmp_path / "output"+ result = subprocess.run(+ [sys.executable, str(Path(__file__).with_name("rust_test_shards.py"))],+ env={**os.environ, "AFFECTED_CRATES": '["capture", "new-crate"]', "GITHUB_OUTPUT": str(output)},+ check=True,+ capture_output=True,+ text=True,+ )+ assert "using 60s default): new-crate" in result.stdout+ assert "Selective build: testing 2 of affected crates" in result.stdout+ assert "Total: 162s across 1 shards (target ≤480s/shard)" in result.stdout+ assert json.loads(output.read_text().removeprefix("matrix=")) == {"include": [{"packages": "capture new-crate"}]}diff --git a/.github/workflows/ci-rust.yml b/.github/workflows/ci-rust.ymlindex 3cf0b6033051..0b20c8d57fd6 100644--- a/.github/workflows/ci-rust.yml+++ b/.github/workflows/ci-rust.yml@@ -75,6 +75,7 @@ jobs:- 'rust/**'- 'proto/**'- '.github/workflows/ci-rust.yml'+ - '.github/scripts/rust_test_shards.py'- '.github/actions/setup-uv/**'- '.github/workflows/rust.yml'- '.github/workflows/rust-docker-build.yml'@@ -98,6 +99,7 @@ jobs:- 'nodejs/src/ingestion/pipelines/sessionreplay/ml-mirror/keys/encryption-vector*.json'- 'nodejs/src/ingestion/pipelines/sessionreplay/ai-research/v4-keys/encryption-vector*.json'all_crates:+ - '.github/scripts/rust_test_shards.py'# Pins the service images that every crate's tests run against.# A determinator rule would also rebuild and deploy every Rust image.- 'docker-compose.base.yml'@@ -124,6 +126,7 @@ jobs:rust/proto/.github/rust-images.yml+ .github/scripts/rust_test_shards.py.github/actions/rust-compute-affected/.github/actions/setup-rust/sparse-checkout-cone-mode: false@@ -148,143 +151,7 @@ jobs:id: shardsenv:AFFECTED_CRATES: ${{ steps.affected.outputs.crates }}- shell: python3 {0}- run: |- import json, math, os-- affected = set(json.loads(os.environ.get("AFFECTED_CRATES", "[]")))-- packages = {- "affected-services": 37,- "assignment-coordination": 5,- "batch-import-worker": 69,- "capture": 102,- "capture-load-gen": 38,- "capture-logs": 87,- "capture-apm-metrics": 5,- "cohort-core": 22,- "cohort-event-shuffler": 25,- "cohort-seeder": 77,- "cohort-stream-processor": 285,- "common-alloc": 5,- "common-cache": 6,- "common-compression": 5,- "common-continuous-profiling": 9,- "common-cookieless": 103,- "common-database": 35,- "common-dns": 18,- "common-geoip": 5,- "common-hypercache": 36,- "common-ingestion-warnings": 59,- "common-kafka": 5,- "common-liveness": 5,- "common-metrics": 5,- "common-posthog": 18,- "common-profiler": 59,- "common-redis": 12,- "common-s3": 5,- "common-temporal": 12,- "common-types": 16,- "cymbal": 236,- "cymbal-proto": 5,- "deltalite-core": 5,- "deltalite-python": 5,- "embedding-worker": 90,- "feature-flags": 241,- "flags-consumer": 23,- "health": 5,- "hogql_parser_rs": 7,- "hogvm": 5,- "hypercache-server": 40,- "ingestion-consumer": 20,- "ingestion-control-plane": 41,- "pgcollector": 45,- "pgapi": 40,- "k8s-awareness": 116,- "kafka-assigner": 175,- "kafka-assigner-proto": 5,- "kafka-deduplicator": 129,- "keyed-stash": 5,- "lifecycle": 5,- "limiters": 5,- "personhog-common": 7,- "personhog-coordination": 230,- "personhog-identity": 80,- "personhog-leader": 113,- "personhog-proto": 9,- "personhog-replica": 37,- "personhog-router": 114,- "personhog-stateright": 173,- "personhog-test-harness": 33,- "personhog-writer": 63,- "posthog-replay-anonymizer": 147,- "posthog-symbol-data": 15,- "property-defs-rs": 74,- "property-vals-rs": 70,- "replay-anonymizer-node": 5,- "replay-core-v4": 10,- "replay-core-v4-node": 5,- "replay-recorder-v4": 15,- "serve-metrics": 5,- "usage-ingestion": 70,- "usage-ingestion-proto": 5,- }-- # Filter to only affected crates when selective builds are active.- # Unknown crates (not in packages dict) get a default weight so new- # services are never silently dropped from CI.- DEFAULT_WEIGHT = 60- if affected:- unknown = affected - packages.keys()- if unknown:- print(f"WARNING: affected crates missing from weight table (using {DEFAULT_WEIGHT}s default): {', '.join(sorted(unknown))}")- print("Add them to the packages dict in ci-rust.yml with measured weights.")- for crate in unknown:- packages[crate] = DEFAULT_WEIGHT- packages = {k: v for k, v in packages.items() if k in affected}- print(f"Selective build: testing {len(packages)} of affected crates")- else:- print("Full build: testing all crates")-- if not packages:- with open(os.environ["GITHUB_OUTPUT"], "a") as f:- f.write('matrix={"include":[]}\n')- raise SystemExit(0)-- # Weights are approximate marginal durations (compile + test) in seconds,- # fitted by non-negative least squares over 2 weeks of shard job- # durations (job time = fixed overhead + sum of member weights;- # last updated: 2026-08-18).- # TARGET_MINUTES controls max wall-clock per shard; shard count is derived.- TARGET_MINUTES = 8- target_seconds = TARGET_MINUTES * 60- total = sum(packages.values())- num_shards = max(1, math.ceil(total / target_seconds))-- # Greedy bin-packing: largest first into lightest bucket- buckets = [[] for _ in range(num_shards)]- times = [0] * num_shards-- for pkg, t in sorted(packages.items(), key=lambda x: -x[1]):- lightest = min(range(num_shards), key=lambda i: times[i])- buckets[lightest].append(pkg)- times[lightest] += t-- matrix = {- "include": [- {"packages": " ".join(sorted(b))}- for b in buckets- if b- ]- }-- for i, b in enumerate(buckets):- if b:- print(f"Shard {i + 1} ({times[i]}s): {', '.join(sorted(b))}")- print(f"\nTotal: {total}s across {num_shards} shards (target ≤{target_seconds}s/shard)")-- with open(os.environ["GITHUB_OUTPUT"], "a") as f:- f.write(f"matrix={json.dumps(matrix)}\n")+ run: python3 .github/scripts/rust_test_shards.pybuild:name: Build Rust (${{ matrix.packages }})