Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

ci: balance test shards and reuse package download caches

morluto40b37aeee10e
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6c81e257d..131f46379 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -118,6 +118,8 @@ jobs:
name: rea-test-runtime
path: .
- run: npm run test:ci:shard:run -- --shard=${{ matrix.shard }}
+ env:
+ REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json
- uses: actions/upload-artifact@v4
if: ${{ !cancelled() }}
with:
@@ -133,6 +135,9 @@ jobs:
needs: changes
if: needs.changes.outputs.code == 'true'
runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ actions: read
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
@@ -141,6 +146,28 @@ jobs:
cache: npm
- run: npm ci
- run: npm run test:prepare
+ - name: Find successful default-branch timing history
+ id: history
+ continue-on-error: true
+ env:
+ GH_TOKEN: ${{ github.token }}
+ CI_REPOSITORY: ${{ github.repository }}
+ CI_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+ run: |
+ run_id="$(gh api --method GET \
+ "repos/${CI_REPOSITORY}/actions/workflows/ci.yml/runs" \
+ -f branch="${CI_DEFAULT_BRANCH}" -f event=push -f status=success -f per_page=1 \
+ --jq '.workflow_runs[0].id // empty')"
+ echo "run-id=${run_id}" >> "${GITHUB_OUTPUT}"
+ - name: Download previous timing report
+ if: steps.history.outputs.run-id != ''
+ continue-on-error: true
+ uses: actions/download-artifact@v4
+ with:
+ name: test-results
+ run-id: ${{ steps.history.outputs.run-id }}
+ github-token: ${{ github.token }}
+ path: .cache/ci-test-history
- uses: actions/upload-artifact@v4
with:
name: rea-test-runtime
@@ -148,6 +175,7 @@ jobs:
dist/
skills/
.cache/mcp-tool-catalog.json
+ .cache/ci-test-history/test-results.json
docs/public/product-catalog.json
docs/verification/managed-conformance-*.json
include-hidden-files: true
@@ -305,7 +333,14 @@ jobs:
name: windows-native-controls
path: native/windows/build/
- run: npm run verify:windows-native:artifact
+ - name: Resolve npm cache before verifier HOME isolation
+ id: npm-cache
+ run: |
+ cache_path="$(npm config get cache)"
+ echo "path=${cache_path}" >> "${GITHUB_OUTPUT}"
- run: npm run verify:package
+ env:
+ npm_config_cache: ${{ steps.npm-cache.outputs.path }}
- name: Verify real Node Inspector discovery and observation
timeout-minutes: 10
run: npm run verify:inspector
diff --git a/docs/testing.md b/docs/testing.md
index 82c6f0263..2d6369d89 100644
--- a/docs/testing.md
+++ b/docs/testing.md
@@ -835,6 +835,12 @@ target-free analysis, configuration backups/recovery, Evidence and owned
lifecycle; Inspector checks execute source-owned loopback targets and special
filename cases.
+POSIX package verification resolves the runner's configured npm cache before
+creating disposable client homes and passes that location to its child installs.
+This reuses the download cache restored by `setup-node` while keeping client
+configuration isolated. Installs retain their existing registry and integrity
+checks.
+
The Apple artifact CLI/MCP lane uses one `macos-15` job to avoid adding demand
for scarce macOS runner capacity. It runs the artifact-format and metadata
verifiers, followed by one consolidated focused invocation containing all
@@ -847,6 +853,15 @@ catalog, and managed-conformance metadata. Each shard still installs the locked
dependencies and runs its assigned coverage-enabled tests; it no longer
regenerates the same metadata independently.
+The build job also retrieves the merged timing report from the last successful
+default-branch push and includes that same report in each shard's snapshot.
+CI assigns discovered files by historical duration, with a deterministic
+fallback weight for new files, rather than dividing only by file count. Every
+discovered file still belongs to exactly one shard; worker limits, project
+ordering, coverage instrumentation, and aggregate thresholds remain unchanged.
+Missing or unusable history falls back to Vitest's default assignment. Only
+timing data is reused; persistent compile caches remain disabled.
+
These native baseline checks complement the Linux source-test shards and the
separate Apple-artifact and real-provider lanes. Actual Hopper, Ghidra, IDA,
browser and managed-tool claims require their corresponding verification lanes.
diff --git a/tests/boundary/filesystem/ciTestSequencer.test.ts b/tests/boundary/filesystem/ciTestSequencer.test.ts
new file mode 100644
index 000000000..6177d9481
--- /dev/null
+++ b/tests/boundary/filesystem/ciTestSequencer.test.ts
@@ -0,0 +1,155 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+
+import { fc, it } from "@fast-check/vitest";
+import { describe, expect } from "vitest";
+
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+
+import {
+ allocateByDuration,
+ parseTestDurations,
+ readTestDurations,
+} from "../../ciTestSequencer.js";
+
+describe("CI test history", () => {
+ it("uses only successful in-root file durations", () => {
+ const root = resolve("/workspace/rea");
+ const durations = parseTestDurations(
+ JSON.stringify({
+ success: true,
+ numFailedTests: 0,
+ testResults: [
+ {
+ name: join(root, "tests/slow.test.ts"),
+ startTime: 10,
+ endTime: 110,
+ },
+ {
+ name: join(root, "tests/slow.test.ts"),
+ startTime: 20,
+ endTime: 80,
+ },
+ {
+ name: resolve(root, "../outside.test.ts"),
+ startTime: 10,
+ endTime: 1000,
+ },
+ null,
+ ],
+ }),
+ root,
+ );
+
+ expect(durations).toEqual(new Map([["tests/slow.test.ts", 100]]));
+ expect(
+ parseTestDurations('{"success": false, "numFailedTests": 1}', root),
+ ).toBeUndefined();
+ expect(parseTestDurations("not json", root)).toBeUndefined();
+ expect(parseTestDurations("null", root)).toBeUndefined();
+ });
+
+ it("falls back for missing history while preserving real filesystem errors", async () => {
+ const temporaryRoot = await createTestTempDirectory("rea-ci-sequencer-");
+ const missing = await readTestDurations(
+ join(temporaryRoot, "missing.json"),
+ temporaryRoot,
+ );
+ expect(missing).toBeUndefined();
+
+ const directory = join(temporaryRoot, "directory");
+ await mkdir(directory);
+ const reportPath = join(temporaryRoot, "report.json");
+ await writeFile(reportPath, "{");
+ expect(await readTestDurations(reportPath, temporaryRoot)).toBeUndefined();
+
+ await expect(
+ readTestDurations(directory, temporaryRoot),
+ ).rejects.toMatchObject({
+ code: "EISDIR",
+ });
+ });
+});
+
+describe("duration shard allocation", () => {
+ it.prop([
+ fc.uniqueArray(fc.integer({ min: 0, max: 10_000 }), {
+ minLength: 1,
+ maxLength: 80,
+ }),
+ fc.integer({ min: 3, max: 8 }),
+ ])(
+ "is complete, disjoint, reference-preserving, and input-order independent",
+ (ids, shardCount) => {
+ const items = ids.map((id, index) => ({
+ id,
+ path: `tests/${id}.test.ts`,
+ project: `project-${id % 3}`,
+ pool: id % 2 === 0 ? "forks" : "threads",
+ duration: index > 0 && id % 5 === 0 ? undefined : (id % 17) + 1,
+ }));
+ const specifications = items.map((item) => ({
+ item,
+ key: `${item.path}\0${item.project}\0${item.pool}`,
+ duration: item.duration,
+ }));
+ const shards = allocateByDuration(specifications, shardCount);
+ const reversedShards = allocateByDuration(
+ [...specifications].reverse(),
+ shardCount,
+ );
+
+ expect(shards).toBeDefined();
+ expect(reversedShards).toEqual(shards);
+ const assigned = shards!.flat();
+ expect(assigned).toHaveLength(items.length);
+ expect(new Set(assigned).size).toBe(items.length);
+ expect(assigned.every((item) => items.includes(item))).toBe(true);
+ },
+ );
+
+ it("keeps tied path, project, pool, and new-file entries deterministic", () => {
+ const items = [
+ {
+ id: "a",
+ path: "tests/shared.test.ts",
+ project: "alpha",
+ pool: "forks",
+ duration: 10,
+ },
+ {
+ id: "b",
+ path: "tests/shared.test.ts",
+ project: "beta",
+ pool: "forks",
+ duration: 10,
+ },
+ {
+ id: "c",
+ path: "tests/shared.test.ts",
+ project: "beta",
+ pool: "threads",
+ duration: 10,
+ },
+ {
+ id: "new",
+ path: "tests/new.test.ts",
+ project: "alpha",
+ pool: "forks",
+ duration: undefined,
+ },
+ ];
+ const specifications = items.map((item) => ({
+ item,
+ key: `${item.path}\0${item.project}\0${item.pool}`,
+ duration: item.duration,
+ }));
+ const shards = allocateByDuration(specifications, 3);
+ const reversedShards = allocateByDuration([...specifications].reverse(), 3);
+
+ expect(shards).toBeDefined();
+ expect(reversedShards).toEqual(shards);
+ expect(shards!.flat()).toHaveLength(items.length);
+ expect(new Set(shards!.flat()).size).toBe(items.length);
+ });
+});
diff --git a/tests/ciTestSequencer.ts b/tests/ciTestSequencer.ts
new file mode 100644
index 000000000..5e8c4fe5f
--- /dev/null
+++ b/tests/ciTestSequencer.ts
@@ -0,0 +1,125 @@
+import { readFile } from "node:fs/promises";
+import { isAbsolute, relative, resolve, sep } from "node:path";
+
+import { BaseSequencer, type TestSpecification } from "vitest/node";
+import { z } from "zod";
+
+const reportSchema = z.object({
+ success: z.literal(true),
+ numFailedTests: z.literal(0),
+ testResults: z.array(z.unknown()),
+});
+const fileSchema = z.object({
+ name: z.string(),
+ startTime: z.number(),
+ endTime: z.number(),
+});
+
+export function parseTestDurations(
+ contents: string,
+ root: string,
+): Map<string, number> | undefined {
+ let value: unknown;
+ try {
+ value = JSON.parse(contents);
+ } catch (error) {
+ if (error instanceof SyntaxError) return undefined;
+ throw error;
+ }
+ const report = reportSchema.safeParse(value);
+ if (!report.success) return undefined;
+
+ const durations = new Map<string, number>();
+ for (const value of report.data.testResults) {
+ const result = fileSchema.safeParse(value);
+ if (!result.success) continue;
+ const { name, startTime, endTime } = result.data;
+ const duration = endTime - startTime;
+ const path = relative(root, resolve(root, name));
+ if (
+ !Number.isFinite(duration) ||
+ duration <= 0 ||
+ path === "" ||
+ path === ".." ||
+ path.startsWith(".." + sep) ||
+ isAbsolute(path)
+ )
+ continue;
+ const key = path.split(sep).join("/");
+ durations.set(key, Math.max(durations.get(key) ?? 0, duration));
+ }
+ return durations.size ? durations : undefined;
+}
+
+export async function readTestDurations(
+ historyPath: string,
+ root: string,
+): Promise<Map<string, number> | undefined> {
+ try {
+ return parseTestDurations(await readFile(historyPath, "utf8"), root);
+ } catch (error) {
+ if (error instanceof Error && "code" in error && error.code === "ENOENT")
+ return undefined;
+ throw error;
+ }
+}
+
+type TimedItem<T> = { item: T; key: string; duration: number | undefined };
+
+/** Distribute the longest files first, retaining every discovered specification. */
+export function allocateByDuration<T>(
+ items: readonly TimedItem<T>[],
+ shardCount: number,
+): T[][] | undefined {
+ if (!Number.isInteger(shardCount) || shardCount < 1)
+ throw new RangeError("shardCount must be a positive integer");
+ const known = items
+ .flatMap(({ duration }) => (duration === undefined ? [] : [duration]))
+ .sort((left, right) => left - right);
+ if (!known.length) return undefined;
+ const lower = known[Math.floor((known.length - 1) / 2)]!;
+ const median = lower + (known[Math.floor(known.length / 2)]! - lower) / 2;
+ const ordered = items.toSorted(
+ (left, right) =>
+ (right.duration ?? median) - (left.duration ?? median) ||
+ (left.key < right.key ? -1 : left.key > right.key ? 1 : 0),
+ );
+ const shards = Array.from({ length: shardCount }, () => ({
+ duration: 0,
+ items: [] as T[],
+ }));
+ for (const entry of ordered) {
+ const lightest = shards.reduce((left, right) =>
+ right.duration < left.duration ? right : left,
+ );
+ lightest.items.push(entry.item);
+ lightest.duration += entry.duration ?? median;
+ }
+ return shards.map(({ items }) => items);
+}
+
+export class CiTestSequencer extends BaseSequencer {
+ override async shard(
+ files: TestSpecification[],
+ ): Promise<TestSpecification[]> {
+ const historyPath = process.env.REA_CI_TEST_HISTORY;
+ const shard = this.ctx.config.shard;
+ if (!historyPath || !shard) return super.shard(files);
+ const root = this.ctx.config.root;
+ const durations = await readTestDurations(historyPath, root);
+ if (!durations) return super.shard(files);
+
+ const weighted = files.map((item) => {
+ const path = relative(root, resolve(root, item.moduleId))
+ .split(sep)
+ .join("/");
+ return {
+ item,
+ key: path + "\0" + item.project.name + "\0" + item.pool,
+ duration: durations.get(path),
+ };
+ });
+ const shards = allocateByDuration(weighted, shard.count);
+ return shards ? shards[shard.index - 1]! : super.shard(files);
+ }
+}
diff --git a/vitest.config.ts b/vitest.config.ts
index eae295484..2171bf7a2 100644
--- a/vitest.config.ts
+++ b/vitest.config.ts
@@ -4,6 +4,8 @@ import { join } from "node:path";
import { defineConfig } from "vitest/config";
+import { CiTestSequencer } from "./tests/ciTestSequencer.js";
+
// Playwright's registry does not derive a cache directory for Termux's
// `android` platform. Match the production CLI fallback before workers import
// browser modules.
@@ -151,6 +153,9 @@ export default defineConfig({
env: { TMPDIR: CANONICAL_TEMPORARY_DIRECTORY },
maxWorkers: MAX_TEST_WORKERS,
projects,
+ ...(process.env.REA_CI_TEST_HISTORY
+ ? { sequence: { sequencer: CiTestSequencer } }
+ : {}),
retry: 0,
reporters: ["default"],
runner: "./tests/processOwnershipRunner.ts",