Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
ci: balance test shards and reuse package download caches
morluto40b37aeee10e
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 6c81e257d..131f46379 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -118,6 +118,8 @@ jobs:name: rea-test-runtimepath: .- run: npm run test:ci:shard:run -- --shard=${{ matrix.shard }}+ env:+ REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json- uses: actions/upload-artifact@v4if: ${{ !cancelled() }}with:@@ -133,6 +135,9 @@ jobs:needs: changesif: needs.changes.outputs.code == 'true'runs-on: ubuntu-latest+ permissions:+ contents: read+ actions: readsteps:- uses: actions/checkout@v4- uses: actions/setup-node@v4@@ -141,6 +146,28 @@ jobs:cache: npm- run: npm ci- run: npm run test:prepare+ - name: Find successful default-branch timing history+ id: history+ continue-on-error: true+ env:+ GH_TOKEN: ${{ github.token }}+ CI_REPOSITORY: ${{ github.repository }}+ CI_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}+ run: |+ run_id="$(gh api --method GET \+ "repos/${CI_REPOSITORY}/actions/workflows/ci.yml/runs" \+ -f branch="${CI_DEFAULT_BRANCH}" -f event=push -f status=success -f per_page=1 \+ --jq '.workflow_runs[0].id // empty')"+ echo "run-id=${run_id}" >> "${GITHUB_OUTPUT}"+ - name: Download previous timing report+ if: steps.history.outputs.run-id != ''+ continue-on-error: true+ uses: actions/download-artifact@v4+ with:+ name: test-results+ run-id: ${{ steps.history.outputs.run-id }}+ github-token: ${{ github.token }}+ path: .cache/ci-test-history- uses: actions/upload-artifact@v4with:name: rea-test-runtime@@ -148,6 +175,7 @@ jobs:dist/skills/.cache/mcp-tool-catalog.json+ .cache/ci-test-history/test-results.jsondocs/public/product-catalog.jsondocs/verification/managed-conformance-*.jsoninclude-hidden-files: true@@ -305,7 +333,14 @@ jobs:name: windows-native-controlspath: native/windows/build/- run: npm run verify:windows-native:artifact+ - name: Resolve npm cache before verifier HOME isolation+ id: npm-cache+ run: |+ cache_path="$(npm config get cache)"+ echo "path=${cache_path}" >> "${GITHUB_OUTPUT}"- run: npm run verify:package+ env:+ npm_config_cache: ${{ steps.npm-cache.outputs.path }}- name: Verify real Node Inspector discovery and observationtimeout-minutes: 10run: npm run verify:inspectordiff --git a/docs/testing.md b/docs/testing.mdindex 82c6f0263..2d6369d89 100644--- a/docs/testing.md+++ b/docs/testing.md@@ -835,6 +835,12 @@ target-free analysis, configuration backups/recovery, Evidence and ownedlifecycle; Inspector checks execute source-owned loopback targets and specialfilename cases.+POSIX package verification resolves the runner's configured npm cache before+creating disposable client homes and passes that location to its child installs.+This reuses the download cache restored by `setup-node` while keeping client+configuration isolated. Installs retain their existing registry and integrity+checks.+The Apple artifact CLI/MCP lane uses one `macos-15` job to avoid adding demandfor scarce macOS runner capacity. It runs the artifact-format and metadataverifiers, followed by one consolidated focused invocation containing all@@ -847,6 +853,15 @@ catalog, and managed-conformance metadata. Each shard still installs the lockeddependencies and runs its assigned coverage-enabled tests; it no longerregenerates the same metadata independently.+The build job also retrieves the merged timing report from the last successful+default-branch push and includes that same report in each shard's snapshot.+CI assigns discovered files by historical duration, with a deterministic+fallback weight for new files, rather than dividing only by file count. Every+discovered file still belongs to exactly one shard; worker limits, project+ordering, coverage instrumentation, and aggregate thresholds remain unchanged.+Missing or unusable history falls back to Vitest's default assignment. Only+timing data is reused; persistent compile caches remain disabled.+These native baseline checks complement the Linux source-test shards and theseparate Apple-artifact and real-provider lanes. Actual Hopper, Ghidra, IDA,browser and managed-tool claims require their corresponding verification lanes.diff --git a/tests/boundary/filesystem/ciTestSequencer.test.ts b/tests/boundary/filesystem/ciTestSequencer.test.tsnew file mode 100644index 000000000..6177d9481--- /dev/null+++ b/tests/boundary/filesystem/ciTestSequencer.test.ts@@ -0,0 +1,155 @@+import { mkdir, writeFile } from "node:fs/promises";+import { join, resolve } from "node:path";++import { fc, it } from "@fast-check/vitest";+import { describe, expect } from "vitest";++import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";++import {+ allocateByDuration,+ parseTestDurations,+ readTestDurations,+} from "../../ciTestSequencer.js";++describe("CI test history", () => {+ it("uses only successful in-root file durations", () => {+ const root = resolve("/workspace/rea");+ const durations = parseTestDurations(+ JSON.stringify({+ success: true,+ numFailedTests: 0,+ testResults: [+ {+ name: join(root, "tests/slow.test.ts"),+ startTime: 10,+ endTime: 110,+ },+ {+ name: join(root, "tests/slow.test.ts"),+ startTime: 20,+ endTime: 80,+ },+ {+ name: resolve(root, "../outside.test.ts"),+ startTime: 10,+ endTime: 1000,+ },+ null,+ ],+ }),+ root,+ );++ expect(durations).toEqual(new Map([["tests/slow.test.ts", 100]]));+ expect(+ parseTestDurations('{"success": false, "numFailedTests": 1}', root),+ ).toBeUndefined();+ expect(parseTestDurations("not json", root)).toBeUndefined();+ expect(parseTestDurations("null", root)).toBeUndefined();+ });++ it("falls back for missing history while preserving real filesystem errors", async () => {+ const temporaryRoot = await createTestTempDirectory("rea-ci-sequencer-");+ const missing = await readTestDurations(+ join(temporaryRoot, "missing.json"),+ temporaryRoot,+ );+ expect(missing).toBeUndefined();++ const directory = join(temporaryRoot, "directory");+ await mkdir(directory);+ const reportPath = join(temporaryRoot, "report.json");+ await writeFile(reportPath, "{");+ expect(await readTestDurations(reportPath, temporaryRoot)).toBeUndefined();++ await expect(+ readTestDurations(directory, temporaryRoot),+ ).rejects.toMatchObject({+ code: "EISDIR",+ });+ });+});++describe("duration shard allocation", () => {+ it.prop([+ fc.uniqueArray(fc.integer({ min: 0, max: 10_000 }), {+ minLength: 1,+ maxLength: 80,+ }),+ fc.integer({ min: 3, max: 8 }),+ ])(+ "is complete, disjoint, reference-preserving, and input-order independent",+ (ids, shardCount) => {+ const items = ids.map((id, index) => ({+ id,+ path: `tests/${id}.test.ts`,+ project: `project-${id % 3}`,+ pool: id % 2 === 0 ? "forks" : "threads",+ duration: index > 0 && id % 5 === 0 ? undefined : (id % 17) + 1,+ }));+ const specifications = items.map((item) => ({+ item,+ key: `${item.path}\0${item.project}\0${item.pool}`,+ duration: item.duration,+ }));+ const shards = allocateByDuration(specifications, shardCount);+ const reversedShards = allocateByDuration(+ [...specifications].reverse(),+ shardCount,+ );++ expect(shards).toBeDefined();+ expect(reversedShards).toEqual(shards);+ const assigned = shards!.flat();+ expect(assigned).toHaveLength(items.length);+ expect(new Set(assigned).size).toBe(items.length);+ expect(assigned.every((item) => items.includes(item))).toBe(true);+ },+ );++ it("keeps tied path, project, pool, and new-file entries deterministic", () => {+ const items = [+ {+ id: "a",+ path: "tests/shared.test.ts",+ project: "alpha",+ pool: "forks",+ duration: 10,+ },+ {+ id: "b",+ path: "tests/shared.test.ts",+ project: "beta",+ pool: "forks",+ duration: 10,+ },+ {+ id: "c",+ path: "tests/shared.test.ts",+ project: "beta",+ pool: "threads",+ duration: 10,+ },+ {+ id: "new",+ path: "tests/new.test.ts",+ project: "alpha",+ pool: "forks",+ duration: undefined,+ },+ ];+ const specifications = items.map((item) => ({+ item,+ key: `${item.path}\0${item.project}\0${item.pool}`,+ duration: item.duration,+ }));+ const shards = allocateByDuration(specifications, 3);+ const reversedShards = allocateByDuration([...specifications].reverse(), 3);++ expect(shards).toBeDefined();+ expect(reversedShards).toEqual(shards);+ expect(shards!.flat()).toHaveLength(items.length);+ expect(new Set(shards!.flat()).size).toBe(items.length);+ });+});diff --git a/tests/ciTestSequencer.ts b/tests/ciTestSequencer.tsnew file mode 100644index 000000000..5e8c4fe5f--- /dev/null+++ b/tests/ciTestSequencer.ts@@ -0,0 +1,125 @@+import { readFile } from "node:fs/promises";+import { isAbsolute, relative, resolve, sep } from "node:path";++import { BaseSequencer, type TestSpecification } from "vitest/node";+import { z } from "zod";++const reportSchema = z.object({+ success: z.literal(true),+ numFailedTests: z.literal(0),+ testResults: z.array(z.unknown()),+});+const fileSchema = z.object({+ name: z.string(),+ startTime: z.number(),+ endTime: z.number(),+});++export function parseTestDurations(+ contents: string,+ root: string,+): Map<string, number> | undefined {+ let value: unknown;+ try {+ value = JSON.parse(contents);+ } catch (error) {+ if (error instanceof SyntaxError) return undefined;+ throw error;+ }+ const report = reportSchema.safeParse(value);+ if (!report.success) return undefined;++ const durations = new Map<string, number>();+ for (const value of report.data.testResults) {+ const result = fileSchema.safeParse(value);+ if (!result.success) continue;+ const { name, startTime, endTime } = result.data;+ const duration = endTime - startTime;+ const path = relative(root, resolve(root, name));+ if (+ !Number.isFinite(duration) ||+ duration <= 0 ||+ path === "" ||+ path === ".." ||+ path.startsWith(".." + sep) ||+ isAbsolute(path)+ )+ continue;+ const key = path.split(sep).join("/");+ durations.set(key, Math.max(durations.get(key) ?? 0, duration));+ }+ return durations.size ? durations : undefined;+}++export async function readTestDurations(+ historyPath: string,+ root: string,+): Promise<Map<string, number> | undefined> {+ try {+ return parseTestDurations(await readFile(historyPath, "utf8"), root);+ } catch (error) {+ if (error instanceof Error && "code" in error && error.code === "ENOENT")+ return undefined;+ throw error;+ }+}++type TimedItem<T> = { item: T; key: string; duration: number | undefined };++/** Distribute the longest files first, retaining every discovered specification. */+export function allocateByDuration<T>(+ items: readonly TimedItem<T>[],+ shardCount: number,+): T[][] | undefined {+ if (!Number.isInteger(shardCount) || shardCount < 1)+ throw new RangeError("shardCount must be a positive integer");+ const known = items+ .flatMap(({ duration }) => (duration === undefined ? [] : [duration]))+ .sort((left, right) => left - right);+ if (!known.length) return undefined;+ const lower = known[Math.floor((known.length - 1) / 2)]!;+ const median = lower + (known[Math.floor(known.length / 2)]! - lower) / 2;+ const ordered = items.toSorted(+ (left, right) =>+ (right.duration ?? median) - (left.duration ?? median) ||+ (left.key < right.key ? -1 : left.key > right.key ? 1 : 0),+ );+ const shards = Array.from({ length: shardCount }, () => ({+ duration: 0,+ items: [] as T[],+ }));+ for (const entry of ordered) {+ const lightest = shards.reduce((left, right) =>+ right.duration < left.duration ? right : left,+ );+ lightest.items.push(entry.item);+ lightest.duration += entry.duration ?? median;+ }+ return shards.map(({ items }) => items);+}++export class CiTestSequencer extends BaseSequencer {+ override async shard(+ files: TestSpecification[],+ ): Promise<TestSpecification[]> {+ const historyPath = process.env.REA_CI_TEST_HISTORY;+ const shard = this.ctx.config.shard;+ if (!historyPath || !shard) return super.shard(files);+ const root = this.ctx.config.root;+ const durations = await readTestDurations(historyPath, root);+ if (!durations) return super.shard(files);++ const weighted = files.map((item) => {+ const path = relative(root, resolve(root, item.moduleId))+ .split(sep)+ .join("/");+ return {+ item,+ key: path + "\0" + item.project.name + "\0" + item.pool,+ duration: durations.get(path),+ };+ });+ const shards = allocateByDuration(weighted, shard.count);+ return shards ? shards[shard.index - 1]! : super.shard(files);+ }+}diff --git a/vitest.config.ts b/vitest.config.tsindex eae295484..2171bf7a2 100644--- a/vitest.config.ts+++ b/vitest.config.ts@@ -4,6 +4,8 @@ import { join } from "node:path";import { defineConfig } from "vitest/config";+import { CiTestSequencer } from "./tests/ciTestSequencer.js";+// Playwright's registry does not derive a cache directory for Termux's// `android` platform. Match the production CLI fallback before workers import// browser modules.@@ -151,6 +153,9 @@ export default defineConfig({env: { TMPDIR: CANONICAL_TEMPORARY_DIRECTORY },maxWorkers: MAX_TEST_WORKERS,projects,+ ...(process.env.REA_CI_TEST_HISTORY+ ? { sequence: { sequencer: CiTestSequencer } }+ : {}),retry: 0,reporters: ["default"],runner: "./tests/processOwnershipRunner.ts",