Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

ci: reuse current-run builds across provider lanes

morluto490a28507026
diff --git a/.github/actions/portable-build-cache/action.yml b/.github/actions/portable-build-cache/action.yml
index ad127ce3d..cee775ec5 100644
--- a/.github/actions/portable-build-cache/action.yml
+++ b/.github/actions/portable-build-cache/action.yml
@@ -1,14 +1,24 @@
name: Portable build cache
-description: Reuse Turbo build outputs without serializing verification jobs.
+description: Restore portable Turbo build outputs from the current run or cross-run cache.
inputs:
save:
description: Save newly built outputs after the job succeeds.
default: "false"
+ build_cache_artifact:
+ description: Reuse portable Turbo outputs uploaded by the current workflow run.
+ default: ""
runs:
using: composite
steps:
+ - name: Download portable build outputs from this workflow run
+ if: inputs.build_cache_artifact != ''
+ uses: actions/download-artifact@v4
+ with:
+ name: ${{ inputs.build_cache_artifact }}
+ path: .turbo/cache/
- name: Resolve portable build task hashes
id: plan
+ if: inputs.build_cache_artifact == ''
shell: bash
env:
BUILD_CACHE_PREFIX: rea-portable-build-v2-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.npmrc', '.nvmrc', 'package-lock.json') }}
@@ -28,14 +38,14 @@ runs:
console.log(`prefix=${process.env.BUILD_CACHE_PREFIX}-`);
' >> "$GITHUB_OUTPUT"
- name: Restore portable build outputs
- if: inputs.save != 'true'
+ if: inputs.build_cache_artifact == '' && inputs.save != 'true'
uses: actions/cache/restore@v4
with:
path: .turbo/cache/
key: ${{ steps.plan.outputs.key }}
restore-keys: ${{ steps.plan.outputs.prefix }}
- name: Restore and save portable build outputs
- if: inputs.save == 'true'
+ if: inputs.build_cache_artifact == '' && inputs.save == 'true'
uses: actions/cache@v4
with:
path: .turbo/cache/
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 28373f89d..e0516da85 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -190,7 +190,18 @@ jobs:
build:
name: Build package
needs: changes
- if: needs.changes.outputs.code == 'true'
+ # Reusable provider workflows consume this run's portable build artifact,
+ # even when a workflow-only edit selects the lane without portable tests.
+ if: >-
+ needs.changes.outputs.code == 'true' ||
+ needs.changes.outputs.browser == 'true' ||
+ needs.changes.outputs.binary_layout == 'true' ||
+ needs.changes.outputs.evm == 'true' ||
+ needs.changes.outputs.javascript_recovery == 'true' ||
+ needs.changes.outputs.recorded_crash == 'true' ||
+ needs.changes.outputs.web_network == 'true' ||
+ needs.changes.outputs.web_runtime == 'true' ||
+ needs.changes.outputs.web_source_map == 'true'
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
@@ -673,44 +684,60 @@ jobs:
if-no-files-found: error
browser:
- needs: [changes, static]
+ needs: [changes, static, build]
if: needs.changes.outputs.browser == 'true'
uses: ./.github/workflows/real-browser.yml
+ with:
+ build_cache_artifact: rea-build-cache
binary_layout:
- needs: [changes, static]
+ needs: [changes, static, build]
if: needs.changes.outputs.binary_layout == 'true'
uses: ./.github/workflows/real-binary-layout.yml
+ with:
+ build_cache_artifact: rea-build-cache
evm:
- needs: [changes, static]
+ needs: [changes, static, build]
if: needs.changes.outputs.evm == 'true'
uses: ./.github/workflows/real-evm-interface.yml
+ with:
+ build_cache_artifact: rea-build-cache
javascript_recovery:
- needs: [changes, static]
+ needs: [changes, static, build]
if: needs.changes.outputs.javascript_recovery == 'true'
uses: ./.github/workflows/real-javascript-recovery.yml
+ with:
+ build_cache_artifact: rea-build-cache
recorded_crash:
- needs: [changes, static]
+ needs: [changes, static, build]
if: needs.changes.outputs.recorded_crash == 'true'
uses: ./.github/workflows/real-recorded-crash.yml
+ with:
+ build_cache_artifact: rea-build-cache
web_network:
- needs: [changes, static]
+ needs: [changes, static, build]
if: needs.changes.outputs.web_network == 'true'
uses: ./.github/workflows/real-web-network-captures.yml
+ with:
+ build_cache_artifact: rea-build-cache
web_runtime:
- needs: [changes, static]
+ needs: [changes, static, build]
if: needs.changes.outputs.web_runtime == 'true'
uses: ./.github/workflows/real-web-runtime.yml
+ with:
+ build_cache_artifact: rea-build-cache
web_source_map:
- needs: [changes, static]
+ needs: [changes, static, build]
if: needs.changes.outputs.web_source_map == 'true'
uses: ./.github/workflows/real-web-source-map.yml
+ with:
+ build_cache_artifact: rea-build-cache
nativeaot:
needs: [changes, static]
diff --git a/.github/workflows/real-binary-layout.yml b/.github/workflows/real-binary-layout.yml
index 8dd6f9866..5cec952ce 100644
--- a/.github/workflows/real-binary-layout.yml
+++ b/.github/workflows/real-binary-layout.yml
@@ -8,6 +8,12 @@ on:
type: boolean
default: false
workflow_call:
+ inputs:
+ build_cache_artifact:
+ description: Portable build cache artifact from the calling CI run.
+ required: false
+ type: string
+ default: ""
permissions:
contents: read
@@ -29,6 +35,8 @@ jobs:
cache: npm
- run: npm ci
- uses: ./.github/actions/portable-build-cache
+ with:
+ build_cache_artifact: ${{ inputs.build_cache_artifact }}
- name: Acquire exact upstream libraries in an isolated runner environment
shell: bash
run: |
diff --git a/.github/workflows/real-browser.yml b/.github/workflows/real-browser.yml
index 11a8b7a53..912fdafb1 100644
--- a/.github/workflows/real-browser.yml
+++ b/.github/workflows/real-browser.yml
@@ -2,6 +2,12 @@ name: Real browser verification
on:
workflow_call:
+ inputs:
+ build_cache_artifact:
+ description: Portable build cache artifact from the calling CI run.
+ required: false
+ type: string
+ default: ""
workflow_dispatch:
env:
@@ -22,6 +28,8 @@ jobs:
cache: npm
- run: npm ci
- uses: ./.github/actions/portable-build-cache
+ with:
+ build_cache_artifact: ${{ inputs.build_cache_artifact }}
- run: npm run verify:browser
env:
REA_BROWSER_EXECUTABLE: /usr/bin/google-chrome
diff --git a/.github/workflows/real-evm-interface.yml b/.github/workflows/real-evm-interface.yml
index b335f1391..dfdd3970f 100644
--- a/.github/workflows/real-evm-interface.yml
+++ b/.github/workflows/real-evm-interface.yml
@@ -3,6 +3,12 @@ name: Real offline EVM interface
on:
workflow_dispatch:
workflow_call:
+ inputs:
+ build_cache_artifact:
+ description: Portable build cache artifact from the calling CI run.
+ required: false
+ type: string
+ default: ""
permissions:
contents: read
@@ -23,6 +29,8 @@ jobs:
cache: npm
- run: npm ci
- uses: ./.github/actions/portable-build-cache
+ with:
+ build_cache_artifact: ${{ inputs.build_cache_artifact }}
- name: Acquire unchanged fixture compiler in isolated runner storage
shell: bash
run: |
diff --git a/.github/workflows/real-javascript-recovery.yml b/.github/workflows/real-javascript-recovery.yml
index d94127d63..817dc4dc0 100644
--- a/.github/workflows/real-javascript-recovery.yml
+++ b/.github/workflows/real-javascript-recovery.yml
@@ -2,6 +2,12 @@ name: Real JavaScript recovery
on:
workflow_call:
+ inputs:
+ build_cache_artifact:
+ description: Portable build cache artifact from the calling CI run.
+ required: false
+ type: string
+ default: ""
workflow_dispatch:
permissions:
@@ -24,6 +30,8 @@ jobs:
cache: npm
- run: npm ci
- uses: ./.github/actions/portable-build-cache
+ with:
+ build_cache_artifact: ${{ inputs.build_cache_artifact }}
- name: Fetch pinned Wakaru release into the job workspace
shell: bash
run: |
diff --git a/.github/workflows/real-recorded-crash.yml b/.github/workflows/real-recorded-crash.yml
index dc56034c8..04467a608 100644
--- a/.github/workflows/real-recorded-crash.yml
+++ b/.github/workflows/real-recorded-crash.yml
@@ -3,6 +3,12 @@ name: Real recorded crash evidence
on:
workflow_dispatch:
workflow_call:
+ inputs:
+ build_cache_artifact:
+ description: Portable build cache artifact from the calling CI run.
+ required: false
+ type: string
+ default: ""
permissions:
contents: read
@@ -26,6 +32,8 @@ jobs:
cache: npm
- run: npm ci
- uses: ./.github/actions/portable-build-cache
+ with:
+ build_cache_artifact: ${{ inputs.build_cache_artifact }}
- name: Install GDB on the disposable CI runner
shell: bash
run: |
diff --git a/.github/workflows/real-web-network-captures.yml b/.github/workflows/real-web-network-captures.yml
index df340520d..d41bb37a5 100644
--- a/.github/workflows/real-web-network-captures.yml
+++ b/.github/workflows/real-web-network-captures.yml
@@ -3,6 +3,12 @@ name: Real historical web network captures
on:
workflow_dispatch:
workflow_call:
+ inputs:
+ build_cache_artifact:
+ description: Portable build cache artifact from the calling CI run.
+ required: false
+ type: string
+ default: ""
permissions:
contents: read
@@ -25,6 +31,8 @@ jobs:
run: npm install --global [email protected]
- run: npm ci
- uses: ./.github/actions/portable-build-cache
+ with:
+ build_cache_artifact: ${{ inputs.build_cache_artifact }}
- name: Acquire fixed upstream native profile in runner temporary storage
shell: bash
run: |
diff --git a/.github/workflows/real-web-runtime.yml b/.github/workflows/real-web-runtime.yml
index 0ecdb92c6..2575c55f9 100644
--- a/.github/workflows/real-web-runtime.yml
+++ b/.github/workflows/real-web-runtime.yml
@@ -1,6 +1,12 @@
name: Real website runtime attribution
on:
workflow_call:
+ inputs:
+ build_cache_artifact:
+ description: Portable build cache artifact from the calling CI run.
+ required: false
+ type: string
+ default: ""
workflow_dispatch:
permissions:
contents: read
@@ -18,6 +24,8 @@ jobs:
cache: npm
- run: npm ci
- uses: ./.github/actions/portable-build-cache
+ with:
+ build_cache_artifact: ${{ inputs.build_cache_artifact }}
- run: npm run verify:browser:runtime
env:
REA_BROWSER_EXECUTABLE: /usr/bin/google-chrome
diff --git a/.github/workflows/real-web-source-map.yml b/.github/workflows/real-web-source-map.yml
index b94662129..8e0869eb7 100644
--- a/.github/workflows/real-web-source-map.yml
+++ b/.github/workflows/real-web-source-map.yml
@@ -1,6 +1,12 @@
name: Real website source-map verification
on:
workflow_call:
+ inputs:
+ build_cache_artifact:
+ description: Portable build cache artifact from the calling CI run.
+ required: false
+ type: string
+ default: ""
workflow_dispatch:
permissions:
contents: read
@@ -18,6 +24,8 @@ jobs:
cache: npm
- run: npm ci
- uses: ./.github/actions/portable-build-cache
+ with:
+ build_cache_artifact: ${{ inputs.build_cache_artifact }}
- name: Install the pinned compiler into an isolated fixture directory
run: npm install --prefix "$RUNNER_TEMP/rea-web-source-map-compiler" --no-audit --no-fund --package-lock=false [email protected]
- run: npm run verify:browser:source-maps
diff --git a/docs/testing.md b/docs/testing.md
index 03121e172..1288a954a 100644
--- a/docs/testing.md
+++ b/docs/testing.md
@@ -982,6 +982,16 @@ timing data is reused; live provider results are never cached. Turbo outputs are
restored by task hash, with trusted default-branch runs as cache writers and PRs
as restore-only consumers. npm caches hold downloads, not `node_modules`.
+The eight automatic browser, binary-layout, EVM, JavaScript-recovery,
+recorded-crash, and web provider lanes download `rea-build-cache` from their own
+run after the build and static jobs succeed. Their existing build commands still
+validate task hashes and rebuild mismatched inputs; provider observations always
+run. Provider-workflow-only edits also admit the build without enabling source
+test shards. Direct provider dispatches retain the cross-run restore-only cache.
+This removes repeated compilation on fresh source revisions, but adds the build
+as a prerequisite: a delayed build can delay provider admission. Compare queue
+and execution times before claiming an end-to-end speedup.
+
These native baseline checks complement the Linux source-test shards and the
separate Apple-artifact and real-provider lanes. Actual Hopper, Ghidra, IDA,
browser and managed-tool claims require their corresponding verification lanes.
diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts
index fa4b71ebf..163219c39 100644
--- a/tests/boundary/filesystem/ciChangeScope.test.ts
+++ b/tests/boundary/filesystem/ciChangeScope.test.ts
@@ -680,7 +680,9 @@ it.each([
async (scope, name) => {
const workflow = z
.object({
- on: z.object({ workflow_call: z.null() }).passthrough(),
+ on: z
+ .object({ workflow_call: z.object({}).passthrough().nullable() })
+ .passthrough(),
})
.parse(
parse(
diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts
index e460bb52b..b6e02b54a 100644
--- a/tests/boundary/filesystem/ciRequiredLanes.test.ts
+++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts
@@ -168,6 +168,115 @@ it("holds native and provider fan-out behind static validation while portable fe
expect(jobs["test-shard"]?.needs).not.toContain("static");
});
+it("binds provider build reuse to the current run while keeping standalone dispatches independent", async () => {
+ const { jobs } = await workflow();
+ const consumers = [
+ "browser",
+ "binary_layout",
+ "evm",
+ "javascript_recovery",
+ "recorded_crash",
+ "web_network",
+ "web_runtime",
+ "web_source_map",
+ ];
+ const buildTerms = z
+ .string()
+ .parse(jobs.build?.if)
+ .split("||")
+ .map((term) => term.trim());
+ // Provider workflow edits select their lane without code=true. The build
+ // producer must still run, without selecting the portable test shards.
+ expect(new Set(buildTerms)).toEqual(
+ new Set(
+ ["code", ...consumers].map(
+ (scope) => `needs.changes.outputs.${scope} == 'true'`,
+ ),
+ ),
+ );
+ expect(jobs["test-shard"]?.if).toBe("needs.changes.outputs.code == 'true'");
+ for (const name of consumers) {
+ const caller = jobs[name];
+ expect(caller?.needs).toContain("build");
+ expect(caller?.with).toEqual({ build_cache_artifact: "rea-build-cache" });
+ const path = z.string().parse(caller?.uses);
+ const called = z
+ .object({
+ on: z.object({
+ workflow_call: z.object({
+ inputs: z.record(z.string(), z.unknown()),
+ }),
+ workflow_dispatch: z.unknown(),
+ }),
+ jobs: z.record(
+ z.string(),
+ z.object({
+ steps: z.array(
+ z.object({
+ uses: z.string().optional(),
+ with: z.record(z.string(), z.unknown()).optional(),
+ }),
+ ),
+ }),
+ ),
+ })
+ .parse(
+ parse(
+ await readFile(new URL(`../../../${path}`, import.meta.url), "utf8"),
+ ),
+ );
+ expect(called.on.workflow_call.inputs.build_cache_artifact).toMatchObject({
+ type: "string",
+ default: "",
+ });
+ expect(Object.hasOwn(called.on, "workflow_dispatch")).toBe(true);
+ for (const job of Object.values(called.jobs))
+ expect(
+ job.steps.find(
+ (step) => step.uses === "./.github/actions/portable-build-cache",
+ )?.with,
+ ).toEqual({ build_cache_artifact: "${{ inputs.build_cache_artifact }}" });
+ }
+ const action = z
+ .object({
+ runs: z.object({
+ steps: z.array(
+ z.object({
+ id: z.string().optional(),
+ uses: z.string().optional(),
+ if: z.string().optional(),
+ with: z.record(z.string(), z.unknown()).optional(),
+ "continue-on-error": z.boolean().optional(),
+ }),
+ ),
+ }),
+ })
+ .parse(
+ parse(
+ await readFile(
+ new URL(
+ "../../../.github/actions/portable-build-cache/action.yml",
+ import.meta.url,
+ ),
+ "utf8",
+ ),
+ ),
+ );
+ const download = action.runs.steps.find(
+ (step) => step.uses === "actions/download-artifact@v4",
+ );
+ expect(download?.with).toEqual({
+ name: "${{ inputs.build_cache_artifact }}",
+ path: ".turbo/cache/",
+ });
+ expect(download?.if).toBe("inputs.build_cache_artifact != ''");
+ expect(download?.["continue-on-error"]).not.toBe(true);
+ for (const step of action.runs.steps.filter(
+ (step) => step.id === "plan" || step.uses?.startsWith("actions/cache"),
+ ))
+ expect(step.if).toContain("inputs.build_cache_artifact == ''");
+});
+
it("runs workflow validation inside required static checks when selected", async () => {
const { jobs } = await workflow();
const steps = z
diff --git a/tests/boundary/filesystem/testTaskCache.test.ts b/tests/boundary/filesystem/testTaskCache.test.ts
index 7e06ce1a1..5d877e829 100644
--- a/tests/boundary/filesystem/testTaskCache.test.ts
+++ b/tests/boundary/filesystem/testTaskCache.test.ts
@@ -1,6 +1,6 @@
import { execFile } from "node:child_process";
-import { readFile, writeFile } from "node:fs/promises";
-import { join } from "node:path";
+import { cp, readFile, writeFile } from "node:fs/promises";
+import { basename, join } from "node:path";
import { promisify } from "node:util";
import { expect, it } from "vitest";
@@ -78,7 +78,7 @@ it.skipIf(process.platform === "android")(
await workflowGit(root, ["init", "--initial-branch=main"]);
await workflowGit(root, ["add", "."]);
await workflowGit(root, ["commit", "-m", "test: seed cache fixture"]);
- const run = () =>
+ const run = (cwd = root) =>
execute(
process.execPath,
[
@@ -87,7 +87,7 @@ it.skipIf(process.platform === "android")(
"test:run",
"--output-logs=errors-only",
],
- { cwd: root, env: { ...process.env, TURBO_DAEMON: "false" } },
+ { cwd, env: { ...process.env, TURBO_DAEMON: "false" } },
);
await run();
await writeFile(join(root, "README_zh.md"), "wrong command");
@@ -104,5 +104,32 @@ it.skipIf(process.platform === "android")(
"prepared\n",
);
}
+ // Provider jobs receive the producer's cache in a fresh checkout, not its
+ // outputs. Turbo must restore identical inputs and rebuild changed ones.
+ const consumer = await createTestTempDirectory("rea-cache-consumer-");
+ await cp(root, consumer, {
+ recursive: true,
+ filter: (path) =>
+ basename(path) !== ".turbo" &&
+ !/^prepared-\d+\.txt$/u.test(basename(path)),
+ });
+ await cp(join(root, ".turbo/cache"), join(consumer, ".turbo/cache"), {
+ recursive: true,
+ });
+ const restored = await run(consumer);
+ expect(restored.stdout).toContain(`${dependencies.length} cached`);
+ for (const index of dependencies.keys())
+ expect(
+ await readFile(join(consumer, `prepared-${index}.txt`), "utf8"),
+ ).toBe("prepared\n");
+ await writeFile(
+ join(consumer, "artifact-input.txt"),
+ "changed producer input",
+ );
+ await run(consumer);
+ for (const index of dependencies.keys())
+ expect(
+ await readFile(join(consumer, `prepared-${index}.txt`), "utf8"),
+ ).toBe("prepared\nprepared\n");
},
);