Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

test(smoke): a gate that a passing frame can no longer sneak past

yhhae21a265f832
diff --git a/.github/workflows/android-device.yml b/.github/workflows/android-device.yml
index 98826b363..6890ae32b 100644
--- a/.github/workflows/android-device.yml
+++ b/.github/workflows/android-device.yml
@@ -29,6 +29,21 @@ name: Android real devices
# Snapdragon Galaxy (Adreno) and an Exynos Galaxy (Xclipse) cover the three
# drivers between them, and a fourth phone from the same vendor as one of those
# is a device minute spent on a question already answered.
+#
+# Pick the EXAMPLES the same way. One run per example, sequential, and each one
+# costs real device minutes — so the default is not "all of them" (the emulator
+# matrix already does that) and not one (which only ever exercised the plainest
+# renderer path). It is the smallest set that asks a different question of the
+# driver:
+#
+# space-shooter sprites, batching — the baseline that must never break
+# postprocess-effects render targets and float textures, the most
+# driver-divergent thing the engine does
+# tilemap-demo large atlases and chunked geometry
+#
+# Physics, audio and video are CPU or wasm on every device; an emulator answers
+# for them as well as a phone does, so they stay in the emulator matrix. Pass
+# `examples` by hand for anything else.
on:
workflow_dispatch:
@@ -36,7 +51,7 @@ on:
examples:
description: 'Comma-separated example names to package and run.'
required: false
- default: 'space-shooter'
+ default: 'space-shooter,postprocess-effects,tilemap-demo'
type: string
template-source:
description: 'Which runtime template to wrap: the latest release, or a build of this branch (~35 min).'
@@ -65,7 +80,31 @@ jobs:
runs-on: ubuntu-latest
outputs:
configured: ${{ steps.cfg.outputs.configured }}
+ # The example list as a JSON array, so the device job can be one run PER
+ # example. `appArn` names a single file: interpolating the raw input into
+ # it turned "a,b" — which the input documents and the APK job honours —
+ # into a hunt for `apks/a,b.apk`.
+ examples: ${{ steps.list.outputs.json }}
steps:
+ - id: list
+ env:
+ NAMES: ${{ inputs.examples || 'space-shooter,postprocess-effects,tilemap-demo' }}
+ run: |
+ set -euo pipefail
+ # Split the way the APK job splits it (android-canary-apk.yml), so the
+ # two cannot disagree about what "a, b" means.
+ IFS=',' read -ra NAMES_ARR <<< "$NAMES"
+ json='['
+ sep=''
+ for raw in "${NAMES_ARR[@]}"; do
+ name="$(echo "$raw" | tr -d '[:space:]')"
+ [ -z "$name" ] && continue
+ json="$json$sep\"$name\""
+ sep=','
+ done
+ json="$json]"
+ [ "$json" = '[]' ] && { echo '::error::no example names given'; exit 1; }
+ echo "json=$json" >> "$GITHUB_OUTPUT"
- id: cfg
env:
ROLE: ${{ secrets.AWS_DEVICEFARM_ROLE_ARN }}
@@ -100,7 +139,7 @@ jobs:
contents: read
pull-requests: read
with:
- examples: ${{ inputs.examples || 'space-shooter' }}
+ examples: ${{ inputs.examples || 'space-shooter,postprocess-effects,tilemap-demo' }}
template-source: ${{ inputs.template-source || 'release' }}
devices:
@@ -112,6 +151,14 @@ jobs:
permissions:
contents: read
id-token: write # OIDC, so no long-lived AWS keys live in this repo
+ strategy:
+ # One Device Farm run per example. Not parallel by default and not
+ # fail-fast: device minutes cost money, and a second example's result is
+ # still worth having when the first one failed.
+ max-parallel: 1
+ fail-fast: false
+ matrix:
+ example: ${{ fromJSON(needs.preflight.outputs.examples) }}
steps:
- uses: actions/checkout@v7
@@ -145,10 +192,10 @@ jobs:
# File paths (rather than ARNs) tell the action to upload them first.
run-settings-json: |
{
- "name": "estella canary ${{ github.ref_name }} ${{ github.sha }}",
+ "name": "estella canary ${{ matrix.example }} ${{ github.ref_name }} ${{ github.sha }}",
"projectArn": "${{ secrets.DEVICEFARM_PROJECT_ARN }}",
"devicePoolArn": "${{ secrets.DEVICEFARM_DEVICE_POOL_ARN }}",
- "appArn": "apks/${{ inputs.examples || 'space-shooter' }}.apk",
+ "appArn": "apks/${{ matrix.example }}.apk",
"test": {
"type": "APPIUM_NODE",
"testPackageArn": "build/devicefarm/tests.zip",
@@ -166,12 +213,15 @@ jobs:
if: always() && steps.farm.outputs.artifact-folder != ''
uses: actions/upload-artifact@v7
with:
- name: device-farm-artifacts
+ # Per example: one name across the matrix is a collision, and the
+ # upload fails rather than merging.
+ name: device-farm-artifacts-${{ matrix.example }}
path: ${{ steps.farm.outputs.artifact-folder }}
if-no-files-found: warn
- name: Where to look
if: always()
run: |
+ echo "example: ${{ matrix.example }}"
echo "result: ${{ steps.farm.outputs.result }}"
echo "console: ${{ steps.farm.outputs.console-url }}"
diff --git a/tools/verify-native-boot.mjs b/tools/verify-native-boot.mjs
index 97a636157..7e9892c45 100644
--- a/tools/verify-native-boot.mjs
+++ b/tools/verify-native-boot.mjs
@@ -51,6 +51,9 @@ const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const APP_ID = 'com.estella.game';
const LOG_NAME = 'estella-boot.log';
const READY = 'ready in';
+/** Captures per app. See the comment at the capture site for why >1 and why
+ * the richest one wins. */
+const FRAME_SAMPLES = 3;
/**
* Examples that do not start or draw on a device yet, and why.
@@ -231,8 +234,22 @@ function androidDriver(opts) {
foreground() {
const out = trySh(adb, ['shell', 'dumpsys', 'activity', 'activities']).stdout ?? '';
const line = out.split('\n').find((l) => /ResumedActivity/.test(l));
- if (line) return line.includes(APP_ID) ? null : `on screen instead: ${line.trim().slice(0, 100)}`;
- return trySh(adb, ['shell', 'pidof', APP_ID]).stdout.trim() ? null : 'the app process is gone';
+ if (line && !line.includes(APP_ID)) return `on screen instead: ${line.trim().slice(0, 100)}`;
+ if (!line && !trySh(adb, ['shell', 'pidof', APP_ID]).stdout.trim()) return 'the app process is gone';
+ // The resumed activity is still OURS while a dialog sits on top of
+ // it, which is the hole this check had: the frame judge counts
+ // colours on the SCREEN, and a system dialog is hundreds of them —
+ // so a game that had gone black passed on the dialog's antialiased
+ // text. Focus is what actually moves when something covers us.
+ const focus = (trySh(adb, ['shell', 'dumpsys', 'window']).stdout ?? '')
+ .split('\n').find((l) => /mCurrentFocus/.test(l));
+ // No focus line at all is not evidence of anything — some versions
+ // word it differently, and inventing a failure from a parse miss is
+ // how a gate teaches people to ignore it.
+ if (focus && !focus.includes(APP_ID) && !/mCurrentFocus=null/.test(focus)) {
+ return `something is on top of it: ${focus.trim().slice(0, 100)}`;
+ }
+ return null;
},
clearScreen() {
trySh(adb, ['shell', 'am', 'broadcast', '-a', 'android.intent.action.CLOSE_SYSTEM_DIALOGS']);
@@ -520,7 +537,29 @@ async function verifyApp(driver, artifact, label, opts, judgeFrame = true) {
}
driver.clearScreen();
- const frame = driver.screenshot();
+ // More than one, and keep the richest.
+ //
+ // A single capture is a coin flip against everything transient: a toast
+ // fading in, a frame taken between two the game drew, a compositor that
+ // handed back the previous buffer. Every one of those makes a WORKING game
+ // look broken, which is the failure that gets a gate ignored. Three is
+ // enough to make that unlucky rather than likely, and they cost a
+ // screencap each.
+ //
+ // Richest, not median: the question is "did it ever draw", so one good
+ // frame is proof and the others cannot un-prove it. The dialog case runs
+ // the other way and is not this check's to catch — `foreground` above is,
+ // because a dialog's colours would win exactly this comparison.
+ let frame = driver.screenshot();
+ let image = decodePng(frame);
+ let colors = distinctColors(image);
+ for (let i = 1; i < FRAME_SAMPLES; i++) {
+ await sleep(120);
+ const again = driver.screenshot();
+ const decoded = decodePng(again);
+ const n = distinctColors(decoded);
+ if (n > colors) { frame = again; image = decoded; colors = n; }
+ }
const shot = path.join(opts.out, `${driver.name}-${label}.png`);
writeFileSync(shot, frame);
writeFileSync(path.join(opts.out, `${driver.name}-${label}.log`), log);
@@ -552,8 +591,8 @@ async function verifyApp(driver, artifact, label, opts, judgeFrame = true) {
}
driver.stop();
- const image = decodePng(frame);
- const colors = distinctColors(image);
+ // `image` and `colors` come from the sampling above — decoding the kept
+ // frame a second time would be two sources for one number.
const ready = log.includes(READY);
const readyLine = log.split('\n').find((l) => l.includes(READY))?.trim() ?? '';
if (metrics) metrics.startup.readyMs = firstNumber(/ready in (\d+) ms/, readyLine);