Free, open-source 3D sculpting for Linux, macOS and Windows: dense meshes, remeshing, painting, built-in AI (image/text to 3D), Send to Unreal, game-ready LODs, Gaussian splats and OpenUSD.
C++ 3d3d-sculptingcppdear-imguidigital-sculptingfbxgame-developmentgaussian-splattinggltfimage-to-3dmcpopenglopenusdsculptingtext-to-3dunreal-engine
Tiered test flow (T0 impacted tests, T1 PC suite with test cache, T2 staging merge queue, T3 nightly), release CI reuse, test budget
islandmed87f59321725b
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex d7d66dd9..1eb86c5c 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -15,9 +15,11 @@ on:permissions:contents: read+# A newer push cancels a branch's run, but never main's: a commit's finished run on main is what the next commits reuse+# (a release bump right after a merge reused nothing when the bump cancelled its parent's run, 0.10.10).concurrency:group: ci-${{ github.ref }}- cancel-in-progress: true+ cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}# Compiler cache (ccache). Every job compiles through ccache and starts from the newest cache of the same job; ccache# hashes the compiler and the preprocessed source with every flag, so a hit is the object file that compiler would@@ -44,10 +46,16 @@ jobs:changes:name: Changesruns-on: ubuntu-24.04+ permissions:+ contents: read+ actions: read # the staging run this commit may reuseoutputs:code: ${{ steps.diff.outputs.code }}full: ${{ steps.diff.outputs.full }}steps:+ - uses: actions/checkout@v7+ if: github.event_name == 'push'+ with: { fetch-depth: 20 } # the pushed range, for the paperwork check below- id: diffenv:GH_TOKEN: ${{ github.token }}@@ -79,6 +87,32 @@ jobs:done <<< "$files"fifi+ # T2 (docs/testing-flow.md): main only ever fast-forwards to a commit the merge queue tested on the branch+ # staging (scripts/merge-queue.sh staging). When this exact commit already passed a full run of this workflow+ # there (dispatched, full=true: its VFX Reference Platform jobs ran and passed), main's run reuses it.+ if [ "$code" = true ] && [ "$EVENT" = push ] && [ "$GITHUB_REF" = refs/heads/main ]; then+ for id in $(gh api "repos/$REPO/actions/workflows/ci.yml/runs?head_sha=$HEAD&event=workflow_dispatch&status=success&per_page=5" \+ --jq '.workflow_runs[] | select(.head_sha == "'"$HEAD"'" and .path == ".github/workflows/ci.yml" and .head_repository.full_name == "'"$REPO"'") | .id'); do+ vfx=$(gh api "repos/$REPO/actions/runs/$id/jobs?per_page=100" \+ --jq '[.jobs[] | select((.name | startswith("VFX Reference Platform")) and .conclusion == "success")] | length')+ if [ "${vfx:-0}" -ge 2 ]; then+ echo "::notice::$HEAD passed the full CI on staging (run $id, https://github.com/$REPO/actions/runs/$id): reused, not rebuilt"+ code=false+ break+ fi+ done+ fi+ # A release bump (scripts/release.sh step bump: only version lines, notes, docs, site; checked line by line by+ # tools/release/paperwork_only.py) on top of a main commit whose CI passed: that result stands for it.+ if [ "$code" = true ] && [ "$EVENT" = push ] && [ "$GITHUB_REF" = refs/heads/main ] && [ -f tools/release/paperwork_only.py ] &&+ python3 tools/release/paperwork_only.py "$BEFORE" "$HEAD"; then+ id=$(gh api "repos/$REPO/actions/workflows/ci.yml/runs?head_sha=$BEFORE&event=push&branch=main&status=success&per_page=5" \+ --jq '[.workflow_runs[] | select(.head_sha == "'"$BEFORE"'" and .path == ".github/workflows/ci.yml")][0].id // empty')+ if [ -n "$id" ]; then+ echo "::notice::only release paperwork since $BEFORE, whose CI passed (run $id): reused"+ code=false+ fi+ fiecho "code=$code" | tee -a "$GITHUB_OUTPUT"@@ -114,6 +148,7 @@ jobs:CCACHE_BASEDIR: ${{ github.workspace }}CCACHE_MAXSIZE: 500MCCACHE_DEPEND: 'true' # include files from the compiler's own dependency list (-MD, /showIncludes): no extra preprocessor run+ SF_TEST_BUDGET: 150 # seconds per test (tools/test/run.py); software OpenGL is several times slower than the PC's GPUsteps:- uses: actions/checkout@v7- uses: actions/setup-python@v7@@ -202,15 +237,17 @@ jobs:path: .ccachekey: ccache-ci-${{ matrix.key }}.${{ github.sha }}# Every test in one parallel ctest run (UI tests too, where there is a display), so no core idles while the- # last tests of one group finish.+ # last tests of one group finish. tools/test/run.py (docs/testing-flow.md): leaves out label nightly, reruns a+ # failed test once (a pass on the rerun is reported as flaky and doesn't fail the job), and fails a test slower+ # than SF_TEST_BUDGET seconds: 20 on the PC's GPU, scaled here for software OpenGL on 4-core runners.- name: Test (C++, Python API, UI tests with Xvfb + Mesa)if: runner.os == 'Linux'run: |xvfb-run -a -s "-screen 0 1920x1080x24" env LIBGL_ALWAYS_SOFTWARE=1 \- ctest --test-dir build -C Release --output-on-failure --output-junit ctest.xml --test-output-size-passed 262144 -j 4 -I '${{ matrix.shard }},,${{ matrix.shards }}'+ python3 tools/test/run.py --build build --junit build/ctest.xml -- -C Release -j 4 -I '${{ matrix.shard }},,${{ matrix.shards }}'- name: Test (C++ and Python API; UI tests need a display)if: runner.os == 'macOS'- run: ctest --test-dir build -C Release --output-on-failure --output-junit ctest.xml --test-output-size-passed 262144 -LE ui -j 4 -I '${{ matrix.shard }},,${{ matrix.shards }}'+ run: python3 tools/test/run.py --build build --junit build/ctest.xml -- -C Release -LE ui -j 4 -I '${{ matrix.shard }},,${{ matrix.shards }}'- name: Test (C++, Python API, UI tests with Mesa llvmpipe)if: runner.os == 'Windows'shell: pwsh@@ -223,7 +260,7 @@ jobs:}7z e mesa.7z x64/opengl32.dll x64/libgallium_wgl.dll -obuild/Release -y | Out-Null$env:GALLIUM_DRIVER = 'llvmpipe'- ctest --test-dir build -C Release --output-on-failure --output-junit ctest.xml --test-output-size-passed 262144 -j 4 -I '${{ matrix.shard }},,${{ matrix.shards }}'+ python tools/test/run.py --build build --junit build/ctest.xml -- -C Release -j 4 -I '${{ matrix.shard }},,${{ matrix.shards }}'if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }- name: Slowest tests (job summary; tells which test to split when a shard is the long pole)if: always()@@ -348,7 +385,7 @@ jobs:path: .ccachekey: ccache-ci-${{ matrix.cy }}.${{ github.sha }}- name: Test- run: ctest --test-dir build --output-on-failure -LE ui -j 4+ run: ctest --test-dir build --output-on-failure -LE 'ui|nightly' -j 4- name: Check binary compatibility (glibc <= ${{ matrix.glibc }})run: |for f in build/sculptforge build/sculptforge-cli build/sculptforge*.so; dodiff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.ymlnew file mode 100644index 00000000..ff2d1157--- /dev/null+++ b/.github/workflows/nightly.yml@@ -0,0 +1,40 @@+name: Nightly++# T3 of the test flow (docs/testing-flow.md): the ctest entries labelled `nightly` (TEST(nightly_*) cases, stress and+# benchmark checks), which CI on main and the merge queue leave out, once a day on Linux x86_64. The GPU-bound ones+# (label ui as well: heavy meshes, perf gates) need real graphics and run on the maintainers' PC instead+# (~/Work/.sf/nightly.sh); this job runs the rest. No secrets, read-only token, nothing is saved.+on:+ schedule:+ - cron: '17 9 * * *' # 09:17 UTC, after the US evening's merges+ workflow_dispatch:++permissions:+ contents: read++jobs:+ nightly:+ name: Nightly tests (Linux x86_64)+ runs-on: ubuntu-24.04+ timeout-minutes: 120+ env:+ CCACHE_DIR: ${{ github.workspace }}/.ccache+ CCACHE_BASEDIR: ${{ github.workspace }}+ steps:+ - uses: actions/checkout@v7+ - run: |+ sudo ./packaging/linux/install-build-deps.sh+ sudo apt-get install -y --no-install-recommends ccache+ - name: Compiler cache (restore main's)+ uses: actions/cache/restore@v5+ with:+ path: .ccache+ key: ccache-ci-linux-x86_64.${{ github.sha }}+ restore-keys: ccache-ci-linux-x86_64.+ - name: Build+ run: |+ cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DSCULPTFORGE_USE_SYSTEM_SDL=OFF \+ -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache+ cmake --build build --parallel 4+ - name: Nightly tests (not the GPU-bound ones)+ run: python3 tools/test/run.py --build build --budget 0 -- -L nightly -LE ui -j 2diff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex 7f2c89e9..28b1dd69 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -53,7 +53,7 @@ jobs:tag: ${{ steps.find.outputs.tag }}steps:- uses: actions/checkout@v7- with: { ref: '${{ env.TAG }}' }+ with: { ref: '${{ env.TAG }}', fetch-depth: 20 } # the commits before the bump, for the paperwork check- id: findenv:GH_TOKEN: ${{ github.token }}@@ -93,6 +93,15 @@ jobs:}ci=$(runs ci.yml || true)pkg=$(runs packages.yml || true)+ if [ -z "$ci" ] && [ -n "$pkg" ]; then+ # The bump commit adds only release paperwork (tools/release/paperwork_only.py: version lines, CHANGELOG,+ # docs, site, fragments) to a main commit whose CI passed: that result stands (scripts/release.sh step ci).+ for c in $(git rev-list --first-parent -n 8 "$sha~1"); do+ python3 tools/release/paperwork_only.py "$c" "$sha" || break+ ci=$(sha=$c; runs ci.yml || true)+ [ -n "$ci" ] && { echo "CI run $ci on $c stands for $sha (only release paperwork since)"; break; }+ done+ fiif [ -n "$ci" ] && [ -n "$pkg" ]; thenecho "Promoting the packages of Packages run $pkg (CI run $ci passed for $sha)"echo "run-id=$pkg" >> "$GITHUB_OUTPUT"diff --git a/CMakeLists.txt b/CMakeLists.txtindex ea9f2a18..c44c0bad 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -607,6 +607,12 @@ if(SCULPTFORGE_BUILD_TESTS)set_tests_properties(sculptforge_tests_${k}of${SCULPTFORGE_TEST_SHARDS} PROPERTIES COST 1000 # the longest: start firstENVIRONMENT "TMPDIR=${shard_tmp};TMP=${shard_tmp};TEMP=${shard_tmp}")endforeach()+ # T3 (docs/testing-flow.md): the TEST(nightly_*) cases, which no shard runs. Label nightly: tools/test/run.py and CI+ # leave it out; the nightly workflow and tools/test/nightly.sh run `ctest -L nightly`.+ add_test(NAME sculptforge_tests_nightly COMMAND sculptforge_tests --nightly)+ set_tests_properties(sculptforge_tests_nightly PROPERTIES LABELS nightly TIMEOUT 3600+ ENVIRONMENT "TMPDIR=${CMAKE_CURRENT_BINARY_DIR}/test-tmp/nightly;TMP=${CMAKE_CURRENT_BINARY_DIR}/test-tmp/nightly;TEMP=${CMAKE_CURRENT_BINARY_DIR}/test-tmp/nightly")+ file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test-tmp/nightly)# A stand-in for Brush, COLMAP and ffmpeg (tests/tools/fake_splat_tools.cpp): splat training is tested without a GPU.add_executable(sf_fake_splat_tools tests/tools/fake_splat_tools.cpp)target_include_directories(sf_fake_splat_tools PRIVATE third_party/stb)@@ -675,6 +681,7 @@ if(SCULPTFORGE_BUILD_TESTS)add_test(NAME marks_self_test COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/check_marks.py --self-test)add_test(NAME release_notes COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/release/assemble_notes.py --self-test)add_test(NAME release_notes_fragments COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/release/assemble_notes.py --check)+ add_test(NAME release_paperwork COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/release/paperwork_only.py --self-test)# scripts/release.sh's beta (X.Y.Z-beta.N) and its promotion to stable, in a sandbox with stand-ins for gh and curl.if(UNIX AND NOT APPLE AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/.git)add_test(NAME release_script_beta COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/tests/release_beta_test.sh)@@ -683,6 +690,8 @@ if(SCULPTFORGE_BUILD_TESTS)# The Actions cache pruner keeps the newest cache of each kind (tools/ci/prune_caches.py).add_test(NAME ci_prune_caches COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/ci/prune_caches.py --self-test)add_test(NAME ci_slowest_tests COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/ci/slowest_tests.py --self-test)+ add_test(NAME test_impact COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/test/impacted.py --self-test)+ add_test(NAME test_run COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/test/run.py --self-test)add_test(NAME benchmark_submit COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/benchmark/submit.py selftest)# The upstream watch (tools/upstream-watch, .github/workflows/upstream-watch.yml) against recorded API responses.add_test(NAME upstream_watch COMMAND ${Python3_EXECUTABLE} -I ${CMAKE_CURRENT_SOURCE_DIR}/tests/upstream/test_upstream_watch.py)@@ -708,6 +717,13 @@ if(SCULPTFORGE_BUILD_TESTS)add_test(NAME ui_${name} COMMAND sculptforge --ui-test ${script} --ui-out ${CMAKE_CURRENT_BINARY_DIR}/ui-shots/${name})set_tests_properties(ui_${name} PROPERTIES LABELS ui TIMEOUT 300)endforeach()+ # Heavy-mesh and benchmark scripts (tests/perf/*.sfui, e.g. 33M-triangle precision): nightly only (T3).+ file(GLOB SCULPTFORGE_PERF_TESTS CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/tests/perf/*.sfui)+ foreach(script ${SCULPTFORGE_PERF_TESTS})+ get_filename_component(name ${script} NAME_WE)+ add_test(NAME perf_${name} COMMAND sculptforge --ui-test ${script} --ui-out ${CMAKE_CURRENT_BINARY_DIR}/ui-shots/perf_${name})+ set_tests_properties(perf_${name} PROPERTIES LABELS "ui;nightly" TIMEOUT 3600 RUN_SERIAL TRUE)+ endforeach()# Splat training (also started by a phone scan) runs the fake Brush / COLMAP / ffmpeg.foreach(t ui_splat_train ui_phone_link)if(TEST ${t})@@ -715,6 +731,13 @@ if(SCULPTFORGE_BUILD_TESTS)"SCULPTFORGE_BRUSH=$<TARGET_FILE:sf_fake_splat_tools>;SCULPTFORGE_COLMAP=$<TARGET_FILE:sf_fake_splat_tools>;SCULPTFORGE_FFMPEG=$<TARGET_FILE:sf_fake_splat_tools>")endif()endforeach()+ # Label no_d3d12: tests that the Windows PC's WSL GPU path (Mesa's OpenGL on D3D12, ~/Work/.sf/pc.sh) can't run and+ # pc.sh leaves out. ui_micro_levels: its first micro-displacement frame makes D3D12 remove the device (a segfault,+ # and every UI test running beside it loses its GL context); it passes on CI's llvmpipe and on the same PC's native+ # NVIDIA driver (2026-11-03, v0.10.11, 128 frames), so it is the translation layer, not SculptForge.+ if(TEST ui_micro_levels)+ set_tests_properties(ui_micro_levels PROPERTIES LABELS "ui;no_d3d12")+ endif()# The hardware profile panel with a real calibration (tests/ui/hardware_profile.sfui).if(TEST ui_hardware_profile)set_tests_properties(ui_hardware_profile PROPERTIES ENVIRONMENT "SF_HW_PROFILE_IN_TESTS=1")diff --git a/REPORT.md b/REPORT.mdnew file mode 100644index 00000000..d3dbd546--- /dev/null+++ b/REPORT.md@@ -0,0 +1,87 @@+# feat/test-flow: tiered testing, test cache, faster release++Branch `feat/test-flow` (from main 058dba75). Plan and rules: docs/testing-flow.md. Machine scripts in ~/Work/.sf+(not in git; old versions in ~/Work/.sf/archive): pc.sh, t0.sh, t1.sh, merge-batch.sh, sfwatch.sh, nightly.sh.+Delegated to herdr agents (own worktrees, own REPORTs): **warmui** (feat/warm-ui-runner, persistent UI runner, still+working) and **linkspeed** (feat/link-speed, pushed 79f77aa8: mold, PCH, test-binary linking).++## Numbers (PC = RTX 3060 in WSL, ctest -j10; laptop = aarch64 Arch)++| | before | after |+|---|---|---|+| **One-file edit → test result** (PC free) | full suite: build + 240 s of tests (282 s in your run), or ~33 min on the laptop (software GL) | **T0: 31 s** for an app .cpp (build 9 s + 4 tests 12 s); **71 s** for App.h (build 49 s); core .cpp: 24 tests, 49 s of tests cold, 5 s with the cache |+| Waiting for the PC (6+ sessions share it) | 107-1320 s observed tonight | the same queue: the biggest cost left. sfwatch uses only idle time, and T0/the cache make each job short |+| **Full suite (T1)** | 240 s of tests, 175 tests, 4 failing (all GPU-loss related) | cold **248 s**, green (172 tests; 2,235 test-seconds, the PC is busy the whole time). **Warm (nothing changed): 7 s**, 161 of 171 skipped by the cache. A batch reruns only what it changed |+| **Merge batch** | laptop build + full ctest -j3 (~33 min), then main CI (16.6-20 min) | T1 on the PC (≈4 min) **in parallel with** full CI on `staging` (≈17-20 min); main fast-forwards; main's CI reuses the staging run (≈1 min). **≈50 → ≈20 min** |+| **Release, bump → published** | 0.10.11: 24.7 min, 20.3 of it waiting for CI on the bump | CI wait **20.3 min → 0.8 s** (reuse check against real 0.10.11 data). Packages on the bump commit (17-20 min) is now the long pole: **≈20 min** (-4 to -5). Proposal below for ≈5 min |++## What changed++- **T0** `tools/test/impacted.py`: changed files → `ctest -R` regex, from the build graph (Ninja `-t deps` or the+ Makefile `.o.d`/`link.txt`), a test's own files, UI data, a hand map, and for app files name words + co-change+ history + a smoke set. 0.2 s. `~/Work/.sf/t0.sh` (PC, else laptop); `SF_T0_BASE=HEAD` = only uncommitted edits.+- **tools/test/run.py**, the one ctest runner for every tier:+ - **content-hash test cache** (Tony's item 1): local on the PC, never on CI.+ - **one rerun**, with flakes logged to tests/flaky.md (one real flake so far: sculptforge_tests_9of16).+ - **20 s budget** with the tests/slow_tests.txt ratchet (34 entries, each with its highest measured seconds and a plan; near 20 s a test varies by ±5 s with the PC's load).+ - **nightly exclusion**, and **GPU-lost** classification (exit 4).+ - `--threads-per-test`: each test process gets its share of the cores.+- **T1/T2** `scripts/merge-queue.sh staging --with ~/Work/.sf/t1.sh HEAD`: the batch goes to `staging`, the full CI runs+ there and T1 runs on the PC; main fast-forwards when both pass. I tested it in a sandbox (green lands, red CI holds,+ red T1 holds), but **not live yet**: the first real batch will be merge0109's. merge-batch.sh now uses it (old path+ kept for older main). ci.yml: main reuses a full staging run of the same commit; main's runs are never cancelled.+- **Release**: `tools/release/paperwork_only.py` (exact files and lines, fails safe) lets release.sh step `ci` and+ release.yml `locate` reuse the parent's green CI. Step `check` runs the bump's own tests. ci.yml's run on the bump+ also reuses it, which frees runners for Packages.+- **T3**: `TEST(nightly_*)` → `sculptforge_tests_nightly`; tests/perf/*.sfui → `perf_*`; .github/workflows/nightly.yml+ (daily, the non-GPU part); ~/Work/.sf/nightly.sh (by hand, no cron: PC GPU, no-cache full run, floor tier on the+ laptop at load < 2, Windows desktop idle and calibrate). 6 stress and perf-gate cases moved to nightly (list in+ slow_tests.txt).+- **Speculative CI** (Tony's item 3): ~/Work/.sf/sfwatch.sh, **running in herdr tab `sfwatch`**. It runs T0 on every+ new worktree commit and T1 on a new main, only while the PC is idle. Results go to ~/Work/.sf/watch/<wt>.md; a red+ result is sent to that worktree's herdr agent. t0.sh reuses its result for the same clean commit.+- **ccache** (Tony's item 5): the PC now shares ccache across all worktrees (`CCACHE_BASEDIR=/root/wt`; before, every+ worktree missed: 10 % hits); linkspeed measured a warm clean rebuild at 7 s with 868/868 hits. Laptop ↔ PC can't+ share: aarch64 vs x86_64. **CI ↔ PC: not done.** It needs your OK, because objects built off GitHub would flow+ into release packages (a cache-poisoning surface).+- pc.sh: per-worktree dependencies (the shared /root/fc build was rebuilt or clobbered by other worktrees), one PC job+ at a time, a per-worktree lock, a fixed `-R '^x$'` quoting bug (my earlier edit), times/JUnit copied back,+ `SF_PC_ENV`, wait time printed.+- `SF_UI_PROFILE=1`: a UI test prints its slowest script lines. ui_fill_layers now uses 1024 textures (was 2048).+- IP: the UI test renames you asked for were already on main (fix/ip-purge-2: classic_feel, forgespheres, library,+ parts, parts_files, custom_litspheres), so this branch keeps those and adds none of its own.+- ~/Work/CLAUDE.md Gate and Lean lines updated (59 lines); docs/releasing.md and docs/merge-queue.md updated.++## Your questions++- **ui_micro_levels SEGFAULT: D3D12/WSL-specific.** Its first micro-displacement frame makes D3D12 remove the device.+ It passes on the same PC's native NVIDIA driver (v0.10.11, `PASS (128 frames)`) and on CI's llvmpipe. It now has+ the label `no_d3d12`, which pc.sh leaves out, with the reason in CMakeLists.txt.+- **ui_update_check: not a bug.** It fails only when the GPU is gone and passes on the rerun (run.py logs+ "gpu: … not a flake"). ui_texture_start, ui_timeline and ui_retopo failed the same way: they ran beside the device+ loss.+- **Real bug found, fixed here:** after a failed init (no window or no OpenGL), the app crashed in shutdown instead of+ exiting with its error (seen on the PC during a GPU outage). Fixed in App.cpp; reasoned from the code, not+ reproduced (SDL falls back to software GL).++## Watch++- **The PC is unstable under heavy load.** Three compiler crashes tonight ("cc1: Bus error", "internal compiler+ error: Segmentation fault" on trivial lines) and several GPU losses, all while builds overlapped tests or another+ agent's build (load 56-65). Now serialized; if crashes come back at normal load, check its RAM, heat and+ overclock.+- slow_tests.txt is a to-do list with a plan per line. ui_uv_editor (116 s) and insider_drops (100 s) are the top+ two.++## Next (proposal)++- **Release ≈5 min after the release batch lands.** Run Packages on `staging` too, and let release.yml promote+ that run (same commit). Include the bump commit in the release batch. Then Packages runs in parallel with CI+ instead of after the bump. This changes the coordinator's bump step, so it's their call.++Tests: T1 on the PC green at 74d8e424 (143 run, 28 cached, 0 failed; the 2 tests over budget there were then listed+in a222de07). Full CI on the branch with full=true is green: run 41807683356, 20.9 min, every platform plus the VFX+Reference Platform and Blender jobs. run.py ran on every runner: no flakes, nothing over the 150 s CI budget.+Self-tests test_impact, test_run and release_paperwork pass; release_script_beta passes.+Rebased as **feat/test-flow-2** on origin/main d614f533 (after fix/ip-purge-2; feat/test-flow is superseded).+~/Work/.sf scripts are already live. Merge after 0.11.0-beta.1 is cut (coordinator), as a batch of its own.diff --git a/changes/feat-test-flow.md b/changes/feat-test-flow.mdnew file mode 100644index 00000000..ae313dba--- /dev/null+++ b/changes/feat-test-flow.md@@ -0,0 +1,9 @@+---+changed: **Testing flow** (docs/testing-flow.md): each change is tested once per tier instead of four times. T0 is+ `tools/test/impacted.py`, the tests a change can break, from the build graph. T1 is the whole suite once per merge+ batch on a GPU machine, in parallel with T2, CI on the `staging` branch; main fast-forwards only when both pass, and+ CI on main reuses that run. T3 is a nightly label and workflow for benchmarks and heavy meshes. `tools/test/run.py`+ wraps ctest with a content-hash test cache, one rerun for flaky tests (logged in tests/flaky.md), and a 20-second+ budget per test. A release bump that only adds paperwork reuses CI instead of waiting for it again.+---+Contributor-facing only; nothing in the app changes.diff --git a/docs/merge-queue.md b/docs/merge-queue.mdindex df9662a6..61955a8d 100644--- a/docs/merge-queue.md+++ b/docs/merge-queue.md@@ -8,7 +8,7 @@ same job.)1. Branch from `origin/main`, one note fragment `changes/<branch>.md` instead of CHANGELOG / What's New edits([changes/README.md](../changes/README.md)).-2. Full ctest green locally (`~/Work/.sf/heavy.sh`), push the branch, `gh pr create --fill`.+2. T0 green (`~/Work/.sf/t0.sh`, docs/testing-flow.md), push the branch, `gh pr create --fill`.3. The PR's CI runs the fast set (Linux x86_64 and ARM64, macOS, Windows over 4 runners; about 10 minutes). Add thelabel `full-ci` for the VFX Reference Platform and Blender add-on jobs too (build or packaging changes).4. `scripts/merge-queue.sh add <pr>`, write REPORT.md, tell the coordinator.@@ -22,6 +22,11 @@ comment and the `queue-failed` label; its author fixes the branch and queues it`--serial` (update each PR from main, wait for its checks, merge), which isolates the PR that broke it.`scripts/merge-queue.sh list` shows the queue.+The merger's own batch (branches merged without pull requests, `~/Work/.sf/merge-batch.sh`) lands the same way with+`scripts/merge-queue.sh staging --with ~/Work/.sf/t1.sh HEAD`. The batch goes to the branch `staging`, the full CI runs+there while the PC runs the whole suite, and main fast-forwards when both pass. Either way, CI's run on main finds the+full run of the same commit and reuses it rather than building everything a second time (docs/testing-flow.md).+Cut, don't wait: what is green at cut time lands; the rest rides the next batch.## Timing checks on CIdiff --git a/docs/releasing.md b/docs/releasing.mdindex 82e697d4..c3c7c783 100644--- a/docs/releasing.md+++ b/docs/releasing.md@@ -11,8 +11,10 @@ the gates, what each step does, and how to recover.## Gates (in this order; a release never skips one)-1. **Merged tree, tested locally**: on main with the branches for this release merged, `ctest --test-dir build -j3`- all green.+1. **Merged tree, tested once on each machine** (docs/testing-flow.md): the merger's batch goes to the branch+ `staging` (`scripts/merge-queue.sh staging --with ~/Work/.sf/t1.sh HEAD`). The whole suite runs on the PC (T1) while+ the full CI runs on staging (T2), and main fast-forwards to the batch only when both are green. Nobody runs a+ second local full ctest.2. **Version and words**:- written by people, on main before the release: one note fragment per branch, `changes/<branch>.md`(changes/README.md: the What's New bullet, CHANGELOG entry and section, highlight, #testers bullets); in@@ -27,7 +29,11 @@ the gates, what each step does, and how to recover.Step `check` verifies all of this, plus what the steps after the tag need (PKGBUILD format, the Homebrew tap, theOmarchy PR is open, the home page's version text), so nothing stops after the tag for a reason known before it.-3. **CI and Packages green on main** for exactly that commit:+3. **CI and Packages green on main** for exactly that commit. CI's run on main reuses the staging run of the same+ commit. For the bump commit, step `ci` and release.yml reuse the CI result of the commit before it when the bump+ only added release paperwork (`tools/release/paperwork_only.py`: exact files and lines, fails safe). Step `check` has+ already run the bump's own tests (What's New, the guide, notes) on the bumped tree. Packages still has to pass for the+ bump commit itself:- **CI** (ci.yml) runs every test on Linux x86_64 and ARM64, macOS, Windows, the VFX platforms and Blender.- **Packages** (packages.yml) builds every release package the way it ships, with its tests.4. **Packaged-artifact smoke tests** (release.yml `smoke`): every package is installed or unpacked on a fresh runner.diff --git a/docs/testing-flow.md b/docs/testing-flow.mdnew file mode 100644index 00000000..99d613b2--- /dev/null+++ b/docs/testing-flow.md@@ -0,0 +1,102 @@+# Testing flow++Each assurance is paid for once, by the cheapest machine that can give it. Before this flow, one change was tested+four times: by the agent, by the merger's full local ctest, by CI on main, and by CI again on the release bump.++| tier | when | where | what | target |+|---|---|---|---|---|+| **T0 impact** | every agent commit | the PC (laptop if the PC sleeps) | build + the tests the change can break | < 1 min |+| **T1 batch** | once per merge batch | the PC, while T2 runs | the whole suite, -j10 on the GPU, test cache on | < 5 min |+| **T2 platforms** | once per merge batch | CI on the branch `staging` | every platform, VFX Reference Platform, Blender | the gate for main |+| **T3 nightly** | once a day | CI (scheduled) + the PC | label `nightly`: benchmarks, stress, heavy meshes, floor-tier perf | |++## T0: what can this change break?++`tools/test/impacted.py` turns the changed files (branch vs `origin/main`, plus uncommitted ones) into a+`ctest -R` regex:++1. **build graph**: `build.ninja` plus `ninja -t deps`, so headers count. A file maps to the objects that include it,+ then to the libraries and programs that link them, then to the tests that run those programs. A core header+ reaches every `sculptforge_tests` shard, `sculptforge-cli` and the Python API.+2. **a test's own files**: the script it runs, any file its command names, and that file's folder.+3. **UI data**: scripts that name a changed `tests/ui/data` file.+4. **hand map** (top of impacted.py): docs, site, release scripts, note fragments, packaging.+5. **app changes** (every UI test runs the app, so the graph alone would pick all of them): UI tests whose name+ shares a word with the changed file (`PanelMicro.cpp` picks `ui_micro_levels`), UI tests changed in the same+ commits as it in the last 600 commits, and a smoke set.++`marks` (the IP check) always runs. CMake files mean everything runs (`.`). `-v` explains each pick.+Run it with `~/Work/.sf/t0.sh`: `pc.sh` builds on the PC, runs impacted.py there against the PC's build graph, then+runs the tests through `tools/test/run.py`.++## T1 and T2: one batch, two machines at once++The merger (`~/Work/.sf/merge-batch.sh`) merges the green branches, then calls+`scripts/merge-queue.sh staging --with ~/Work/.sf/t1.sh HEAD`:++- the batch commit is pushed to `staging` and the full CI runs there (`workflow_dispatch`, `full=true`);+- at the same time, `t1.sh` runs the whole suite on the PC;+- when both are green, main fast-forwards to exactly that commit. When either is red, main doesn't move.++The PC runs one job at a time, build and tests together. A build overlapping another job's tests drove the+load to 65 on 12 threads and the compiler crashed. Each worktree fetches and builds its own dependencies+(`/root/wt/<name>/fc`); ccache (`CCACHE_BASEDIR=/root/wt`) is shared, so another worktree's objects are cache hits.++CI on main then doesn't build it all again. The `Changes` job sees that this exact commit passed a full run on+staging, with both VFX jobs green, and passes as reused. The merger no longer runs a local full ctest.++## Release: the bump reuses CI++`scripts/release.sh` step `ci` used to wait about 20 minutes for CI on the bump commit. That commit only adds+paperwork to a commit CI had already passed. `tools/release/paperwork_only.py BASE HEAD` checks this file by file+and line by line: the version lines, CHANGELOG, Markdown docs, the generated site, deleted fragments. Anything else+fails safe. When the check passes, step `ci` and release.yml (job `locate`) take the earlier commit's CI result.+On main, a newer push never cancels a run that is still going (ci.yml `concurrency`), so that earlier result exists. The+bump's own Packages run still has to pass, because the version is built into the app. Step `check` runs the bump's+own tests (What's New, the guide, notes) on the bumped tree.++## tools/test/run.py: cache, flakes, budget++Every tier runs ctest through `tools/test/run.py --build build [-- ctest args]`:++- **Nightly** tests (label `nightly`) run only when the ctest arguments select them (`-L nightly`).+- **Test cache** (`--cache DIR`, `$SF_TEST_CACHE`; the PC keeps `/root/.cache/sf-test`): every test gets a+ fingerprint, a hash of everything it can read. That covers its program, every file or folder its command names, the app's assets and+ test data for app tests, `tests/data` for the unit tests, its ENVIRONMENT, and the `SF_`, `SCULPTFORGE_`,+ `SDL_` and GL driver variables. Script tests that nobody listed in `READS` hash every tracked file. A fingerprint+ that passed on the first try is skipped next time, so a rerun of the full suite runs only what changed. The cache+ is never used on CI, and the nightly run on the PC runs everything without it once.+- **Flakes**: failed tests run once more. A test that passes on the rerun is logged as `flaky: name`, added to+ `tests/flaky.md` (by pc.sh, t0.sh and t1.sh), and doesn't fail the run. A test listed there twice gets the label+ `flaky`, and a failure then never blocks a release. Its owner fixes it or deletes it.+- **Budget**: a test slower than `--budget` seconds fails the run. For the `sculptforge_tests` shards, each TEST+ case counts on its own. The default is 20 s on the PC's GPU; CI sets `SF_TEST_BUDGET=150` for software OpenGL on+ 4-core runners. `tests/slow_tests.txt` lists the tests over budget today, with their PC seconds and a plan for+ each. A listed test may grow to 1.5× its seconds (scaled by budget ÷ 20 on CI), so the list can only shrink. A+ slow test should wait on events rather than fixed frames, use a smaller mesh or texture, be split, or be labelled+ nightly.+- **Where the time goes**: `SF_UI_PROFILE=1` makes a UI test print its slowest script lines with real seconds+ (`SF_PC_ENV=SF_UI_PROFILE=1 ~/Work/.sf/pc.sh . -R ui_bake`).+- **Parallel runs** (`ctest -jN`, N > 1): timing checks get CI's slack (`SF_TIME_LIMIT_SCALE=4`, `timeLimit()` in+ tests/TestHarness.h). With `--threads-per-test auto` (pc.sh), each test process gets its share of the cores+ (`SF_THREADS`, core/ThreadPool.cpp), instead of ten pools each the size of the machine.+- **GPU lost**: a failure whose output says the graphics device went away is the machine, not the code (exit 4).+ pc.sh says so, and such a failure is never logged as a flake. Tests that the PC's WSL GPU path (Mesa on D3D12) can't+ run carry the label `no_d3d12`, which pc.sh leaves out. ui_micro_levels makes D3D12 remove the device, yet passes on+ CI and on the same PC's native driver.++## T3 nightly++- `TEST(nightly_*)` cases (tests/test_main.cpp) are left out of every shard. They run in+ `sculptforge_tests_nightly`. The `tests/perf/*.sfui` scripts are `perf_*` (labels ui and nightly).+- `.github/workflows/nightly.yml` runs the nightly tests that don't need a GPU (`-L nightly -LE ui`) once a day.+- `~/Work/.sf/nightly.sh` is run by hand at the end of the day, not from cron. It runs `ctest -L nightly` on the PC's GPU, the whole suite with no+ cache, floor-tier stroke benchmarks on the laptop (`SF_HW_SIMULATE=floor`, only at load < 2), and an idle and+ calibration check on the Windows desktop (`win.sh`). The summary goes to `~/Work/.sf/nightly/<date>.md`.++## Speculative CI++`~/Work/.sf/sfwatch.sh` runs in the background and tests every new commit as it lands. A worktree's HEAD gets T0+and a new `origin/main` gets T1. Results go to `~/Work/.sf/watch/<worktree>.md`. A red result is sent to the+worktree's herdr agent. `t0.sh` prints that result instead of running again when the commit matches and the tree is+clean. By the time an agent asks, the answer is usually there.diff --git a/scripts/merge-queue.sh b/scripts/merge-queue.shindex b1df6e9f..4f766e9a 100755--- a/scripts/merge-queue.sh+++ b/scripts/merge-queue.sh@@ -9,6 +9,11 @@# scripts/merge-queue.sh add <pr>... queue pull requests (label "queue"); any session may do this# scripts/merge-queue.sh list the queue, oldest first, with each PR's check state# scripts/merge-queue.sh run [--serial] land the queue (the coordinator, in the background)+# scripts/merge-queue.sh staging [--with CMD] [REF] the merger's batch (T2, docs/testing-flow.md): REF (default HEAD,+# main merged into it) goes to the branch `staging`, the full CI runs there+# while CMD runs here (e.g. '~/Work/.sf/pc.sh .', T1), and when both pass main+# fast-forwards to exactly that commit. CI's run on main then reuses the+# staging result (ci.yml, job Changes) instead of building it all again.## run, batch mode (the default): main plus every queued PR merged in queue order on a branch queue/batch-<time># (a PR that conflicts, or whose own checks are red, is dropped with a comment and the "queue-failed" label), the@@ -132,7 +137,40 @@ run_batch() {done}+land_staging() { # land_staging [--with CMD] [REF]+ local with="" ref sha pid="" ci t0 local_rc=0+ if [ "${1:-}" = --with ]; then with=$2; shift 2; fi+ ref=${1:-HEAD}+ sha=$(git rev-parse "$ref^{commit}") || die "no commit $ref"+ git fetch -q origin main+ git merge-base --is-ancestor origin/main "$sha" ||+ die "$ref does not contain origin/main: merge origin/main into it first (main must fast-forward to the tested commit)"+ [ "$sha" != "$(git rev-parse origin/main)" ] || { echo "main is already at ${sha:0:8}"; return 0; }+ t0=$(date +%s)+ say "staging ${sha:0:8} ($(git log -1 --format=%s "$sha" | cut -c1-60)) on main $(git rev-parse --short origin/main)"+ git push -q --force origin "$sha:refs/heads/staging"+ git fetch -q origin staging+ if [ -n "$with" ]; then+ bash -c "$with" > "/tmp/merge-queue-with-${sha:0:8}.log" 2>&1 &+ pid=$!+ echo " alongside: $with (log /tmp/merge-queue-with-${sha:0:8}.log)"+ fi+ ci=$(wait_ci staging)+ if [ -n "$pid" ]; then wait "$pid" || local_rc=$?; tail -5 "/tmp/merge-queue-with-${sha:0:8}.log" | sed 's/^/ /'; fi+ echo " CI on staging: $ci; local: $([ $local_rc = 0 ] && echo ok || echo "failed ($local_rc)"); $(( $(date +%s) - t0 )) s"+ [ "$ci" = success ] && [ $local_rc = 0 ] || die "staging is red: main stays at $(git rev-parse --short origin/main); fix the batch and run again"+ git fetch -q origin main+ git merge-base --is-ancestor origin/main "$sha" ||+ die "main moved while staging ran ($(git rev-parse --short origin/main)): merge it in and run again"+ git push -q origin "$sha:refs/heads/main" # a fast-forward: exactly the tree CI passed+ say "main is now ${sha:0:8}"+}+case ${1:-} in+staging)+ shift+ land_staging "$@"+ ;;add)shift[ $# -gt 0 ] || die "usage: scripts/merge-queue.sh add <pr>..."@@ -152,7 +190,7 @@ run)if [ "${2:-}" = --serial ]; then run_serial; else run_batch; fi;;*)- sed -n '5,18p' "$0" | sed 's/^# \{0,1\}//'+ sed -n '5,23p' "$0" | sed 's/^# \{0,1\}//'exit 2;;esacdiff --git a/scripts/release.sh b/scripts/release.shindex 255e82ac..84c64371 100755--- a/scripts/release.sh+++ b/scripts/release.sh@@ -132,6 +132,24 @@ wait_run() {"fix it on a branch, merge to main, then start again from step check (or re-run the run if it was a runner hiccup: gh run rerun $id --failed -R $REPO)"echo "$id"}+# ci_reused SHA: prints "<ancestor> <run id>" when SHA differs from a main commit with a successful CI push run only by+# release paperwork (the bump commit: tools/release/paperwork_only.py checks the exact files and lines), so its CI+# stands for SHA and the release doesn't wait 20 minutes for the same answer. Fails safe: anything else, keep waiting.+# The same rule is in release.yml (job locate), which then promotes SHA's own Packages run.+ci_reused() {+ local sha=$1 c id+ for c in $(git rev-list --first-parent -n 8 "$sha~1" 2>/dev/null); do+ python3 tools/release/paperwork_only.py "$c" "$sha" >/dev/null 2>&1 || return 1+ id=$(gh run list -R "$REPO" -w ci.yml -c "$c" -e push -b main -L 1 --json databaseId -q '.[0].databaseId' 2>/dev/null)+ [ -n "$id" ] || continue+ if [ "$(gh run view "$id" -R "$REPO" --json status -q .status)" != completed ]; then+ echo " waiting for ci.yml run $id on ${c:0:8} (the commit before the bump) ..." >&2+ gh run watch "$id" -R "$REPO" --exit-status >/dev/null 2>&1 || true+ fi+ [ "$(gh run view "$id" -R "$REPO" --json conclusion -q .conclusion)" = success ] && { echo "${c:0:8} $id"; return 0; }+ done+ return 1+}# wait_live URL TEXT SECONDS: true once the live page contains TEXT. Pages deploys lag, so it polls every 15 s.# grep reads the whole page: grep -q would close the pipe early and curl would fail under pipefail.wait_live() {@@ -185,6 +203,18 @@ dpi_ok() {"${heavy[@]}" cmake --build "$b" --target sculptforge -j3 >/dev/null || die "the app doesn't build in $b" "cmake --build $b --target sculptforge and fix the error"SDL_APP_ID=sculptforge-agent "${heavy[@]}" ctest --test-dir "$b" -L dpi --output-on-failure >"$b/release-dpi.log" 2>&1 ||die "the UI scale / DPI tests fail (ctest -L dpi; log: $b/release-dpi.log)" "fix them on a branch, merge to main, run again from step check"+ # What the bump changed (What's New, guide, notes) is tested here, on the bumped tree: step ci may reuse the CI+ # result of the commit before the bump (ci_reused), which never saw these files.+ local re green+ if green=$(git rev-list --first-parent -n 1 --grep '^Release \|^Beta ' --invert-grep HEAD 2>/dev/null) && [ -n "$green" ]; then+ re=$(python3 tools/test/impacted.py --build "$b" --base "$green" 2>/dev/null || true)+ if [ -n "$re" ] && [ "$re" != . ]; then+ "${heavy[@]}" cmake --build "$b" -j3 >/dev/null || die "the build in $b fails" "cmake --build $b and fix the error"+ SDL_APP_ID=sculptforge-agent "${heavy[@]}" ctest --test-dir "$b" -R "$re" --output-on-failure >"$b/release-bump.log" 2>&1 ||+ die "the tests of what the bump changed fail (ctest -R '$re'; log: $b/release-bump.log)" "fix the notes or the guide, commit, run again from step check"+ ok "the bump's own tests pass on the bumped tree ($re)"+ fi+ fiok "UI scale tests (100-200 %, display-scale event) pass; Windows manifest declares PerMonitorV2"}# Everything the steps after the tag need that doesn't depend on the release existing. The home page card is@@ -381,9 +411,14 @@ step_ci() {local shasha=$(git rev-parse HEAD)if git rev-parse -q --verify "refs/tags/$T" >/dev/null; then ok "$T is tagged: CI was checked before tagging"; return; fi- # bump may have pushed a moment ago: give the run 3 minutes to appear- wait_run ci.yml "$sha" 18 >/dev/null || die "no CI run on main for $sha after 3 minutes" "push main first (git push origin main), or check Actions, then run again"- ok "CI passed on $sha"+ local green+ if green=$(ci_reused "$sha"); then+ ok "CI: ${sha:0:8} only adds release paperwork to ${green%% *}, whose CI run ${green##* } passed (tools/release/paperwork_only.py): reused"+ else+ # bump may have pushed a moment ago: give the run 3 minutes to appear+ wait_run ci.yml "$sha" 18 >/dev/null || die "no CI run on main for $sha after 3 minutes" "push main first (git push origin main), or check Actions, then run again"+ ok "CI passed on $sha"+ fiif wait_run packages.yml "$sha" >/dev/null; thenok "the Packages run passed: the release will promote its tested packages"elsediff --git a/src/app/App.cpp b/src/app/App.cppindex b8d5f790..c15a9318 100644--- a/src/app/App.cpp+++ b/src/app/App.cpp@@ -256,10 +256,17 @@ bool App::init(const AppOptions& options) {ai::defaultGeneratorCommand(std::string(base) + cliName).c_str());}initHardwareProfile(); // the cached hardware profile, or calibration over the first frames (PanelPerformanceProfile.cpp)+ initialised_ = true;return true;}void App::shutdown() {+ if (!initialised_) { // init stopped early (no window or no OpenGL): nothing but SDL to take down+ if (glContext_) SDL_GL_DestroyContext(glContext_);+ if (window_) SDL_DestroyWindow(window_);+ SDL_Quit();+ return;+ }saveWindowState();if (doc_.sculptor.strokeActive()) endStroke();solver_.reset(); // (its thread uses the renderer)@@ -404,6 +411,7 @@ int App::runUiTest(int w, int h, int fps) {if (!img.savePng(shot, err)) SDL_Log("%s", err.c_str());else SDL_Log("Screenshot %s", shot.c_str());}+ demo_->printProfile();for (const auto& f : demo_->failures()) std::fprintf(stderr, "FAIL %s\n", f.c_str());if (demo_->failures().empty())std::printf("PASS %s (%d frames)\n", options_.uiTest.c_str(), frames);diff --git a/src/app/App.h b/src/app/App.hindex 5996c37a..dddb1ec0 100644--- a/src/app/App.h+++ b/src/app/App.h@@ -236,6 +236,7 @@ private:SDL_Window* window_ = nullptr;SDL_GLContext glContext_ = nullptr;+ bool initialised_ = false; // init() got to the end (shutdown() takes down only what it made otherwise)bool running_ = true;std::string iniPath_;diff --git a/src/app/Demo.cpp b/src/app/Demo.cppindex e249a761..42ca653a 100644--- a/src/app/Demo.cpp+++ b/src/app/Demo.cpp@@ -8,6 +8,7 @@#include <imgui_internal.h>#include <algorithm>+#include <cstdio>#include <cstdlib>#include <cmath>#include <cstdarg>@@ -211,6 +212,12 @@ void Demo::update(float dt) {SDL_Log("t=%.2f step=%zu brushlab=(%.0f,%.0f) mouse=(%.0f,%.0f) down=%d", clock_, step_, w->Pos.x, w->Pos.y, mouse_.x,mouse_.y, int(buttonDown_));stride_ = 1;+ if (!stepLines_.empty()) { // SF_UI_PROFILE: the frame just drawn belongs to the step that was running+ const auto now = std::chrono::steady_clock::now();+ if (lastUpdate_.time_since_epoch().count() != 0 && step_ < stepLines_.size())+ lineSeconds_[size_t(stepLines_[step_])] += std::chrono::duration<double>(now - lastUpdate_).count();+ lastUpdate_ = now;+ }while (step_ < steps_.size()) {if (!steps_[step_](stepTime_)) {stepTime_ += dt;@@ -349,6 +356,23 @@ float Demo::fade() const {// ---------------------------------------------------------------- building blocks+void Demo::printProfile() const {+ if (stepLines_.empty()) return;+ std::vector<int> order;+ double total = 0;+ for (size_t i = 0; i < lineSeconds_.size(); ++i) {+ total += lineSeconds_[i];+ if (lineSeconds_[i] > 0) order.push_back(int(i));+ }+ std::sort(order.begin(), order.end(), [this](int a, int b) { return lineSeconds_[size_t(a)] > lineSeconds_[size_t(b)]; });+ std::printf("profile %s: %.2f s in script lines; slowest:\n", scriptName_.c_str(), total);+ for (size_t k = 0; k < order.size() && k < 12; ++k) {+ const int n = order[k];+ std::printf("profile %6.2f s line %d: %s\n", lineSeconds_[size_t(n)], n,+ size_t(n) < scriptLines_.size() ? scriptLines_[size_t(n)].c_str() : "(start or end)");+ }+}+void Demo::wait(float seconds) {steps_.push_back([seconds](float t) { return t >= seconds; });}diff --git a/src/app/Demo.h b/src/app/Demo.hindex a2c9b102..8a63ec36 100644--- a/src/app/Demo.h+++ b/src/app/Demo.h@@ -20,6 +20,7 @@#include <SDL3/SDL.h>+#include <chrono>#include <functional>#include <map>#include <string>@@ -74,6 +75,8 @@ public:lastShotH_ = h;}+ void printProfile() const; // the slowest script lines (SF_UI_PROFILE=1), after the script ran+private:using Step = std::function<bool(float t)>; // returns true when done; t = seconds since the step began@@ -132,6 +135,11 @@ private:std::vector<Step> steps_;size_t step_ = 0;float stepTime_ = 0.0f;+ // SF_UI_PROFILE=1 (docs/testing-flow.md): the real time each script line took, printed when the script ends.+ std::vector<int> stepLines_; // the script line each step came from+ std::vector<std::string> scriptLines_; // the script's lines, for the report+ std::vector<double> lineSeconds_; // real seconds spent on each line+ std::chrono::steady_clock::time_point lastUpdate_{};int stride_ = 1;ImVec2 mouse_{960, 540};bool buttonDown_ = false;diff --git a/src/app/UiTest.cpp b/src/app/UiTest.cppindex 1fff7956..5666eabd 100644--- a/src/app/UiTest.cpp+++ b/src/app/UiTest.cpp@@ -2433,12 +2433,15 @@ bool Demo::loadScript(const std::string& path, std::string& error) {std::filesystem::create_directories(std::filesystem::path(std::u8string(shotDir_.begin(), shotDir_.end())), ec);}mouse_ = ImVec2(float(app_.options_.width) * 0.5f, float(app_.options_.height) * 0.5f);+ const bool profile = std::getenv("SF_UI_PROFILE") != nullptr;wait(0.2f); // first frames: windows find their placesstd::string line;int n = 0;+ if (profile) scriptLines_.push_back("");while (std::getline(f, line)) {++n;if (!line.empty() && line.back() == '\r') line.pop_back();+ if (profile) scriptLines_.push_back(line);auto words = splitWords(line);if (words.empty()) continue;lineText_ = line;@@ -2447,8 +2450,13 @@ bool Demo::loadScript(const std::string& path, std::string& error) {error = scriptName_ + ":" + std::to_string(n) + ": " + err;return false;}+ if (profile) stepLines_.resize(steps_.size(), n); // (steps the line added; the first wait counts as line 0)}wait(0.1f);+ if (profile) {+ stepLines_.resize(steps_.size(), 0);+ lineSeconds_.assign(scriptLines_.size(), 0.0);+ }return true;}diff --git a/src/core/ThreadPool.cpp b/src/core/ThreadPool.cppindex 92e6f31d..7af0a6e0 100644--- a/src/core/ThreadPool.cpp+++ b/src/core/ThreadPool.cpp@@ -2,6 +2,8 @@// SPDX-License-Identifier: GPL-3.0-or-later#include "core/ThreadPool.h"+#include <cstdlib>+namespace sf {namespace {@@ -34,7 +36,10 @@ bool ThreadPool::setSharedThreadCount(int threads) {ThreadPool& ThreadPool::shared() {static ThreadPool pool([] {g_sharedMade.store(true);- const int wanted = g_sharedThreads.load();+ int wanted = g_sharedThreads.load();+ // SF_THREADS: a parallel test run (tools/test/run.py --threads-per-test) gives each process its share of+ // the cores; ten processes with a pool as big as the machine each spend their time waiting for a core.+ if (const char* env = std::getenv("SF_THREADS"); wanted <= 0 && env) wanted = std::clamp(std::atoi(env), 0, 64);return wanted > 0 ? wanted : int(std::clamp(std::thread::hardware_concurrency(), 1u, 16u));}());return pool;diff --git a/tests/TestHarness.h b/tests/TestHarness.hindex 4044c45f..0625f76a 100644--- a/tests/TestHarness.h+++ b/tests/TestHarness.h@@ -59,7 +59,11 @@ inline bool inShard(const char* name, int shard, int shards) {inline double timeLimit(double local) {const char* strict = std::getenv("SF_BENCH_LIMITS");if (strict && std::string(strict) == "1") return local;- return std::getenv("CI") || std::getenv("GITHUB_ACTIONS") ? local * 4.0 : local;+ if (std::getenv("CI") || std::getenv("GITHUB_ACTIONS")) return local * 4.0;+ // tools/test/run.py sets SF_TIME_LIMIT_SCALE for a parallel run (ctest -jN): the machine is as loaded as CI's.+ const char* scale = std::getenv("SF_TIME_LIMIT_SCALE");+ const double k = scale ? std::atof(scale) : 0.0;+ return k > 1.0 ? local * k : local;}inline std::string tempPath(const char* name) { return (std::filesystem::temp_directory_path() / name).string(); }diff --git a/tests/flaky.md b/tests/flaky.mdnew file mode 100644index 00000000..5f89c6e2--- /dev/null+++ b/tests/flaky.md@@ -0,0 +1,9 @@+# Flaky tests++Written by tools/test/run.py (docs/testing-flow.md): a test that failed, then passed on its one automatic rerun.+A test here twice gets the `flaky` label in CMakeLists.txt (it then never blocks a release) and an owner who fixes or+deletes it; remove its lines once fixed.++| date | test | machine | commit |+|---|---|---|---|+| 2026-11-04 | sculptforge_tests_9of16 | PC (WSL, RTX 3060), ctest -j10 | 76e4be24 |diff --git a/tests/perf/precision_33m.sfui b/tests/perf/precision_33m.sfuiindex 19e996c8..a926d668 100644--- a/tests/perf/precision_33m.sfui+++ b/tests/perf/precision_33m.sfui@@ -1,4 +1,4 @@-# Precision at extreme zoom on 33M triangles (by hand, not in ctest: tools/precision_check.sh). See+# Precision at extreme zoom on 33M triangles (nightly: ctest -L nightly, or by hand: tools/precision_check.sh). See# tests/ui/precision_dense.sfui.run new_bench_sphere millions=33expect_value pixel_density >= 1diff --git a/tests/slow_tests.txt b/tests/slow_tests.txtnew file mode 100644index 00000000..5be1243c--- /dev/null+++ b/tests/slow_tests.txt@@ -0,0 +1,42 @@+# Tests over the 20 s budget (tools/test/run.py, docs/testing-flow.md), measured on the PC (RTX 3060, WSL, ctest -j10,+# full suite, 2026-11-04). A listed test passes up to 1.5x its seconds here; an unlisted one over 20 s fails the run.+# Make it fast (wait on events, a smaller mesh or texture, split it) or label it nightly, then delete its line.+# name seconds plan+ui_uv_editor 116 unwrap of the full lion head (35 s in one step): a smaller model for T1, the lion nightly+insider_drops 100 tools/drops selftest: profile it; split the network-free checks from the packaging ones+ui_fill_layers 71 now 1024 textures (was 2048): re-measure; Paint Layers bakes Mesh Maps at open (29 s)+ui_texture_transfer 71 transfers on a dense sphere: smaller source mesh+ui_make_sculptable 66 two make-sculptable runs on a 150k capture (19 s + 15 s): a 30k capture+ui_retopo 63 one wait_idle of 23 s (line 103: QRemesh of the sculpt): a lower target+ui_first_sculpt 48 the lesson end to end (225 lines, no single long step): split into two scripts+ui_texture_paint 40 profile with SF_UI_PROFILE=1+ui_bake 40 2048 AO bake (22 s): the default size is what it checks; split the default check from the AO bake+ui_quick_tour 36 long tail of 1-2 s steps: re-measure with per-test threads (SF_THREADS)+ui_bake_levels 31 one 2048 bake (19 s): 1024+ui_gizmo 26 long tail of 2-3 s drags: re-measure with SF_THREADS+ui_ctrl_brushes 27 re-measure with SF_THREADS (24-27 s)+ui_ui_scale 35 ui scale steps 100-200 %: one size per step is enough (24-35 s)+ui_pbr_viewport 23 re-measure with SF_THREADS+ui_stroke_background_preview 22 re-measure with SF_THREADS+ui_face_shapes 22 re-measure with SF_THREADS+ui_parts 22 re-measure with SF_THREADS+ui_pbr_channels 22 re-measure with SF_THREADS+ui_vdm 21 re-measure with SF_THREADS+ui_shell_layout 21 re-measure with SF_THREADS+ui_windows_dpi_2560 21 re-measure with SF_THREADS+ui_ui_scale_900 21 re-measure with SF_THREADS+ui_dyntopo 21 re-measure with SF_THREADS+ui_multires 20 re-measure with SF_THREADS+sculptforge_tests: qremesh_guide_loop_round_an_eye 49 guide loops on a full face: a coarser face+sculptforge_tests: texture_layer_undo_stores_only_what_changed 42 4k texture layers: 1k shows the same+sculptforge_tests: inbox0106_smooth_keeps_quad_remesh_regular_lion 29 the lion head: a smaller quad mesh+sculptforge_tests: texture_transfer_carries_material_channels 28 smaller textures+sculptforge_tests: texture_transfer_remesh_and_qremesh_keep_the_texture 27 smaller textures+sculptforge_tests: qremesh_sculpt_target_counts 26 fewer target counts per run+sculptforge_tests: texture_transfer_auto_uv_of_a_dense_sphere_does_not_fold 24 a less dense sphere+sculptforge_tests: fill_layer_effect_chain_and_channels 24 re-measure with SF_THREADS (21-24 s)+sculptforge_tests: qremesh_guide_loop_across_the_symmetry_plane 22 a coarser mesh (21-22 s)+# Moved to nightly (TEST(nightly_*)), 11-04: inbox0106_keep_masked_voxel_on_a_real_model (82 s),+# uv_unwrap_lion_head_by_sharp_edges (54 s), mesh_maps_live_update_is_faster_than_a_full_bake (30 s),+# uv_editor_million_triangles_stay_interactive (29 s), face_weights_timing_on_a_500k_head (20 s),+# dyntopo_random_strokes_stay_valid (25 s, stress).diff --git a/tests/test_dyntopo.cpp b/tests/test_dyntopo.cppindex de0530be..1d7157e3 100644--- a/tests/test_dyntopo.cpp+++ b/tests/test_dyntopo.cpp@@ -224,7 +224,7 @@ TEST(dyntopo_adds_detail_under_the_brush_only) {CHECK(d.mesh.positions[0] == far);}-TEST(dyntopo_random_strokes_stay_valid) {+TEST(nightly_dyntopo_random_strokes_stay_valid) {Document d = sphereDoc(3);uint32_t rng = 12345;auto rnd = [&rng] {diff --git a/tests/test_face.cpp b/tests/test_face.cppindex 7e6bf3be..f9cd75fb 100644--- a/tests/test_face.cpp+++ b/tests/test_face.cpp@@ -437,7 +437,7 @@ TEST(face_weights_fast_path_is_exact_and_not_undoable) {CHECK(again.positions == doc.mesh.positions);}-TEST(face_weights_timing_on_a_500k_head) {+TEST(nightly_face_weights_timing_on_a_500k_head) {// A 720k-vertex head (the template skin divided three times) with the 52 shapes: a live frame that changes// every weight costs a few milliseconds and touches only the face.Mesh base = templateHead();diff --git a/tests/test_inbox_0106.cpp b/tests/test_inbox_0106.cppindex 4426beef..51d29c3b 100644--- a/tests/test_inbox_0106.cpp+++ b/tests/test_inbox_0106.cpp@@ -132,7 +132,7 @@ CommandResult lionKeepMasked(const std::function<bool(const glm::vec3&)>& masked} // namespace-TEST(inbox0106_keep_masked_voxel_on_a_real_model) {+TEST(nightly_inbox0106_keep_masked_voxel_on_a_real_model) {// Before the fix every one of these failed with "Couldn't cut the remeshed surface cleanly" (the plain voxel// remesh of the lion meets itself on 150-200 edges in the mane at any resolution).const std::function<bool(const glm::vec3&)> masks[] = {diff --git a/tests/test_main.cpp b/tests/test_main.cppindex 2e1b39e8..a517b92f 100644--- a/tests/test_main.cpp+++ b/tests/test_main.cpp@@ -4,7 +4,8 @@// Runs every TEST registered by the tests/test_*.cpp files (with arguments: only the tests whose names contain// one of them, e.g. `sculptforge_tests qremesh_guide`). `--shard K/N` runs only shard K (0-based) of N, so the// suite can run as N processes at once (CMake registers sculptforge_tests_0of16 ... for ctest -j); every test is-// in exactly one shard (sf::test::inShard).+// in exactly one shard (sf::test::inShard). Tests named nightly_* (benchmarks, stress, huge meshes) run only with+// `--nightly` (ctest entry sculptforge_tests_nightly, label nightly: T3 of docs/testing-flow.md), never in a shard.#include "TestHarness.h"#include <chrono>@@ -16,6 +17,7 @@int main(int argc, char** argv) {using namespace sf::test;int shard = 0, shards = 1;+ bool nightly = false;std::vector<const char*> filters;for (int i = 1; i < argc; ++i) {if (std::strcmp(argv[i], "--shard") == 0 && i + 1 < argc) {@@ -23,6 +25,8 @@ int main(int argc, char** argv) {std::printf("--shard wants K/N with 0 <= K < N, got %s\n", argv[i]);return 2;}+ } else if (std::strcmp(argv[i], "--nightly") == 0) {+ nightly = true;} else {filters.push_back(argv[i]);}@@ -31,6 +35,7 @@ int main(int argc, char** argv) {for (std::size_t index = 0; index < registry().size(); ++index) {const auto& t = registry()[index];if (!inShard(t.name, shard, shards)) continue;+ if ((std::strncmp(t.name, "nightly_", 8) == 0) != nightly) continue;bool wanted = filters.empty();for (const char* f : filters) wanted = wanted || std::strstr(t.name, f) != nullptr;if (!wanted) continue;diff --git a/tests/test_mesh_maps.cpp b/tests/test_mesh_maps.cppindex 82ed76f7..1a18a360 100644--- a/tests/test_mesh_maps.cpp+++ b/tests/test_mesh_maps.cpp@@ -202,7 +202,7 @@ TEST(mesh_maps_live_update_matches_a_full_bake) {CHECK(differ == 0);}-TEST(mesh_maps_live_update_is_faster_than_a_full_bake) {+TEST(nightly_mesh_maps_live_update_is_faster_than_a_full_bake) {// A denser Part: a 2K map of a 130k-triangle sphere, then a small local change.Mesh m = makeIcosphere(6, 1.0f);sphereUvs(m);diff --git a/tests/test_uv_editor.cpp b/tests/test_uv_editor.cppindex b44e952e..6a843cb7 100644--- a/tests/test_uv_editor.cpp+++ b/tests/test_uv_editor.cpp@@ -491,7 +491,7 @@ TEST(uv_project_round_trip_keeps_seams_and_pins) {std::filesystem::remove(path);}-TEST(uv_editor_million_triangles_stay_interactive) {+TEST(nightly_uv_editor_million_triangles_stay_interactive) {// An 8-level icosphere: 1.3M triangles. Topology, measures, a transform and a pack are each well under the// time a frame or a click can take; a full unwrap by face groups (in the background in the app) takes seconds.Mesh m = makeIcosphere(8);@@ -538,7 +538,7 @@ TEST(uv_editor_million_triangles_stay_interactive) {CHECK(flippedTriangles(m) < m.triangleCount() / 1000);}-TEST(uv_unwrap_lion_head_by_sharp_edges) {+TEST(nightly_uv_unwrap_lion_head_by_sharp_edges) {// A real sculpt with holes and thin strands: the islands don't fold, overlap or leave the tile.Document doc;std::string lion = __FILE__;diff --git a/tests/ui/fill_layers.sfui b/tests/ui/fill_layers.sfuiindex 0e05a82f..6b14ef2a 100644--- a/tests/ui/fill_layers.sfui+++ b/tests/ui/fill_layers.sfui@@ -6,9 +6,9 @@ run stroke brush=Crease points="-0.6,0.35,1 0,0.55,1 0.6,0.35,1" radius=0.07 strrun stroke brush=Inflate points="-0.35,-0.3,1" radius=0.3 strength=1run stroke brush=Clay points="0.25,-0.5,1 0.55,-0.05,1" radius=0.1 strength=1run stroke brush=Crease invert=true points="-0.75,-0.1,1 -0.45,-0.65,1" radius=0.05 strength=1-run uv_auto resolution=2048+run uv_auto resolution=1024wait_value auto_uv_running == 0 120-run texture_create size=2048+run texture_create size=1024expect_value fill_layers == 0# A recipe from the Paint Layers panel: four Fill layers; Mesh Maps bake on a worker thread, then they evaluate.diff --git a/tools/release/paperwork_only.py b/tools/release/paperwork_only.pynew file mode 100644index 00000000..a991db7e--- /dev/null+++ b/tools/release/paperwork_only.py@@ -0,0 +1,123 @@+#!/usr/bin/env python3+# SculptForge - open source digital sculpting+# SPDX-License-Identifier: GPL-3.0-or-later+"""Is BASE..HEAD only release paperwork? Then CI's result for BASE stands for HEAD (scripts/release.sh step ci,+.github/workflows/release.yml job locate; docs/testing-flow.md).++ tools/release/paperwork_only.py BASE HEAD exit 0: paperwork only; 1: something else changed (printed); 2: error+ tools/release/paperwork_only.py --self-test++Paperwork is exactly what scripts/release.sh step bump writes, file by file and line by line: the version lines of+CMakeLists.txt, CITATION.cff, packaging/rez/package.py and the metainfo's <release> lines; CHANGELOG.md; Markdown under+docs/ (the guide's What's New included: release.sh step check runs the guide and What's New tests on the bumped tree);+the generated website (website/, not website/link, which is built into the app); deleted note fragments (changes/).+Anything else, a rename, a mode change or an unreadable diff is not paperwork (fail safe).+"""+import re+import subprocess+import sys++# file (exact or prefix/) -> None (any change) or a regex every changed line must match+LINES = {+ "CMakeLists.txt": r'(project\(SculptForge VERSION [0-9.]+[ )].*|set\(SCULPTFORGE_PRERELEASE "[a-z0-9.]*"\))',+ "CITATION.cff": r"(version|date-released): .*",+ "packaging/rez/package.py": r'version = "[0-9.]+"',+ "packaging/linux/org.sculptforge.SculptForge.metainfo.xml": r'\s*<release version="[0-9.]+" date="[0-9-]+"/>',+ "CHANGELOG.md": None,+}+++def allowed(path):+ if path in LINES:+ return True+ if path.startswith("website/") and not path.startswith("website/link/"):+ return True+ if path.startswith("docs/") and path.endswith(".md"):+ return True+ return False+++def git(*args):+ return subprocess.run(["git", *args], capture_output=True, text=True, check=True).stdout+++def check(base, head):+ problems = []+ for line in git("diff", "--no-renames", "--raw", base, head).splitlines():+ meta, path = line.split("\t", 1) # :old_mode new_mode old_sha new_sha status+ om, nm, _, _, status = meta[1:].split()+ if om != nm and om != "000000" and nm != "000000":+ problems.append(f"{path}: mode change")+ continue+ if path.startswith("changes/") and status == "D":+ continue+ if not allowed(path):+ problems.append(f"{path}: not release paperwork")+ continue+ rule = LINES.get(path)+ if rule:+ diff = git("diff", "--no-renames", "-U0", base, head, "--", path)+ for d in diff.splitlines():+ if d.startswith(("+++", "---")) or not d.startswith(("+", "-")):+ continue+ if not re.fullmatch(rule, d[1:]):+ problems.append(f"{path}: {d[:80]}")+ return problems+++def self_test():+ import os+ import tempfile+ with tempfile.TemporaryDirectory() as d:+ run = lambda *a: subprocess.run(a, cwd=d, check=True, capture_output=True)+ run("git", "init", "-q")+ os.makedirs(os.path.join(d, "changes"))+ os.makedirs(os.path.join(d, "src"))+ w = lambda p, t: open(os.path.join(d, p), "w").write(t)+ w("CMakeLists.txt", 'project(SculptForge VERSION 1.2.3 LANGUAGES C CXX)\nset(SCULPTFORGE_PRERELEASE "")\nadd_x()\n')+ w("CHANGELOG.md", "## [Unreleased]\n- a\n")+ w("changes/f.md", "x")+ w("src/a.cpp", "int a;")+ run("git", "add", "-A")+ run("git", "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "a")+ cwd = os.getcwd()+ os.chdir(d)+ try:+ w("CMakeLists.txt", 'project(SculptForge VERSION 1.3.0 LANGUAGES C CXX)\nset(SCULPTFORGE_PRERELEASE "beta.1")\nadd_x()\n')+ w("CHANGELOG.md", "## [Unreleased]\n\n## [1.3.0]\n- a\n")+ os.remove("changes/f.md")+ run("git", "add", "-A")+ run("git", "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "b")+ assert check("HEAD~1", "HEAD") == [], check("HEAD~1", "HEAD")+ w("CMakeLists.txt", 'project(SculptForge VERSION 1.3.0 LANGUAGES C CXX)\nset(SCULPTFORGE_PRERELEASE "beta.1")\nadd_y()\n')+ run("git", "add", "-A")+ run("git", "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "c")+ assert check("HEAD~1", "HEAD") == ["CMakeLists.txt: -add_x()", "CMakeLists.txt: +add_y()"], check("HEAD~1", "HEAD")+ w("src/a.cpp", "int b;")+ run("git", "add", "-A")+ run("git", "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "d")+ assert check("HEAD~1", "HEAD") == ["src/a.cpp: not release paperwork"]+ finally:+ os.chdir(cwd)+ print("paperwork_only.py self-test passed")+ return 0+++def main(argv):+ if argv[:1] == ["--self-test"]:+ return self_test()+ if len(argv) != 2:+ print(__doc__)+ return 2+ try:+ problems = check(*argv)+ except (subprocess.CalledProcessError, ValueError) as e:+ print(f"can't compare {argv[0]} and {argv[1]}: {e}")+ return 2+ for p in problems:+ print(p)+ return 1 if problems else 0+++if __name__ == "__main__":+ sys.exit(main(sys.argv[1:]))diff --git a/tools/test/impacted.py b/tools/test/impacted.pynew file mode 100644index 00000000..5ad8f9fc--- /dev/null+++ b/tools/test/impacted.py@@ -0,0 +1,373 @@+#!/usr/bin/env python3+# SculptForge - open source digital sculpting+# SPDX-License-Identifier: GPL-3.0-or-later+"""T0 of the test flow (docs/testing-flow.md): which ctest entries a change can break, as a `ctest -R` regex.++ tools/test/impacted.py [--build build] [--base origin/main] [-v] changed files from git (base...HEAD + work tree)+ tools/test/impacted.py --build build --files a.cpp tests/ui/b.sfui an explicit file list (pc.sh passes one)+ tools/test/impacted.py --self-test++Prints one line: the regex (`^(ui_bake|marks)$`), or `.` when the change touches the build itself (CMake files) and+everything must run. Nightly tests are never listed (T3 runs them). -v explains every pick on stderr.++How a file maps to tests, most exact first:+ 1. the build graph (build.ninja plus `ninja -t deps`, so headers count): a source or header -> the object files that+ include it -> the libraries and programs linking them -> the tests that run those programs (a test's command or+ arguments name the program; sculptforge_tests is split into shards, all of them run);+ 2. a test's own files: the script it runs, the files its command names, and their folder (tools/guide/*.py);+ 3. UI scripts that name a changed data file (tests/ui/data/x.json -> scripts mentioning x.json);+ 4. the hand map below for files no command names (docs, site, release scripts, fragments);+ 5. when the app itself changed (src/app, src/render ...): UI tests whose name shares a word with the changed file+ (PanelMicro.cpp -> ui_micro_levels), UI tests changed together with it in the last 600 commits, and SMOKE.+T1 (the full suite on the PC) catches anything this misses; files that map to nothing are listed with -v.+"""+import argparse+import fnmatch+import json+import os+import re+import subprocess+import sys+from collections import Counter, defaultdict++# Files no test command names, and the tests that read them. Globs match the repo-relative path.+HAND_MAP = [+ ("CMakeLists.txt", "."), ("cmake/*", "."),+ ("docs/guide/*", "guide_site guide_menu_paths ui_user_guide ui_user_guide_1080 ui_whats_new ui_guide_shortcuts ui_quick_tour"),+ ("docs/guide/whats-new.md", "release_notes ui_whats_new"),+ ("docs/supporters.txt", "ui_support ui_support_card"),+ ("website/*", "guide_site"),+ ("changes/*", "release_notes release_notes_fragments"),+ ("CHANGELOG.md", "release_notes"),+ ("scripts/release.sh", "release_script_beta"),+ ("tests/release_beta_test.sh", "release_script_beta"),+ ("tools/release/*", "release_notes release_notes_fragments release_script_beta"),+ ("packaging/windows/*", "ui_windows_dpi_2560 ui_windows_dpi_2160 ui_ui_scale_900"),+ ("addons/blender/*", "blender_exchange blender_addon_package blender_addon"),+ ("integrations/maya/*", "maya_shelf_script ui_maya_bridge"),+ ("services/link-worker/*", "link_worker ui_phone_link"),+ ("tools/brush-gallery/*", "brush_gallery_index ui_brush_gallery"),+ ("tools/marks_*.txt", "marks"),+ ("tools/test/*", "test_impact"),+ ("tests/ui/data/*", ""), # step 3 handles these+]+# Public text is checked for third-party marks on every change: cheap, and the one check nobody may skip.+ALWAYS = ["marks"]+# The app's own sanity set, whenever the app program changed.+SMOKE = ["ui_shell_layout", "ui_sculpt_basics", "ui_idle_redraw", "ui_release_smoke"]+# Sources whose change only matters to the app's UI (step 5); a core change reaches every test through the graph.+APP_DIRS = ("src/app/", "src/render/", "src/link/")+STOP_WORDS = {"panel", "tool", "tools", "util", "utils", "test", "impl", "main", "app", "core", "data", "file", "ui"}+++def sh(*cmd, cwd=None):+ try:+ return subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, check=True).stdout+ except (OSError, subprocess.CalledProcessError):+ return ""+++def changed_files(root, base):+ mb = sh("git", "merge-base", base, "HEAD", cwd=root).strip() or base+ out = sh("git", "diff", "--name-only", mb, cwd=root) + sh("git", "ls-files", "-o", "--exclude-standard", cwd=root)+ return sorted({f for f in out.split("\n") if f})+++def words(path):+ stem = os.path.splitext(os.path.basename(path))[0]+ parts = re.findall(r"[A-Z]?[a-z]+|[A-Z]+(?![a-z])|[0-9]+", stem)+ return {p.lower() for p in parts if len(p) >= 4 and p.lower() not in STOP_WORDS}+++class Graph:+ """The build graph: which outputs (objects, libraries, programs) each repo file reaches."""++ def __init__(self, build, root):+ self.build, self.root = build, root+ self.users = defaultdict(set) # input path (absolute) -> outputs built from it+ ninja = os.path.join(build, "build.ninja")+ if not os.path.exists(ninja):+ self.ok = self.load_make(build)+ return+ self.ok = True+ text = open(ninja, encoding="utf-8", errors="replace").read().replace("$\n", " ")+ for line in text.split("\n"):+ if not line.startswith("build "):+ continue+ m = re.match(r"build (.+?)(?<!\$): (\S+)(.*)", line)+ if not m or m.group(2) in ("phony", "CUSTOM_COMMAND", "RERUN_CMAKE", "CLEAN", "HELP"):+ continue+ outs = [self.abs(o) for o in self.split(m.group(1)) if o != "|"]+ ins = self.split(m.group(3).split("||")[0])+ for i in ins:+ if i != "|":+ for o in outs:+ self.users[self.abs(i)].add(o)+ # Headers: the dependency lists the compiler wrote (valid after a build).+ obj = None+ for line in sh("ninja", "-C", build, "-t", "deps").split("\n"):+ if line and not line.startswith(" "):+ obj = self.abs(line.split(":")[0])+ elif line.strip() and obj:+ self.users[self.abs(line.strip())].add(obj)++ def load_make(self, build):+ """The Makefile generator's graph: CMakeFiles/<target>.dir/link.txt (what links what) and the compiler's+ .o.d dependency files (which headers each object includes). False when there is no build at all."""+ found = False+ for d, dirs, files in os.walk(build):+ dirs[:] = [x for x in dirs if x not in ("_deps", "Testing", "ui-shots", "test-tmp")]+ if not d.endswith(".dir") or os.path.basename(os.path.dirname(d)) != "CMakeFiles":+ continue+ base = os.path.dirname(os.path.dirname(d)) # where the target's outputs and commands live+ try:+ words = open(os.path.join(d, "link.txt"), encoding="utf-8", errors="replace").read().split()+ except OSError:+ words = []+ out = None+ if "-o" in words and words.index("-o") + 1 < len(words):+ out = words[words.index("-o") + 1]+ elif len(words) > 2 and words[0].endswith("ar") and words[1] in ("qc", "cr", "rcs", "qcs"):+ out = words[2]+ for root, _, fs in os.walk(d):+ for f in fs:+ path = os.path.join(root, f)+ if f.endswith(".o.d"):+ found = True+ text = open(path, encoding="utf-8", errors="replace").read().replace("\\\n", " ")+ head, _, deps = text.partition(": ")+ obj = os.path.normpath(os.path.join(base, head.strip()))+ for dep in deps.split():+ self.users[os.path.normpath(os.path.join(base, dep))].add(obj)+ elif f.endswith(".o") and out:+ self.users[os.path.normpath(path)].add(os.path.normpath(os.path.join(base, out)))+ if out:+ for w in words:+ if w.endswith((".a", ".so", ".lib")) and not os.path.isabs(w):+ self.users[os.path.normpath(os.path.join(base, w))].add(os.path.normpath(os.path.join(base, out)))+ return found++ @staticmethod+ def split(s):+ return [p.replace("$ ", " ").replace("$:", ":") for p in re.split(r"(?<!\$) +", s.strip()) if p]++ def abs(self, p):+ return os.path.normpath(p if os.path.isabs(p) else os.path.join(self.build, p))++ def reach(self, path):+ seen, todo = set(), [os.path.normpath(os.path.join(self.root, path))]+ while todo:+ for o in self.users.get(todo.pop(), ()):+ if o not in seen:+ seen.add(o)+ todo.append(o)+ return seen+++def load_tests(build):+ try:+ out = subprocess.run(["ctest", "--test-dir", build, "--show-only=json-v1"], capture_output=True, text=True,+ check=True).stdout+ data = json.loads(out)+ except (OSError, subprocess.CalledProcessError, ValueError):+ return []+ tests = []+ for t in data.get("tests", []):+ props = {p["name"]: p["value"] for p in t.get("properties", [])}+ if props.get("DISABLED"):+ continue+ tests.append({"name": t["name"], "command": t.get("command", []), "labels": props.get("LABELS", []),+ "cwd": props.get("WORKING_DIRECTORY", "")})+ return tests+++def co_changed(root, files, ui_names, limit=600, top=6, log_file=None):+ """UI tests whose script changed in the same commits as these files (the last `limit` commits). log_file: that+ `git log --no-merges --name-only --format=@@` output, for a copy of the tree without .git (pc.sh)."""+ if log_file:+ log = open(log_file, encoding="utf-8", errors="replace").read()+ else:+ log = sh("git", "log", f"-{limit}", "--no-merges", "--name-only", "--format=tformat:@@", cwd=root)+ pairs = defaultdict(Counter)+ want = set(files)+ for commit in log.split("@@"):+ names = {f for f in commit.split("\n") if f}+ hit = names & want+ if not hit:+ continue+ scripts = {"ui_" + os.path.splitext(os.path.basename(f))[0] for f in names+ if f.startswith("tests/ui/") and f.endswith(".sfui")}+ for f in hit:+ for s in scripts & ui_names:+ pairs[f][s] += 1+ out = {}+ for f, c in pairs.items():+ out[f] = [s for s, n in c.most_common(top) if n >= 2]+ return out+++def is_app(out):+ return os.path.basename(out) in ("sculptforge", "sculptforge.exe")+++def impacted(root, build, files, verbose=False, graph=None, tests=None, history=True, log_file=None):+ log = (lambda *a: print(*a, file=sys.stderr)) if verbose else (lambda *a: None)+ tests = load_tests(build) if tests is None else tests+ graph = Graph(build, root) if graph is None else graph+ names = {t["name"] for t in tests}+ ui_names = {n for n in names if n.startswith("ui_")}+ nightly = {t["name"] for t in tests if "nightly" in t["labels"]}+ pick = {}++ def add(name, why):+ if name in names and name not in nightly and name not in pick:+ pick[name] = why++ # Every file a test's command names, and that file's folder (a script's sibling modules).+ by_file, by_dir, by_output = defaultdict(set), defaultdict(set), defaultdict(set)+ for t in tests:+ for arg in t["command"]:+ for a in re.split(r"[=;]", arg):+ if not a or not (os.path.isabs(a) or "/" in a):+ continue+ p = os.path.normpath(a if os.path.isabs(a) else os.path.join(t["cwd"] or build, a))+ by_output[p].add(t["name"])+ rel = os.path.relpath(p, root).replace(os.sep, "/") # (git's spelling, also on Windows)+ if not rel.startswith(".."):+ by_file[rel].add(t["name"])+ if os.path.isfile(p) and not rel.startswith("tests/ui/"):+ by_dir[os.path.dirname(rel)].add(t["name"])+ elif os.path.isdir(p):+ by_dir[rel].add(t["name"])++ everything, app_files, unmapped = False, [], []+ for f in files:+ before = len(pick)+ for glob, targets in HAND_MAP:+ if fnmatch.fnmatch(f, glob):+ if targets == ".":+ everything = True+ for n in targets.split():+ add(n, f"hand map {glob}")+ for n in by_file.get(f, ()):+ add(n, f"runs {f}")+ d = os.path.dirname(f)+ while d:+ for n in by_dir.get(d, ()):+ add(n, f"same folder as its script ({d})")+ if d.startswith("tools/") and d.count("/") > 1:+ d = os.path.dirname(d)+ else:+ break+ if f.startswith("tests/ui/data/"):+ base = os.path.basename(f)+ for t in tests:+ for arg in t["command"]:+ if arg.endswith(".sfui") and os.path.isfile(arg) and base in open(arg, encoding="utf-8").read():+ add(t["name"], f"script names {base}")+ if graph.ok:+ for out in graph.reach(f):+ if is_app(out): # every UI test runs the app: step 5 picks the ones that matter+ app_files.append(f)+ continue+ for n in by_output.get(out, ()):+ add(n, f"build graph: {f} -> {os.path.relpath(out, build)}")+ elif f.startswith(APP_DIRS):+ app_files.append(f)+ if len(pick) == before and not everything and f not in app_files:+ unmapped.append(f)++ if not graph.ok and any(f.startswith("src/") and not f.startswith(APP_DIRS) for f in files):+ # No build graph: a core source change could reach anything.+ log("no build.ninja: every non-nightly test")+ everything = True+ app_files = sorted(set(app_files))+ if app_files:+ for f in app_files:+ w = words(f)+ for n in ui_names:+ if w & set(n[3:].split("_")):+ add(n, f"name shares a word with {f}")+ if history:+ for f, ns in co_changed(root, app_files, ui_names, log_file=log_file).items():+ for n in ns:+ add(n, f"changed together with {f}")+ for n in SMOKE:+ add(n, "smoke (the app changed)")+ if files:+ for n in ALWAYS:+ add(n, "always")+ for n, why in sorted(pick.items()):+ log(f" {n:40} {why}")+ for f in unmapped:+ log(f" (no test reads {f}; T1 still runs everything)")+ if everything:+ return "."+ return "^(" + "|".join(sorted(pick)) + ")$" if pick else ""+++def self_test():+ import tempfile+ with tempfile.TemporaryDirectory() as root:+ b = os.path.join(root, "build")+ os.makedirs(os.path.join(root, "tools/guide"))+ os.makedirs(os.path.join(root, "tests/ui/data"))+ os.makedirs(b)+ for p in ("tools/guide/build_site.py", "tools/guide/lib.py", "tests/ui/micro_levels.sfui", "tests/ui/bake.sfui"):+ open(os.path.join(root, p), "w").write("env F $SCRIPT_DIR/data/feed.json\n" if "bake" in p else "")+ open(os.path.join(root, "tests/ui/data/feed.json"), "w").write("{}")+ open(os.path.join(b, "build.ninja"), "w").write(+ "build CMakeFiles/core.dir/src/core/Mesh.cpp.o: CXX_COMPILER__core ../src/core/Mesh.cpp\n"+ "build libcore.a: CXX_STATIC_LIBRARY_LINKER__core CMakeFiles/core.dir/src/core/Mesh.cpp.o\n"+ "build CMakeFiles/app.dir/src/app/PanelMicro.cpp.o: CXX_COMPILER__app ../src/app/PanelMicro.cpp\n"+ "build sculptforge: CXX_EXECUTABLE_LINKER__app CMakeFiles/app.dir/src/app/PanelMicro.cpp.o | $\n"+ " libcore.a\n"+ "build sculptforge_tests: CXX_EXECUTABLE_LINKER__t libcore.a\n")+ g = Graph(b, root)+ g.users[g.abs("../src/core/Mesh.h")].add(g.abs("CMakeFiles/core.dir/src/core/Mesh.cpp.o")) # a -t deps line+ T = lambda n, *c, labels=(): {"name": n, "command": list(c), "labels": list(labels), "cwd": ""}+ app, ui = os.path.join(b, "sculptforge"), os.path.join(root, "tests/ui")+ tests = [T("sculptforge_tests_0of2", os.path.join(b, "sculptforge_tests"), "--shard", "0/2"),+ T("sculptforge_tests_1of2", os.path.join(b, "sculptforge_tests"), "--shard", "1/2"),+ T("ui_micro_levels", app, "--ui-test", ui + "/micro_levels.sfui"),+ T("ui_bake", app, "--ui-test", ui + "/bake.sfui"),+ T("ui_bench", app, "--bench", labels=["ui", "nightly"]),+ T("ui_shell_layout", app), T("marks", "python3", root + "/tools/check_marks.py"),+ T("guide_site", "python3", root + "/tools/guide/build_site.py", "--check"),+ T("release_notes_fragments", "python3", "x.py")]+ r = lambda *f: impacted(root, b, list(f), graph=g, tests=tests, history=False)+ assert r("src/core/Mesh.h") == "^(marks|sculptforge_tests_0of2|sculptforge_tests_1of2|ui_shell_layout)$", r("src/core/Mesh.h")+ assert r("src/app/PanelMicro.cpp") == "^(marks|ui_micro_levels|ui_shell_layout)$", r("src/app/PanelMicro.cpp")+ assert r("tests/ui/bake.sfui") == "^(marks|ui_bake)$", r("tests/ui/bake.sfui")+ assert r("tests/ui/data/feed.json") == "^(marks|ui_bake)$"+ assert r("tools/guide/lib.py") == "^(guide_site|marks)$", r("tools/guide/lib.py")+ assert r("changes/x.md") == "^(marks|release_notes_fragments)$"+ assert r("CMakeLists.txt") == "."+ assert r() == ""+ assert words("src/app/PanelMicroLevels.cpp") == {"micro", "levels"}+ print("impacted.py self-test passed")+ return 0+++def main():+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("--build", default="build")+ ap.add_argument("--base", default="origin/main")+ ap.add_argument("--files", nargs="*")+ ap.add_argument("--git-log", help="git log --no-merges --name-only --format=@@ output (a tree without .git)")+ ap.add_argument("-v", "--verbose", action="store_true")+ ap.add_argument("--self-test", action="store_true")+ a = ap.parse_args()+ if a.self_test:+ return self_test()+ root = sh("git", "rev-parse", "--show-toplevel").strip() or os.getcwd()+ build = os.path.abspath(a.build)+ files = a.files if a.files is not None else changed_files(root, a.base)+ if a.verbose:+ print(f"{len(files)} changed file(s)", file=sys.stderr)+ print(impacted(root, build, files, a.verbose, log_file=a.git_log))+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tools/test/run.py b/tools/test/run.pynew file mode 100644index 00000000..bf81b414--- /dev/null+++ b/tools/test/run.py@@ -0,0 +1,446 @@+#!/usr/bin/env python3+# SculptForge - open source digital sculpting+# SPDX-License-Identifier: GPL-3.0-or-later+"""The one way every tier runs ctest (docs/testing-flow.md): ctest plus a test cache, a flake rerun and a time budget.++ tools/test/run.py --build build [--cache DIR] [--budget 20] [--flaky-log tests/flaky.md] [-- ctest args...]+ tools/test/run.py --self-test++* Nightly tests (label `nightly`) never run here unless the ctest args select them (-L nightly): T3 runs them.+* Test cache (--cache, or $SF_TEST_CACHE): every test gets a fingerprint, the hash of everything it can read: its+ program (the built file), every file or folder its command names, the app's assets and test data for app tests,+ the test's ENVIRONMENT and the SF_/SCULPTFORGE_/SDL_/GL driver variables of this run. Tests that read the whole tree+ (marks, the guide, release notes...) hash every tracked file. A test that passed with the same fingerprint on this+ machine is skipped and reported as cached; a pass on the first try is recorded. Never on CI (nothing persists).+* Flakes: failed tests run once more (ctest --rerun-failed). A pass on the rerun is a flake: logged (`flaky: name`+ lines, appended to --flaky-log), not cached, and does not fail the run. A test labelled `flaky` that fails twice is+ reported but never fails the run either (it never blocks a release; fix it or delete it).+* Budget: a non-nightly test that took longer than --budget seconds (default $SF_TEST_BUDGET or 20; 0 = off) fails+ the run, with the list. Make it wait on events instead of fixed frames, split it, or label it nightly.+Exit code: 0 green, 1 a real failure or a test over budget, 2 ctest itself could not run, 4 only GPU-lost failures+(the output says the graphics device went away: the machine, not the code).+"""+import argparse+import hashlib+import json+import os+import re+import shutil+import subprocess+import sys+import tempfile+import time+try:+ import xml.etree.ElementTree as ET+ ET.fromstring("<a/>")+except ImportError: # some system Pythons (Rocky 8's 3.12) ship a pyexpat that doesn't load: plain ctest then+ ET = None++ENV_PREFIXES = ("SF_", "SCULPTFORGE_", "SDL_", "GALLIUM_", "LIBGL_", "MESA_", "__GLX_")+ENV_IGNORE = {"SDL_APP_ID"}+# Tests that read files their command doesn't name: hashed as globs (repo-relative). "*" = every tracked file.+READS = {+ "marks": ["*"], "marks_self_test": ["tools/check_marks.py"],+ "guide_site": ["*"], "guide_menu_paths": ["*"], "ui_guide_shortcuts": ["docs/guide"],+ "release_notes": ["tools/release"], "release_notes_fragments": ["changes", "tools/release"],+ "release_script_beta": ["*"], "upstream_watch": ["tools/upstream-watch"],+ "brush_gallery_index": ["tools/brush-gallery"], "insider_drops": ["tools/drops"],+ "blender_exchange": ["addons/blender", "tests/blender"], "blender_addon_package": ["addons/blender"],+ "maya_shelf_script": ["integrations/maya", "tests/maya"], "link_worker": ["services/link-worker"],+ "release_smoke": ["tests/release_smoke.py", "tests/ui/release_smoke.sfui"],+}+APP_READS = ["assets", "tests/ui/data", "docs/guide"] # the app loads these at run time (docs/guide: Help)+UNIT_READS = ["tests/data"]+++MEMO = {"file": {}, "path": {}, "tree": {}}+++def forget():+ for m in MEMO.values():+ m.clear()+++def sha_file(path):+ memo = MEMO["file"]+ try:+ st = os.stat(path)+ except OSError:+ return "missing"+ key = (path, st.st_mtime_ns, st.st_size)+ if key not in memo:+ h = hashlib.sha256()+ with open(path, "rb") as f:+ for chunk in iter(lambda: f.read(1 << 20), b""):+ h.update(chunk)+ memo[key] = h.hexdigest()+ return memo[key]+++def sha_path(path):+ memo = MEMO["path"]+ if path in memo:+ return memo[path]+ if os.path.isdir(path):+ h = hashlib.sha256()+ for d, dirs, files in sorted(os.walk(path)):+ dirs[:] = sorted(x for x in dirs if x not in (".git", "build", "__pycache__", "node_modules"))+ for f in sorted(files):+ p = os.path.join(d, f)+ h.update(f"{os.path.relpath(p, path)}\0{sha_file(p)}\n".encode())+ memo[path] = h.hexdigest()+ else:+ memo[path] = sha_file(path)+ return memo[path]+++def tree_hash(root):+ memo = MEMO["tree"]+ """Every tracked and new file of the work tree (what a whole-tree test can read)."""+ if root not in memo:+ out = subprocess.run(["git", "ls-files", "-co", "--exclude-standard", "-z"], cwd=root, capture_output=True)+ h = hashlib.sha256()+ for f in sorted(x for x in out.stdout.decode(errors="replace").split("\0") if x):+ h.update(f"{f}\0{sha_file(os.path.join(root, f))}\n".encode())+ memo[root] = h.hexdigest() if out.returncode == 0 else f"no-git-{time.time()}" # no git: never cached+ return memo[root]+++def load_tests(build, ctest_args):+ out = subprocess.run(["ctest", "--test-dir", build, "--show-only=json-v1", *ctest_args], capture_output=True, text=True)+ if out.returncode != 0:+ return None+ tests = []+ for t in json.loads(out.stdout).get("tests", []):+ props = {p["name"]: p["value"] for p in t.get("properties", [])}+ tests.append({"name": t["name"], "command": t.get("command", []), "labels": props.get("LABELS", []),+ "env": props.get("ENVIRONMENT", []), "disabled": bool(props.get("DISABLED")),+ "cwd": props.get("WORKING_DIRECTORY", build)})+ return tests+++def fingerprint(t, root, build, run_env):+ h = hashlib.sha256()+ h.update(json.dumps([t["name"], t["command"], sorted(t["env"]), sorted(run_env.items())]).encode())+ is_app = False+ cmd = t["command"]+ for i, arg in enumerate(cmd):+ if i and cmd[i - 1] in ("--ui-out", "--out", "-o"):+ continue # where the test writes (screenshots): an output, not an input+ for a in re.split(r"[=;]", arg):+ if a and (os.path.isabs(a) or "/" in a):+ p = os.path.normpath(a if os.path.isabs(a) else os.path.join(t["cwd"], a))+ if os.path.isdir(p) and not os.path.relpath(p, build).startswith(".."):+ continue # a folder in the build tree is one the tests write (crash-log-test, test-tmp)+ if os.path.exists(p):+ h.update(f"{p}\0{sha_path(p)}\n".encode())+ is_app |= os.path.basename(p) in ("sculptforge", "sculptforge.exe")+ # a script's sibling modules+ if p.endswith(".py") and not os.path.relpath(p, root).startswith(".."):+ h.update(sha_path(os.path.dirname(p)).encode())+ exe = shutil.which(t["command"][0]) if t["command"] else None+ if exe:+ h.update(sha_file(exe).encode())+ reads = READS.get(t["name"], [])+ if is_app:+ reads = reads + APP_READS+ if t["name"].startswith("sculptforge_tests"):+ reads = reads + UNIT_READS+ if not exe or t["command"][0].endswith(("python", "python3", "python3.exe", "python.exe", "bash", "node", "cmake")) \+ and t["name"] not in READS:+ reads = ["*"] # a script test nobody listed: it may read anything+ for r in reads:+ h.update((tree_hash(root) if r == "*" else sha_path(os.path.join(root, r))).encode())+ return h.hexdigest()+++def load_slow_list(path):+ """tests/slow_tests.txt: `name seconds plan...` lines (# comments). Names of sculptforge_tests cases are written+ `sculptforge_tests: case`."""+ out = {}+ try:+ for line in open(path, encoding="utf-8"):+ line = line.split("#", 1)[0].strip()+ m = re.match(r"(sculptforge_tests: \S+|\S+)\s+([0-9.]+)", line)+ if m:+ out[m.group(1)] = float(m.group(2))+ except OSError:+ pass+ return out+++def parallel_jobs(args):+ """N of ctest's -jN / -j N / --parallel N (1 when absent)."""+ for i, x in enumerate(args):+ m = re.fullmatch(r"-j(\d+)", x)+ if m:+ return int(m.group(1))+ if x in ("-j", "--parallel") and i + 1 < len(args) and args[i + 1].isdigit():+ return int(args[i + 1])+ return 1+++def run_env():+ return {k: v for k, v in os.environ.items() if k.startswith(ENV_PREFIXES) and k not in ENV_IGNORE}+++def ctest(build, args, junit):+ cmd = ["ctest", "--test-dir", build, "--output-on-failure", "--output-junit", junit,+ "--test-output-size-passed", "1048576", *args]+ print("+ " + " ".join(cmd), flush=True)+ return subprocess.run(cmd).returncode+++# A failure whose output says the GPU went away is the machine, not the code (the PC's WSL GPU path can lose its+# device: a test that crashes the driver, or the Windows desktop resetting it). Reported apart, exit code 4 (3 is pc.sh's+# "PC unreachable").+GPU_LOST = re.compile(r"Could not find a valid EGL device|D3D12: Removing Device|Offscreen framebuffer is not supported"+ r"|failed to create dri2 screen|GL context lost")+++def outputs(junit):+ if ET is None:+ return {}+ try:+ root = ET.parse(junit).getroot()+ except (OSError, ET.ParseError):+ return {}+ return {tc.get("name"): "".join(so.text or "" for so in tc.iter("system-out")) for tc in root.iter("testcase")}+++INNER = re.compile(r"^\[ OK \] (\S+) \((\d+) ms\)$", re.M)+++def inner_times(junit):+ """The single TEST cases inside the sculptforge_tests shards: {case: seconds} (their "[ OK ] name (N ms)" lines)."""+ if ET is None:+ return {}+ try:+ root = ET.parse(junit).getroot()+ except (OSError, ET.ParseError):+ return {}+ out = {}+ for tc in root.iter("testcase"):+ if (tc.get("name") or "").startswith("sculptforge_tests"):+ for so in tc.iter("system-out"):+ for name, ms in INNER.findall(so.text or ""):+ out[name] = int(ms) / 1000+ return out+++def results(junit):+ if ET is None:+ return {}+ try:+ root = ET.parse(junit).getroot()+ except (OSError, ET.ParseError):+ return {}+ out = {}+ for tc in root.iter("testcase"):+ failed = tc.find("failure") is not None or tc.find("error") is not None or tc.get("status") == "fail"+ skipped = tc.find("skipped") is not None or tc.get("status") in ("notrun", "disabled")+ out[tc.get("name")] = ("fail" if failed else "skip" if skipped else "pass", float(tc.get("time") or 0))+ return out+++def main(argv=None):+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("--build", default="build")+ ap.add_argument("--cache", default=os.environ.get("SF_TEST_CACHE", ""))+ ap.add_argument("--budget", type=float, default=float(os.environ.get("SF_TEST_BUDGET", "20")))+ ap.add_argument("--flaky-log", default="")+ ap.add_argument("--slow-list", default="tests/slow_tests.txt",+ help="known slow tests (name seconds plan): over budget but within 1.5x their seconds is allowed")+ ap.add_argument("--times", default="", help="write name<TAB>seconds<TAB>result for every test run here")+ ap.add_argument("--threads-per-test", default="", help="'auto': SF_THREADS = cores / ctest -j for each test process")+ ap.add_argument("--junit", default="", help="also keep the first run's JUnit file here (CI: build/ctest.xml)")+ ap.add_argument("--self-test", action="store_true")+ ap.add_argument("ctest_args", nargs=argparse.REMAINDER)+ a = ap.parse_args(argv)+ if a.self_test:+ return self_test()+ args = [x for x in a.ctest_args if x != "--"]+ build = os.path.abspath(a.build)+ if ET is None: # no JUnit to read: no cache, rerun or budget, just the tests+ print("run.py: no XML parser in this Python: plain ctest")+ return subprocess.run(["ctest", "--test-dir", build, "--output-on-failure", *args]).returncode+ root = subprocess.run(["git", "rev-parse", "--show-toplevel"], capture_output=True, text=True).stdout.strip() or os.getcwd()+ if os.environ.get("CI"):+ a.cache = ""+ selects_nightly = any(x in ("-L", "--label-regex") and i + 1 < len(args) and "nightly" in args[i + 1]+ for i, x in enumerate(args))+ if not selects_nightly: # one -LE (ctest keeps only the last): fold nightly into the caller's+ le = [i for i, x in enumerate(args) if x in ("-LE", "--label-exclude") and i + 1 < len(args)]+ if le:+ args[le[-1] + 1] = f"({args[le[-1] + 1]})|nightly"+ else:+ args += ["-LE", "nightly"]+ # A parallel run loads the machine like CI: timing checks (timeLimit, tests/TestHarness.h) get CI's slack.+ if parallel_jobs(args) > 1:+ os.environ.setdefault("SF_TIME_LIMIT_SCALE", "4")+ if a.threads_per_test == "auto":+ os.environ.setdefault("SF_THREADS", str(max(2, (os.cpu_count() or 4) // parallel_jobs(args))))+ tests = load_tests(build, args)+ if tests is None:+ print(f"run.py: ctest can't list the tests in {build} (configured?)")+ return 2+ labels = {t["name"]: t["labels"] for t in tests}+ t0 = time.time()++ # Cache: skip tests whose fingerprint passed before.+ cached, prints = [], {}+ if a.cache:+ os.makedirs(a.cache, exist_ok=True)+ env = run_env()+ for t in tests:+ if t["disabled"]:+ continue+ fp = prints[t["name"]] = fingerprint(t, root, build, env)+ if os.path.exists(os.path.join(a.cache, fp)):+ cached.append(t["name"])+ print(f"test cache: {len(cached)} of {len(prints)} tests passed before with these exact inputs: skipped "+ f"({time.time() - t0:.1f} s to fingerprint)")+ todo = [t["name"] for t in tests if not t["disabled"] and t["name"] not in cached]+ res, inner, gpu_lost = {}, {}, []+ rc = 0+ with tempfile.TemporaryDirectory() as tmp:+ if todo:+ run_args = list(args)+ if cached: # only the rest: their names (a subset of what the args selected, so -R gives way)+ run_args = [x for i, x in enumerate(run_args) if x not in ("-R", "--tests-regex")+ and (i == 0 or run_args[i - 1] not in ("-R", "--tests-regex"))]+ run_args += ["-R", "^(" + "|".join(re.escape(n) for n in todo) + ")$"]+ rc = ctest(build, run_args, os.path.join(tmp, "a.xml"))+ res = results(os.path.join(tmp, "a.xml"))+ inner = inner_times(os.path.join(tmp, "a.xml"))+ if a.junit and os.path.exists(os.path.join(tmp, "a.xml")):+ shutil.copy(os.path.join(tmp, "a.xml"), a.junit)+ if not res and rc != 0:+ print("run.py: ctest failed before running tests")+ return 2+ failed = [n for n, (s, _) in res.items() if s == "fail"]+ flaky = []+ if failed:+ print(f"\nrerunning {len(failed)} failed test(s) once: {' '.join(failed)}", flush=True)+ ctest(build, ["--rerun-failed", *[x for x in args if x.startswith("-j")]], os.path.join(tmp, "b.xml"))+ again = results(os.path.join(tmp, "b.xml"))+ said = outputs(os.path.join(tmp, "b.xml"))+ gpu_lost = [n for n in failed if again.get(n, ("fail",))[0] != "pass" and GPU_LOST.search(said.get(n, ""))]+ first = outputs(os.path.join(tmp, "a.xml"))+ for n in failed:+ if again.get(n, ("fail",))[0] == "pass":+ res[n] = ("pass", again[n][1])+ if GPU_LOST.search(first.get(n, "")):+ print(f"gpu: {n} failed with the graphics device gone, then passed: not a flake")+ else:+ flaky.append(n)+ res[n] = ("flaky", res[n][1])+ for n in flaky:+ print(f"flaky: {n} (failed, then passed on the rerun)")+ if flaky and a.flaky_log:+ head = "" if os.path.exists(a.flaky_log) else FLAKY_HEADER+ sha = subprocess.run(["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True).stdout.strip()+ with open(a.flaky_log, "a", encoding="utf-8") as f:+ f.write(head + "".join(f"| {time.strftime('%Y-%m-%d')} | {n} | {os.uname().nodename} | {sha} |\n" for n in flaky))+ # Record first-try passes in the cache.+ if a.cache:+ for n, (s, _) in res.items():+ if s == "pass" and n in prints:+ open(os.path.join(a.cache, prints[n]), "w").write(f"{n} {time.strftime('%Y-%m-%dT%H:%M:%S')}\n")+ real = [n for n, (s, _) in res.items() if s == "fail" and "flaky" not in labels.get(n, []) and n not in gpu_lost]+ tolerated = [n for n, (s, _) in res.items() if s == "fail" and "flaky" in labels.get(n, [])]+ # The budget is per test: a ctest entry, or one TEST case inside a sculptforge_tests shard (a shard is many tests).+ slow = sorted(((d, n) for n, (s, d) in res.items() if a.budget > 0 and d > a.budget and s != "skip"+ and "nightly" not in labels.get(n, []) and not n.startswith("sculptforge_tests")), reverse=True)+ slow += sorted(((d, f"sculptforge_tests: {n}") for n, d in inner.items() if a.budget > 0 and d > a.budget+ and not n.startswith("nightly_")), reverse=True)+ if a.times:+ with open(a.times, "w") as f:+ f.write("".join(f"{n}\t{d:.2f}\t{s}\n" for n, (s, d) in sorted(res.items())))+ f.write("".join(f"{n}\t0\tcached\n" for n in cached))+ f.write("".join(f"case:{n}\t{d:.2f}\tpass\n" for n, d in sorted(inner.items())))+ print(f"\n== {len(res)} run, {len(cached)} cached, {len(real)} failed, {sum(s == 'flaky' for s, _ in res.values())} flaky, "+ f"{len(slow)} over the {a.budget:g} s budget; {time.time() - t0:.0f} s")+ for n in tolerated:+ print(f"failed but labelled flaky (does not block): {n}")+ known = load_slow_list(os.path.join(root, a.slow_list) if not os.path.isabs(a.slow_list) else a.slow_list)+ # The list's seconds are the PC's, where the budget is 20 s; a machine with a bigger budget (CI's software+ # OpenGL: 150 s) scales them the same way.+ scale = max(1.0, a.budget / 20.0)+ listed = [(d, n) for d, n in slow if n in known and d <= 1.5 * known[n] * scale]+ slow = [(d, n) for d, n in slow if (d, n) not in listed]+ for d, n in listed:+ print(f"known slow: {n} {d:.1f} s (tests/slow_tests.txt: {known[n]:g} s; make it fast, then delete its line)")+ for d, n in slow:+ why = f"1.5x its {known[n] * scale:g} s in tests/slow_tests.txt" if n in known else f"the {a.budget:g} s budget"+ print(f"over budget: {n} {d:.1f} s, over {why} (wait on events, use a smaller mesh, split it, or label it nightly)")+ for n in real:+ print(f"FAILED: {n}")+ if gpu_lost:+ print(f"GPU lost: {len(gpu_lost)} test(s) failed twice with the graphics device gone ({' '.join(gpu_lost[:8])}): "+ "the machine, not the code; run them again (ctest --rerun-failed)")+ return 1 if real or slow else 4 if gpu_lost else 0+++FLAKY_HEADER = """# Flaky tests++Written by tools/test/run.py (docs/testing-flow.md): a test that failed, then passed on its one automatic rerun.+A test here twice gets the `flaky` label in CMakeLists.txt (it then never blocks a release) and an owner who fixes or+deletes it; remove its lines once fixed.++| date | test | machine | commit |+|---|---|---|---|+"""+++def self_test():+ if ET is None:+ print("run.py self-test: no XML parser in this Python (pyexpat): skipped")+ return 0+ with tempfile.TemporaryDirectory() as d:+ src = os.path.join(d, "src")+ os.makedirs(os.path.join(src, "assets"))+ subprocess.run(["git", "init", "-q", src], check=True)+ open(os.path.join(src, "assets", "a.txt"), "w").write("1")+ script = os.path.join(src, "s.sfui")+ open(script, "w").write("x")+ app = os.path.join(d, "sculptforge")+ open(app, "w").write("bin")+ shots = os.path.join(d, "ui-shots", "s")+ os.makedirs(shots)+ t = {"name": "ui_s", "command": [app, "--ui-test", script, "--ui-out", shots], "labels": ["ui"], "env": [], "cwd": d}+ f1 = fingerprint(t, src, d, {})+ open(os.path.join(shots, "shot.png"), "w").write("new")+ forget()+ assert f1 == fingerprint(t, src, d, {}), "screenshots the test wrote are not inputs"+ open(script, "w").write("y")+ forget()+ f2 = fingerprint(t, src, d, {})+ assert f2 != f1, "the script is an input"+ open(os.path.join(src, "assets", "a.txt"), "w").write("2")+ forget()+ f3 = fingerprint(t, src, d, {})+ assert f3 != f2, "the app's assets are inputs"+ assert fingerprint(t, src, d, {"SF_X": "1"}) != f3, "the environment is an input"+ py = {"name": "unknown_script", "command": [sys.executable, os.path.join(src, "t.py")], "labels": [], "env": [], "cwd": d}+ open(os.path.join(src, "t.py"), "w").write("")+ g1 = fingerprint(py, src, d, {})+ open(os.path.join(src, "other.md"), "w").write("new") # an unlisted script test may read any file+ forget()+ assert fingerprint(py, src, d, {}) != g1, "unlisted script tests hash the whole tree"+ junit = os.path.join(d, "j.xml")+ open(junit, "w").write('<testsuite><testcase name="a" time="1.5" status="run"/>'+ '<testcase name="b" time="2" status="fail"><failure message="x"/></testcase></testsuite>')+ assert results(junit) == {"a": ("pass", 1.5), "b": ("fail", 2.0)}+ open(junit, "w").write('<testsuite><testcase name="sculptforge_tests_0of2" time="9" status="run">'+ '<system-out>[ RUN] x\n[ OK ] mesh_io (1200 ms)\n[FAIL] y (3 ms)\n</system-out></testcase></testsuite>')+ assert inner_times(junit) == {"mesh_io": 1.2}+ open(os.path.join(d, "slow.txt"), "w").write("# x\nui_bake 40 smaller mesh\nsculptforge_tests: lion 50.5 nightly\n")+ assert load_slow_list(os.path.join(d, "slow.txt")) == {"ui_bake": 40.0, "sculptforge_tests: lion": 50.5}+ assert parallel_jobs(["-j10"]) == 10 and parallel_jobs(["-C", "Release", "-j", "4"]) == 4 and parallel_jobs([]) == 1+ print("run.py self-test passed")+ return 0+++if __name__ == "__main__":+ sys.exit(main())