Free, open-source 3D sculpting for Linux, macOS and Windows: dense meshes, remeshing, painting, built-in AI (image/text to 3D), Send to Unreal, game-ready LODs, Gaussian splats and OpenUSD.
C++ 3d3d-sculptingcppdear-imguidigital-sculptingfbxgame-developmentgaussian-splattinggltfimage-to-3dmcpopenglopenusdsculptingtext-to-3dunreal-engine
#!/usr/bin/env python3
# SculptForge - open source digital sculpting
# SPDX-License-Identifier: GPL-3.0-or-later
"""The one way every tier runs ctest (docs/testing-flow.md): ctest plus a test cache, a flake rerun and a time budget.
tools/test/run.py --build build [--cache DIR] [--budget 20] [--flaky-log tests/flaky.md] [-- ctest args...]
tools/test/run.py --self-test
* Nightly tests (label `nightly`) never run here unless the ctest args select them (-L nightly): T3 runs them.
* Test cache (--cache, or $SF_TEST_CACHE): every test gets a fingerprint, the hash of everything it can read: its
program (the built file), every file or folder its command names, the app's assets and test data for app tests,
the test's ENVIRONMENT and the SF_/SCULPTFORGE_/SDL_/GL driver variables of this run. Tests that read the whole tree
(marks, the guide, release notes...) hash every tracked file. A test that passed with the same fingerprint on this
machine is skipped and reported as cached; a pass on the first try is recorded. Never on CI (nothing persists).
* Flakes: failed tests run once more (ctest --rerun-failed). A pass on the rerun is a flake: logged (`flaky: name`
lines, appended to --flaky-log), not cached, and does not fail the run. A test labelled `flaky` that fails twice is
reported but never fails the run either (it never blocks a release; fix it or delete it).
* Budget: a non-nightly test that took longer than --budget seconds (default $SF_TEST_BUDGET or 20; 0 = off) fails
the run, with the list. Make it wait on events instead of fixed frames, split it, or label it nightly. An
over-budget ctest entry runs once more alone first (--no-alone: not): only a test over budget alone counts, so a
busy machine's parallel run doesn't fail an innocent UI test.
Exit code: 0 green, 1 a real failure or a test over budget, 2 ctest itself could not run, 4 only GPU-lost failures
(the output says the graphics device went away: the machine, not the code).
"""
import argparse
import hashlib
import json
import os
import platform
import re
import shutil
import subprocess
import sys
import tempfile
import time
try:
import xml.etree.ElementTree as ET
ET.fromstring("<a/>")
except ImportError: # some system Pythons (Rocky 8's 3.12) ship a pyexpat that doesn't load: plain ctest then
ET = None
ENV_PREFIXES = ("SF_", "SCULPTFORGE_", "SDL_", "GALLIUM_", "LIBGL_", "MESA_", "__GLX_")
ENV_IGNORE = {"SDL_APP_ID"}
# Tests that read files their command doesn't name: hashed as globs (repo-relative). "*" = every tracked file.
READS = {
"marks": ["*"], "marks_self_test": ["tools/check_marks.py"], "public_email": ["*"], "public_email_self_test": ["tools/check_public_email.py"],
"guide_site": ["*"], "guide_menu_paths": ["*"], "ui_guide_shortcuts": ["docs/guide"],
"release_notes": ["tools/release"], "release_notes_fragments": ["changes", "tools/release"],
"release_script_beta": ["*"], "upstream_watch": ["tools/upstream-watch"],
"brush_gallery_index": ["tools/brush-gallery"], "insider_drops": ["tools/drops"],
"blender_exchange": ["addons/blender", "tests/blender"], "blender_addon_package": ["addons/blender"],
"maya_shelf_script": ["integrations/maya", "tests/maya"], "link_worker": ["services/link-worker"],
"release_smoke": ["tests/release_smoke.py", "tests/ui/release_smoke.sfui"],
}
APP_READS = ["assets", "tests/ui/data", "docs/guide"] # the app loads these at run time (docs/guide: Help)
UNIT_READS = ["tests/data"]
MEMO = {"file": {}, "path": {}, "tree": {}}
def forget():
for m in MEMO.values():
m.clear()
def sha_file(path):
memo = MEMO["file"]
try:
st = os.stat(path)
except OSError:
return "missing"
key = (path, st.st_mtime_ns, st.st_size)
if key not in memo:
h = hashlib.sha256()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
memo[key] = h.hexdigest()
return memo[key]
def sha_path(path):
memo = MEMO["path"]
if path in memo:
return memo[path]
if os.path.isdir(path):
h = hashlib.sha256()
for d, dirs, files in sorted(os.walk(path)):
dirs[:] = sorted(x for x in dirs if x not in (".git", "build", "__pycache__", "node_modules"))
for f in sorted(files):
p = os.path.join(d, f)
h.update(f"{os.path.relpath(p, path)}\0{sha_file(p)}\n".encode())
memo[path] = h.hexdigest()
else:
memo[path] = sha_file(path)
return memo[path]
def tree_hash(root):
memo = MEMO["tree"]
"""Every tracked and new file of the work tree (what a whole-tree test can read)."""
if root not in memo:
out = subprocess.run(["git", "ls-files", "-co", "--exclude-standard", "-z"], cwd=root, capture_output=True)
h = hashlib.sha256()
for f in sorted(x for x in out.stdout.decode(errors="replace").split("\0") if x):
h.update(f"{f}\0{sha_file(os.path.join(root, f))}\n".encode())
memo[root] = h.hexdigest() if out.returncode == 0 else f"no-git-{time.time()}" # no git: never cached
return memo[root]
def load_tests(build, ctest_args):
out = subprocess.run(["ctest", "--test-dir", build, "--show-only=json-v1", *ctest_args], capture_output=True, text=True)
if out.returncode != 0:
return None
tests = []
for t in json.loads(out.stdout).get("tests", []):
props = {p["name"]: p["value"] for p in t.get("properties", [])}
tests.append({"name": t["name"], "command": t.get("command", []), "labels": props.get("LABELS", []),
"env": props.get("ENVIRONMENT", []), "disabled": bool(props.get("DISABLED")),
"cwd": props.get("WORKING_DIRECTORY", build)})
return tests
def fingerprint(t, root, build, run_env):
h = hashlib.sha256()
h.update(json.dumps([t["name"], t["command"], sorted(t["env"]), sorted(run_env.items())]).encode())
is_app = False
cmd = t["command"]
for i, arg in enumerate(cmd):
if i and cmd[i - 1] in ("--ui-out", "--out", "-o"):
continue # where the test writes (screenshots): an output, not an input
for a in re.split(r"[=;]", arg):
if a and (os.path.isabs(a) or "/" in a):
p = os.path.normpath(a if os.path.isabs(a) else os.path.join(t["cwd"], a))
if os.path.isdir(p) and not os.path.relpath(p, build).startswith(".."):
continue # a folder in the build tree is one the tests write (crash-log-test, test-tmp)
if os.path.exists(p):
h.update(f"{p}\0{sha_path(p)}\n".encode())
is_app |= os.path.basename(p) in ("sculptforge", "sculptforge.exe")
# a script's sibling modules
if p.endswith(".py") and not os.path.relpath(p, root).startswith(".."):
h.update(sha_path(os.path.dirname(p)).encode())
exe = shutil.which(t["command"][0]) if t["command"] else None
if exe:
h.update(sha_file(exe).encode())
reads = READS.get(t["name"], [])
if is_app:
reads = reads + APP_READS
if t["name"].startswith("sculptforge_tests"):
reads = reads + UNIT_READS
if not exe or t["command"][0].endswith(("python", "python3", "python3.exe", "python.exe", "bash", "node", "cmake")) \
and t["name"] not in READS:
reads = ["*"] # a script test nobody listed: it may read anything
for r in reads:
h.update((tree_hash(root) if r == "*" else sha_path(os.path.join(root, r))).encode())
return h.hexdigest()
def load_slow_list(path):
"""tests/slow_tests.txt: `name seconds plan...` lines (# comments). Names of sculptforge_tests cases are written
`sculptforge_tests: case`."""
out = {}
try:
for line in open(path, encoding="utf-8"):
line = line.split("#", 1)[0].strip()
m = re.match(r"(sculptforge_tests: \S+|\S+)\s+([0-9.]+)", line)
if m:
out[m.group(1)] = float(m.group(2))
except OSError:
pass
return out
def parallel_jobs(args):
"""N of ctest's -jN / -j N / --parallel N (1 when absent)."""
for i, x in enumerate(args):
m = re.fullmatch(r"-j(\d+)", x)
if m:
return int(m.group(1))
if x in ("-j", "--parallel") and i + 1 < len(args) and args[i + 1].isdigit():
return int(args[i + 1])
return 1
def rerun_args(args):
"""The options a rerun keeps: the configuration (-C X: a multi-config build runs nothing without it) and -j."""
keep, i = [], 0
while i < len(args):
x = args[i]
if x in ("-C", "--build-config", "-j", "--parallel") and i + 1 < len(args) and not args[i + 1].startswith("-"):
keep += [x, args[i + 1]]
i += 2
continue
if re.fullmatch(r"-C.+|-j\d*", x):
keep.append(x)
i += 1
return keep
def run_env():
return {k: v for k, v in os.environ.items() if k.startswith(ENV_PREFIXES) and k not in ENV_IGNORE}
def ctest(build, args, junit):
cmd = ["ctest", "--test-dir", build, "--output-on-failure", "--output-junit", junit,
"--test-output-size-passed", "1048576", *args]
print("+ " + " ".join(cmd), flush=True)
return subprocess.run(cmd).returncode
# A failure whose output says the GPU went away is the machine, not the code (the PC's WSL GPU path can lose its
# device: a test that crashes the driver, or the Windows desktop resetting it). Reported apart, exit code 4 (3 is pc.sh's
# "PC unreachable").
GPU_LOST = re.compile(r"Could not find a valid EGL device|D3D12: Removing Device|Offscreen framebuffer is not supported"
r"|failed to create dri2 screen|GL context lost")
def outputs(junit):
if ET is None:
return {}
try:
root = ET.parse(junit).getroot()
except (OSError, ET.ParseError):
return {}
return {tc.get("name"): "".join(so.text or "" for so in tc.iter("system-out")) for tc in root.iter("testcase")}
INNER = re.compile(r"^\[ OK \] (\S+) \((\d+) ms\)$", re.M)
def inner_times(junit):
"""The single TEST cases inside the sculptforge_tests shards: {case: seconds} (their "[ OK ] name (N ms)" lines)."""
if ET is None:
return {}
try:
root = ET.parse(junit).getroot()
except (OSError, ET.ParseError):
return {}
out = {}
for tc in root.iter("testcase"):
if (tc.get("name") or "").startswith("sculptforge_tests"):
for so in tc.iter("system-out"):
for name, ms in INNER.findall(so.text or ""):
out[name] = int(ms) / 1000
return out
RUN_LINE = re.compile(r"^\[ RUN\] (\S+)$", re.M)
DONE_LINE = re.compile(r"^\[(?: OK |FAIL)\] (\S+) \(", re.M)
def timeouts(junit):
"""The tests ctest stopped at their TIMEOUT: {test: the sculptforge_tests case still running ("" if none)}."""
if ET is None:
return {}
try:
root = ET.parse(junit).getroot()
except (OSError, ET.ParseError):
return {}
out = {}
for tc in root.iter("testcase"):
if not any((f.get("message") or "").startswith("Timeout") for f in tc.iter("failure")):
continue
text = "".join(so.text or "" for so in tc.iter("system-out"))
done = set(DONE_LINE.findall(text))
out[tc.get("name")] = next((n for n in reversed(RUN_LINE.findall(text)) if n not in done), "")
return out
def results(junit):
if ET is None:
return {}
try:
root = ET.parse(junit).getroot()
except (OSError, ET.ParseError):
return {}
out = {}
for tc in root.iter("testcase"):
failed = tc.find("failure") is not None or tc.find("error") is not None or tc.get("status") == "fail"
skipped = tc.find("skipped") is not None or tc.get("status") in ("notrun", "disabled")
out[tc.get("name")] = ("fail" if failed else "skip" if skipped else "pass", float(tc.get("time") or 0))
return out
def run_tests(argv=None):
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--build", default="build")
ap.add_argument("--cache", default=os.environ.get("SF_TEST_CACHE", ""))
ap.add_argument("--no-alone", action="store_true", help="count over-budget tests without rerunning them alone")
ap.add_argument("--budget", type=float, default=float(os.environ.get("SF_TEST_BUDGET", "20")))
ap.add_argument("--flaky-log", default="")
ap.add_argument("--slow-list", default="tests/slow_tests.txt",
help="known slow tests (name seconds plan): over budget but within 1.5x their seconds is allowed")
ap.add_argument("--times", default="", help="write name<TAB>seconds<TAB>result for every test run here")
ap.add_argument("--threads-per-test", default="", help="'auto': SF_THREADS = cores / ctest -j for each test process")
ap.add_argument("--junit", default="", help="also keep the first run's JUnit file here (CI: build/ctest.xml)")
ap.add_argument("--self-test", action="store_true")
ap.add_argument("ctest_args", nargs=argparse.REMAINDER)
a = ap.parse_args(argv)
if a.self_test:
return self_test()
args = [x for x in a.ctest_args if x != "--"]
build = os.path.abspath(a.build)
if ET is None: # no JUnit to read: no cache, rerun or budget, just the tests
print("run.py: no XML parser in this Python: plain ctest")
return subprocess.run(["ctest", "--test-dir", build, "--output-on-failure", *args]).returncode
root = subprocess.run(["git", "rev-parse", "--show-toplevel"], capture_output=True, text=True).stdout.strip() or os.getcwd()
if os.environ.get("CI"):
a.cache = ""
selects_nightly = any(x in ("-L", "--label-regex") and i + 1 < len(args) and "nightly" in args[i + 1]
for i, x in enumerate(args))
if not selects_nightly: # one -LE (ctest keeps only the last): fold nightly into the caller's
le = [i for i, x in enumerate(args) if x in ("-LE", "--label-exclude") and i + 1 < len(args)]
if le:
args[le[-1] + 1] = f"({args[le[-1] + 1]})|nightly"
else:
args += ["-LE", "nightly"]
# A parallel run loads the machine like CI: timing checks (timeLimit, tests/TestHarness.h) get CI's slack.
if parallel_jobs(args) > 1:
os.environ.setdefault("SF_TIME_LIMIT_SCALE", "4")
if a.threads_per_test == "auto":
os.environ.setdefault("SF_THREADS", str(max(2, (os.cpu_count() or 4) // parallel_jobs(args))))
tests = load_tests(build, args)
if tests is None:
print(f"run.py: ctest can't list the tests in {build} (configured?)")
return 2
labels = {t["name"]: t["labels"] for t in tests}
t0 = time.time()
# Cache: skip tests whose fingerprint passed before.
cached, prints = [], {}
if a.cache:
os.makedirs(a.cache, exist_ok=True)
env = run_env()
for t in tests:
if t["disabled"]:
continue
fp = prints[t["name"]] = fingerprint(t, root, build, env)
if os.path.exists(os.path.join(a.cache, fp)):
cached.append(t["name"])
print(f"test cache: {len(cached)} of {len(prints)} tests passed before with these exact inputs: skipped "
f"({time.time() - t0:.1f} s to fingerprint)")
todo = [t["name"] for t in tests if not t["disabled"] and t["name"] not in cached]
res, inner, gpu_lost, hung = {}, {}, [], {}
rc = 0
with tempfile.TemporaryDirectory() as tmp:
if todo:
run_args = list(args)
if cached: # only the rest: their names (a subset of what the args selected, so -R gives way)
run_args = [x for i, x in enumerate(run_args) if x not in ("-R", "--tests-regex")
and (i == 0 or run_args[i - 1] not in ("-R", "--tests-regex"))]
run_args += ["-R", "^(" + "|".join(re.escape(n) for n in todo) + ")$"]
rc = ctest(build, run_args, os.path.join(tmp, "a.xml"))
res = results(os.path.join(tmp, "a.xml"))
inner = inner_times(os.path.join(tmp, "a.xml"))
if a.junit and os.path.exists(os.path.join(tmp, "a.xml")):
shutil.copy(os.path.join(tmp, "a.xml"), a.junit)
if not res and rc != 0:
print("run.py: ctest failed before running tests")
return 2
# A hang is no flake, and its rerun would hold the machine as long again: named, never rerun.
hung = timeouts(os.path.join(tmp, "a.xml"))
for n, case in hung.items():
print(f"TIMEOUT: {n}" + (f" hung in {case}" if case else ""), flush=True)
failed = [n for n, (s, _) in res.items() if s == "fail" and n not in hung]
flaky = []
if failed:
print(f"\nrerunning {len(failed)} failed test(s) once: {' '.join(failed)}", flush=True)
ctest(build, ["--rerun-failed", *rerun_args(args)], os.path.join(tmp, "b.xml"))
again = results(os.path.join(tmp, "b.xml"))
said = outputs(os.path.join(tmp, "b.xml"))
gpu_lost = [n for n in failed if again.get(n, ("fail",))[0] != "pass" and GPU_LOST.search(said.get(n, ""))]
first = outputs(os.path.join(tmp, "a.xml"))
for n in failed:
if again.get(n, ("fail",))[0] == "pass":
res[n] = ("pass", again[n][1])
if GPU_LOST.search(first.get(n, "")):
print(f"gpu: {n} failed with the graphics device gone, then passed: not a flake")
else:
flaky.append(n)
res[n] = ("flaky", res[n][1])
for n in flaky:
print(f"flaky: {n} (failed, then passed on the rerun)")
if flaky and a.flaky_log:
sha = subprocess.run(["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True).stdout.strip()
# The machine as a description (SF_MACHINE, else OS and CPU), never its host name: the file is public.
machine = os.environ.get("SF_MACHINE") or f"{platform.system()} {platform.machine()}, -j{parallel_jobs(args)}"
rows = "".join(f"| {time.strftime('%Y-%m-%d')} | {n} | {machine} | {sha} |\n" for n in flaky)
text = open(a.flaky_log, encoding="utf-8").read() if os.path.exists(a.flaky_log) else FLAKY_HEADER
at = text.find("\n## ") # (into the table, above the notes that follow it)
text = text.rstrip("\n") + "\n" + rows if at < 0 else text[:at].rstrip("\n") + "\n" + rows + text[at:]
with open(a.flaky_log, "w", encoding="utf-8") as f:
f.write(text)
# Record first-try passes in the cache.
if a.cache:
for n, (s, _) in res.items():
if s == "pass" and n in prints:
open(os.path.join(a.cache, prints[n]), "w").write(f"{n} {time.strftime('%Y-%m-%dT%H:%M:%S')}\n")
real = [n for n, (s, _) in res.items() if s == "fail" and "flaky" not in labels.get(n, []) and n not in gpu_lost]
tolerated = [n for n, (s, _) in res.items() if s == "fail" and "flaky" in labels.get(n, [])]
# The budget is per test: a ctest entry, or one TEST case inside a sculptforge_tests shard (a shard is many tests).
slow = sorted(((d, n) for n, (s, d) in res.items() if a.budget > 0 and d > a.budget and s != "skip"
and "nightly" not in labels.get(n, []) and not n.startswith("sculptforge_tests")), reverse=True)
slow += sorted(((d, f"sculptforge_tests: {n}") for n, d in inner.items() if a.budget > 0 and d > a.budget
and not n.startswith("nightly_")), reverse=True)
if a.times:
with open(a.times, "w") as f:
f.write("".join(f"{n}\t{d:.2f}\t{s}\n" for n, (s, d) in sorted(res.items())))
f.write("".join(f"{n}\t0\tcached\n" for n in cached))
f.write("".join(f"case:{n}\t{d:.2f}\tpass\n" for n, d in sorted(inner.items())))
print(f"\n== {len(res)} run, {len(cached)} cached, {len(real)} failed, {sum(s == 'flaky' for s, _ in res.values())} flaky, "
f"{len(slow)} over the {a.budget:g} s budget; {time.time() - t0:.0f} s")
for n in tolerated:
print(f"failed but labelled flaky (does not block): {n}")
known = load_slow_list(os.path.join(root, a.slow_list) if not os.path.isabs(a.slow_list) else a.slow_list)
# The list's seconds are the PC's, where the budget is 20 s; a machine with a bigger budget (CI's software
# OpenGL: 150 s) scales them the same way.
scale = max(1.0, a.budget / 20.0)
listed = [(d, n) for d, n in slow if n in known and d <= 1.5 * known[n] * scale]
slow = [(d, n) for d, n in slow if (d, n) not in listed]
for d, n in listed:
print(f"known slow: {n} {d:.1f} s (tests/slow_tests.txt: {known[n]:g} s; make it fast, then delete its line)")
# Over budget in a parallel run can be the machine's load (each full run a different UI test): once more, alone.
# Only a test still over its budget alone counts. (sculptforge_tests cases are timed inside their shard: not rerun.)
if slow and not a.no_alone:
with tempfile.TemporaryDirectory() as tmp2:
for d, n in list(slow):
if n.startswith("sculptforge_tests"):
continue
ctest(build, ["-R", f"^{re.escape(n)}$"], os.path.join(tmp2, "alone.xml"))
s2, d2 = results(os.path.join(tmp2, "alone.xml")).get(n, ("fail", d))
limit = 1.5 * known[n] * scale if n in known else a.budget
if s2 == "pass" and d2 <= limit:
slow.remove((d, n))
print(f"over budget under load only: {n} {d:.1f} s in the run, {d2:.1f} s alone (not counted)")
for d, n in slow:
why = f"1.5x its {known[n] * scale:g} s in tests/slow_tests.txt" if n in known else f"the {a.budget:g} s budget"
print(f"over budget: {n} {d:.1f} s, over {why} (wait on events, use a smaller mesh, split it, or label it nightly)")
for n in real:
print(f"FAILED: {n}" + (f" (timed out in {hung[n]})" if hung.get(n) else " (timed out)" if n in hung else ""))
if gpu_lost:
print(f"GPU lost: {len(gpu_lost)} test(s) failed twice with the graphics device gone ({' '.join(gpu_lost[:8])}): "
"the machine, not the code; run them again (ctest --rerun-failed)")
return 1 if real or slow else 4 if gpu_lost else 0
FLAKY_HEADER = """# Flaky tests
Written by tools/test/run.py (docs/testing-flow.md): a test that failed, then passed on its one automatic rerun.
A test here twice gets the `flaky` label in CMakeLists.txt (it then never blocks a release) and an owner who fixes or
deletes it; remove its lines once fixed.
| date | test | machine | commit |
|---|---|---|---|
"""
def self_test():
if ET is None:
print("run.py self-test: no XML parser in this Python (pyexpat): skipped")
return 0
with tempfile.TemporaryDirectory() as d:
src = os.path.join(d, "src")
os.makedirs(os.path.join(src, "assets"))
subprocess.run(["git", "init", "-q", src], check=True)
open(os.path.join(src, "assets", "a.txt"), "w").write("1")
script = os.path.join(src, "s.sfui")
open(script, "w").write("x")
app = os.path.join(d, "sculptforge")
open(app, "w").write("bin")
shots = os.path.join(d, "ui-shots", "s")
os.makedirs(shots)
t = {"name": "ui_s", "command": [app, "--ui-test", script, "--ui-out", shots], "labels": ["ui"], "env": [], "cwd": d}
f1 = fingerprint(t, src, d, {})
open(os.path.join(shots, "shot.png"), "w").write("new")
forget()
assert f1 == fingerprint(t, src, d, {}), "screenshots the test wrote are not inputs"
open(script, "w").write("y")
forget()
f2 = fingerprint(t, src, d, {})
assert f2 != f1, "the script is an input"
open(os.path.join(src, "assets", "a.txt"), "w").write("2")
forget()
f3 = fingerprint(t, src, d, {})
assert f3 != f2, "the app's assets are inputs"
assert fingerprint(t, src, d, {"SF_X": "1"}) != f3, "the environment is an input"
py = {"name": "unknown_script", "command": [sys.executable, os.path.join(src, "t.py")], "labels": [], "env": [], "cwd": d}
open(os.path.join(src, "t.py"), "w").write("")
g1 = fingerprint(py, src, d, {})
open(os.path.join(src, "other.md"), "w").write("new") # an unlisted script test may read any file
forget()
assert fingerprint(py, src, d, {}) != g1, "unlisted script tests hash the whole tree"
junit = os.path.join(d, "j.xml")
open(junit, "w").write('<testsuite><testcase name="a" time="1.5" status="run"/>'
'<testcase name="b" time="2" status="fail"><failure message="x"/></testcase></testsuite>')
assert results(junit) == {"a": ("pass", 1.5), "b": ("fail", 2.0)}
open(junit, "w").write('<testsuite><testcase name="sculptforge_tests_0of2" time="9" status="run">'
'<system-out>[ RUN] x\n[ OK ] mesh_io (1200 ms)\n[FAIL] y (3 ms)\n</system-out></testcase></testsuite>')
assert inner_times(junit) == {"mesh_io": 1.2}
open(junit, "w").write('<testsuite><testcase name="sculptforge_tests_1of2" time="600" status="fail">'
'<failure message="Timeout"/><system-out>[ RUN] a\n[ OK ] a (1 ms)\n[ RUN] spin\n'
'</system-out></testcase><testcase name="ui_x" status="fail"><failure message="Failed"/>'
'</testcase><testcase name="ui_y" status="fail"><failure message="Timeout"/></testcase></testsuite>')
assert timeouts(junit) == {"sculptforge_tests_1of2": "spin", "ui_y": ""}
open(os.path.join(d, "slow.txt"), "w").write("# x\nui_bake 40 smaller mesh\nsculptforge_tests: lion 50.5 nightly\n")
assert load_slow_list(os.path.join(d, "slow.txt")) == {"ui_bake": 40.0, "sculptforge_tests: lion": 50.5}
assert parallel_jobs(["-j10"]) == 10 and parallel_jobs(["-C", "Release", "-j", "4"]) == 4 and parallel_jobs([]) == 1
assert rerun_args(["-C", "Release", "-j", "4", "-L", "ui"]) == ["-C", "Release", "-j", "4"]
assert rerun_args(["-j10", "-CRelease", "-R", "x"]) == ["-j10", "-CRelease"] and rerun_args(["-j"]) == ["-j"]
print("run.py self-test passed")
return 0
def disk_tmp():
"""A temp folder on disk: /tmp is RAM (tmpfs) on some machines, and UI tests write hundreds of MB each."""
for d in ("/var/tmp",):
if os.path.isdir(d) and os.access(d, os.W_OK):
return d
return tempfile.gettempdir()
def main(argv=None):
# Every test of this run writes its temp files (the app's sculptforge-ui-test-* folders: settings, logs, autosaves of
# big meshes) under one private folder, removed when the run passes and kept, with its path printed, when it fails
# (11-06: thousands of leftover folders in a RAM /tmp helped run the laptop out of memory).
tmp = tempfile.mkdtemp(prefix="sf-tests-", dir=disk_tmp())
saved = {k: os.environ.get(k) for k in ("TMPDIR", "TMP", "TEMP")}
os.environ.update(TMPDIR=tmp, TMP=tmp, TEMP=tmp)
rc = 1
try:
rc = run_tests(argv)
return rc
finally:
for k, v in saved.items():
if v is None: os.environ.pop(k, None)
else: os.environ[k] = v
if rc == 0:
shutil.rmtree(tmp, ignore_errors=True)
elif os.path.isdir(tmp) and os.listdir(tmp):
print(f"run.py: the tests' temp files are kept for a look: {tmp}")
else:
shutil.rmtree(tmp, ignore_errors=True)
if __name__ == "__main__":
sys.exit(main())