Skip to content

Free, open-source 3D sculpting for Linux, macOS and Windows: dense meshes, remeshing, painting, built-in AI (image/text to 3D), Send to Unreal, game-ready LODs, Gaussian splats and OpenUSD.

21.8 KB
Raw Blame
#!/usr/bin/env python3
# SculptForge - open source digital sculpting
# SPDX-License-Identifier: GPL-3.0-or-later
"""Warm UI tests: a pool of `sculptforge --ui-serve` servers (src/app/UiServe.h) runs tests/ui/*.sfui scripts
without starting a process, SDL, a window and an OpenGL context for each one.

  warm_ui.py start  --dir D --app APP      start the pool in the background (ctest fixture ui_warm)
  warm_ui.py client --dir D --app APP -- ARGS...   run one test through the pool, as `APP ARGS...` would run it
                                           cold: same output, same exit code (each ctest ui_* test)
  warm_ui.py stop   --dir D                stop the pool (ctest fixture cleanup)
  warm_ui.py suite  --build B [--cold] [--repeat K] [--shuffle] [--json F]
                                           every ui test ctest knows, through a pool of its own (or cold), with
                                           wall time and screenshot hashes: to compare warm with cold
  warm_ui.py compare COLD.json WARM.json   verdicts and screenshots of WARM's runs that differ from what every run
                                           in COLD agrees on

The pool talks to its servers over their standard input and output, and clients reach the pool through files in
D (made private to this user, 0700): no port, no socket. A server that crashes or hangs is restarted and its script
rerun cold, once. A script with a `# cold` line, or SF_UI_COLD=1 in the environment, always runs cold. Servers:
SF_UI_SERVERS or half the CPU threads.
"""
import argparse
import hashlib
import json
import os
import queue
import random
import re
import shutil
import signal
import subprocess
import sys
import threading
import time
import uuid

TIMEOUT = 290  # seconds a script may take (ctest gives a ui test 300)
IDLE_EXIT = 900  # a pool nobody used for this long stops (a ctest run that never reached the cleanup)


def servers_wanted():
    n = os.environ.get("SF_UI_SERVERS")
    return max(1, int(n)) if n else max(1, (os.cpu_count() or 2) // 2)


def script_of(args):
    return args[args.index("--ui-test") + 1] if "--ui-test" in args[:-1] else None


def wants_cold(args):
    """Runs that stay cold: anything but a UI test script, scripts marked `# cold`, and SF_UI_COLD=1."""
    if os.environ.get("SF_UI_COLD", "0") not in ("", "0"):
        return True
    script = script_of(args)
    if not script:
        return True
    try:
        with open(script, encoding="utf-8") as f:
            return any(re.match(r"\s*#\s*cold\b", line) for line in f)
    except OSError:
        return True


def run_cold(app, args, env, cwd, timeout=TIMEOUT):
    """(exit code, output) of one cold run, as ctest would see it."""
    try:
        p = subprocess.run([app] + args, env=env, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                           stdin=subprocess.DEVNULL, timeout=timeout)
        return p.returncode, p.stdout.decode("utf-8", "replace")
    except subprocess.TimeoutExpired as e:
        return 124, (e.stdout or b"").decode("utf-8", "replace") + f"\nwarm_ui: cold run timed out after {timeout} s\n"


class Server:
    """One `APP --ui-serve`; run() sends it a script and waits for the answer."""

    def __init__(self, app, logdir, index, threads):
        self.app, self.index, self.threads = app, index, threads
        self.errlog = os.path.join(logdir, f"server-{index}.log")
        self.proc = None
        self.lines = None
        self.runs = 0

    def start(self):
        self.lines = queue.Queue()
        err = open(self.errlog, "ab")
        self.proc = subprocess.Popen([self.app, "--ui-serve", "--threads", str(self.threads)], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                                     stderr=err, bufsize=0)
        err.close()
        self.runs = 0
        out, lines = self.proc.stdout, self.lines

        def read():
            for raw in iter(out.readline, b""):
                lines.put(raw.decode("utf-8", "replace").rstrip("\r\n"))
            lines.put(None)  # the server is gone

        threading.Thread(target=read, daemon=True).start()
        if self._answer(120) == "ready":
            return True
        self.stop(kill=True)
        return False

    def start_retrying(self, tries=4):
        """start(), again after a pause when it fails: an OpenGL context can fail to come up while many others start
        (WSL's d3d12: "Could not find a valid EGL device")."""
        for i in range(tries):
            if self.start():
                return True
            time.sleep(0.5 + random.random() * (i + 1))
        return False

    def _answer(self, timeout):
        try:
            return self.lines.get(timeout=timeout)
        except queue.Empty:
            return None

    def stop(self, kill=False):
        if not self.proc:
            return
        try:
            if kill:
                self.proc.kill()
            else:
                self.proc.stdin.write(b"quit\n")
                self.proc.stdin.flush()
            self.proc.wait(timeout=10)
        except (OSError, subprocess.TimeoutExpired):
            self.proc.kill()
        self.proc = None

    def run(self, args, env_changes, cwd, log, timeout):
        """Exit code, or None when the server crashed, hung or lost its GPU device during a failed script (it is
        then gone)."""
        if not self.proc and not self.start_retrying():
            return None
        fields = ["run", log]
        for k, v in env_changes.items():
            fields += ["-e", f"{k}={v}"]
        fields += ["-C", cwd] + args
        if any("\t" in f or "\n" in f for f in fields):
            return None
        try:
            self.proc.stdin.write(("\t".join(fields) + "\n").encode("utf-8"))
            self.proc.stdin.flush()
        except OSError:
            self.stop(kill=True)
            return None
        answer = self._answer(timeout)
        if not answer or not answer.startswith("done\t"):
            self.stop(kill=True)
            return None
        self.runs += 1
        fields = answer.split("\t")
        code = int(fields[1])
        if "lost" in fields[3:]:  # the GPU dropped its device: the server quit; a failure may be that, not the script
            self.stop(kill=True)
            return None if code != 0 else code
        return code


class Pool:
    """N servers; run() takes a free one (blocking) and falls back to a cold run when a server fails."""

    def __init__(self, app, workdir, n):
        self.app, self.workdir = app, workdir
        os.makedirs(os.path.join(workdir, "logs"), exist_ok=True)
        self.free = queue.Queue()
        # The CPU threads shared out: each server's brush thread pool gets its part (at least 2).
        threads = max(2, (os.cpu_count() or 2) // n)
        self.servers = [Server(app, workdir, i, threads) for i in range(n)]
        for s in self.servers:
            self.free.put(s)
        self.base_env = dict(os.environ)
        self.cold_reruns = 0

    def warm_up(self):
        threads = [threading.Thread(target=s.start_retrying) for s in self.servers]
        for t in threads:
            t.start()
        for t in threads:
            t.join()

    def env_changes(self, env):
        return {k: v for k, v in env.items() if self.base_env.get(k) != v}

    def run(self, args, env, cwd, timeout=TIMEOUT):
        """(exit code, output, how): how is warm, cold (asked for) or rerun (a server crashed or hung)."""
        if wants_cold(args) or env.get("SF_UI_COLD", "0") not in ("", "0"):
            return run_cold(self.app, args, env, cwd, timeout) + ("cold",)
        server = self.free.get()
        log = os.path.join(self.workdir, "logs", uuid.uuid4().hex + ".log")
        try:
            code = server.run(args, self.env_changes(env), cwd, log, timeout)
        finally:
            self.free.put(server)
        output = ""
        try:
            with open(log, encoding="utf-8", errors="replace") as f:
                output = f.read()
            os.remove(log)
        except OSError:
            pass
        if code is not None:
            return code, output, "warm"
        self.cold_reruns += 1
        code, cold = run_cold(self.app, args, env, cwd, timeout)
        note = f"warm_ui: the warm server failed on {script_of(args)} (crash, timeout or lost GPU device); rerun cold:\n"
        return code, output + "\n" + note + cold, "rerun"

    def stop(self):
        for s in self.servers:
            s.stop()


# --- the pool as a background process, for ctest -------------------------------------------------------------------

def private_dir(d):
    os.makedirs(d, mode=0o700, exist_ok=True)
    if os.name != "nt":
        os.chmod(d, 0o700)
    for sub in ("queue", "done"):
        os.makedirs(os.path.join(d, sub), mode=0o700, exist_ok=True)


def pid_alive(pid):
    if os.name == "nt":  # (os.kill would end it)
        import ctypes
        h = ctypes.windll.kernel32.OpenProcess(0x1000, False, pid)  # PROCESS_QUERY_LIMITED_INFORMATION
        if not h:
            return False
        code = ctypes.c_ulong()
        ctypes.windll.kernel32.GetExitCodeProcess(h, ctypes.byref(code))
        ctypes.windll.kernel32.CloseHandle(h)
        return code.value == 259  # STILL_ACTIVE
    try:
        os.kill(pid, 0)
        return True
    except OSError:
        return False


def pool_alive(d):
    try:
        with open(os.path.join(d, "pool.pid")) as f:
            return pid_alive(int(f.read()))
    except (OSError, ValueError):
        return False


def cmd_start(a):
    if os.environ.get("SF_UI_COLD", "0") not in ("", "0"):
        print("SF_UI_COLD=1: UI tests run cold")
        return 0
    d = os.path.abspath(a.dir)
    cmd_stop(a, quiet=True)
    shutil.rmtree(d, ignore_errors=True)
    private_dir(d)
    log = open(os.path.join(d, "pool.log"), "ab")
    kw = {"start_new_session": True} if os.name != "nt" else {"creationflags": 0x00000008}  # DETACHED_PROCESS
    subprocess.Popen([sys.executable, os.path.abspath(__file__), "serve", "--dir", d, "--app", a.app],
                     stdin=subprocess.DEVNULL, stdout=log, stderr=log, close_fds=True, **kw)
    for _ in range(100):  # until its pid file is there (the servers warm up while the first tests queue)
        if pool_alive(d):
            print(f"warm UI pool started: {servers_wanted()} servers ({d})")
            return 0
        time.sleep(0.05)
    print("warm UI pool did not start; UI tests run cold")
    return 0


def cmd_serve(a):
    d = a.dir
    pool = Pool(a.app, d, servers_wanted())
    with open(os.path.join(d, "pool.pid.tmp"), "w") as f:
        f.write(str(os.getpid()))
    os.replace(os.path.join(d, "pool.pid.tmp"), os.path.join(d, "pool.pid"))
    threading.Thread(target=pool.warm_up, daemon=True).start()
    jobs = queue.Queue()
    taken = set()
    stop = threading.Event()

    def work():
        while not stop.is_set():
            try:
                name = jobs.get(timeout=0.2)
            except queue.Empty:
                continue
            path = os.path.join(d, "queue", name)
            try:
                with open(path, encoding="utf-8") as f:
                    req = json.load(f)
                os.remove(path)
                code, output, how = pool.run(req["args"], req["env"], req["cwd"], req.get("timeout", TIMEOUT))
            except Exception as e:  # a broken request: answer it rather than leave its client waiting
                code, output, how = 1, f"warm_ui: {e}\n", "error"
            tmp = os.path.join(d, "done", name + ".tmp")
            with open(tmp, "w", encoding="utf-8") as f:
                json.dump({"code": code, "output": output, "how": how}, f)
            os.replace(tmp, os.path.join(d, "done", name))

    workers = [threading.Thread(target=work, daemon=True) for _ in pool.servers]
    for w in workers:
        w.start()
    last = time.time()
    while not os.path.exists(os.path.join(d, "stop")) and time.time() - last < IDLE_EXIT:
        names = sorted(n for n in os.listdir(os.path.join(d, "queue")) if n.endswith(".json") and n not in taken)
        for n in names:
            taken.add(n)
            jobs.put(n)
            last = time.time()
        time.sleep(0.01)
    stop.set()
    for w in workers:
        w.join(timeout=TIMEOUT)
    pool.stop()
    try:
        os.remove(os.path.join(d, "pool.pid"))
    except OSError:
        pass
    print(f"warm UI pool stopped; {pool.cold_reruns} cold reruns")
    return 0


def cmd_client(a):
    args = a.args[1:] if a.args[:1] == ["--"] else a.args
    d = os.path.abspath(a.dir)
    if wants_cold(args) or not pool_alive(d):
        code, output = run_cold(a.app, args, dict(os.environ), os.getcwd())
        sys.stdout.write(output)
        return code
    name = f"{time.time():.6f}-{os.getpid()}-{uuid.uuid4().hex[:6]}.json"
    tmp = os.path.join(d, "queue", name + ".tmp")
    with open(tmp, "w", encoding="utf-8") as f:
        json.dump({"args": args, "env": dict(os.environ), "cwd": os.getcwd(), "timeout": TIMEOUT}, f)
    os.replace(tmp, os.path.join(d, "queue", name))
    done = os.path.join(d, "done", name)
    checked = time.time()
    while not os.path.exists(done):
        time.sleep(0.02)
        if time.time() - checked > 5:  # the pool died with our request: run it ourselves
            checked = time.time()
            if not pool_alive(d) and not os.path.exists(done):
                code, output = run_cold(a.app, args, dict(os.environ), os.getcwd())
                sys.stdout.write("warm_ui: the pool stopped; ran cold\n" + output)
                return code
    with open(done, encoding="utf-8") as f:
        res = json.load(f)
    os.remove(done)
    sys.stdout.write(res["output"])
    if res["how"] == "rerun":
        sys.stdout.write("warm_ui: [rerun cold]\n")
    return res["code"]


def cmd_stop(a, quiet=False):
    d = os.path.abspath(a.dir)
    if not os.path.isdir(d):
        return 0
    open(os.path.join(d, "stop"), "w").close()
    for _ in range(600):
        if not pool_alive(d):
            break
        time.sleep(0.05)
    if not quiet:
        try:
            with open(os.path.join(d, "pool.log"), encoding="utf-8", errors="replace") as f:
                sys.stdout.write(f.read())
        except OSError:
            pass
    return 0


# --- the whole UI suite, for comparing warm with cold ---------------------------------------------------------------

def ui_tests(build, label_regex):
    out = subprocess.run(["ctest", "--test-dir", build, "--show-only=json-v1", "-L", label_regex],
                         capture_output=True, text=True, check=True).stdout
    tests = []
    for t in json.loads(out)["tests"]:
        props = {p["name"]: p["value"] for p in t.get("properties", [])}
        if props.get("DISABLED") or "command" not in t:
            continue
        cmd = t["command"]
        if "client" in cmd and "--" in cmd:  # a warm client: the app and its arguments
            cmd = [cmd[cmd.index("--app") + 1]] + cmd[cmd.index("--") + 1:]
        env = dict(os.environ)
        for kv in props.get("ENVIRONMENT", []):
            k, _, v = kv.partition("=")
            env[k] = v
        tests.append({"name": t["name"], "app": cmd[0], "args": cmd[1:], "env": env,
                      "cwd": props.get("WORKING_DIRECTORY", build)})
    return tests


def shot_hashes(folder):
    hashes = {}
    for root, _, files in os.walk(folder):
        for n in files:
            if n.endswith(".png"):
                p = os.path.join(root, n)
                with open(p, "rb") as f:
                    hashes[os.path.relpath(p, folder)] = hashlib.sha256(f.read()).hexdigest()[:16]
    return hashes


def cmd_suite(a):
    build = os.path.abspath(a.build)
    tests = [t for t in ui_tests(build, a.label) if script_of(t["args"])]
    if a.regex:
        tests = [t for t in tests if re.search(a.regex, t["name"])]
    shots = os.path.abspath(a.shots or os.path.join(build, "ui-shots-" + ("cold" if a.cold else "warm")))
    for t in tests:  # each run's screenshots in a folder of its own
        i = t["args"].index("--ui-out") + 1 if "--ui-out" in t["args"] else None
        if i:
            t["args"][i] = os.path.join(shots, os.path.basename(t["args"][i]))
    n = a.servers or servers_wanted()
    pool = None
    if not a.cold:
        pool = Pool(tests[0]["app"], os.path.join(build, "ui-warm-suite"), n)
        t0 = time.time()
        pool.warm_up()
        print(f"{n} servers ready in {time.time() - t0:.1f} s")
    results = {}
    for rep in range(a.repeat):
        order = list(tests)
        if a.shuffle:
            random.Random(a.seed + rep).shuffle(order)
        shutil.rmtree(shots, ignore_errors=True)
        work = queue.Queue()
        for t in order:
            work.put(t)
        lock = threading.Lock()
        rep_results = {}

        def go():
            while True:
                try:
                    t = work.get_nowait()
                except queue.Empty:
                    return
                s = time.time()
                if pool:
                    code, out, how = pool.run(t["args"], t["env"], t["cwd"])
                else:
                    code, out = run_cold(t["app"], t["args"], t["env"], t["cwd"])
                    how = "cold"
                with lock:
                    rep_results[t["name"]] = {"code": code, "seconds": round(time.time() - s, 2), "how": how,
                                              "fail": [l for l in out.splitlines() if l.startswith("FAIL")][:5]}
                    if code != 0:
                        rep_results[t["name"]]["tail"] = out.splitlines()[-12:]
                        if a.verbose:
                            print(f"--- {t['name']} ({how}) exit {code}:\n" + "\n".join(out.splitlines()[-12:]))

        t0 = time.time()
        threads = [threading.Thread(target=go) for _ in range(a.jobs or n)]
        for th in threads:
            th.start()
        for th in threads:
            th.join()
        wall = time.time() - t0
        failed = sorted(k for k, r in rep_results.items() if r["code"] != 0)
        print(f"run {rep + 1}: {len(order)} tests in {wall:.1f} s, {len(failed)} failed"
              + (f": {' '.join(failed)}" if failed else ""))
        results[f"run{rep + 1}"] = {"wall": round(wall, 1), "tests": rep_results, "shots": shot_hashes(shots),
                                    "failed": failed}
    if pool:
        print(f"cold reruns: {pool.cold_reruns}")
        pool.stop()
    if a.json:
        with open(a.json, "w") as f:
            json.dump(results, f, indent=1, sort_keys=True)
    return 0 if all(not r["failed"] for r in results.values()) else 1


def cmd_compare(a):
    """The first file's runs are the reference (cold ones, say): a verdict or screenshot counts only where they
    all agree, since some screenshots differ from run to run anyway (GPU timing). Each run of the other files is
    checked against that."""
    def runs_of(path):
        with open(path) as f:
            return [(f"{os.path.basename(path)}:{k}", r) for k, r in sorted(json.load(f).items())]
    ref = runs_of(a.files[0])
    others = [r for path in a.files[1:] for r in runs_of(path)] or ref[1:]
    ref_runs = [r for _, r in ref]
    ok = lambda r, t: r["tests"].get(t, {}).get("code") == 0
    tests = set().union(*(r["tests"] for _, r in ref))
    stable_verdict = {t: ok(ref_runs[0], t) for t in tests if len({ok(r, t) for r in ref_runs}) == 1}
    shots = set().union(*(r["shots"] for r in ref_runs))
    stable_shot = {s: ref_runs[0]["shots"].get(s) for s in shots if len({r["shots"].get(s) for r in ref_runs}) == 1}
    print(f"reference {a.files[0]}: {len(ref)} runs ({', '.join(str(r['wall']) + ' s' for r in ref_runs)}); "
          f"{len(stable_verdict)} of {len(tests)} verdicts and {len(stable_shot)} of {len(shots)} screenshots the same in each")
    differ = 0
    for name, r in others:
        verdicts = sorted(t for t, v in stable_verdict.items() if ok(r, t) != v)
        diff_shots = sorted(s for s, h in stable_shot.items() if r["shots"].get(s) != h)
        differ += len(verdicts) + len(diff_shots)
        print(f"{name} ({r['wall']} s): {len(verdicts)} verdicts and {len(diff_shots)} stable screenshots differ")
        for t in verdicts:
            print(f"  verdict {t}: {r['tests'].get(t, {}).get('code')} {r['tests'].get(t, {}).get('fail')}")
        for s in diff_shots:
            print(f"  shot {s}")
    return 1 if differ else 0


def main():
    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    sub = p.add_subparsers(dest="cmd", required=True)
    for name in ("start", "serve", "client", "stop"):
        s = sub.add_parser(name)
        s.add_argument("--dir", required=True)
        if name != "stop":
            s.add_argument("--app", required=name != "stop")
        if name == "client":
            s.add_argument("args", nargs=argparse.REMAINDER)
    s = sub.add_parser("suite")
    s.add_argument("--build", required=True)
    s.add_argument("--label", default="^ui$")
    s.add_argument("--regex", help="only the tests whose name matches")
    s.add_argument("--cold", action="store_true", help="every test in a process of its own")
    s.add_argument("--servers", type=int, help="warm servers (default SF_UI_SERVERS or half the CPU threads)")
    s.add_argument("--jobs", type=int, help="tests at once (default: as many as servers)")
    s.add_argument("--repeat", type=int, default=1)
    s.add_argument("--shuffle", action="store_true")
    s.add_argument("--seed", type=int, default=1)
    s.add_argument("--shots", help="screenshot folder (default BUILD/ui-shots-warm or -cold)")
    s.add_argument("--json", help="write every run's results and screenshot hashes here")
    s.add_argument("--verbose", action="store_true", help="print the end of each failed test's output")
    s = sub.add_parser("compare")
    s.add_argument("files", nargs="+")
    a = p.parse_args()
    if os.name != "nt":
        signal.signal(signal.SIGTERM, lambda *_: sys.exit(1))
    return {"start": cmd_start, "serve": cmd_serve, "client": cmd_client, "stop": cmd_stop,
            "suite": cmd_suite, "compare": cmd_compare}[a.cmd](a)


if __name__ == "__main__":
    sys.exit(main())