Skip to content

Free, open-source 3D sculpting for Linux, macOS and Windows: dense meshes, remeshing, painting, built-in AI (image/text to 3D), Send to Unreal, game-ready LODs, Gaussian splats and OpenUSD.

Splat training from captures: Brush/COLMAP/ffmpeg job with live preview, splat_train commands, tests

islandmedba7876bdd7cd
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 79828697..36e182f6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -52,6 +52,7 @@ add_library(sculptforge_core STATIC
src/core/Image.cpp
src/core/ImageTo3D.cpp
src/core/Splats.cpp
+ src/core/SplatCapture.cpp
src/core/Voxel.cpp
src/core/EnginePresets.cpp
src/core/Masking.cpp
@@ -119,7 +120,7 @@ endif()
# --- Bring-your-own-AI (brush assistant). HTTPS via WinHTTP or libcurl. ---
add_library(sculptforge_ai STATIC src/ai/Http.cpp src/ai/BrushAssistant.cpp src/ai/Generator3D.cpp
src/ai/ImageTo3DService.cpp src/ai/GradioClient.cpp src/ai/ComfyUI.cpp src/ai/AiCommands.cpp src/ai/BrushGalleryClient.cpp src/link/Socket.cpp
- src/link/UnrealLink.cpp)
+ src/link/UnrealLink.cpp src/ai/Process.cpp src/ai/SplatTrainer.cpp)
# Lets a build run from its tree find the bundled ComfyUI workflows (integrations/comfyui/workflows).
target_compile_definitions(sculptforge_ai PRIVATE SF_SOURCE_DIR="${CMAKE_CURRENT_SOURCE_DIR}")
find_package(Threads REQUIRED)
@@ -193,6 +194,7 @@ if(SCULPTFORGE_BUILD_APP)
src/app/Gizmo.cpp
src/app/PanelBake.cpp
src/app/PanelRetopo.cpp
+ src/app/PanelSplatTrain.cpp
src/app/PanelPerformance.cpp
src/app/PanelGoz.cpp
src/app/PanelBlender.cpp
@@ -395,6 +397,12 @@ if(SCULPTFORGE_BUILD_TESTS)
target_compile_options(sculptforge_tests PRIVATE /utf-8)
endif()
add_test(NAME sculptforge_tests COMMAND sculptforge_tests)
+ # A stand-in for Brush, COLMAP and ffmpeg (tests/tools/fake_splat_tools.cpp): splat training is tested without a GPU.
+ add_executable(sf_fake_splat_tools tests/tools/fake_splat_tools.cpp)
+ target_include_directories(sf_fake_splat_tools PRIVATE third_party/stb)
+ target_link_libraries(sf_fake_splat_tools PRIVATE sculptforge_core)
+ add_dependencies(sculptforge_tests sf_fake_splat_tools)
+ target_compile_definitions(sculptforge_tests PRIVATE SF_FAKE_SPLAT_TOOLS="$<TARGET_FILE:sf_fake_splat_tools>")
# Tests that launch our programs from Python get a path they can run directly. When cross-compiling
# (e.g. a Windows build tested under Wine, tools/wintest.sh) that is a script calling the emulator.
@@ -451,6 +459,11 @@ if(SCULPTFORGE_BUILD_TESTS)
add_test(NAME ui_${name} COMMAND sculptforge --ui-test ${script} --ui-out ${CMAKE_CURRENT_BINARY_DIR}/ui-shots/${name})
set_tests_properties(ui_${name} PROPERTIES LABELS ui TIMEOUT 300)
endforeach()
+ # Splat training runs the fake Brush / COLMAP / ffmpeg.
+ if(TEST ui_splat_train)
+ set_tests_properties(ui_splat_train PROPERTIES ENVIRONMENT
+ "SCULPTFORGE_BRUSH=$<TARGET_FILE:sf_fake_splat_tools>;SCULPTFORGE_COLMAP=$<TARGET_FILE:sf_fake_splat_tools>;SCULPTFORGE_FFMPEG=$<TARGET_FILE:sf_fake_splat_tools>")
+ endif()
# The shell layout again on a 1920x1080 window (docs/ui-design.md §2.3).
add_test(NAME ui_shell_layout_1080 COMMAND sculptforge --ui-test ${CMAKE_CURRENT_SOURCE_DIR}/tests/ui/shell_layout.sfui
--size 1920x1080 --ui-out ${CMAKE_CURRENT_BINARY_DIR}/ui-shots/shell_layout_1080)
diff --git a/src/ai/Process.cpp b/src/ai/Process.cpp
new file mode 100644
index 00000000..2e86c2b7
--- /dev/null
+++ b/src/ai/Process.cpp
@@ -0,0 +1,292 @@
+// SculptForge - open source digital sculpting
+// SPDX-License-Identifier: GPL-3.0-or-later
+#include "ai/Process.h"
+
+#include <atomic>
+#include <chrono>
+#include <cstdlib>
+#include <filesystem>
+#include <fstream>
+#include <sstream>
+#include <thread>
+
+#if defined(_WIN32)
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#else
+#include <cerrno>
+#include <cstring>
+#include <fcntl.h>
+#include <signal.h>
+#include <spawn.h>
+#include <sys/wait.h>
+#include <unistd.h>
+extern char** environ;
+#endif
+
+namespace sf::ai {
+
+namespace fs = std::filesystem;
+
+namespace {
+fs::path u8path(const std::string& s) { return fs::path(std::u8string(s.begin(), s.end())); }
+std::string pathString(const fs::path& p) {
+ const auto s = p.u8string();
+ return std::string(s.begin(), s.end());
+}
+} // namespace
+
+std::string findOnPath(const std::string& name) {
+ if (name.empty()) return {};
+ std::error_code ec;
+ if (name.find('/') != std::string::npos || name.find('\\') != std::string::npos)
+ return fs::exists(u8path(name), ec) ? name : std::string();
+ const char* path = std::getenv("PATH");
+ if (!path) return {};
+#if defined(_WIN32)
+ const char sep = ';';
+ const std::vector<std::string> exts = {".exe", ".bat", ""};
+#else
+ const char sep = ':';
+ const std::vector<std::string> exts = {""};
+#endif
+ std::stringstream ss(path);
+ std::string dir;
+ while (std::getline(ss, dir, sep)) {
+ if (dir.empty()) continue;
+ for (const std::string& ext : exts) {
+ const fs::path candidate = u8path(dir) / u8path(name + ext);
+ if (fs::is_regular_file(candidate, ec)) {
+#if !defined(_WIN32)
+ if (::access(candidate.c_str(), X_OK) != 0) continue;
+#endif
+ return pathString(candidate);
+ }
+ }
+ }
+ return {};
+}
+
+ChildProcess::~ChildProcess() {
+ if (started_ && !finished_) kill(0.5);
+}
+
+#if defined(_WIN32)
+
+namespace {
+std::wstring widen(const std::string& s) {
+ if (s.empty()) return {};
+ const int n = MultiByteToWideChar(CP_UTF8, 0, s.data(), int(s.size()), nullptr, 0);
+ std::wstring w(size_t(n), L'\0');
+ MultiByteToWideChar(CP_UTF8, 0, s.data(), int(s.size()), w.data(), n);
+ return w;
+}
+// Quoting for CommandLineToArgvW / the MSVC runtime.
+void appendQuoted(std::wstring& cmd, const std::wstring& arg) {
+ if (!arg.empty() && arg.find_first_of(L" \t\n\v\"") == std::wstring::npos) {
+ cmd += arg;
+ return;
+ }
+ cmd += L'"';
+ for (size_t i = 0;; ++i) {
+ size_t slashes = 0;
+ while (i < arg.size() && arg[i] == L'\\') {
+ ++i;
+ ++slashes;
+ }
+ if (i == arg.size()) {
+ cmd.append(slashes * 2, L'\\');
+ break;
+ }
+ if (arg[i] == L'"') {
+ cmd.append(slashes * 2 + 1, L'\\');
+ cmd += L'"';
+ } else {
+ cmd.append(slashes, L'\\');
+ cmd += arg[i];
+ }
+ }
+ cmd += L'"';
+}
+} // namespace
+
+bool ChildProcess::start(const std::string& program, const std::vector<std::string>& args, const std::string& logPath,
+ const std::vector<std::pair<std::string, std::string>>& env, std::string& error) {
+ const std::string exe = findOnPath(program);
+ if (exe.empty()) {
+ error = "Cannot find " + program;
+ return false;
+ }
+ std::wstring cmd;
+ appendQuoted(cmd, widen(exe));
+ for (const std::string& a : args) {
+ cmd += L' ';
+ appendQuoted(cmd, widen(a));
+ }
+ SECURITY_ATTRIBUTES sa{sizeof sa, nullptr, TRUE};
+ HANDLE log = INVALID_HANDLE_VALUE;
+ if (!logPath.empty()) {
+ log = CreateFileW(widen(logPath).c_str(), FILE_APPEND_DATA, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+ &sa, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr);
+ } else {
+ log = CreateFileW(L"NUL", GENERIC_WRITE, FILE_SHARE_WRITE, &sa, OPEN_EXISTING, 0, nullptr);
+ }
+ // The child inherits our environment; the overrides are only tool settings (RUST_LOG), so setting them
+ // on ourselves is harmless and avoids building an environment block.
+ for (const auto& [k, v] : env) _wputenv_s(widen(k).c_str(), widen(v).c_str());
+ STARTUPINFOW si{};
+ si.cb = sizeof si;
+ si.dwFlags = STARTF_USESTDHANDLES;
+ si.hStdInput = GetStdHandle(STD_INPUT_HANDLE);
+ si.hStdOutput = log;
+ si.hStdError = log;
+ PROCESS_INFORMATION pi{};
+ const BOOL ok = CreateProcessW(nullptr, cmd.data(), nullptr, nullptr, TRUE, CREATE_NO_WINDOW, nullptr, nullptr, &si, &pi);
+ if (log != INVALID_HANDLE_VALUE) CloseHandle(log);
+ if (!ok) {
+ error = "Cannot start " + exe + " (error " + std::to_string(GetLastError()) + ")";
+ return false;
+ }
+ CloseHandle(pi.hThread);
+ process_ = pi.hProcess;
+ started_ = true;
+ finished_ = false;
+ exitCode_ = -1;
+ return true;
+}
+
+bool ChildProcess::running() {
+ if (!started_ || finished_) return false;
+ if (WaitForSingleObject(process_, 0) == WAIT_TIMEOUT) return true;
+ DWORD code = 0;
+ GetExitCodeProcess(process_, &code);
+ exitCode_ = int(code);
+ CloseHandle(process_);
+ process_ = nullptr;
+ finished_ = true;
+ return false;
+}
+
+void ChildProcess::kill(double) {
+ if (!running()) return;
+ TerminateProcess(process_, 1);
+ wait();
+}
+
+int ChildProcess::wait() {
+ if (!started_) return -1;
+ if (!finished_) WaitForSingleObject(process_, INFINITE);
+ running();
+ return exitCode_;
+}
+
+#else
+
+bool ChildProcess::start(const std::string& program, const std::vector<std::string>& args, const std::string& logPath,
+ const std::vector<std::pair<std::string, std::string>>& env, std::string& error) {
+ const std::string exe = findOnPath(program);
+ if (exe.empty()) {
+ error = "Cannot find " + program;
+ return false;
+ }
+ std::vector<std::string> argStore;
+ argStore.push_back(exe);
+ argStore.insert(argStore.end(), args.begin(), args.end());
+ std::vector<char*> argv;
+ for (std::string& a : argStore) argv.push_back(a.data());
+ argv.push_back(nullptr);
+
+ std::vector<std::string> envStore;
+ for (char** e = environ; e && *e; ++e) {
+ const std::string entry(*e);
+ bool replaced = false;
+ for (const auto& kv : env)
+ if (entry.compare(0, kv.first.size() + 1, kv.first + "=") == 0) replaced = true;
+ if (!replaced) envStore.push_back(entry);
+ }
+ for (const auto& [k, v] : env) envStore.push_back(k + "=" + v);
+ std::vector<char*> envp;
+ for (std::string& e : envStore) envp.push_back(e.data());
+ envp.push_back(nullptr);
+
+ posix_spawn_file_actions_t actions;
+ posix_spawn_file_actions_init(&actions);
+ const std::string out = logPath.empty() ? std::string("/dev/null") : logPath;
+ posix_spawn_file_actions_addopen(&actions, 1, out.c_str(), O_WRONLY | O_CREAT | O_APPEND, 0644);
+ posix_spawn_file_actions_adddup2(&actions, 1, 2);
+ posix_spawn_file_actions_addopen(&actions, 0, "/dev/null", O_RDONLY, 0);
+ posix_spawnattr_t attr;
+ posix_spawnattr_init(&attr);
+ // Its own process group, so stopping it also stops what it started.
+ posix_spawnattr_setflags(&attr, POSIX_SPAWN_SETPGROUP);
+ posix_spawnattr_setpgroup(&attr, 0);
+ pid_t pid = -1;
+ const int rc = posix_spawn(&pid, exe.c_str(), &actions, &attr, argv.data(), envp.data());
+ posix_spawn_file_actions_destroy(&actions);
+ posix_spawnattr_destroy(&attr);
+ if (rc != 0) {
+ error = "Cannot start " + exe + ": " + std::strerror(rc);
+ return false;
+ }
+ pid_ = pid;
+ started_ = true;
+ finished_ = false;
+ exitCode_ = -1;
+ return true;
+}
+
+bool ChildProcess::running() {
+ if (!started_ || finished_) return false;
+ int status = 0;
+ const pid_t r = ::waitpid(pid_, &status, WNOHANG);
+ if (r == 0) return true;
+ if (r == pid_) exitCode_ = WIFEXITED(status) ? WEXITSTATUS(status) : 128 + (WIFSIGNALED(status) ? WTERMSIG(status) : 0);
+ finished_ = true;
+ return false;
+}
+
+void ChildProcess::kill(double graceSeconds) {
+ if (!running()) return;
+ ::kill(-pid_, SIGTERM);
+ const auto until = std::chrono::steady_clock::now() + std::chrono::milliseconds(int(graceSeconds * 1000));
+ while (running() && std::chrono::steady_clock::now() < until) std::this_thread::sleep_for(std::chrono::milliseconds(20));
+ if (running()) {
+ ::kill(-pid_, SIGKILL);
+ wait();
+ }
+}
+
+int ChildProcess::wait() {
+ if (!started_) return -1;
+ if (!finished_) {
+ int status = 0;
+ while (::waitpid(pid_, &status, 0) < 0 && errno == EINTR) {
+ }
+ exitCode_ = WIFEXITED(status) ? WEXITSTATUS(status) : 128 + (WIFSIGNALED(status) ? WTERMSIG(status) : 0);
+ finished_ = true;
+ }
+ return exitCode_;
+}
+
+#endif
+
+int runProcess(const std::string& program, const std::vector<std::string>& args, std::string& output,
+ const std::string& scratchDir) {
+ std::error_code ec;
+ fs::create_directories(u8path(scratchDir), ec);
+ static std::atomic<int> counter{0};
+ const std::string log = pathString(u8path(scratchDir) / ("run-" + std::to_string(++counter) + ".log"));
+ fs::remove(u8path(log), ec);
+ ChildProcess p;
+ if (!p.start(program, args, log, {}, output)) return -1;
+ const int code = p.wait();
+ std::ifstream f(u8path(log), std::ios::binary);
+ std::stringstream ss;
+ ss << f.rdbuf();
+ output = ss.str();
+ f.close();
+ fs::remove(u8path(log), ec);
+ return code;
+}
+
+} // namespace sf::ai
diff --git a/src/ai/Process.h b/src/ai/Process.h
new file mode 100644
index 00000000..31ffc535
--- /dev/null
+++ b/src/ai/Process.h
@@ -0,0 +1,56 @@
+// SculptForge - open source digital sculpting
+// SPDX-License-Identifier: GPL-3.0-or-later
+//
+// A child process with its output going to a log file: external tools that run
+// for minutes (the Brush splat trainer, COLMAP, ffmpeg) without a shell, so
+// paths need no quoting and the process can be stopped. posix_spawn on
+// Linux/macOS, CreateProcessW on Windows.
+#pragma once
+
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace sf::ai {
+
+class ChildProcess {
+public:
+ ChildProcess() = default;
+ ~ChildProcess(); // kills a process that is still running
+ ChildProcess(const ChildProcess&) = delete;
+ ChildProcess& operator=(const ChildProcess&) = delete;
+
+ // Starts `program` (a path, or a name looked up on PATH) with `args`. stdout
+ // and stderr are appended to `logPath` ("" = discarded). `env` adds or
+ // replaces environment variables for the child.
+ bool start(const std::string& program, const std::vector<std::string>& args, const std::string& logPath,
+ const std::vector<std::pair<std::string, std::string>>& env, std::string& error);
+ // True while it runs; once it has ended, exitCode() is valid.
+ bool running();
+ int exitCode() const { return exitCode_; }
+ // Stops it (SIGTERM, then SIGKILL after `graceSeconds`; TerminateProcess on Windows) and waits.
+ void kill(double graceSeconds = 2.0);
+ // Waits for it to end; returns the exit code.
+ int wait();
+ bool started() const { return started_; }
+
+private:
+ bool started_ = false;
+ bool finished_ = false;
+ int exitCode_ = -1;
+#if defined(_WIN32)
+ void* process_ = nullptr;
+#else
+ int pid_ = -1;
+#endif
+};
+
+// The full path of `name` on PATH (with .exe on Windows), or "" when it isn't there.
+std::string findOnPath(const std::string& name);
+
+// Runs a program to completion with its output in `output` (both streams). Returns the exit code, or -1 if it
+// could not start (`output` then holds the reason). For short tools only (version checks, tar).
+int runProcess(const std::string& program, const std::vector<std::string>& args, std::string& output,
+ const std::string& scratchDir);
+
+} // namespace sf::ai
diff --git a/src/ai/SplatTrainer.cpp b/src/ai/SplatTrainer.cpp
new file mode 100644
index 00000000..5b8f1dd5
--- /dev/null
+++ b/src/ai/SplatTrainer.cpp
@@ -0,0 +1,767 @@
+// SculptForge - open source digital sculpting
+// SPDX-License-Identifier: GPL-3.0-or-later
+#include "ai/SplatTrainer.h"
+
+#include "ai/Http.h"
+#include "ai/Process.h"
+#include "core/BrushLibrary.h"
+#include "core/Commands.h"
+#include "core/Json.h"
+#include "core/Sha256.h"
+#include "io/PlyIO.h"
+
+#include <algorithm>
+#include <chrono>
+#include <cstdlib>
+#include <filesystem>
+#include <fstream>
+#include <sstream>
+
+namespace sf::ai {
+
+namespace fs = std::filesystem;
+using namespace sf::capture;
+
+namespace {
+
+fs::path u8path(const std::string& s) { return fs::path(std::u8string(s.begin(), s.end())); }
+std::string pathString(const fs::path& p) {
+ const auto s = p.u8string();
+ return std::string(s.begin(), s.end());
+}
+double nowSeconds() {
+ return std::chrono::duration<double>(std::chrono::steady_clock::now().time_since_epoch()).count();
+}
+void sleepSeconds(double s) { std::this_thread::sleep_for(std::chrono::milliseconds(int(s * 1000.0))); }
+std::string envValue(const char* name) {
+ const char* v = std::getenv(name);
+ return v ? std::string(v) : std::string();
+}
+bool readFile(const std::string& path, std::string& out) {
+ std::ifstream f(u8path(path), std::ios::binary);
+ if (!f) return false;
+ std::stringstream ss;
+ ss << f.rdbuf();
+ out = ss.str();
+ return true;
+}
+// The last `lines` lines of a log, for error messages.
+std::string logTail(const std::string& path, int lines) {
+ std::string text;
+ if (!readFile(path, text)) return {};
+ while (!text.empty() && (text.back() == '\n' || text.back() == '\r')) text.pop_back();
+ size_t pos = text.size();
+ for (int i = 0; i < lines && pos != std::string::npos && pos > 0; ++i) pos = text.rfind('\n', pos - 1);
+ return pos == std::string::npos ? text : text.substr(pos + 1);
+}
+// "[12/80]" style progress anywhere in the text: the last one.
+float bracketProgress(const std::string& text) {
+ for (size_t p = text.rfind('['); p != std::string::npos; p = p ? text.rfind('[', p - 1) : std::string::npos) {
+ int a = 0, b = 0;
+ if (std::sscanf(text.c_str() + p, "[%d/%d]", &a, &b) == 2 && b > 0) return std::clamp(float(a) / float(b), 0.0f, 1.0f);
+ if (p == 0) break;
+ }
+ return -1.0f;
+}
+std::string toolInDir(const fs::path& dir, const std::string& exe) {
+ std::error_code ec;
+ if (!fs::is_directory(dir, ec)) return {};
+ for (fs::recursive_directory_iterator it(dir, ec), end; !ec && it != end; it.increment(ec))
+ if (it->is_regular_file(ec) && it->path().filename() == u8path(exe)) return pathString(it->path());
+ return {};
+}
+std::string firstOnPath(std::initializer_list<const char*> names) {
+ for (const char* n : names)
+ if (std::string p = findOnPath(n); !p.empty()) return p;
+ return {};
+}
+std::string captureName(const std::string& source) {
+ fs::path p = u8path(source);
+ while (!p.empty() && p.filename().empty()) p = p.parent_path(); // trailing separator
+ std::error_code ec;
+ const std::string name = pathString(fs::is_directory(p, ec) ? p.filename() : p.stem());
+ return name.empty() ? std::string("capture") : name;
+}
+
+} // namespace
+
+// --- Settings ---
+
+std::string defaultSplatTrainSettingsPath() { return pathString(u8path(appDataDir()) / "splat_train.json"); }
+std::string splatToolsDir() { return pathString(u8path(appDataDir()) / "tools"); }
+
+SplatTrainSettings SplatTrainSettings::load(const std::string& path) {
+ SplatTrainSettings s;
+ std::string text, error;
+ Json j;
+ if (!readFile(path, text) || !Json::parse(text, j, error) || !j.isObject()) return s;
+ if (j["brush"].isString()) s.brushPath = j["brush"].asString();
+ if (j["colmap"].isString()) s.colmapPath = j["colmap"].asString();
+ if (j["ffmpeg"].isString()) s.ffmpegPath = j["ffmpeg"].asString();
+ if (j["preset"].isString()) s.preset = j["preset"].asString();
+ s.copyToLibrary = j["copy_to_library"].asBool(true);
+ return s;
+}
+
+bool SplatTrainSettings::save(const std::string& path) const {
+ Json j = Json::object();
+ j.set("brush", brushPath);
+ j.set("colmap", colmapPath);
+ j.set("ffmpeg", ffmpegPath);
+ j.set("preset", preset);
+ j.set("copy_to_library", copyToLibrary);
+ std::error_code ec;
+ fs::create_directories(u8path(path).parent_path(), ec);
+ std::ofstream f(u8path(path), std::ios::binary);
+ f << j.dump(true);
+ return bool(f);
+}
+
+// --- Tools ---
+
+const ToolDownload* brushDownload() {
+ // Brush v0.3.0 (2025-09-14); hashes from the release's sha256.sum.
+#if defined(_WIN32) && (defined(_M_X64) || defined(__x86_64__))
+ static const ToolDownload d{"Brush", "0.3.0",
+ "https://github.com/ArthurBrussee/brush/releases/download/v0.3.0/brush-app-x86_64-pc-windows-msvc.zip",
+ "b68e3e9cf052d51bf3ee30776fa5a364de7f2ba13b58443128ff797bb7bcfcd6", 158791348, "brush_app.exe",
+ "brush-0.3.0", "Apache-2.0", "https://github.com/ArthurBrussee/brush"};
+ return &d;
+#elif defined(__APPLE__) && defined(__aarch64__)
+ static const ToolDownload d{"Brush", "0.3.0",
+ "https://github.com/ArthurBrussee/brush/releases/download/v0.3.0/brush-app-aarch64-apple-darwin.tar.xz",
+ "65b2631398c839be3c1d4d7160fe2326389dec87830aac0710985e6690a1048c", 41447012, "brush_app",
+ "brush-0.3.0", "Apache-2.0", "https://github.com/ArthurBrussee/brush"};
+ return &d;
+#elif defined(__linux__) && defined(__x86_64__)
+ static const ToolDownload d{"Brush", "0.3.0",
+ "https://github.com/ArthurBrussee/brush/releases/download/v0.3.0/brush-app-x86_64-unknown-linux-gnu.tar.xz",
+ "4f0f9a8785d1951c62df26aae247c02c5bba32b00f40b06df4e1c9b867399e20", 44242224, "brush_app",
+ "brush-0.3.0", "Apache-2.0", "https://github.com/ArthurBrussee/brush"};
+ return &d;
+#else
+ return nullptr; // no prebuilt release (Linux on ARM, Intel Macs): build it with cargo and set the path
+#endif
+}
+
+const ToolDownload* colmapDownload() {
+#if defined(_WIN32) && (defined(_M_X64) || defined(__x86_64__))
+ // COLMAP 4.2.1 (2026-09-29) without CUDA (runs everywhere); hash from GitHub's asset digest.
+ static const ToolDownload d{"COLMAP", "4.2.1",
+ "https://github.com/colmap/colmap/releases/download/4.2.1/colmap-x64-windows-nocuda.zip",
+ "c493d88cb4a43f21afba42cb55bf786e5434a0420e73db0ba1454c5d8b3686ae", 127731595, "colmap.exe",
+ "colmap-4.2.1", "BSD-3-Clause", "https://github.com/colmap/colmap"};
+ return &d;
+#else
+ return nullptr; // Linux: the distribution's colmap package; macOS: Homebrew (brew install colmap)
+#endif
+}
+
+std::string findBrush(const SplatTrainSettings& settings) {
+ if (std::string e = envValue("SCULPTFORGE_BRUSH"); !e.empty()) return findOnPath(e);
+ if (!settings.brushPath.empty()) return findOnPath(settings.brushPath);
+ if (const ToolDownload* d = brushDownload())
+ if (std::string p = toolInDir(u8path(splatToolsDir()) / d->folder, d->exe); !p.empty()) return p;
+ return firstOnPath({"brush_app", "brush"});
+}
+
+std::string findColmap(const SplatTrainSettings& settings) {
+ if (std::string e = envValue("SCULPTFORGE_COLMAP"); !e.empty()) return findOnPath(e);
+ if (!settings.colmapPath.empty()) return findOnPath(settings.colmapPath);
+ if (const ToolDownload* d = colmapDownload())
+ if (std::string p = toolInDir(u8path(splatToolsDir()) / d->folder, d->exe); !p.empty()) return p;
+ return firstOnPath({"colmap"});
+}
+
+std::string findFfmpeg(const SplatTrainSettings& settings) {
+ if (std::string e = envValue("SCULPTFORGE_FFMPEG"); !e.empty()) return findOnPath(e);
+ if (!settings.ffmpegPath.empty()) return findOnPath(settings.ffmpegPath);
+ return firstOnPath({"ffmpeg"});
+}
+
+bool downloadTool(const ToolDownload& tool, std::string& exePath, std::string& error, std::atomic<float>* progress,
+ std::atomic<bool>* cancel) {
+ const fs::path dir = u8path(splatToolsDir()) / tool.folder;
+ std::error_code ec;
+ fs::create_directories(dir, ec);
+ if (ec) {
+ error = "Cannot create " + pathString(dir);
+ return false;
+ }
+ std::string data;
+ data.reserve(size_t(tool.bytes));
+ const HttpResponse r = httpRequestStream("GET", tool.url, {}, {}, 1800, [&](const char* p, size_t n) {
+ data.append(p, n);
+ if (progress) progress->store(std::min(0.99f, float(double(data.size()) / double(tool.bytes))));
+ return !(cancel && cancel->load());
+ });
+ if (cancel && cancel->load()) {
+ error = "Download cancelled";
+ return false;
+ }
+ if (r.status != 200) {
+ error = "Downloading " + std::string(tool.tool) + " failed: " +
+ (r.error.empty() ? "HTTP " + std::to_string(r.status) : r.error);
+ return false;
+ }
+ if (sha256Hex(data) != tool.sha256) {
+ error = std::string(tool.tool) + " download did not match its checksum (not installed)";
+ return false;
+ }
+ const std::string name = std::string(tool.url).substr(std::string(tool.url).rfind('/') + 1);
+ const fs::path archive = dir / u8path(name);
+ {
+ std::ofstream f(archive, std::ios::binary);
+ f.write(data.data(), std::streamsize(data.size()));
+ if (!f) {
+ error = "Cannot write " + pathString(archive);
+ return false;
+ }
+ }
+ data = std::string();
+ std::string output;
+ const int code = runProcess("tar", {"-xf", pathString(archive), "-C", pathString(dir)}, output,
+ pathString(dir / "unpack"));
+ fs::remove(archive, ec);
+ fs::remove_all(dir / "unpack", ec);
+ if (code != 0) {
+ error = "Unpacking " + std::string(tool.tool) + " failed (tar): " + output;
+ return false;
+ }
+ exePath = toolInDir(dir, tool.exe);
+ if (exePath.empty()) {
+ error = std::string(tool.tool) + " unpacked, but " + tool.exe + " is not in it";
+ return false;
+ }
+#if !defined(_WIN32)
+ fs::permissions(u8path(exePath), fs::perms::owner_exec | fs::perms::group_exec | fs::perms::others_exec,
+ fs::perm_options::add, ec);
+#endif
+ if (progress) progress->store(1.0f);
+ return true;
+}
+
+// --- The job ---
+
+const char* trainStageName(TrainStage stage) {
+ switch (stage) {
+ case TrainStage::Idle: return "idle";
+ case TrainStage::Frames: return "frames";
+ case TrainStage::Poses: return "poses";
+ case TrainStage::Training: return "training";
+ case TrainStage::Done: return "done";
+ case TrainStage::Failed: return "failed";
+ case TrainStage::Cancelled: return "cancelled";
+ }
+ return "?";
+}
+
+std::string defaultTrainOutDir(const std::string& source) {
+ const fs::path p = u8path(source);
+ std::error_code ec;
+ if (fs::is_directory(p, ec)) return pathString(p / "splat");
+ return pathString(p.parent_path() / u8path(captureName(source) + "-splat"));
+}
+
+SplatTrainJob::~SplatTrainJob() {
+ cancel();
+ wait();
+}
+
+void SplatTrainJob::wait() {
+ if (worker_.joinable()) worker_.join();
+}
+
+void SplatTrainJob::cancel() { cancel_ = true; }
+void SplatTrainJob::finishNow() { finish_ = true; }
+
+TrainStatus SplatTrainJob::status() const {
+ std::lock_guard lock(mutex_);
+ TrainStatus s = status_;
+ if (running_) s.elapsed = nowSeconds() - startTime_;
+ return s;
+}
+
+std::shared_ptr<SplatCloud> SplatTrainJob::takePreview() {
+ std::lock_guard lock(mutex_);
+ return std::move(preview_);
+}
+
+void SplatTrainJob::fail(const std::string& error) {
+ update([&](TrainStatus& s) {
+ s.stage = TrainStage::Failed;
+ s.error = error;
+ s.message = error;
+ });
+}
+
+bool SplatTrainJob::start(const TrainRequest& request, std::string& error) {
+ if (running_) {
+ error = "A splat is already training (cancel it first)";
+ return false;
+ }
+ wait(); // the previous worker has ended; join it
+ TrainStatus s;
+ s.stage = TrainStage::Failed;
+ const SourceKind kind = classifySource(request.source);
+ bool posed = kind == SourceKind::NerfstudioDataset || kind == SourceKind::ColmapDataset;
+ if (kind == SourceKind::Images) {
+ CaptureInfo info;
+ std::string ignored;
+ const std::string json = pathString(u8path(request.source) / "capture.json");
+ posed = readCaptureJson(json, info, ignored) && !info.poses.empty();
+ }
+ if (kind == SourceKind::Missing) error = "Capture not found: " + request.source;
+ else if (kind == SourceKind::Unsupported)
+ error = "Not a capture: give a video, a folder of photos, or a COLMAP / nerfstudio dataset";
+ else if (request.brush.empty()) {
+ error = "Brush (the splat trainer) is not installed";
+ s.missingTrainer = true;
+ } else if ((kind == SourceKind::Video || kind == SourceKind::VideoFolder) && request.ffmpeg.empty()) {
+ error = "ffmpeg is needed to read frames from a video: install it, or train from a folder of photos";
+ s.missingFfmpeg = true;
+ } else if (!posed && request.colmap.empty()) {
+ error = "COLMAP is needed to find the camera positions of the photos";
+ s.missingPoses = true;
+ }
+ if (!error.empty()) {
+ s.error = s.message = error;
+ std::lock_guard lock(mutex_);
+ status_ = s;
+ return false;
+ }
+ cancel_ = false;
+ finish_ = false;
+ {
+ std::lock_guard lock(mutex_);
+ status_ = TrainStatus();
+ status_.stage = TrainStage::Frames;
+ status_.message = "Starting";
+ status_.outDir = request.outDir.empty() ? defaultTrainOutDir(request.source) : request.outDir;
+ preview_.reset();
+ }
+ framing_ = glm::mat4(1.0f);
+ startTime_ = nowSeconds();
+ running_ = true;
+ TrainRequest r = request;
+ if (r.outDir.empty()) r.outDir = defaultTrainOutDir(request.source);
+ worker_ = std::thread([this, r] {
+ run(r);
+ update([&](TrainStatus& st) { st.elapsed = nowSeconds() - startTime_; });
+ running_ = false;
+ });
+ return true;
+}
+
+void SplatTrainJob::run(TrainRequest r) {
+ std::error_code ec;
+ const fs::path out = u8path(r.outDir), work = out / "work";
+ fs::create_directories(work, ec);
+ if (ec) return fail("Cannot create " + pathString(work));
+ const SourceKind kind = classifySource(r.source);
+ std::string dataset;
+ bool needPoses = false;
+ if (!prepareFrames(r, kind, dataset, needPoses)) return;
+ if (needPoses && !findPoses(r, dataset)) return;
+ if (stopRequested()) {
+ update([](TrainStatus& s) {
+ s.stage = TrainStage::Cancelled;
+ s.message = "Cancelled";
+ });
+ return;
+ }
+ train(r, dataset);
+}
+
+bool SplatTrainJob::prepareFrames(const TrainRequest& r, SourceKind kind, std::string& dataset, bool& needPoses) {
+ const fs::path work = u8path(r.outDir) / "work";
+ const TrainPreset& preset = trainPreset(r.preset);
+ std::error_code ec;
+ needPoses = false;
+ if (kind == SourceKind::NerfstudioDataset || kind == SourceKind::ColmapDataset) {
+ dataset = r.source;
+ std::vector<CameraView> views;
+ std::string err;
+ const bool ok = kind == SourceKind::NerfstudioDataset
+ ? readTransformsJson(pathString(u8path(r.source) / "transforms.json"), views, err)
+ : readColmapImages(pathString(u8path(r.source) / "sparse" / "0"), views, err);
+ if (ok) framing_ = orbitFraming(views);
+ update([&](TrainStatus& s) { s.frames = int(views.size()); });
+ return true;
+ }
+ const fs::path ds = work / "dataset", images = ds / "images";
+ fs::remove_all(ds, ec);
+ fs::create_directories(images, ec);
+ dataset = pathString(ds);
+
+ std::vector<std::string> frames; // candidate photos
+ fs::path framesDir;
+ CaptureInfo info;
+ if (kind == SourceKind::Video || kind == SourceKind::VideoFolder) {
+ const std::string video = kind == SourceKind::Video ? r.source : findVideo(r.source);
+ framesDir = work / "frames";
+ fs::remove_all(framesDir, ec);
+ fs::create_directories(framesDir, ec);
+ update([&](TrainStatus& s) {
+ s.stage = TrainStage::Frames;
+ s.message = "Reading frames from the video";
+ s.stageProgress = -1.0f;
+ });
+ const std::string log = pathString(work / "ffmpeg.log");
+ fs::remove(u8path(log), ec);
+ ChildProcess ff;
+ std::string err;
+ // 5 frames a second, at most 1600 px (phones film 4K; Brush trains at <= 1920 anyway).
+ if (!ff.start(r.ffmpeg,
+ {"-hide_banner", "-loglevel", "error", "-y", "-i", video, "-vf",
+ "fps=5,scale=1600:1600:force_original_aspect_ratio=decrease", "-q:v", "2",
+ pathString(framesDir / "%05d.jpg")},
+ log, {}, err)) {
+ fail(err);
+ return false;
+ }
+ while (ff.running()) {
+ if (stopRequested()) {
+ ff.kill();
+ break;
+ }
+ const size_t n = listImages(pathString(framesDir)).size();
+ update([&](TrainStatus& s) { s.message = "Reading frames from the video (" + std::to_string(n) + ")"; });
+ sleepSeconds(pollSeconds);
+ }
+ if (stopRequested()) return true; // run() reports the cancel
+ frames = listImages(pathString(framesDir));
+ if (ff.exitCode() != 0 || frames.empty()) {
+ fail("ffmpeg could not read the video: " + logTail(log, 3));
+ return false;
+ }
+ } else { // photos
+ const fs::path src = u8path(r.source);
+ frames = listImages(r.source);
+ if (frames.empty()) frames = listImages(pathString(src / "images"));
+ std::string ignored;
+ readCaptureJson(pathString(src / "capture.json"), info, ignored);
+ }
+
+ // The sharpest frame of each stretch (videos), or all photos up to twice the preset's frame count.
+ const size_t keep = kind == SourceKind::Images ? size_t(preset.maxFrames) * 2 : size_t(preset.maxFrames);
+ std::vector<size_t> chosen;
+ const bool posedPhotos = !info.poses.empty();
+ if (frames.size() > keep && !posedPhotos) {
+ std::vector<double> scores(frames.size(), 0.0);
+ for (size_t i = 0; i < frames.size(); ++i) {
+ if (stopRequested()) return true;
+ Image img;
+ std::string err;
+ if (Image::load(frames[i], img, err)) scores[i] = sharpness(img);
+ update([&](TrainStatus& s) {
+ s.message = "Picking the sharpest frames";
+ s.stageProgress = float(i + 1) / float(frames.size());
+ });
+ }
+ chosen = selectFrames(scores, keep);
+ } else {
+ for (size_t i = 0; i < frames.size(); ++i) chosen.push_back(i);
+ }
+ std::vector<std::string> names;
+ for (size_t k = 0; k < chosen.size(); ++k) {
+ const fs::path from = u8path(frames[chosen[k]]);
+ // Video frames are renumbered; photos keep their names (AR poses refer to them).
+ const fs::path name = framesDir.empty() ? from.filename() : u8path(std::to_string(100000 + k).substr(1) + ".jpg");
+ if (!framesDir.empty()) fs::rename(from, images / name, ec);
+ if (framesDir.empty() || ec) fs::copy_file(from, images / name, fs::copy_options::overwrite_existing, ec);
+ if (ec) {
+ fail("Cannot copy " + pathString(from) + " into the dataset");
+ return false;
+ }
+ names.push_back(pathString(name));
+ }
+ if (!framesDir.empty()) fs::remove_all(framesDir, ec);
+ update([&](TrainStatus& s) { s.frames = int(names.size()); });
+ if (names.size() < 3) {
+ fail("Only " + std::to_string(names.size()) + " usable photos: a splat needs many views (walk around the object)");
+ return false;
+ }
+
+ if (posedPhotos) { // the phone's AR tracking already knows where each photo was taken
+ std::vector<CameraView> views;
+ for (CameraView v : info.poses) {
+ const std::string base = pathString(u8path(v.file).filename());
+ if (std::find(names.begin(), names.end(), base) == names.end()) continue;
+ v.file = "images/" + base;
+ views.push_back(std::move(v));
+ }
+ std::string err;
+ if (views.size() >= 3 && writeTransformsJson(pathString(ds / "transforms.json"), views, err)) {
+ framing_ = orbitFraming(views);
+ return true;
+ }
+ if (r.colmap.empty()) {
+ fail("The capture's camera poses don't match its photos, and COLMAP is not installed");
+ return false;
+ }
+ }
+ needPoses = true;
+ return true;
+}
+
+bool SplatTrainJob::findPoses(const TrainRequest& r, const std::string& dataset) {
+ const fs::path ds = u8path(dataset), work = u8path(r.outDir) / "work";
+ const TrainPreset& preset = trainPreset(r.preset);
+ std::error_code ec;
+ update([&](TrainStatus& s) {
+ s.stage = TrainStage::Poses;
+ s.message = "Finding the camera positions (COLMAP)";
+ s.stageProgress = -1.0f;
+ });
+ const std::string log = pathString(work / "colmap.log");
+ fs::remove(u8path(log), ec);
+ const std::vector<std::string> base = {"automatic_reconstructor", "--workspace_path", dataset,
+ "--image_path", pathString(ds / "images"),
+ "--data_type", "video", // frames in order: sequential matching
+ "--quality", std::string(preset.steps > 10000 ? "medium" : "low"),
+ "--single_camera", "1", "--camera_model", "PINHOLE", "--dense", "0"};
+ // COLMAP 3.12+ has the global mapper (formerly GLOMAP, much faster); older versions reject the option, and a
+ // build without a usable GPU may need --use_gpu 0.
+ const std::vector<std::vector<std::string>> attempts = {{"--mapper", "global"}, {}, {"--use_gpu", "0"}};
+ bool ok = false;
+ for (const auto& extra : attempts) {
+ fs::remove_all(ds / "sparse", ec);
+ fs::remove(ds / "database.db", ec);
+ std::vector<std::string> args = base;
+ args.insert(args.end(), extra.begin(), extra.end());
+ ChildProcess p;
+ std::string err;
+ {
+ std::ofstream f(u8path(log), std::ios::app);
+ f << "\n$ colmap";
+ for (const auto& a : args) f << ' ' << a;
+ f << '\n';
+ }
+ if (!p.start(r.colmap, args, log, {}, err)) {
+ fail(err);
+ return false;
+ }
+ while (p.running()) {
+ if (stopRequested()) {
+ p.kill();
+ return true;
+ }
+ const float f = bracketProgress(logTail(log, 5));
+ update([&](TrainStatus& s) { s.stageProgress = f; });
+ sleepSeconds(pollSeconds);
+ }
+ // The model with the most photos goes to sparse/0, where Brush looks.
+ int best = -1;
+ size_t bestCount = 0;
+ for (int k = 0; k < 16; ++k) {
+ std::vector<CameraView> views;
+ std::string ignored;
+ if (readColmapImages(pathString(ds / "sparse" / std::to_string(k)), views, ignored) && views.size() > bestCount) {
+ best = k;
+ bestCount = views.size();
+ }
+ }
+ if (p.exitCode() == 0 && best >= 0 && bestCount >= 3) {
+ if (best != 0) {
+ fs::rename(ds / "sparse" / "0", ds / "sparse" / "unused-0", ec);
+ fs::rename(ds / "sparse" / std::to_string(best), ds / "sparse" / "0", ec);
+ }
+ ok = true;
+ break;
+ }
+ }
+ if (!ok) {
+ fail("COLMAP could not place the cameras (too few overlapping views, blur, or a shiny/plain object). "
+ "Log: " + log + "\n" + logTail(log, 3));
+ return false;
+ }
+ std::vector<CameraView> views;
+ std::string err;
+ if (readColmapImages(pathString(ds / "sparse" / "0"), views, err)) framing_ = orbitFraming(views);
+ update([&](TrainStatus& s) {
+ s.message = "Placed " + std::to_string(views.size()) + " of " + std::to_string(s.frames) + " photos";
+ });
+ return true;
+}
+
+bool SplatTrainJob::loadCheckpoint(const std::string& path, int step) {
+ Mesh mesh;
+ auto cloud = std::make_shared<SplatCloud>();
+ PlyContent content;
+ std::string err;
+ if (!loadPly(path, mesh, *cloud, content, err) || cloud->empty()) return false;
+ cloud->transform = framing_;
+ cloud->name = "Training";
+ std::lock_guard lock(mutex_);
+ preview_ = std::move(cloud);
+ ++status_.previewVersion;
+ status_.previewStep = step;
+ status_.splats = int(preview_->size());
+ return true;
+}
+
+bool SplatTrainJob::train(const TrainRequest& r, const std::string& dataset) {
+ const fs::path work = u8path(r.outDir) / "work", checkpoints = work / "checkpoints";
+ const TrainPreset& preset = trainPreset(r.preset);
+ std::error_code ec;
+ fs::remove_all(checkpoints, ec);
+ fs::create_directories(checkpoints, ec);
+ const std::string log = pathString(work / "brush.log");
+ fs::remove(u8path(log), ec);
+ update([&](TrainStatus& s) {
+ s.stage = TrainStage::Training;
+ s.message = "Starting Brush";
+ s.totalSteps = preset.steps;
+ s.stageProgress = -1.0f;
+ s.logPath = log;
+ });
+ ChildProcess p;
+ std::string err;
+ if (!p.start(r.brush, brushArguments(dataset, pathString(checkpoints), preset, r.evaluate), log,
+ {{"RUST_LOG", "brush_cli=info,brush_process=info"}, {"NO_COLOR", "1"}}, err)) {
+ fail(err);
+ return false;
+ }
+ const double trainStart = nowSeconds();
+ size_t logOffset = 0;
+ std::string partial;
+ int loadedStep = -1, pendingStep = -1;
+ uintmax_t pendingSize = 0;
+ bool sawError = false;
+ auto readLog = [&] {
+ std::ifstream f(u8path(log), std::ios::binary);
+ if (!f) return;
+ f.seekg(std::streamoff(logOffset));
+ std::string chunk((std::istreambuf_iterator<char>(f)), std::istreambuf_iterator<char>());
+ logOffset += chunk.size();
+ partial += chunk;
+ size_t nl;
+ while ((nl = partial.find('\n')) != std::string::npos) {
+ const std::string line = partial.substr(0, nl);
+ partial.erase(0, nl + 1);
+ TrainerLog t;
+ if (!parseTrainerLog(line, t)) continue;
+ update([&](TrainStatus& s) {
+ if (t.refineIter >= 0) s.step = std::max(s.step, t.refineIter);
+ if (t.splats >= 0) s.splats = t.splats;
+ if (t.evalIter >= 0) s.step = std::max(s.step, t.evalIter);
+ if (t.psnr > 0) s.psnr = t.psnr;
+ });
+ sawError = sawError || t.error;
+ }
+ };
+ // A checkpoint is loaded once its size stops changing between two checks (Brush may still be writing it).
+ auto pollCheckpoints = [&](bool final) {
+ int step = -1;
+ const std::string path = latestCheckpoint(pathString(checkpoints), step);
+ if (path.empty() || step <= loadedStep) return;
+ const uintmax_t size = fs::file_size(u8path(path), ec);
+ if (ec) return;
+ if (final || (step == pendingStep && size == pendingSize && size > 0)) {
+ if (loadCheckpoint(path, step)) loadedStep = step;
+ pendingStep = -1;
+ } else {
+ pendingStep = step;
+ pendingSize = size;
+ }
+ update([&](TrainStatus& s) { s.step = std::max(s.step, step); });
+ };
+ while (p.running()) {
+ if (stopRequested()) {
+ p.kill();
+ break;
+ }
+ readLog();
+ pollCheckpoints(false);
+ const double t = nowSeconds() - trainStart;
+ update([&](TrainStatus& s) {
+ s.message = s.step > 0 ? "Training" : "Loading the photos";
+ if (s.step > 0) s.eta = t / double(s.step) * double(std::max(0, s.totalSteps - s.step));
+ s.stageProgress = s.totalSteps > 0 ? float(s.step) / float(s.totalSteps) : -1.0f;
+ });
+ sleepSeconds(pollSeconds);
+ }
+ readLog();
+ if (cancel_) {
+ update([](TrainStatus& s) {
+ s.stage = TrainStage::Cancelled;
+ s.message = "Cancelled";
+ });
+ return false;
+ }
+ pollCheckpoints(true);
+ if (!finish_ && (p.exitCode() != 0 || sawError)) {
+ fail("Brush stopped with an error (exit " + std::to_string(p.exitCode()) + "). Log: " + log + "\n" +
+ logTail(log, 4));
+ return false;
+ }
+ int step = -1;
+ const std::string last = latestCheckpoint(pathString(checkpoints), step);
+ if (last.empty()) {
+ if (finish_) {
+ update([](TrainStatus& s) {
+ s.stage = TrainStage::Cancelled;
+ s.message = "Stopped before the first checkpoint";
+ });
+ } else {
+ fail("Brush finished without writing a splat. Log: " + log);
+ }
+ return false;
+ }
+ const fs::path result = u8path(r.outDir) / u8path(captureName(r.source) + ".ply");
+ fs::copy_file(u8path(last), result, fs::copy_options::overwrite_existing, ec);
+ if (ec) {
+ fail("Cannot write " + pathString(result));
+ return false;
+ }
+ update([&](TrainStatus& s) {
+ s.stage = TrainStage::Done;
+ s.result = pathString(result);
+ s.step = std::max(s.step, step);
+ s.eta = 0.0;
+ s.stageProgress = 1.0f;
+ s.message = "Trained " + std::to_string(s.splats) + " splats in " + std::to_string(step) + " steps";
+ });
+ return true;
+}
+
+// --- Commands for sculptforge-cli / Python / MCP (blocking) ---
+
+void registerSplatTrainCommands(CommandRegistry& registry, std::function<SplatTrainSettings()> settings) {
+ using T = ParamSpec::Type;
+ registry.add({"splat_train",
+ "Train a Gaussian splat from a capture: a video, a folder of photos, a phone capture folder or a "
+ "COLMAP / nerfstudio dataset. Uses Brush (and COLMAP for camera positions, ffmpeg for video). Here "
+ "it waits until training ends and loads the result; the app trains in the background with a live "
+ "preview.",
+ {{"source", T::String, "The capture (video file or folder).", true},
+ {"out", T::String, "Folder for the result and work files. Default: next to the capture.", false},
+ {"preset", T::String, "fast (a quick preview) or quality. Default: the saved preset.", false}},
+ [settings](Document& doc, const Args& a) -> CommandResult {
+ const SplatTrainSettings s = settings();
+ TrainRequest req;
+ req.source = argString(a, "source");
+ req.outDir = argString(a, "out");
+ req.preset = argString(a, "preset", s.preset);
+ req.brush = findBrush(s);
+ req.colmap = findColmap(s);
+ req.ffmpeg = findFfmpeg(s);
+ SplatTrainJob job;
+ std::string err;
+ if (!job.start(req, err)) return {false, err};
+ job.wait();
+ const TrainStatus st = job.status();
+ if (st.stage != TrainStage::Done) return {false, st.error.empty() ? st.message : st.error};
+ if (auto cloud = job.takePreview()) {
+ doc.splats = std::move(*cloud);
+ doc.splats.name = captureName(req.source);
+ doc.splatsChanged();
+ }
+ return {true, st.message + "; saved " + st.result};
+ }});
+ registry.add({"splat_train_cancel", "Stop splat training (in the app; sculptforge-cli trains in the foreground).", {},
+ [](Document&, const Args&) -> CommandResult { return {true, "No splat is training"}; }});
+}
+
+} // namespace sf::ai
diff --git a/src/ai/SplatTrainer.h b/src/ai/SplatTrainer.h
new file mode 100644
index 00000000..cc007c9b
--- /dev/null
+++ b/src/ai/SplatTrainer.h
@@ -0,0 +1,153 @@
+// SculptForge - open source digital sculpting
+// SPDX-License-Identifier: GPL-3.0-or-later
+//
+// Training a Gaussian splat from a phone capture, in the background, with a
+// live preview. The heavy lifting is done by external, permissively licensed
+// tools that SculptForge runs as separate processes (nothing is linked in):
+// - Brush (Apache-2.0, github.com/ArthurBrussee/brush): the trainer. WebGPU
+// via wgpu, so NVIDIA, AMD, Intel and Apple GPUs without CUDA. Downloaded on
+// request from its GitHub release (checksum pinned below) or a path you set.
+// - COLMAP (BSD-3-Clause, github.com/colmap/colmap): camera positions when the
+// capture has none (its global mapper is the former GLOMAP). Downloaded on
+// request on Windows/macOS; on Linux from the distribution or a path.
+// - ffmpeg (on PATH, or a path): frames from a video.
+//
+// The job: frames (video -> the sharpest frame of each stretch) -> camera
+// positions (COLMAP, skipped for datasets and AR-posed phone captures) ->
+// Brush, which writes a checkpoint every few hundred steps. The worker thread
+// loads each new checkpoint, so the main thread only swaps it in.
+#pragma once
+
+#include "core/Commands.h"
+#include "core/SplatCapture.h"
+#include "core/Splats.h"
+
+#include <atomic>
+#include <functional>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <vector>
+
+namespace sf::ai {
+
+// Tool paths and the preset, kept in <app data>/splat_train.json (shared by the app and sculptforge-cli).
+struct SplatTrainSettings {
+ std::string brushPath; // "" = find it (environment, download, PATH)
+ std::string colmapPath;
+ std::string ffmpegPath;
+ std::string preset = "fast";
+ bool copyToLibrary = true; // also put the result in the LightBox Splats tab
+
+ static SplatTrainSettings load(const std::string& path);
+ bool save(const std::string& path) const;
+};
+std::string defaultSplatTrainSettingsPath();
+std::string splatToolsDir(); // <app data>/tools
+
+// A tool release SculptForge can download for this platform.
+struct ToolDownload {
+ const char* tool; // "Brush", "COLMAP"
+ const char* version; // "0.3.0"
+ const char* url;
+ const char* sha256;
+ long long bytes;
+ const char* exe; // the program inside the archive, relative to its top folder
+ const char* folder; // where it goes under splatToolsDir()
+ const char* licence;
+ const char* home; // project page shown in the prompt
+};
+const ToolDownload* brushDownload(); // nullptr: no prebuilt Brush for this platform (e.g. Linux on ARM)
+const ToolDownload* colmapDownload(); // nullptr on Linux (install it from the distribution)
+
+// Where the tools are: the SCULPTFORGE_BRUSH / SCULPTFORGE_COLMAP / SCULPTFORGE_FFMPEG environment variables,
+// then the path in the settings, then a download in splatToolsDir(), then PATH. "" when missing.
+std::string findBrush(const SplatTrainSettings& settings);
+std::string findColmap(const SplatTrainSettings& settings);
+std::string findFfmpeg(const SplatTrainSettings& settings);
+
+// Downloads, checks the SHA-256 and unpacks (with the system's tar, which also reads .zip on Windows 10+).
+// Blocking; run on a worker thread. `exePath` receives the unpacked program.
+bool downloadTool(const ToolDownload& tool, std::string& exePath, std::string& error,
+ std::atomic<float>* progress = nullptr, std::atomic<bool>* cancel = nullptr);
+
+enum class TrainStage { Idle, Frames, Poses, Training, Done, Failed, Cancelled };
+const char* trainStageName(TrainStage stage);
+
+struct TrainRequest {
+ std::string source; // a video, a folder of photos, a phone capture folder, or a COLMAP/nerfstudio dataset
+ std::string outDir; // "" = next to the capture (see defaultTrainOutDir)
+ std::string preset = "fast";
+ std::string brush, colmap, ffmpeg; // resolved tool paths
+ bool evaluate = true; // hold back views to measure PSNR
+};
+// <capture folder>/splat for a folder, <folder>/<video name>-splat for a video.
+std::string defaultTrainOutDir(const std::string& source);
+
+struct TrainStatus {
+ TrainStage stage = TrainStage::Idle;
+ std::string message; // what it is doing now
+ std::string error; // why it failed
+ bool missingTrainer = false, missingPoses = false, missingFfmpeg = false;
+ int step = 0, totalSteps = 0;
+ int splats = 0;
+ double psnr = -1.0; // dB on the held-back views, -1 before the first evaluation
+ double elapsed = 0.0; // seconds since start
+ double eta = -1.0; // seconds left in training, -1 unknown
+ float stageProgress = -1.0f; // 0..1 inside Frames / Poses, -1 unknown
+ int frames = 0; // photos used
+ int previewVersion = 0; // bumps with each loaded checkpoint
+ int previewStep = -1;
+ std::string result; // the final .ply
+ std::string outDir, logPath;
+};
+
+class SplatTrainJob {
+public:
+ SplatTrainJob() = default;
+ ~SplatTrainJob(); // cancels and joins
+ SplatTrainJob(const SplatTrainJob&) = delete;
+ SplatTrainJob& operator=(const SplatTrainJob&) = delete;
+
+ // Starts the worker; false (and `error`) when one is running or the source can't be used.
+ bool start(const TrainRequest& request, std::string& error);
+ void cancel(); // stop; the job ends Cancelled
+ void finishNow(); // "Good enough": stop training and keep the newest checkpoint as the result
+ bool busy() const { return running_.load(); }
+ TrainStatus status() const;
+ // The newest loaded checkpoint not taken yet, placed by the capture's framing (nullptr when none). Taking
+ // hands it over, so the caller can move it into the scene without a copy.
+ std::shared_ptr<SplatCloud> takePreview();
+ void wait(); // joins the worker
+
+ // Seconds between checks of the trainer's output (tests make it faster).
+ double pollSeconds = 0.25;
+
+private:
+ void run(TrainRequest request);
+ bool prepareFrames(const TrainRequest& r, capture::SourceKind kind, std::string& dataset, bool& needPoses);
+ bool findPoses(const TrainRequest& r, const std::string& dataset);
+ bool train(const TrainRequest& r, const std::string& dataset);
+ bool loadCheckpoint(const std::string& path, int step);
+ void fail(const std::string& error);
+ template <class F> void update(F&& f) {
+ std::lock_guard lock(mutex_);
+ f(status_);
+ }
+ bool stopRequested() const { return cancel_.load() || finish_.load(); }
+
+ std::thread worker_;
+ std::atomic<bool> running_{false}, cancel_{false}, finish_{false};
+ mutable std::mutex mutex_;
+ TrainStatus status_;
+ std::shared_ptr<SplatCloud> preview_;
+ glm::mat4 framing_{1.0f};
+ double startTime_ = 0.0;
+};
+
+// splat_train / splat_train_cancel for sculptforge-cli, Python and MCP: training blocks until it finishes and
+// the result is loaded as the scene's splats. The app registers its own background versions (live preview).
+void registerSplatTrainCommands(CommandRegistry& registry, std::function<SplatTrainSettings()> settings);
+
+} // namespace sf::ai
diff --git a/src/app/App.cpp b/src/app/App.cpp
index c9ee75b9..29c40d43 100644
--- a/src/app/App.cpp
+++ b/src/app/App.cpp
@@ -344,6 +344,7 @@ int App::runUiTest(int w, int h, int fps) {
pollAutoUv();
pollBake();
pollQRemesh();
+ pollSplatTrain();
updateCameraAspect();
Demo::beginFrame();
@@ -501,6 +502,7 @@ int App::runRecording() {
pollAutoUv();
pollBake();
pollQRemesh();
+ pollSplatTrain();
updateCameraAspect();
Demo::beginFrame();
@@ -568,6 +570,7 @@ int App::run() {
pollAutoUv();
pollBake();
pollQRemesh();
+ pollSplatTrain();
int fbW = 0, fbH = 0, winW = 0, winH = 0;
SDL_GetWindowSizeInPixels(window_, &fbW, &fbH);
@@ -1306,6 +1309,7 @@ void App::drawUi() {
const ImVec2 mouse(mouseX_, mouseY_);
drawGizmo(); // Gizmo 3D handles and panel (Gizmo.cpp)
drawQRemeshGuides(); // QRemesh Guides curves (PanelRetopo.cpp)
+ drawSplatTrainOverlay(); // splat training progress (PanelSplatTrain.cpp)
// In Gizmo mode (and with the QRemesh Guides brush) the brush ring shows only while Ctrl masks or Shift smooths.
const bool ringOff = gizmoActive() || (guidesMode_ && !(inputMods() & SDL_KMOD_SHIFT));
if (((!ImGui::GetIO().WantCaptureMouse && canvasRect().contains(mouse.x, mouse.y)) || drag_ == Drag::Stroke) &&
@@ -2186,6 +2190,8 @@ void App::drawImageTo3D() {
// The Splats palette (it comes into the right tray by itself when splats arrive, see drawShell).
void App::drawPalette_Splats() {
if (doc_.splats.empty()) {
+ drawSplatTrainSection(); // train one from a phone capture (PanelSplatTrain.cpp)
+ ImGui::Spacing();
ImGui::PushStyleColor(ImGuiCol_Text, ImGui::GetStyleColorVec4(ImGuiCol_TextDisabled));
ImGui::TextWrapped("Open or drop a .ply capture (Polycam, Luma, Postshot, Scaniverse...) to sculpt against it.");
ImGui::PopStyleColor();
@@ -2217,6 +2223,7 @@ void App::drawPalette_Splats() {
if (ImGui::Button("Save splats...", ImVec2(half, 0))) saveFileDialog(FileAction::SaveSplats);
ImGui::SameLine();
if (ImGui::Button("Clear##splats", ImVec2(half, 0))) runCommand("splat_clear");
+ drawSplatTrainSection();
}
void App::drawBrushCursor() {
diff --git a/src/app/App.h b/src/app/App.h
index b61b7661..f3ed6492 100644
--- a/src/app/App.h
+++ b/src/app/App.h
@@ -452,6 +452,17 @@ private:
// (1 when the active SubTool shows a baked normal map); NaN for other names.
double bakeTestValue(const std::string& name);
std::shared_ptr<struct BakeUiState> bake_;
+ // Splats > Train from capture (PanelSplatTrain.cpp, ai/SplatTrainer.h): Brush trains a splat from a phone
+ // capture in the background; each checkpoint replaces the splats in the viewport (live preview).
+ void registerSplatTrainUi();
+ void drawSplatTrainSection(); // top of the Splats palette
+ void drawSplatTrainOverlay(); // progress over the canvas, the tool download prompt
+ void pollSplatTrain();
+ bool splatTrainBusy() const;
+ // UI test values: splat_training, splat_previews, splat_train_prompt, splat_train_step, splat_train_stage,
+ // splat_train_psnr; NaN for other names.
+ double splatTrainTestValue(const std::string& name);
+ std::shared_ptr<struct SplatTrainUi> splatTrain_;
std::string lastExportFolder_; // where the last export went: the default folder for baked maps
// Cached UV wireframe of the active SubTool for the UV view: segments in 0..1 UV space.
std::vector<glm::vec2> uvLines_;
diff --git a/src/app/Config.cpp b/src/app/Config.cpp
index cfe48b64..d1b08c8c 100644
--- a/src/app/Config.cpp
+++ b/src/app/Config.cpp
@@ -365,6 +365,7 @@ void App::initActions() {
registerGizmoActions(); // Q W E R: Draw, and the Gizmo 3D's Move, Scale, Rotate (Gizmo.cpp)
registerBakeUiActions(); // Tool > Bake (PanelBake.cpp)
registerRetopoUiActions(); // Geometry > QRemesh, SubTool > ProjectAll (PanelRetopo.cpp)
+ registerSplatTrainUi(); // Splats > Train from capture (PanelSplatTrain.cpp)
registerVdmActions(); // Brush > Vector Displacement (PanelVdm.cpp)
registerPerformanceUiActions(); // Preferences > Performance (PanelPerformance.cpp)
registerInputActions(); // ZBrush keys: U I O, Space, L, V, Ctrl+A / I / H / W..., the input style (Input.cpp)
diff --git a/src/app/PanelProject.cpp b/src/app/PanelProject.cpp
index e1a85cb0..5a52294e 100644
--- a/src/app/PanelProject.cpp
+++ b/src/app/PanelProject.cpp
@@ -6,6 +6,7 @@
// Recent and autosave with crash recovery. Prompts only appear in interactive
// sessions; --render, --record and demos run straight through.
#include "app/App.h"
+#include "core/SplatCapture.h"
#include "app/Ui.h"
#include "ai/Http.h"
@@ -366,6 +367,10 @@ void App::openPathConfirmed(const std::string& path) {
// Images go to the Image to 3D panel and brushes into the library: no scene change.
std::string ext = fsPath(path).extension().string();
for (auto& c : ext) c = char(std::tolower(static_cast<unsigned char>(c)));
+ if (capture::isVideoFile(path)) { // a phone video of an object: train a splat from it (PanelSplatTrain.cpp)
+ runCommand("splat_train", {{"source", path}});
+ return;
+ }
if (Image::isImagePath(path) || ext == ".sfbrush" || ext == ".exr" || ext == ".sfvdm") openPath(path);
else confirmDiscard([this, path] { openPath(path); });
}
diff --git a/src/app/PanelSplatTrain.cpp b/src/app/PanelSplatTrain.cpp
new file mode 100644
index 00000000..e5a137f7
--- /dev/null
+++ b/src/app/PanelSplatTrain.cpp
@@ -0,0 +1,476 @@
+// SculptForge - open source digital sculpting
+// SPDX-License-Identifier: GPL-3.0-or-later
+//
+// Splats > Train from capture: a phone video or a folder of photos becomes a
+// Gaussian splat in the viewport, sharpening while it trains. The job runs on
+// a worker thread (ai/SplatTrainer.h: frames, COLMAP camera positions, Brush),
+// loads each checkpoint there, and this file only swaps the newest one into
+// the scene, so the view stays responsive (orbit while it trains). An overlay
+// over the canvas shows the step, PSNR, elapsed and remaining time, Cancel and
+// "Good enough - stop". Missing tools open a prompt that says what would be
+// downloaded, from where, and how big it is (nothing downloads without a click).
+#include "app/App.h"
+#include "app/Ui.h"
+
+#include "ai/SplatTrainer.h"
+#include "core/SplatCapture.h"
+
+#include <IconsLucide.h>
+
+#include <atomic>
+#include <cmath>
+#include <filesystem>
+#include <future>
+#include <limits>
+#include <mutex>
+
+namespace sf {
+
+namespace {
+
+constexpr const char* kPath = "Splats > Train";
+namespace fs = std::filesystem;
+
+std::string utf8(const fs::path& p) {
+ const std::u8string s = p.u8string();
+ return std::string(s.begin(), s.end());
+}
+fs::path fsPath(const std::string& s) { return fs::path(std::u8string(s.begin(), s.end())); }
+
+std::string clock(double seconds) {
+ if (seconds < 0 || !std::isfinite(seconds)) return "--:--";
+ const int s = int(seconds + 0.5);
+ char buf[32];
+ if (s >= 3600) std::snprintf(buf, sizeof buf, "%d:%02d:%02d", s / 3600, s / 60 % 60, s % 60);
+ else std::snprintf(buf, sizeof buf, "%d:%02d", s / 60, s % 60);
+ return buf;
+}
+
+std::string megabytes(long long bytes) { return std::to_string((bytes + 500'000) / 1'000'000) + " MB"; }
+
+// Native dialog results (any thread).
+enum class Pick { None, Video, Folder, Brush, Colmap };
+std::mutex g_pickMutex;
+std::vector<std::pair<Pick, std::string>> g_picked;
+void SDLCALL onPicked(void* userdata, const char* const* files, int) {
+ if (!files || !files[0]) return;
+ std::lock_guard lock(g_pickMutex);
+ g_picked.emplace_back(Pick(reinterpret_cast<intptr_t>(userdata)), files[0]);
+}
+void* pickTag(Pick p) { return reinterpret_cast<void*>(intptr_t(p)); }
+
+} // namespace
+
+struct SplatTrainUi {
+ ai::SplatTrainJob job;
+ ai::SplatTrainSettings settings;
+ std::string settingsPath; // "" = not saved (UI tests)
+ bool active = false; // started, not yet finished by pollSplatTrain
+ int previews = 0; // checkpoints shown in the viewport by this job
+ std::string source; // the capture being trained
+ std::string preset = "fast";
+ // Tools, looked up when the section is first drawn and after a download or a choice.
+ bool toolsKnown = false;
+ std::string brush, colmap, ffmpeg;
+ // A start waiting for a tool: the download / choose prompt.
+ bool prompt = false;
+ bool promptOpened = false;
+ Pick promptFor = Pick::Brush;
+ std::string pendingSource, pendingOut;
+ std::string promptError;
+ // The running download.
+ std::future<bool> download;
+ std::shared_ptr<std::atomic<float>> downloadProgress = std::make_shared<std::atomic<float>>(0.0f);
+ std::shared_ptr<std::atomic<bool>> downloadCancel = std::make_shared<std::atomic<bool>>(false);
+ std::shared_ptr<std::string> downloadError = std::make_shared<std::string>();
+ const ai::ToolDownload* downloading = nullptr;
+
+ void findTools() {
+ brush = ai::findBrush(settings);
+ colmap = ai::findColmap(settings);
+ ffmpeg = ai::findFfmpeg(settings);
+ toolsKnown = true;
+ }
+ void save() {
+ if (!settingsPath.empty()) settings.save(settingsPath);
+ }
+};
+
+bool App::splatTrainBusy() const {
+ return splatTrain_ && (splatTrain_->job.busy() || splatTrain_->active || splatTrain_->download.valid());
+}
+
+double App::splatTrainTestValue(const std::string& name) {
+ const double none = std::numeric_limits<double>::quiet_NaN();
+ if (!splatTrain_) return none;
+ SplatTrainUi& s = *splatTrain_;
+ if (name == "splat_training") return s.job.busy() || s.active ? 1.0 : 0.0;
+ if (name == "splat_previews") return double(s.previews);
+ if (name == "splat_train_prompt") return s.prompt ? 1.0 : 0.0;
+ const ai::TrainStatus st = s.job.status();
+ if (name == "splat_train_step") return double(st.step);
+ if (name == "splat_train_stage") return double(int(st.stage)); // ai::TrainStage: 4 done, 5 failed, 6 cancelled
+ if (name == "splat_train_psnr") return st.psnr;
+ return none;
+}
+
+// Starts training or, when a tool is missing, opens the prompt for it.
+static CommandResult startSplatTraining(SplatTrainUi& s, const std::string& source, const std::string& out,
+ const std::string& preset) {
+ if (s.job.busy() || s.active) return {false, "A splat is already training - Cancel it first"};
+ s.findTools(); // the user may have installed one since
+ ai::TrainRequest r;
+ r.source = source;
+ r.outDir = out;
+ r.preset = preset.empty() ? s.preset : preset;
+ r.brush = s.brush;
+ r.colmap = s.colmap;
+ r.ffmpeg = s.ffmpeg;
+ std::string err;
+ if (!s.job.start(r, err)) {
+ const ai::TrainStatus st = s.job.status();
+ if (st.missingTrainer || (st.missingPoses && ai::colmapDownload())) {
+ s.prompt = true;
+ s.promptOpened = false;
+ s.promptFor = st.missingTrainer ? Pick::Brush : Pick::Colmap;
+ s.pendingSource = source;
+ s.pendingOut = out;
+ s.promptError.clear();
+ return {true, err + " - see the window that opened"};
+ }
+ if (st.missingPoses)
+ err += ". Install COLMAP (Linux: your distribution's colmap package; macOS: brew install colmap) or "
+ "choose it in Splats > Train > Tools";
+ return {false, err};
+ }
+ s.active = true;
+ s.previews = 0;
+ s.source = source;
+ return {true, "Training a splat from " + utf8(fsPath(source).filename()) + " (" +
+ "the viewport updates as it learns)"};
+}
+
+void App::registerSplatTrainUi() {
+ splatTrain_ = std::make_shared<SplatTrainUi>();
+ SplatTrainUi& s = *splatTrain_;
+ if (!uiConfigDeterministic_) { // UI tests never read or write the artist's settings
+ s.settingsPath = ai::defaultSplatTrainSettingsPath();
+ s.settings = ai::SplatTrainSettings::load(s.settingsPath);
+ }
+ s.preset = capture::trainPreset(s.settings.preset).id;
+
+ using T = ParamSpec::Type;
+ commands_.add({"splat_train",
+ "Train a Gaussian splat from a capture - a phone video, a folder of photos, a phone-link capture "
+ "folder, or a COLMAP / nerfstudio dataset - in the background. The splat appears in the viewport "
+ "after the first few hundred steps and sharpens as it trains; the result is saved next to the "
+ "capture and in the LightBox Splats tab. Uses Brush (downloaded on request), COLMAP for camera "
+ "positions and ffmpeg for video.",
+ {{"source", T::String, "The capture: a video file or a folder.", true},
+ {"out", T::String, "Folder for the result and work files. Default: next to the capture.", false},
+ {"preset", T::String, "fast (a quick preview, minutes) or quality (sharper, longer).", false}},
+ [this](Document&, const Args& a) -> CommandResult {
+ return startSplatTraining(*splatTrain_, argString(a, "source"), argString(a, "out"),
+ argString(a, "preset"));
+ }});
+ commands_.add({"splat_train_cancel",
+ "Stop splat training. finish=true keeps the newest checkpoint as the result (Good enough - stop); "
+ "otherwise nothing is saved.",
+ {{"finish", T::Bool, "Keep what it has learned so far. Default false.", false}},
+ [this](Document&, const Args& a) -> CommandResult {
+ SplatTrainUi& st = *splatTrain_;
+ if (!st.job.busy()) return {true, "No splat is training"};
+ if (argBool(a, "finish", false)) {
+ st.job.finishNow();
+ return {true, "Stopping: keeping the newest checkpoint"};
+ }
+ st.job.cancel();
+ return {true, "Cancelling splat training"};
+ }});
+
+ auto add = [this](const char* id, const char* label, const char* icon, std::function<void()> run,
+ std::function<bool()> enabled, const char* description, const char* command) {
+ Action a;
+ a.id = id;
+ a.label = label;
+ a.icon = icon;
+ a.palette = kPath;
+ a.run = std::move(run);
+ a.enabled = std::move(enabled);
+ a.description = description;
+ a.command = command;
+ actions_.add(std::move(a));
+ };
+ auto idle = [this] { return !splatTrainBusy(); };
+ add("splats.train_video", "Video...", ICON_LC_VIDEO,
+ [this] {
+ static const SDL_DialogFileFilter filters[] = {{"Videos", "mp4;mov;m4v;webm;mkv;avi"}};
+ SDL_ShowOpenFileDialog(onPicked, pickTag(Pick::Video), window_, filters, 1, nullptr, false);
+ },
+ idle,
+ "Train a Gaussian splat from a video of an object (walk slowly all the way around it). The sharpest frame "
+ "of each moment is used. The splat shows in the viewport after a few hundred steps and sharpens live.",
+ "splat_train");
+ add("splats.train_folder", "Photos Folder...", ICON_LC_FOLDER_OPEN,
+ [this] { SDL_ShowOpenFolderDialog(onPicked, pickTag(Pick::Folder), window_, nullptr, false); }, idle,
+ "Train a Gaussian splat from a folder: photos taken all around an object, a phone-link capture, or a COLMAP "
+ "/ nerfstudio dataset (which already has camera positions).",
+ "splat_train");
+ add("splats.train_stop", "Good enough - stop", ICON_LC_CHECK,
+ [this] { runCommand("splat_train_cancel", {{"finish", true}}); },
+ [this] { return splatTrain_->job.busy(); },
+ "Stop training now and keep what it has learned (the newest checkpoint) as the result.",
+ "splat_train_cancel");
+ add("splats.train_cancel", "Cancel##splattrain", ICON_LC_X, [this] { runCommand("splat_train_cancel"); },
+ [this] { return splatTrain_->job.busy(); }, "Stop training and keep nothing.", "splat_train_cancel");
+ add("splats.train_choose_brush", "Choose Brush...", nullptr,
+ [this] { SDL_ShowOpenFileDialog(onPicked, pickTag(Pick::Brush), window_, nullptr, 0, nullptr, false); }, nullptr,
+ "Use a Brush trainer you already have (brush_app): for example one you built yourself with cargo on Linux "
+ "ARM, where there is no ready-made download.",
+ "");
+ add("splats.train_choose_colmap", "Choose COLMAP...", nullptr,
+ [this] { SDL_ShowOpenFileDialog(onPicked, pickTag(Pick::Colmap), window_, nullptr, 0, nullptr, false); },
+ nullptr, "Use a COLMAP program you already have (it finds where each photo was taken).", "");
+}
+
+void App::pollSplatTrain() {
+ if (!splatTrain_) return;
+ SplatTrainUi& s = *splatTrain_;
+ // Native dialogs.
+ std::vector<std::pair<Pick, std::string>> picked;
+ {
+ std::lock_guard lock(g_pickMutex);
+ picked.swap(g_picked);
+ }
+ for (auto& [kind, path] : picked) {
+ if (kind == Pick::Video || kind == Pick::Folder) {
+ runCommand("splat_train", {{"source", path}, {"preset", s.preset}});
+ } else {
+ (kind == Pick::Brush ? s.settings.brushPath : s.settings.colmapPath) = path;
+ s.save();
+ s.findTools();
+ status_ = std::string(kind == Pick::Brush ? "Brush" : "COLMAP") + ": " + path;
+ if (s.prompt && kind == s.promptFor) {
+ s.prompt = false;
+ runCommand("splat_train", {{"source", s.pendingSource}, {"out", s.pendingOut}});
+ }
+ }
+ }
+ // A download finishing.
+ if (s.download.valid() && s.download.wait_for(std::chrono::seconds(0)) == std::future_status::ready) {
+ const bool ok = s.download.get();
+ const char* tool = s.downloading ? s.downloading->tool : "tool";
+ s.downloading = nullptr;
+ s.findTools();
+ if (ok) {
+ status_ = std::string(tool) + " downloaded";
+ s.prompt = false;
+ runCommand("splat_train", {{"source", s.pendingSource}, {"out", s.pendingOut}});
+ } else {
+ s.promptError = *s.downloadError;
+ status_ = *s.downloadError;
+ }
+ }
+ if (!s.active) return;
+ // The newest checkpoint: moved in, so swapping costs nothing here (the renderer uploads it next frame).
+ if (std::shared_ptr<SplatCloud> cloud = s.job.takePreview()) {
+ const bool first = s.previews == 0;
+ if (!first) cloud->transform = doc_.splats.transform; // keep a Flip / Fit made while it trains
+ cloud->name = utf8(fsPath(s.source).stem());
+ doc_.splats = std::move(*cloud);
+ doc_.splatsChanged();
+ renderSettings_.showSplats = true;
+ ++s.previews;
+ if (first) camera_.frame(sceneBounds());
+ }
+ if (s.job.busy()) return;
+ s.active = false;
+ const ai::TrainStatus st = s.job.status();
+ if (st.stage == ai::TrainStage::Done) {
+ status_ = "Splat trained: " + std::to_string(doc_.splats.size()) + " splats, saved to " + st.result;
+ if (s.settings.copyToLibrary && !library_.user.empty() && !st.result.empty()) {
+ std::error_code ec;
+ const fs::path dir = fsPath(library_.user) / "Splats";
+ fs::create_directories(dir, ec);
+ fs::copy_file(fsPath(st.result), dir / fsPath(st.result).filename(), fs::copy_options::overwrite_existing, ec);
+ if (!ec) libraryScanner_.invalidate();
+ }
+ } else if (st.stage == ai::TrainStage::Cancelled) {
+ status_ = st.message;
+ } else {
+ status_ = "Splat training failed: " + st.error;
+ }
+}
+
+void App::drawSplatTrainSection() {
+ if (!splatTrain_) return;
+ SplatTrainUi& s = *splatTrain_;
+ if (!s.toolsKnown) s.findTools();
+ ImGui::SeparatorText("Train from capture");
+ const bool busy = splatTrainBusy();
+ ImGui::BeginDisabled(busy);
+ const auto& presets = capture::trainPresets();
+ const float half = (ImGui::GetContentRegionAvail().x - ImGui::GetStyle().ItemSpacing.x) * 0.5f;
+ for (size_t i = 0; i < presets.size(); ++i) {
+ if (i) ImGui::SameLine();
+ const bool on = s.preset == presets[i].id;
+ if (on) ImGui::PushStyleColor(ImGuiCol_Button, ImGui::GetColorU32(ui::color::accent));
+ if (ImGui::Button(presets[i].label, ImVec2(half, 0))) {
+ s.preset = presets[i].id;
+ s.settings.preset = s.preset;
+ s.save();
+ }
+ if (on) ImGui::PopStyleColor();
+ ui::HelpText(presets[i].label, nullptr,
+ i == 0 ? "Fast preview: 5,000 steps at up to 1024 px, about 80 frames from a video. A usable "
+ "splat in minutes on a gaming GPU - good to check the capture."
+ : "Quality: 30,000 steps at up to 1920 px, up to 200 frames, full view-dependent colour. "
+ "Sharper; takes several times longer.",
+ kPath);
+ }
+ ImGui::EndDisabled();
+ const float avail = ImGui::GetContentRegionAvail().x, gap = ImGui::GetStyle().ItemSpacing.x;
+ ui::ActionButton(*this, "splats.train_video", ImVec2((avail - gap) * 0.5f, 0), ui::Look::IconLabel);
+ ImGui::SameLine();
+ ui::ActionButton(*this, "splats.train_folder", ImVec2((avail - gap) * 0.5f, 0), ui::Look::IconLabel);
+ if (s.job.busy()) {
+ ImGui::TextDisabled("Training - progress is over the viewport");
+ } else if (s.brush.empty()) {
+ ImGui::PushStyleColor(ImGuiCol_Text, ImGui::GetStyleColorVec4(ImGuiCol_TextDisabled));
+ ImGui::TextWrapped("Trainer: Brush is not installed yet (it is offered when you start).");
+ ImGui::PopStyleColor();
+ }
+ if (ImGui::TreeNode("Tools##splattrain")) {
+ auto row = [](const char* name, const std::string& path, const char* missing) {
+ ImGui::TextUnformatted(name);
+ ImGui::SameLine();
+ if (path.empty()) ImGui::TextDisabled("%s", missing);
+ else ImGui::TextDisabled("%s", utf8(fsPath(path).filename()).c_str());
+ if (!path.empty() && ImGui::IsItemHovered()) ImGui::SetTooltip("%s", path.c_str());
+ };
+ row("Brush", s.brush, "not installed");
+ row("COLMAP", s.colmap, "not found (photos without positions need it)");
+ row("ffmpeg", s.ffmpeg, "not found (videos need it)");
+ ui::ActionButton(*this, "splats.train_choose_brush", ImVec2((avail - gap) * 0.5f, 0));
+ ImGui::SameLine();
+ ui::ActionButton(*this, "splats.train_choose_colmap", ImVec2(-1, 0));
+ ImGui::TreePop();
+ }
+}
+
+void App::drawSplatTrainOverlay() {
+ if (!splatTrain_) return;
+ SplatTrainUi& s = *splatTrain_;
+ const CanvasRect c = canvasRect();
+ const float scale = uiScale_;
+ const ImGuiWindowFlags flags = ImGuiWindowFlags_NoDecoration | ImGuiWindowFlags_AlwaysAutoResize |
+ ImGuiWindowFlags_NoSavedSettings | ImGuiWindowFlags_NoFocusOnAppearing |
+ ImGuiWindowFlags_NoNav | ImGuiWindowFlags_NoMove;
+ // The tool prompt: what is downloaded, from where, how big. Nothing happens without a click.
+ if (s.prompt) {
+ const ai::ToolDownload* d = s.promptFor == Pick::Brush ? ai::brushDownload() : ai::colmapDownload();
+ const char* tool = s.promptFor == Pick::Brush ? "Brush" : "COLMAP";
+ ImGui::SetNextWindowPos(ImVec2(c.x + c.w * 0.5f, c.y + c.h * 0.4f), ImGuiCond_Always, ImVec2(0.5f, 0.5f));
+ ImGui::SetNextWindowSize(ImVec2(440 * scale, 0), ImGuiCond_Always);
+ if (ImGui::Begin("##splat_tool_prompt", nullptr, flags & ~ImGuiWindowFlags_AlwaysAutoResize)) {
+ ImGui::TextUnformatted(s.promptFor == Pick::Brush ? "Splat training needs Brush" : "Splat training needs COLMAP");
+ ImGui::Spacing();
+ if (s.promptFor == Pick::Brush)
+ ImGui::TextWrapped("Brush is a free, open-source Gaussian splat trainer (Apache-2.0) that runs on your "
+ "graphics card - NVIDIA, AMD, Intel or Apple, no CUDA needed. SculptForge runs it "
+ "as a separate program.");
+ else
+ ImGui::TextWrapped("COLMAP (BSD licence) works out where each photo was taken, which training needs. "
+ "SculptForge runs it as a separate program.");
+ ImGui::Spacing();
+ if (d) {
+ ImGui::TextWrapped("Download %s %s (%s) from %s into %s. Its SHA-256 checksum is checked before it is "
+ "used.",
+ d->tool, d->version, megabytes(d->bytes).c_str(), d->url, ai::splatToolsDir().c_str());
+ } else {
+ ImGui::TextWrapped("There is no ready-made %s for this computer. %s", tool,
+ s.promptFor == Pick::Brush
+ ? "Build it with Rust (cargo build --release in a Brush checkout) and choose "
+ "brush_app below, or set SCULPTFORGE_BRUSH."
+ : "Install it (Linux: the colmap package; macOS: brew install colmap) or choose it.");
+ }
+ if (!s.promptError.empty()) ImGui::TextColored(ImVec4(0.95f, 0.45f, 0.35f, 1), "%s", s.promptError.c_str());
+ ImGui::Spacing();
+ if (s.download.valid()) {
+ ImGui::ProgressBar(s.downloadProgress->load(), ImVec2(-1, 0), "Downloading...");
+ if (ImGui::Button("Stop download")) s.downloadCancel->store(true);
+ } else {
+ if (d) {
+ const std::string label = "Download " + std::string(d->tool) + " (" + megabytes(d->bytes) + ")";
+ if (ImGui::Button(label.c_str())) {
+ s.downloading = d;
+ s.downloadProgress->store(0.0f);
+ s.downloadCancel->store(false);
+ s.promptError.clear();
+ auto progress = s.downloadProgress;
+ auto cancel = s.downloadCancel;
+ auto error = s.downloadError;
+ s.download = std::async(std::launch::async, [d, progress, cancel, error] {
+ std::string exe;
+ return ai::downloadTool(*d, exe, *error, progress.get(), cancel.get());
+ });
+ }
+ ui::HelpText(label.c_str(), nullptr, "Download it once from its GitHub release; nothing else is sent.",
+ kPath);
+ ImGui::SameLine();
+ }
+ ui::ActionButton(*this, s.promptFor == Pick::Brush ? "splats.train_choose_brush" : "splats.train_choose_colmap");
+ ImGui::SameLine();
+ if (ImGui::Button("Project page")) SDL_OpenURL(d ? d->home
+ : s.promptFor == Pick::Brush ? "https://github.com/ArthurBrussee/brush"
+ : "https://colmap.github.io/install.html");
+ ImGui::SameLine();
+ if (ImGui::Button("Not now")) s.prompt = false;
+ }
+ }
+ ImGui::End();
+ }
+
+ if (!s.active) return;
+ const ai::TrainStatus st = s.job.status();
+ ImGui::SetNextWindowPos(ImVec2(c.x + c.w * 0.5f, c.y + c.h - 16.0f * scale), ImGuiCond_Always, ImVec2(0.5f, 1.0f));
+ ImGui::SetNextWindowBgAlpha(0.92f);
+ ImGui::SetNextWindowSize(ImVec2(400 * scale, 0), ImGuiCond_Always);
+ if (ImGui::Begin("##splat_train_overlay", nullptr, flags & ~ImGuiWindowFlags_AlwaysAutoResize)) {
+ const capture::TrainPreset& preset = capture::trainPreset(s.preset);
+ ImGui::Text("Training splat - %s", preset.label);
+ ImGui::SameLine();
+ ImGui::TextDisabled("%s", utf8(fsPath(s.source).filename()).c_str());
+ if (st.stage == ai::TrainStage::Training && st.step > 0) {
+ char bar[64];
+ std::snprintf(bar, sizeof bar, "%d / %d steps", st.step, st.totalSteps);
+ ImGui::ProgressBar(st.totalSteps ? float(st.step) / float(st.totalSteps) : 0.0f, ImVec2(-1, 0), bar);
+ std::string line;
+ if (st.psnr > 0) {
+ char p[64];
+ std::snprintf(p, sizeof p, "PSNR %.1f dB (%s)", st.psnr, capture::psnrQuality(st.psnr));
+ line = p;
+ } else {
+ line = "Quality: measured soon";
+ }
+ if (st.splats > 0) line += " " + std::to_string((st.splats + 500) / 1000) + "k splats";
+ ImGui::TextUnformatted(line.c_str());
+ ImGui::TextDisabled("%s elapsed, about %s left", clock(st.elapsed).c_str(), clock(st.eta).c_str());
+ } else {
+ const float f = st.stageProgress;
+ ImGui::ProgressBar(f >= 0 ? f : float(std::fmod(ImGui::GetTime() * 0.4, 1.0)), ImVec2(-1, 0),
+ st.message.c_str());
+ const std::string photos = st.frames ? " - " + std::to_string(st.frames) + " photos" : std::string();
+ ImGui::TextDisabled("%s elapsed%s", clock(st.elapsed).c_str(), photos.c_str());
+ }
+ const float avail = ImGui::GetContentRegionAvail().x, gap = ImGui::GetStyle().ItemSpacing.x;
+ ImGui::BeginDisabled(st.stage != ai::TrainStage::Training || s.previews == 0);
+ ui::ActionButton(*this, "splats.train_stop", ImVec2((avail - gap) * 0.6f, 0), ui::Look::IconLabel);
+ ImGui::EndDisabled();
+ ImGui::SameLine();
+ ui::ActionButton(*this, "splats.train_cancel", ImVec2(-1, 0), ui::Look::IconLabel);
+ }
+ ImGui::End();
+}
+
+} // namespace sf
diff --git a/src/app/UiTest.cpp b/src/app/UiTest.cpp
index d559a97d..56501490 100644
--- a/src/app/UiTest.cpp
+++ b/src/app/UiTest.cpp
@@ -32,6 +32,7 @@
// copy SRC DST copy a file or folder (SRC relative to the script's folder, DST to the working
// directory), e.g. to stage what another program would write (the Blender bridge)
// wait SECONDS / wait_idle [TIMEOUT] (AI requests, thumbnails, update checks and other background jobs) / wait_for LABEL [TIMEOUT]
+// wait_value NAME OP N [TIMEOUT] wait (real time, default 30 s) until expect_value NAME OP N would pass
// expect LABEL / expect_not LABEL
// expect_text TEXT / expect_no_text TEXT any text drawn this frame contains TEXT
// expect_status TEXT the status bar message contains TEXT
@@ -84,6 +85,8 @@
// center_x / center_y / center_z and size_x / size_y / size_z (active mesh bounds)
// bake_running, baked_maps (maps in the last bake), bake_size, bake_files (files it wrote),
// normal_map_preview (1 when the active SubTool shows its baked normal map)
+// splat_training (1 while a splat trains), splat_previews (checkpoints shown), splat_train_step,
+// splat_train_stage (4 done, 5 failed, 6 cancelled), splat_train_psnr, splat_train_prompt
// (mesh values are the active SubTool's)
// snapshot remember the active mesh's vertices (for the moved* values)
// remember VALUE NAME keep VALUE's current value as $NAME
@@ -260,6 +263,7 @@ double Demo::query(const std::string& name, bool& known) const {
if (name == "active_texture_layer") return hasTexture(m) && !m.texture->layers.empty() ? double(m.texture->activeLayer) : -1.0;
if (name == "auto_uv_running") return app_.autoUv_ ? 1.0 : 0.0;
if (const double v = app_.bakeTestValue(name); !std::isnan(v)) return v;
+ if (const double v = app_.splatTrainTestValue(name); !std::isnan(v)) return v; // splat_training, ...
if (name == "qremesh_running") return app_.qremesh_ ? 1.0 : 0.0;
if (name == "qremesh_guides") return double(app_.doc_.qremeshGuides.size());
if (name == "qremesh_guide_loops") {
@@ -838,7 +842,8 @@ bool Demo::parseStep(int line, const std::vector<std::string>& w, std::string& e
const bool idle = !busy(app_.pendingBrush_) && !busy(app_.pendingGenerator_) && !busy(app_.pendingModels_) &&
!busy(app_.pendingUnreal_) && !busy(app_.pendingComfyCheck_) && !app_.galleryBusy() &&
!app_.thumbs_.busy() && !app_.libraryScanner_.busy() && !app_.autoUv_ &&
- !app_.bakeBusy() && !app_.qremesh_ && !app_.voxelRemesh_ && !app_.updateJob_;
+ !app_.bakeBusy() && !app_.qremesh_ && !app_.voxelRemesh_ && !app_.updateJob_ &&
+ !app_.splatTrainBusy();
*idleFrames = idle ? *idleFrames + 1 : 0;
if (*idleFrames >= 4) return true;
if (idle) return false;
@@ -939,6 +944,25 @@ bool Demo::parseStep(int line, const std::vector<std::string>& w, std::string& e
const double v = query(name, k);
if (!compare(v, op, want)) fail(line, name + " is " + number(v) + ", expected " + op + " " + number(want) + " (" + wantText + ")");
});
+ } else if (verb == "wait_value") { // wait (in real time) until a value holds, e.g. a background job's progress
+ if (!need(3)) return false;
+ bool known = false;
+ query(w[1], known);
+ if (!known) {
+ error = "unknown value '" + w[1] + "'";
+ return false;
+ }
+ const std::string name = w[1], op = w[2];
+ const double want = std::strtod(w[3].c_str(), nullptr);
+ const float timeout = num(4, 30.0f);
+ steps_.push_back([this, line, name, op, want, timeout](float t) {
+ bool k = false;
+ const double v = query(name, k);
+ if (compare(v, op, want)) return true;
+ SDL_Delay(10);
+ if (t > timeout) fail(line, name + " is " + number(v) + ", still not " + op + " " + number(want) + " after " + number(timeout) + " s");
+ return t > timeout;
+ });
} else if (verb == "expect_guide") {
if (!need(1)) return false;
const std::string file = w[1];
diff --git a/src/cli/main.cpp b/src/cli/main.cpp
index 4b2d38aa..dcd9bc4e 100644
--- a/src/cli/main.cpp
+++ b/src/cli/main.cpp
@@ -4,6 +4,7 @@
// sculptforge-cli: run SculptForge commands headless, for build farms, asset
// pipelines and AI agents.
#include "ai/AiCommands.h"
+#include "ai/SplatTrainer.h"
#include "cli/McpServer.h"
#include "core/Benchmark.h"
#include "core/BrushLibrary.h"
@@ -53,6 +54,8 @@ int main(int argc, char** argv) {
sf::CommandRegistry registry;
// AI commands use the same settings file as the app (keys can also come from the environment).
sf::ai::registerAiCommands(registry, [] { return sf::ai::AiSettings::load(sf::ai::defaultAiSettingsPath()); });
+ // splat_train: trains in the foreground with the app's tool settings (ai/SplatTrainer.h).
+ sf::ai::registerSplatTrainCommands(registry, [] { return sf::ai::SplatTrainSettings::load(sf::ai::defaultSplatTrainSettingsPath()); });
std::string script;
bool quiet = false;
std::map<std::string, std::string> vars;
diff --git a/src/core/SplatCapture.cpp b/src/core/SplatCapture.cpp
new file mode 100644
index 00000000..bc9593ba
--- /dev/null
+++ b/src/core/SplatCapture.cpp
@@ -0,0 +1,476 @@
+// SculptForge - open source digital sculpting
+// SPDX-License-Identifier: GPL-3.0-or-later
+#include "core/SplatCapture.h"
+
+#include "core/Json.h"
+
+#include <glm/gtc/quaternion.hpp>
+#include <glm/gtc/matrix_transform.hpp>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdio>
+#include <cstring>
+#include <filesystem>
+#include <fstream>
+#include <sstream>
+
+namespace sf::capture {
+
+namespace fs = std::filesystem;
+
+namespace {
+
+fs::path u8path(const std::string& s) { return fs::path(std::u8string(s.begin(), s.end())); }
+std::string pathString(const fs::path& p) {
+ const auto s = p.u8string();
+ return std::string(s.begin(), s.end());
+}
+std::string lowerExt(const fs::path& p) {
+ std::string e = p.extension().string();
+ std::transform(e.begin(), e.end(), e.begin(), [](unsigned char c) { return char(std::tolower(c)); });
+ return e;
+}
+bool readText(const std::string& path, std::string& out) {
+ std::ifstream f(u8path(path), std::ios::binary);
+ if (!f) return false;
+ std::stringstream ss;
+ ss << f.rdbuf();
+ out = ss.str();
+ return true;
+}
+
+// A 4x4 matrix from JSON: nested rows [[..],[..],[..],[..]] or 16 numbers, row-major.
+bool readMatrix(const Json& j, glm::mat4& m) {
+ float v[16];
+ if (j.isArray() && j.size() == 4) {
+ for (int r = 0; r < 4; ++r) {
+ const Json& row = j[size_t(r)];
+ if (!row.isArray() || row.size() != 4) return false;
+ for (int c = 0; c < 4; ++c) v[r * 4 + c] = float(row[size_t(c)].asNumber());
+ }
+ } else if (j.isArray() && j.size() == 16) {
+ for (int i = 0; i < 16; ++i) v[i] = float(j[size_t(i)].asNumber());
+ } else {
+ return false;
+ }
+ for (int r = 0; r < 4; ++r)
+ for (int c = 0; c < 4; ++c) m[c][r] = v[r * 4 + c]; // glm is column-major
+ return true;
+}
+
+Json matrixRows(const glm::mat4& m) {
+ Json rows = Json::array();
+ for (int r = 0; r < 4; ++r) {
+ Json row = Json::array();
+ for (int c = 0; c < 4; ++c) row.push(double(m[c][r]));
+ rows.push(std::move(row));
+ }
+ return rows;
+}
+
+void readIntrinsics(const Json& f, CameraView& v) {
+ v.fx = float(f["fl_x"].asNumber(v.fx));
+ v.fy = float(f["fl_y"].asNumber(v.fx));
+ v.cx = float(f["cx"].asNumber(v.cx));
+ v.cy = float(f["cy"].asNumber(v.cy));
+ v.width = int(f["w"].asNumber(v.width));
+ v.height = int(f["h"].asNumber(v.height));
+}
+
+// COLMAP pose (world-to-camera rotation as a quaternion and translation, OpenCV axes) to an OpenGL camToWorld.
+glm::mat4 colmapToCamToWorld(double qw, double qx, double qy, double qz, double tx, double ty, double tz) {
+ const glm::mat3 r = glm::mat3_cast(glm::quat(float(qw), float(qx), float(qy), float(qz)));
+ const glm::mat3 rt = glm::transpose(r);
+ const glm::vec3 centre = -(rt * glm::vec3(float(tx), float(ty), float(tz)));
+ glm::mat4 m(1.0f);
+ m[0] = glm::vec4(rt[0], 0.0f);
+ m[1] = glm::vec4(-rt[1], 0.0f); // OpenCV y down -> OpenGL y up
+ m[2] = glm::vec4(-rt[2], 0.0f); // OpenCV looks down +z -> OpenGL -z
+ m[3] = glm::vec4(centre, 1.0f);
+ return m;
+}
+
+bool readColmapImagesBin(const std::string& path, std::vector<CameraView>& views) {
+ std::ifstream f(u8path(path), std::ios::binary);
+ if (!f) return false;
+ auto rd = [&](void* p, size_t n) { return bool(f.read(static_cast<char*>(p), std::streamsize(n))); };
+ uint64_t count = 0;
+ if (!rd(&count, 8) || count > 10'000'000) return false;
+ for (uint64_t i = 0; i < count; ++i) {
+ uint32_t id = 0, camera = 0;
+ double q[4], t[3];
+ if (!rd(&id, 4) || !rd(q, sizeof q) || !rd(t, sizeof t) || !rd(&camera, 4)) return false;
+ std::string name;
+ for (char c; f.get(c) && c != '\0';) name += c;
+ uint64_t points = 0;
+ if (!rd(&points, 8) || points > 100'000'000) return false;
+ f.seekg(std::streamoff(points * 24), std::ios::cur); // x, y (double) + point3D id (int64)
+ if (!f) return false;
+ CameraView v;
+ v.file = "images/" + name;
+ v.camToWorld = colmapToCamToWorld(q[0], q[1], q[2], q[3], t[0], t[1], t[2]);
+ views.push_back(std::move(v));
+ }
+ return true;
+}
+
+bool readColmapImagesTxt(const std::string& path, std::vector<CameraView>& views) {
+ std::ifstream f(u8path(path));
+ if (!f) return false;
+ std::string line;
+ bool poseLine = true;
+ while (std::getline(f, line)) {
+ if (line.empty() || line[0] == '#') continue;
+ if (poseLine) {
+ std::istringstream ss(line);
+ long id = 0, camera = 0;
+ double q[4], t[3];
+ std::string name;
+ if (!(ss >> id >> q[0] >> q[1] >> q[2] >> q[3] >> t[0] >> t[1] >> t[2] >> camera >> name)) return false;
+ CameraView v;
+ v.file = "images/" + name;
+ v.camToWorld = colmapToCamToWorld(q[0], q[1], q[2], q[3], t[0], t[1], t[2]);
+ views.push_back(std::move(v));
+ }
+ poseLine = !poseLine; // every pose line is followed by its 2D points line
+ }
+ return true;
+}
+
+} // namespace
+
+double sharpness(const Image& image) {
+ if (image.empty()) return 0.0;
+ const Image small = image.downscaled(512);
+ const int w = small.width, h = small.height;
+ if (w < 3 || h < 3) return 0.0;
+ std::vector<float> lum(size_t(w) * size_t(h));
+ for (size_t i = 0; i < lum.size(); ++i) {
+ const uint8_t* p = &small.rgba[i * 4];
+ lum[i] = (0.299f * p[0] + 0.587f * p[1] + 0.114f * p[2]) / 255.0f;
+ }
+ double sum = 0.0, sum2 = 0.0;
+ size_t n = 0;
+ for (int y = 1; y < h - 1; ++y)
+ for (int x = 1; x < w - 1; ++x) {
+ const size_t i = size_t(y) * size_t(w) + size_t(x);
+ const double lap = double(lum[i - 1]) + lum[i + 1] + lum[i - size_t(w)] + lum[i + size_t(w)] - 4.0 * lum[i];
+ sum += lap;
+ sum2 += lap * lap;
+ ++n;
+ }
+ const double mean = sum / double(n);
+ return std::max(0.0, sum2 / double(n) - mean * mean) * 1000.0;
+}
+
+std::vector<size_t> selectFrames(const std::vector<double>& scores, size_t count) {
+ std::vector<size_t> out;
+ const size_t n = scores.size();
+ if (count == 0 || n == 0) return out;
+ if (count >= n) {
+ for (size_t i = 0; i < n; ++i) out.push_back(i);
+ return out;
+ }
+ for (size_t k = 0; k < count; ++k) {
+ const size_t begin = k * n / count, end = std::max(begin + 1, (k + 1) * n / count);
+ size_t best = begin;
+ for (size_t i = begin + 1; i < end; ++i)
+ if (scores[i] > scores[best]) best = i;
+ out.push_back(best);
+ }
+ return out;
+}
+
+bool writeTransformsJson(const std::string& path, const std::vector<CameraView>& views, std::string& error) {
+ Json root = Json::object();
+ root.set("camera_model", "PINHOLE");
+ Json frames = Json::array();
+ for (const CameraView& v : views) {
+ Json f = Json::object();
+ f.set("file_path", v.file);
+ f.set("transform_matrix", matrixRows(v.camToWorld));
+ if (v.fx > 0) {
+ f.set("fl_x", double(v.fx));
+ f.set("fl_y", double(v.fy > 0 ? v.fy : v.fx));
+ }
+ if (v.width > 0 && v.height > 0) {
+ f.set("cx", double(v.cx > 0 ? v.cx : v.width * 0.5f));
+ f.set("cy", double(v.cy > 0 ? v.cy : v.height * 0.5f));
+ f.set("w", v.width);
+ f.set("h", v.height);
+ }
+ frames.push(std::move(f));
+ }
+ root.set("frames", std::move(frames));
+ std::ofstream out(u8path(path), std::ios::binary);
+ if (!out) {
+ error = "Cannot write " + path;
+ return false;
+ }
+ out << root.dump(true);
+ if (!out) {
+ error = "Cannot write " + path;
+ return false;
+ }
+ return true;
+}
+
+bool readTransformsJson(const std::string& path, std::vector<CameraView>& views, std::string& error) {
+ std::string text;
+ if (!readText(path, text)) {
+ error = "Cannot read " + path;
+ return false;
+ }
+ Json root;
+ if (!Json::parse(text, root, error)) return false;
+ CameraView shared;
+ readIntrinsics(root, shared);
+ for (const Json& f : root["frames"].items()) {
+ CameraView v = shared;
+ v.file = f["file_path"].isString() ? f["file_path"].asString() : std::string();
+ if (!readMatrix(f["transform_matrix"], v.camToWorld)) {
+ error = "A frame in " + path + " has no 4x4 transform_matrix";
+ return false;
+ }
+ readIntrinsics(f, v);
+ views.push_back(std::move(v));
+ }
+ return true;
+}
+
+bool readColmapImages(const std::string& sparseDir, std::vector<CameraView>& views, std::string& error) {
+ const fs::path dir = u8path(sparseDir);
+ std::error_code ec;
+ if (fs::exists(dir / "images.bin", ec)) {
+ if (readColmapImagesBin(pathString(dir / "images.bin"), views)) return true;
+ error = "Cannot read " + pathString(dir / "images.bin");
+ return false;
+ }
+ if (fs::exists(dir / "images.txt", ec)) {
+ if (readColmapImagesTxt(pathString(dir / "images.txt"), views)) return true;
+ error = "Cannot read " + pathString(dir / "images.txt");
+ return false;
+ }
+ error = "No COLMAP images.bin or images.txt in " + sparseDir;
+ return false;
+}
+
+glm::mat4 orbitFraming(const std::vector<CameraView>& views) {
+ if (views.size() < 2) return glm::mat4(1.0f);
+ // The point closest to every viewing ray: sum (I - d d^T) p = sum (I - d d^T) c.
+ glm::dmat3 a(0.0);
+ glm::dvec3 b(0.0), up(0.0), mean(0.0);
+ for (const CameraView& v : views) {
+ const glm::dvec3 c(v.camToWorld[3]);
+ const glm::dvec3 d = glm::normalize(-glm::dvec3(v.camToWorld[2]));
+ const glm::dmat3 p = glm::dmat3(1.0) - glm::outerProduct(d, d);
+ a += p;
+ b += p * c;
+ up += glm::dvec3(v.camToWorld[1]);
+ mean += c;
+ }
+ mean /= double(views.size());
+ glm::dvec3 target = mean;
+ if (std::abs(glm::determinant(a)) > 1e-6 * double(views.size())) target = glm::inverse(a) * b;
+ // Cameras all looking the same way (a straight walk past something) give no
+ // sensible crossing: fall back to the middle of the cameras.
+ if (!std::isfinite(target.x) || !std::isfinite(target.y) || !std::isfinite(target.z)) target = mean;
+ std::vector<double> dist;
+ for (const CameraView& v : views) dist.push_back(glm::length(glm::dvec3(v.camToWorld[3]) - target));
+ std::nth_element(dist.begin(), dist.begin() + std::ptrdiff_t(dist.size() / 2), dist.end());
+ const double median = dist[dist.size() / 2];
+ const double scale = median > 1e-9 ? 3.0 / median : 1.0;
+
+ glm::quat turn(1.0f, 0.0f, 0.0f, 0.0f);
+ if (glm::length(up) > 1e-9) {
+ const glm::vec3 u = glm::normalize(glm::vec3(up)), y(0.0f, 1.0f, 0.0f);
+ const float cosA = glm::clamp(glm::dot(u, y), -1.0f, 1.0f);
+ glm::vec3 axis = glm::cross(u, y);
+ if (glm::length(axis) < 1e-6f) axis = cosA > 0.0f ? y : glm::vec3(1.0f, 0.0f, 0.0f); // aligned or opposite
+ turn = glm::angleAxis(std::acos(cosA), glm::normalize(axis));
+ }
+ glm::mat4 m = glm::scale(glm::mat4(1.0f), glm::vec3(float(scale)));
+ m = m * glm::mat4_cast(turn);
+ m = glm::translate(m, -glm::vec3(target));
+ return m;
+}
+
+bool readCaptureJson(const std::string& path, CaptureInfo& info, std::string& error) {
+ std::string text;
+ if (!readText(path, text)) {
+ error = "Cannot read " + path;
+ return false;
+ }
+ Json root;
+ if (!Json::parse(text, root, error)) return false;
+ if (!root.isObject()) {
+ error = path + " is not a JSON object";
+ return false;
+ }
+ if (root["kind"].isString()) info.kind = root["kind"].asString();
+ if (root["device"].isString()) info.device = root["device"].asString();
+ for (const Json& f : root["files"].items()) {
+ if (f.isString()) info.files.push_back(f.asString());
+ else if (f["name"].isString()) info.files.push_back(f["name"].asString()); // {"name":..,"t":..}
+ }
+ CameraView shared;
+ readIntrinsics(root, shared);
+ for (const Json& f : root["frames"].items()) {
+ CameraView v = shared;
+ const Json& name = f["file"].isString() ? f["file"] : f["file_path"];
+ if (!name.isString() || !readMatrix(f["transform_matrix"], v.camToWorld)) continue;
+ v.file = name.asString();
+ readIntrinsics(f, v);
+ info.poses.push_back(std::move(v));
+ }
+ return true;
+}
+
+bool isVideoFile(const std::string& path) {
+ const std::string e = lowerExt(u8path(path));
+ return e == ".mp4" || e == ".mov" || e == ".m4v" || e == ".webm" || e == ".mkv" || e == ".avi";
+}
+
+std::vector<std::string> listImages(const std::string& dir) {
+ std::vector<std::string> out;
+ std::error_code ec;
+ for (fs::directory_iterator it(u8path(dir), ec), end; !ec && it != end; it.increment(ec)) {
+ if (!it->is_regular_file(ec)) continue;
+ const std::string e = lowerExt(it->path());
+ if (e == ".jpg" || e == ".jpeg" || e == ".png") out.push_back(pathString(it->path()));
+ }
+ std::sort(out.begin(), out.end());
+ return out;
+}
+
+std::string findVideo(const std::string& dir) {
+ std::vector<std::string> found;
+ std::error_code ec;
+ for (fs::directory_iterator it(u8path(dir), ec), end; !ec && it != end; it.increment(ec))
+ if (it->is_regular_file(ec) && isVideoFile(pathString(it->path()))) found.push_back(pathString(it->path()));
+ std::sort(found.begin(), found.end());
+ return found.empty() ? std::string() : found.front();
+}
+
+SourceKind classifySource(const std::string& path) {
+ std::error_code ec;
+ const fs::path p = u8path(path);
+ if (path.empty() || !fs::exists(p, ec)) return SourceKind::Missing;
+ if (fs::is_regular_file(p, ec)) return isVideoFile(path) ? SourceKind::Video : SourceKind::Unsupported;
+ if (!fs::is_directory(p, ec)) return SourceKind::Unsupported;
+ if (fs::exists(p / "transforms.json", ec)) return SourceKind::NerfstudioDataset;
+ if (fs::is_directory(p / "images", ec) && fs::is_directory(p / "sparse" / "0", ec)) return SourceKind::ColmapDataset;
+ if (!listImages(path).empty()) return SourceKind::Images;
+ if (fs::is_directory(p / "images", ec) && !listImages(pathString(p / "images")).empty()) return SourceKind::Images;
+ if (!findVideo(path).empty()) return SourceKind::VideoFolder;
+ return SourceKind::Unsupported;
+}
+
+const char* sourceKindName(SourceKind kind) {
+ switch (kind) {
+ case SourceKind::Missing: return "missing";
+ case SourceKind::Video: return "video";
+ case SourceKind::Images: return "photos";
+ case SourceKind::VideoFolder: return "video";
+ case SourceKind::NerfstudioDataset: return "nerfstudio dataset";
+ case SourceKind::ColmapDataset: return "COLMAP dataset";
+ case SourceKind::Unsupported: break;
+ }
+ return "unsupported";
+}
+
+const std::vector<TrainPreset>& trainPresets() {
+ static const std::vector<TrainPreset> presets = {
+ {"fast", "Fast preview", 5000, 1024, 1, 250, 600'000, 80},
+ {"quality", "Quality", 30000, 1920, 3, 1000, 3'000'000, 200},
+ };
+ return presets;
+}
+
+const TrainPreset& trainPreset(const std::string& id) {
+ for (const TrainPreset& p : trainPresets())
+ if (id == p.id) return p;
+ return trainPresets().front();
+}
+
+std::vector<std::string> brushArguments(const std::string& dataset, const std::string& exportDir,
+ const TrainPreset& preset, bool evaluate) {
+ std::vector<std::string> a = {dataset,
+ "--total-steps", std::to_string(preset.steps),
+ "--max-resolution", std::to_string(preset.maxResolution),
+ "--sh-degree", std::to_string(preset.shDegree),
+ "--max-splats", std::to_string(preset.maxSplats),
+ "--export-every", std::to_string(preset.exportEvery),
+ "--export-path", exportDir,
+ "--export-name", "export_{iter}.ply"};
+ if (evaluate) { // hold back every 8th view to measure PSNR on
+ a.insert(a.end(), {"--eval-split-every", "8", "--eval-every", std::to_string(preset.exportEvery * 2)});
+ }
+ return a;
+}
+
+bool parseTrainerLog(const std::string& line, TrainerLog& out) {
+ out = TrainerLog();
+ if (const size_t p = line.find("Refine iter "); p != std::string::npos) {
+ int iter = -1, splats = -1;
+ if (std::sscanf(line.c_str() + p, "Refine iter %d, %d splats", &iter, &splats) >= 1) {
+ out.refineIter = iter;
+ out.splats = splats;
+ return true;
+ }
+ }
+ if (const size_t p = line.find("Eval iter "); p != std::string::npos) {
+ int iter = -1;
+ double psnr = -1.0, ssim = -1.0;
+ if (std::sscanf(line.c_str() + p, "Eval iter %d: PSNR %lf, ssim %lf", &iter, &psnr, &ssim) >= 2) {
+ out.evalIter = iter;
+ out.psnr = psnr;
+ out.ssim = ssim;
+ return true;
+ }
+ }
+ if (line.find("panicked at") != std::string::npos || line.rfind("Error:", 0) == 0 ||
+ line.find(" ERROR ") != std::string::npos) {
+ out.error = true;
+ return true;
+ }
+ return false;
+}
+
+int checkpointStep(const std::string& fileName) {
+ const fs::path p = u8path(fileName);
+ if (lowerExt(p) != ".ply") return -1;
+ const std::string stem = pathString(p.stem());
+ size_t end = stem.size(), begin = end;
+ while (begin > 0 && std::isdigit(static_cast<unsigned char>(stem[begin - 1]))) --begin;
+ if (begin == end || end - begin > 9) return -1;
+ return std::stoi(stem.substr(begin, end - begin));
+}
+
+std::string latestCheckpoint(const std::string& dir, int& step) {
+ step = -1;
+ std::string best;
+ std::error_code ec;
+ for (fs::directory_iterator it(u8path(dir), ec), end; !ec && it != end; it.increment(ec)) {
+ if (!it->is_regular_file(ec)) continue;
+ const std::string name = pathString(it->path().filename());
+ const int s = checkpointStep(name);
+ if (s > step) {
+ step = s;
+ best = pathString(it->path());
+ }
+ }
+ return best;
+}
+
+const char* psnrQuality(double psnr) {
+ if (psnr <= 0.0) return "";
+ if (psnr < 18.0) return "rough";
+ if (psnr < 22.0) return "taking shape";
+ if (psnr < 26.0) return "good";
+ if (psnr < 30.0) return "sharp";
+ return "very sharp";
+}
+
+} // namespace sf::capture
diff --git a/src/core/SplatCapture.h b/src/core/SplatCapture.h
new file mode 100644
index 00000000..bba50870
--- /dev/null
+++ b/src/core/SplatCapture.h
@@ -0,0 +1,122 @@
+// SculptForge - open source digital sculpting
+// SPDX-License-Identifier: GPL-3.0-or-later
+//
+// Turning a phone capture (a video or a folder of photos) into a dataset a
+// Gaussian-splat trainer can read, and reading back what the trainer says.
+// No processes or networking here (those are in ai/SplatTrainer.h), so all of
+// it is unit-tested:
+// - frame selection: the sharpest frame in each stretch of the video, so the
+// views stay spread around the object and motion blur is dropped
+// - capture.json from the phone link (kind, device, files, and optional
+// camera poses from the phone's AR tracking, which skip COLMAP)
+// - nerfstudio transforms.json writing and COLMAP images.bin/txt reading
+// - framing: the object the cameras orbit becomes the centre, their average
+// up becomes +Y (captures otherwise arrive tilted or upside down)
+// - Brush's command line and its log lines (refine and eval steps, PSNR)
+#pragma once
+
+#include "core/Image.h"
+
+#include <glm/glm.hpp>
+
+#include <string>
+#include <vector>
+
+namespace sf::capture {
+
+// Variance of the Laplacian of the luminance (higher = sharper). The image is
+// first reduced to at most 512 px so scores of different sizes are comparable.
+double sharpness(const Image& image);
+
+// Picks up to `count` frames in time order: the frames are split into `count`
+// equal stretches and the sharpest of each is kept (so a blurry second costs
+// one view, not a gap). Returns indices into `scores`, ascending.
+std::vector<size_t> selectFrames(const std::vector<double>& scores, size_t count);
+
+// One posed photo. `camToWorld` uses the OpenGL / nerfstudio convention
+// (x right, y up, the camera looks down -z). Intrinsics in pixels.
+struct CameraView {
+ std::string file; // relative to the dataset folder, e.g. "images/00001.jpg"
+ glm::mat4 camToWorld{1.0f};
+ float fx = 0, fy = 0, cx = 0, cy = 0;
+ int width = 0, height = 0;
+};
+
+// Writes a nerfstudio transforms.json (read by Brush, nerfstudio, gsplat...).
+bool writeTransformsJson(const std::string& path, const std::vector<CameraView>& views, std::string& error);
+bool readTransformsJson(const std::string& path, std::vector<CameraView>& views, std::string& error);
+
+// COLMAP sparse model images (world-to-camera, OpenCV axes) converted to CameraView
+// (intrinsics are left at 0: only the poses are read).
+bool readColmapImages(const std::string& sparseDir, std::vector<CameraView>& views, std::string& error);
+
+// The placement for a capture's splats: moves the point the cameras look at to
+// the origin, turns the cameras' average up to +Y and scales so the cameras sit
+// about 3 units away (a new sphere has radius 1). Identity for fewer than 2 views.
+glm::mat4 orbitFraming(const std::vector<CameraView>& views);
+
+// capture.json written by the phone link next to the uploaded files.
+struct CaptureInfo {
+ std::string kind; // "splat", "photo", "ply"
+ std::string device; // free text, e.g. "iPhone 15 Pro"
+ std::vector<std::string> files; // relative to the capture folder
+ std::vector<CameraView> poses; // optional "frames": AR poses per photo (then no COLMAP is needed)
+};
+bool readCaptureJson(const std::string& path, CaptureInfo& info, std::string& error);
+
+// What a source path is.
+enum class SourceKind {
+ Missing,
+ Video, // .mp4 .mov .m4v .webm .mkv .avi
+ Images, // a folder of photos (or a capture folder with photos)
+ VideoFolder, // a capture folder whose capture.json / contents is one video
+ NerfstudioDataset, // a folder with transforms.json
+ ColmapDataset, // a folder with images/ and sparse/0
+ Unsupported,
+};
+SourceKind classifySource(const std::string& path);
+const char* sourceKindName(SourceKind kind);
+bool isVideoFile(const std::string& path);
+// Photos in a folder (sorted by name; not recursive).
+std::vector<std::string> listImages(const std::string& dir);
+// The first video in a folder ("" if none).
+std::string findVideo(const std::string& dir);
+
+// Training presets.
+struct TrainPreset {
+ const char* id; // "fast", "quality"
+ const char* label; // "Fast preview"
+ int steps;
+ int maxResolution; // longest image side Brush loads
+ int shDegree;
+ int exportEvery; // a checkpoint (live preview) every this many steps
+ int maxSplats;
+ int maxFrames; // frames kept from a video
+};
+const std::vector<TrainPreset>& trainPresets();
+const TrainPreset& trainPreset(const std::string& id); // unknown -> the first (fast)
+
+// Brush's arguments (after the program name) to train `dataset` into `exportDir`.
+std::vector<std::string> brushArguments(const std::string& dataset, const std::string& exportDir,
+ const TrainPreset& preset, bool evaluate);
+
+// One line of Brush's log (RUST_LOG=info on stdout). Fields stay -1 / negative when absent.
+struct TrainerLog {
+ int refineIter = -1; // "Refine iter 400, 51234 splats."
+ int splats = -1;
+ int evalIter = -1; // "Eval iter 1000: PSNR 24.31, ssim 0.81"
+ double psnr = -1.0;
+ double ssim = -1.0;
+ bool error = false; // a panic or "Error:" line
+};
+bool parseTrainerLog(const std::string& line, TrainerLog& out);
+
+// The step number in a checkpoint name such as "export_1200.ply" (-1 if none).
+int checkpointStep(const std::string& fileName);
+// The newest checkpoint (highest step) in `dir`; "" when none. `step` receives its step.
+std::string latestCheckpoint(const std::string& dir, int& step);
+
+// A rough quality word for a PSNR in dB (for the progress overlay).
+const char* psnrQuality(double psnr);
+
+} // namespace sf::capture
diff --git a/src/python/module.cpp b/src/python/module.cpp
index adfc87de..2c2f6dba 100644
--- a/src/python/module.cpp
+++ b/src/python/module.cpp
@@ -11,6 +11,7 @@
// verts, tris = doc.vertices, doc.triangles # (N, 3) float32, (M, 3) uint32
// doc.mask, doc.colors, doc.polygroups # optional attributes, None when absent
#include "ai/AiCommands.h"
+#include "ai/SplatTrainer.h"
#include "core/BrushLibrary.h"
#include "core/Commands.h"
#include "core/Script.h"
@@ -31,6 +32,7 @@ const CommandRegistry& registry() {
static CommandRegistry r = [] {
CommandRegistry reg;
ai::registerAiCommands(reg, [] { return ai::AiSettings::load(ai::defaultAiSettingsPath()); });
+ ai::registerSplatTrainCommands(reg, [] { return ai::SplatTrainSettings::load(ai::defaultSplatTrainSettingsPath()); });
return reg;
}();
return r;
diff --git a/tests/test_splat_train.cpp b/tests/test_splat_train.cpp
new file mode 100644
index 00000000..8d15e9c4
--- /dev/null
+++ b/tests/test_splat_train.cpp
@@ -0,0 +1,500 @@
+// SculptForge - open source digital sculpting
+// SPDX-License-Identifier: GPL-3.0-or-later
+//
+// Splat training from captures: frame selection, dataset and pose files,
+// framing, Brush's log, and the training job driven by a fake Brush / COLMAP /
+// ffmpeg (tests/tools/fake_splat_tools.cpp).
+#include "TestHarness.h"
+
+#include "ai/SplatTrainer.h"
+#include "core/SplatCapture.h"
+#include "io/PlyIO.h"
+
+#include <glm/gtc/quaternion.hpp>
+
+#include <chrono>
+#include <cmath>
+#include <cstdlib>
+#include <filesystem>
+#include <fstream>
+#include <thread>
+
+using namespace sf;
+using namespace sf::capture;
+namespace fs = std::filesystem;
+
+namespace {
+
+void setEnv(const char* name, const char* value) {
+#if defined(_WIN32)
+ _putenv_s(name, value ? value : "");
+#else
+ if (value) setenv(name, value, 1);
+ else unsetenv(name);
+#endif
+}
+
+Image checker(int w, int h, int cell, bool flat = false) {
+ Image img;
+ img.width = w;
+ img.height = h;
+ img.rgba.resize(size_t(w) * size_t(h) * 4, 255);
+ for (int y = 0; y < h; ++y)
+ for (int x = 0; x < w; ++x) {
+ const uint8_t v = flat ? 128 : ((x / cell + y / cell) % 2 ? 230 : 30);
+ for (int c = 0; c < 3; ++c) img.rgba[(size_t(y) * size_t(w) + size_t(x)) * 4 + size_t(c)] = v;
+ }
+ return img;
+}
+
+// A clean folder under the temp directory.
+fs::path freshDir(const char* name) {
+ const fs::path d = fs::temp_directory_path() / name;
+ std::error_code ec;
+ fs::remove_all(d, ec);
+ fs::create_directories(d);
+ return d;
+}
+
+fs::path photoFolder(const char* name, int count) {
+ const fs::path d = freshDir(name);
+ for (int i = 0; i < count; ++i) {
+ std::string err;
+ char file[32];
+ std::snprintf(file, sizeof file, "IMG_%04d.png", i);
+ checker(64, 48, 4 + i % 3).savePng((d / file).string(), err);
+ }
+ return d;
+}
+
+CameraView lookAt(const glm::vec3& eye, const glm::vec3& target, const glm::vec3& up) {
+ // OpenGL camera: looks down -z, y up.
+ const glm::vec3 f = glm::normalize(target - eye), r = glm::normalize(glm::cross(f, up)), u = glm::cross(r, f);
+ CameraView v;
+ v.camToWorld = glm::mat4(glm::vec4(r, 0), glm::vec4(u, 0), glm::vec4(-f, 0), glm::vec4(eye, 1));
+ return v;
+}
+
+const char* fakeTools() {
+#ifdef SF_FAKE_SPLAT_TOOLS
+ return SF_FAKE_SPLAT_TOOLS;
+#else
+ return "";
+#endif
+}
+
+ai::TrainRequest fakeRequest(const fs::path& source, const fs::path& out) {
+ ai::TrainRequest r;
+ r.source = source.string();
+ r.outDir = out.string();
+ r.brush = r.colmap = r.ffmpeg = fakeTools();
+ return r;
+}
+
+bool waitFor(ai::SplatTrainJob& job, const std::function<bool(const ai::TrainStatus&)>& cond, double seconds = 30.0) {
+ const auto until = std::chrono::steady_clock::now() + std::chrono::milliseconds(int(seconds * 1000));
+ while (std::chrono::steady_clock::now() < until) {
+ if (cond(job.status())) return true;
+ std::this_thread::sleep_for(std::chrono::milliseconds(5));
+ }
+ return cond(job.status());
+}
+
+} // namespace
+
+TEST(splat_capture_sharpness_prefers_sharp_frames) {
+ const double sharp = sharpness(checker(320, 240, 8)), flat = sharpness(checker(320, 240, 8, true));
+ CHECK(sharp > 1.0);
+ CHECK(flat < 1e-6);
+ CHECK(sharpness(checker(320, 240, 4)) > sharpness(checker(320, 240, 40))); // finer detail scores higher
+ CHECK(sharpness(Image()) == 0.0);
+}
+
+TEST(splat_capture_selects_sharpest_frame_per_stretch) {
+ // 12 frames, 4 picks: one per stretch of 3, the sharpest of each.
+ const std::vector<double> s = {1, 5, 2, 9, 1, 1, 0, 0, 3, 2, 8, 2};
+ const std::vector<size_t> pick = selectFrames(s, 4);
+ CHECK(pick == std::vector<size_t>({1, 3, 8, 10}));
+ CHECK(selectFrames(s, 20).size() == s.size()); // fewer frames than wanted: all of them
+ CHECK(selectFrames(s, 0).empty());
+ CHECK(selectFrames({}, 3).empty());
+ // Ties keep the earliest frame, so the result never depends on anything but the scores.
+ CHECK(selectFrames({2, 2, 2, 2}, 2) == std::vector<size_t>({0, 2}));
+}
+
+TEST(splat_capture_transforms_json_round_trip) {
+ std::vector<CameraView> views;
+ for (int i = 0; i < 3; ++i) {
+ CameraView v = lookAt(glm::vec3(3.0f * std::cos(i * 2.0f), 0.5f, 3.0f * std::sin(i * 2.0f)), glm::vec3(0), glm::vec3(0, 1, 0));
+ v.file = "images/" + std::to_string(i) + ".jpg";
+ v.fx = v.fy = 500;
+ v.width = 640;
+ v.height = 480;
+ views.push_back(v);
+ }
+ const std::string path = tempPath("sf_transforms.json");
+ std::string err;
+ CHECK(writeTransformsJson(path, views, err));
+ std::vector<CameraView> back;
+ CHECK(readTransformsJson(path, back, err));
+ CHECK(back.size() == 3);
+ if (back.size() == 3) {
+ CHECK(back[2].file == "images/2.jpg");
+ CHECK(back[1].fx == 500.0f && back[1].width == 640 && back[1].cx == 320.0f);
+ float diff = 0;
+ for (int c = 0; c < 4; ++c)
+ for (int r = 0; r < 4; ++r) diff = std::max(diff, std::abs(back[1].camToWorld[c][r] - views[1].camToWorld[c][r]));
+ CHECK(diff < 1e-5f);
+ }
+ CHECK(!readTransformsJson(tempPath("sf_no_such_transforms.json"), back, err));
+}
+
+TEST(splat_capture_reads_colmap_poses) {
+ // A camera at (0,0,-5) looking at the origin, COLMAP style (world-to-camera, identity rotation).
+ const fs::path dir = freshDir("sf_colmap_sparse");
+ std::ofstream(dir / "images.txt") << "# comment\n1 1 0 0 0 0 0 5 1 a.jpg\n1.0 2.0 -1\n2 1 0 0 0 1 0 5 1 b.jpg\n\n";
+ std::vector<CameraView> views;
+ std::string err;
+ CHECK(readColmapImages(dir.string(), views, err));
+ CHECK(views.size() == 2);
+ if (views.size() == 2) {
+ CHECK(views[0].file == "images/a.jpg" && views[1].file == "images/b.jpg");
+ const glm::vec3 c(views[0].camToWorld[3]);
+ CHECK(glm::length(c - glm::vec3(0, 0, -5)) < 1e-5f);
+ const glm::vec3 forward = -glm::vec3(views[0].camToWorld[2]); // OpenGL cameras look down -z
+ CHECK(glm::dot(forward, glm::vec3(0, 0, 1)) > 0.999f);
+ const glm::vec3 up(views[0].camToWorld[1]); // OpenCV y is down: up is -y
+ CHECK(glm::dot(up, glm::vec3(0, -1, 0)) > 0.999f);
+ CHECK(glm::length(glm::vec3(views[1].camToWorld[3]) - glm::vec3(-1, 0, -5)) < 1e-5f);
+ }
+ CHECK(!readColmapImages(freshDir("sf_colmap_empty").string(), views, err));
+}
+
+TEST(splat_capture_orbit_framing_centres_and_levels) {
+ // Cameras circling (1,2,3) in a world whose up is +Z: the object comes to the origin, Z up becomes Y up.
+ std::vector<CameraView> views;
+ for (int i = 0; i < 12; ++i) {
+ const float a = 6.2831853f * float(i) / 12.0f;
+ views.push_back(lookAt(glm::vec3(1, 2, 3) + glm::vec3(4 * std::cos(a), 4 * std::sin(a), 1.5f), glm::vec3(1, 2, 3),
+ glm::vec3(0, 0, 1)));
+ }
+ const glm::mat4 m = orbitFraming(views);
+ const glm::vec3 target(m * glm::vec4(1, 2, 3, 1));
+ CHECK(glm::length(target) < 1e-3f);
+ const glm::vec3 up = glm::normalize(glm::vec3(m * glm::vec4(0, 0, 1, 0)));
+ CHECK(up.y > 0.95f);
+ const glm::vec3 cam(m * views[0].camToWorld[3]);
+ CHECK(std::abs(glm::length(cam) - 3.0f) < 0.01f); // cameras about 3 units away
+ CHECK(orbitFraming({}) == glm::mat4(1.0f));
+}
+
+TEST(splat_capture_reads_phone_capture_json) {
+ const fs::path dir = freshDir("sf_capture_json");
+ std::ofstream(dir / "capture.json")
+ << R"({"kind":"splat","device":"Test Phone","files":["a.jpg",{"name":"b.jpg","t":12}],)"
+ << R"("fl_x":600,"w":640,"h":480,"frames":[{"file":"a.jpg","transform_matrix":[[1,0,0,0],[0,1,0,0],[0,0,1,2],[0,0,0,1]]},)"
+ << R"({"file":"b.jpg","transform_matrix":[1,0,0,1, 0,1,0,0, 0,0,1,2, 0,0,0,1],"fl_x":700},{"file":"bad.jpg"}]})";
+ CaptureInfo info;
+ std::string err;
+ CHECK(readCaptureJson((dir / "capture.json").string(), info, err));
+ CHECK(info.kind == "splat" && info.device == "Test Phone");
+ CHECK(info.files == std::vector<std::string>({"a.jpg", "b.jpg"}));
+ CHECK(info.poses.size() == 2); // the frame without a matrix is skipped
+ if (info.poses.size() == 2) {
+ CHECK(info.poses[0].fx == 600.0f && info.poses[1].fx == 700.0f && info.poses[0].width == 640);
+ CHECK(glm::vec3(info.poses[0].camToWorld[3]) == glm::vec3(0, 0, 2)); // row-major in JSON
+ CHECK(glm::vec3(info.poses[1].camToWorld[3]) == glm::vec3(1, 0, 2));
+ }
+ std::ofstream(dir / "broken.json") << "{nope";
+ CHECK(!readCaptureJson((dir / "broken.json").string(), info, err));
+}
+
+TEST(splat_capture_classifies_sources) {
+ const fs::path root = freshDir("sf_capture_kinds");
+ CHECK(classifySource((root / "missing").string()) == SourceKind::Missing);
+ std::ofstream(root / "clip.MOV") << "x";
+ std::ofstream(root / "notes.txt") << "x";
+ CHECK(classifySource((root / "clip.MOV").string()) == SourceKind::Video);
+ CHECK(classifySource((root / "notes.txt").string()) == SourceKind::Unsupported);
+ fs::create_directories(root / "video_capture");
+ std::ofstream(root / "video_capture" / "walkaround.mp4") << "x";
+ CHECK(classifySource((root / "video_capture").string()) == SourceKind::VideoFolder);
+ const fs::path photos = photoFolder("sf_capture_kinds_photos", 2);
+ CHECK(classifySource(photos.string()) == SourceKind::Images);
+ fs::create_directories(root / "colmap" / "images");
+ fs::create_directories(root / "colmap" / "sparse" / "0");
+ CHECK(classifySource((root / "colmap").string()) == SourceKind::ColmapDataset);
+ fs::create_directories(root / "ns");
+ std::ofstream(root / "ns" / "transforms.json") << "{}";
+ CHECK(classifySource((root / "ns").string()) == SourceKind::NerfstudioDataset);
+ fs::create_directories(root / "empty");
+ CHECK(classifySource((root / "empty").string()) == SourceKind::Unsupported);
+ CHECK(ai::defaultTrainOutDir((root / "clip.MOV").string()) == (root / "clip-splat").string());
+ CHECK(ai::defaultTrainOutDir(photos.string()) == (photos / "splat").string());
+}
+
+TEST(splat_capture_brush_log_and_checkpoints) {
+ TrainerLog t;
+ CHECK(parseTrainerLog("[2026-10-26T10:00:00Z INFO brush_cli] Refine iter 400, 51234 splats.", t));
+ CHECK(t.refineIter == 400 && t.splats == 51234 && t.evalIter == -1);
+ CHECK(parseTrainerLog("[x INFO brush_cli] Eval iter 1000: PSNR 24.31, ssim 0.812", t));
+ CHECK(t.evalIter == 1000 && std::abs(t.psnr - 24.31) < 1e-9 && std::abs(t.ssim - 0.812) < 1e-9);
+ CHECK(parseTrainerLog("thread 'main' panicked at src/main.rs:3:5:", t) && t.error);
+ CHECK(!parseTrainerLog("[x INFO brush_cli] Loading dataset", t));
+ CHECK(checkpointStep("export_1200.ply") == 1200);
+ CHECK(checkpointStep("export_7.PLY") == 7);
+ CHECK(checkpointStep("export_1200.ply.part") == -1);
+ CHECK(checkpointStep("export.ply") == -1);
+ const fs::path dir = freshDir("sf_checkpoints");
+ for (const char* n : {"export_500.ply", "export_2000.ply", "export_1500.ply", "export_9000.ply.part"})
+ std::ofstream(dir / n) << "x";
+ int step = 0;
+ CHECK(fs::path(latestCheckpoint(dir.string(), step)).filename() == "export_2000.ply" && step == 2000);
+ CHECK(latestCheckpoint(freshDir("sf_checkpoints_none").string(), step).empty() && step == -1);
+ CHECK(std::string(psnrQuality(24.0)) == "good" && std::string(psnrQuality(-1)).empty());
+}
+
+TEST(splat_capture_brush_arguments) {
+ const TrainPreset& fast = trainPreset("fast");
+ CHECK(std::string(trainPreset("nonsense").id) == "fast");
+ CHECK(trainPreset("quality").steps > fast.steps);
+ const auto a = brushArguments("/data/set", "/out/ckpt", fast, true);
+ CHECK(a.front() == "/data/set");
+ auto value = [&](const char* flag) {
+ for (size_t i = 0; i + 1 < a.size(); ++i)
+ if (a[i] == flag) return a[i + 1];
+ return std::string();
+ };
+ CHECK(value("--total-steps") == std::to_string(fast.steps));
+ CHECK(value("--export-every") == std::to_string(fast.exportEvery));
+ CHECK(value("--export-path") == "/out/ckpt");
+ CHECK(value("--export-name") == "export_{iter}.ply");
+ CHECK(value("--eval-split-every") == "8");
+ CHECK(value("--eval-every") == std::to_string(fast.exportEvery * 2));
+ CHECK(brushArguments("d", "o", fast, false).size() + 4 == a.size());
+}
+
+TEST(splat_train_job_reports_missing_tools) {
+ const fs::path photos = photoFolder("sf_train_missing", 4);
+ ai::SplatTrainJob job;
+ ai::TrainRequest r = fakeRequest(photos, freshDir("sf_train_missing_out"));
+ std::string err;
+ r.brush.clear();
+ CHECK(!job.start(r, err));
+ CHECK(job.status().missingTrainer && job.status().stage == ai::TrainStage::Failed && !err.empty());
+ r.brush = fakeTools();
+ r.colmap.clear();
+ CHECK(!job.start(r, err)); // photos without camera positions need COLMAP
+ CHECK(job.status().missingPoses && !job.status().missingTrainer);
+ const fs::path video = freshDir("sf_train_missing_video") / "walk.mp4";
+ std::ofstream(video) << "x";
+ r = fakeRequest(video, freshDir("sf_train_missing_out2"));
+ r.ffmpeg.clear();
+ CHECK(!job.start(r, err));
+ CHECK(job.status().missingFfmpeg);
+ r = fakeRequest(fs::temp_directory_path() / "sf_no_such_capture", freshDir("sf_train_missing_out3"));
+ CHECK(!job.start(r, err) && err.find("not found") != std::string::npos);
+ CHECK(!job.busy());
+}
+
+TEST(splat_train_job_photos_to_splat_with_live_previews) {
+ if (!*fakeTools()) return;
+ setEnv("FAKE_BRUSH_MS_PER_EXPORT", "15");
+ setEnv("FAKE_BRUSH_FAIL_AT", nullptr);
+ setEnv("FAKE_COLMAP_FAIL", nullptr);
+ const fs::path photos = photoFolder("sf_train_photos", 10), out = freshDir("sf_train_photos_out");
+ ai::SplatTrainJob job;
+ job.pollSeconds = 0.01;
+ std::string err;
+ CHECK(job.start(fakeRequest(photos, out), err));
+ CHECK(job.busy());
+ CHECK(!job.start(fakeRequest(photos, out), err)); // one at a time
+ // Previews arrive while it trains; each is taken once and placed by the cameras' framing.
+ int previews = 0;
+ std::shared_ptr<SplatCloud> last;
+ while (job.busy() || previews == 0) {
+ if (auto p = job.takePreview()) {
+ ++previews;
+ last = p;
+ }
+ if (!job.busy() && !last) break;
+ std::this_thread::sleep_for(std::chrono::milliseconds(2));
+ }
+ job.wait();
+ if (auto p = job.takePreview()) last = p, ++previews;
+ const ai::TrainStatus st = job.status();
+ CHECK(st.stage == ai::TrainStage::Done);
+ CHECK(previews >= 2);
+ CHECK(st.frames == 10);
+ CHECK(st.step == st.totalSteps);
+ CHECK(st.psnr > 25.0); // the fake's last evaluation
+ CHECK(fs::exists(st.result) && fs::path(st.result).filename() == "sf_train_photos.ply");
+ CHECK(fs::exists(out / "work" / "dataset" / "sparse" / "0" / "images.txt"));
+ CHECK(fs::exists(out / "work" / "dataset" / "images" / "IMG_0003.png")); // photos keep their names
+ CHECK(last && last->size() > 100);
+ if (last) {
+ // The fake trains a sphere around (1,2,3) with cameras 4-4.5 units away: framed, it sits at the origin.
+ const AABB b = last->bounds(0.98f);
+ CHECK(glm::length(b.center()) < 0.3f);
+ CHECK(b.extent().y < 1.5f);
+ }
+ CHECK(job.takePreview() == nullptr); // nothing new
+}
+
+TEST(splat_train_job_video_drops_blurred_frames) {
+ if (!*fakeTools()) return;
+ setEnv("FAKE_FFMPEG_FRAMES", "200");
+ setEnv("FAKE_BRUSH_MS_PER_EXPORT", "1");
+ const fs::path video = freshDir("sf_train_video") / "turntable.mov", out = freshDir("sf_train_video_out");
+ std::ofstream(video) << "not really a video";
+ ai::SplatTrainJob job;
+ job.pollSeconds = 0.01;
+ std::string err;
+ CHECK(job.start(fakeRequest(video, out), err));
+ job.wait();
+ setEnv("FAKE_FFMPEG_FRAMES", nullptr);
+ const ai::TrainStatus st = job.status();
+ CHECK(st.stage == ai::TrainStage::Done);
+ const std::vector<std::string> kept = listImages((out / "work" / "dataset" / "images").string());
+ CHECK(int(kept.size()) == trainPreset("fast").maxFrames);
+ CHECK(st.frames == int(kept.size()));
+ int blurred = 0;
+ for (const std::string& f : kept) {
+ Image img;
+ std::string e;
+ if (!Image::load(f, img, e) || sharpness(img) < 1.0) ++blurred;
+ }
+ CHECK(blurred == 0);
+ CHECK(!fs::exists(out / "work" / "frames")); // the extracted frames are cleaned up
+ CHECK(fs::exists(out / "turntable.ply"));
+}
+
+TEST(splat_train_job_posed_capture_skips_colmap) {
+ if (!*fakeTools()) return;
+ setEnv("FAKE_BRUSH_MS_PER_EXPORT", "1");
+ setEnv("FAKE_COLMAP_FAIL", "1"); // would fail if it were called
+ const fs::path photos = photoFolder("sf_train_posed", 6);
+ {
+ std::ofstream j(photos / "capture.json");
+ j << R"({"kind":"splat","device":"test","fl_x":60,"w":64,"h":48,"frames":[)";
+ for (int i = 0; i < 6; ++i) {
+ const float a = 6.2831853f * float(i) / 6.0f;
+ const CameraView v = lookAt(glm::vec3(3 * std::cos(a), 0.5f, 3 * std::sin(a)), glm::vec3(0), glm::vec3(0, 1, 0));
+ char name[32];
+ std::snprintf(name, sizeof name, "IMG_%04d.png", i);
+ j << (i ? "," : "") << R"({"file":")" << name << R"(","transform_matrix":[)";
+ for (int r = 0; r < 4; ++r)
+ for (int c = 0; c < 4; ++c) j << (r || c ? "," : "") << v.camToWorld[c][r];
+ j << "]}";
+ }
+ j << "]}";
+ }
+ ai::SplatTrainJob job;
+ job.pollSeconds = 0.01;
+ ai::TrainRequest r = fakeRequest(photos, freshDir("sf_train_posed_out"));
+ r.colmap.clear(); // not installed, and not needed
+ std::string err;
+ CHECK(job.start(r, err));
+ job.wait();
+ setEnv("FAKE_COLMAP_FAIL", nullptr);
+ CHECK(job.status().stage == ai::TrainStage::Done);
+ std::vector<CameraView> views;
+ CHECK(readTransformsJson((fs::path(r.outDir) / "work" / "dataset" / "transforms.json").string(), views, err));
+ CHECK(views.size() == 6 && views[0].file == "images/IMG_0000.png" && views[0].fx == 60.0f);
+}
+
+TEST(splat_train_job_cancel_and_good_enough) {
+ if (!*fakeTools()) return;
+ setEnv("FAKE_BRUSH_MS_PER_EXPORT", "150");
+ const fs::path photos = photoFolder("sf_train_cancel", 5);
+ {
+ ai::SplatTrainJob job;
+ job.pollSeconds = 0.01;
+ std::string err;
+ const fs::path out = freshDir("sf_train_cancel_out");
+ CHECK(job.start(fakeRequest(photos, out), err));
+ CHECK(waitFor(job, [](const ai::TrainStatus& s) { return s.stage == ai::TrainStage::Training; }));
+ job.cancel();
+ job.wait();
+ CHECK(job.status().stage == ai::TrainStage::Cancelled);
+ CHECK(job.status().result.empty());
+ CHECK(!fs::exists(out / "sf_train_cancel.ply"));
+ }
+ {
+ ai::SplatTrainJob job;
+ job.pollSeconds = 0.01;
+ std::string err;
+ const fs::path out = freshDir("sf_train_finish_out");
+ CHECK(job.start(fakeRequest(photos, out), err));
+ CHECK(waitFor(job, [](const ai::TrainStatus& s) { return s.previewVersion >= 1; }));
+ job.finishNow(); // Good enough - stop: the newest checkpoint is the result
+ job.wait();
+ const ai::TrainStatus st = job.status();
+ CHECK(st.stage == ai::TrainStage::Done);
+ CHECK(st.step < st.totalSteps);
+ CHECK(fs::exists(out / "sf_train_cancel.ply"));
+ }
+ { // the destructor stops a running job
+ auto job = std::make_unique<ai::SplatTrainJob>();
+ std::string err;
+ CHECK(job->start(fakeRequest(photos, freshDir("sf_train_drop_out")), err));
+ job.reset();
+ }
+ setEnv("FAKE_BRUSH_MS_PER_EXPORT", nullptr);
+}
+
+TEST(splat_train_job_failures) {
+ if (!*fakeTools()) return;
+ setEnv("FAKE_BRUSH_MS_PER_EXPORT", "1");
+ const fs::path photos = photoFolder("sf_train_fail", 5);
+ std::string err;
+ {
+ setEnv("FAKE_BRUSH_FAIL_AT", "1000");
+ ai::SplatTrainJob job;
+ job.pollSeconds = 0.01;
+ CHECK(job.start(fakeRequest(photos, freshDir("sf_train_fail_out")), err));
+ job.wait();
+ setEnv("FAKE_BRUSH_FAIL_AT", nullptr);
+ const ai::TrainStatus st = job.status();
+ CHECK(st.stage == ai::TrainStage::Failed);
+ CHECK(st.error.find("Brush stopped") != std::string::npos);
+ CHECK(st.error.find("panicked") != std::string::npos); // the log's last lines say why
+ }
+ {
+ setEnv("FAKE_COLMAP_FAIL", "1");
+ ai::SplatTrainJob job;
+ job.pollSeconds = 0.01;
+ CHECK(job.start(fakeRequest(photos, freshDir("sf_train_fail_out2")), err));
+ job.wait();
+ setEnv("FAKE_COLMAP_FAIL", nullptr);
+ CHECK(job.status().stage == ai::TrainStage::Failed);
+ CHECK(job.status().error.find("COLMAP could not place") != std::string::npos);
+ }
+ { // too few photos
+ ai::SplatTrainJob job;
+ CHECK(job.start(fakeRequest(photoFolder("sf_train_two", 2), freshDir("sf_train_two_out")), err));
+ job.wait();
+ CHECK(job.status().stage == ai::TrainStage::Failed);
+ CHECK(job.status().error.find("Only 2 usable photos") != std::string::npos);
+ }
+}
+
+TEST(splat_train_settings_round_trip) {
+ ai::SplatTrainSettings s;
+ s.brushPath = "/opt/brush/brush_app";
+ s.preset = "quality";
+ s.copyToLibrary = false;
+ const std::string path = tempPath("sf_splat_train.json");
+ CHECK(s.save(path));
+ const ai::SplatTrainSettings b = ai::SplatTrainSettings::load(path);
+ CHECK(b.brushPath == s.brushPath && b.preset == "quality" && !b.copyToLibrary && b.colmapPath.empty());
+ CHECK(ai::SplatTrainSettings::load(tempPath("sf_no_such_settings.json")).preset == "fast");
+ // The environment wins over the settings (studios and tests point at their own tools).
+ setEnv("SCULPTFORGE_BRUSH", fakeTools());
+ if (*fakeTools()) CHECK(ai::findBrush(s) == fakeTools());
+ setEnv("SCULPTFORGE_BRUSH", nullptr);
+ if (const ai::ToolDownload* d = ai::brushDownload()) {
+ CHECK(std::string(d->sha256).size() == 64);
+ CHECK(std::string(d->url).rfind("https://github.com/ArthurBrussee/brush/releases/download/", 0) == 0);
+ }
+}
diff --git a/tests/tools/fake_splat_tools.cpp b/tests/tools/fake_splat_tools.cpp
new file mode 100644
index 00000000..f991dfb9
--- /dev/null
+++ b/tests/tools/fake_splat_tools.cpp
@@ -0,0 +1,198 @@
+// SculptForge - open source digital sculpting
+// SPDX-License-Identifier: GPL-3.0-or-later
+//
+// A stand-in for the external splat tools, so training is tested without a
+// GPU, COLMAP or ffmpeg (tests/test_splat_train.cpp, tests/ui/splat_train.sfui).
+// One program, three roles, chosen by its arguments the way SculptForge calls
+// the real ones:
+// - ffmpeg (-i VIDEO ... OUT/%05d.jpg): writes FAKE_FFMPEG_FRAMES (40) frames;
+// every 4th one is blurred, so frame selection has something to drop
+// - colmap (automatic_reconstructor --workspace_path W --image_path I ...):
+// writes W/sparse/0/images.txt with the cameras on a circle around (1,2,3),
+// world up = +Z (as arbitrary as a real reconstruction); FAKE_COLMAP_FAIL=1 fails
+// - Brush (DATASET --total-steps N --export-every K --export-path DIR ...):
+// checks the dataset, prints Brush's log lines and writes export_<step>.ply
+// checkpoints of a sphere around (1,2,3) that gets sharper;
+// FAKE_BRUSH_MS_PER_EXPORT (20) slows it down, FAKE_BRUSH_FAIL_AT=<step> crashes
+#include "core/Splats.h"
+#include "io/PlyIO.h"
+
+#define STBI_WRITE_NO_STDIO
+#include "stb_image_write.h"
+
+#include <glm/gtc/quaternion.hpp>
+
+#include <algorithm>
+#include <chrono>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <filesystem>
+#include <fstream>
+#include <map>
+#include <random>
+#include <string>
+#include <thread>
+#include <vector>
+
+namespace fs = std::filesystem;
+
+namespace {
+
+int envInt(const char* name, int fallback) {
+ const char* v = std::getenv(name);
+ return v && *v ? std::atoi(v) : fallback;
+}
+
+std::map<std::string, std::string> options(int argc, char** argv, int first, std::vector<std::string>& positional) {
+ std::map<std::string, std::string> o;
+ for (int i = first; i < argc; ++i) {
+ const std::string a = argv[i];
+ if (a.rfind("--", 0) == 0 && i + 1 < argc) o[a.substr(2)] = argv[++i];
+ else positional.push_back(a);
+ }
+ return o;
+}
+
+void writeJpg(const std::string& path, int w, int h, const std::vector<uint8_t>& rgb) {
+ std::vector<uint8_t> bytes;
+ stbi_write_jpg_to_func(
+ [](void* ctx, void* data, int size) {
+ auto* out = static_cast<std::vector<uint8_t>*>(ctx);
+ out->insert(out->end(), static_cast<uint8_t*>(data), static_cast<uint8_t*>(data) + size);
+ },
+ &bytes, w, h, 3, rgb.data(), 90);
+ std::ofstream(path, std::ios::binary).write(reinterpret_cast<const char*>(bytes.data()), std::streamsize(bytes.size()));
+}
+
+int fakeFfmpeg(int argc, char** argv) {
+ const std::string pattern = argv[argc - 1];
+ const int frames = envInt("FAKE_FFMPEG_FRAMES", 40);
+ const int w = 96, h = 64;
+ for (int f = 1; f <= frames; ++f) {
+ std::vector<uint8_t> rgb(size_t(w) * h * 3);
+ const bool blurred = f % 4 == 0;
+ for (int y = 0; y < h; ++y)
+ for (int x = 0; x < w; ++x) {
+ // A checkerboard that slides a little each frame; blurred frames are flat grey.
+ const int v = blurred ? 128 : (((x + f) / 6 + y / 6) % 2 ? 220 : 40);
+ for (int c = 0; c < 3; ++c) rgb[(size_t(y) * w + x) * 3 + c] = uint8_t(v);
+ }
+ char name[512];
+ std::snprintf(name, sizeof name, pattern.c_str(), f);
+ writeJpg(name, w, h, rgb);
+ }
+ return 0;
+}
+
+int fakeColmap(int argc, char** argv) {
+ std::vector<std::string> pos;
+ auto o = options(argc, argv, 2, pos);
+ if (envInt("FAKE_COLMAP_FAIL", 0)) {
+ std::printf("E20261026 Not enough matches\n");
+ return 1;
+ }
+ std::vector<std::string> names;
+ for (const auto& e : fs::directory_iterator(o["image_path"])) names.push_back(e.path().filename().string());
+ std::sort(names.begin(), names.end());
+ const fs::path sparse = fs::path(o["workspace_path"]) / "sparse" / "0";
+ fs::create_directories(sparse);
+ std::ofstream cams(sparse / "cameras.txt");
+ cams << "1 PINHOLE 96 64 80 80 48 32\n";
+ std::ofstream pts(sparse / "points3D.txt");
+ std::ofstream img(sparse / "images.txt");
+ img << "# fake reconstruction\n";
+ const glm::vec3 target(1, 2, 3), up(0, 0, 1);
+ for (size_t i = 0; i < names.size(); ++i) {
+ std::printf("Processed file [%zu/%zu]\n", i + 1, names.size());
+ const float a = 6.2831853f * float(i) / float(names.size());
+ const glm::vec3 c = target + 4.0f * glm::vec3(std::cos(a), std::sin(a), 0.5f);
+ const glm::vec3 z = glm::normalize(target - c), x = glm::normalize(glm::cross(z, up)),
+ upCam = glm::cross(x, z), y = -upCam;
+ glm::mat3 r; // world-to-camera: rows x, y, z
+ for (int k = 0; k < 3; ++k) {
+ r[k][0] = x[k];
+ r[k][1] = y[k];
+ r[k][2] = z[k];
+ }
+ const glm::quat q = glm::quat_cast(r);
+ const glm::vec3 t = -(r * c);
+ img << i + 1 << ' ' << q.w << ' ' << q.x << ' ' << q.y << ' ' << q.z << ' ' << t.x << ' ' << t.y << ' ' << t.z
+ << " 1 " << names[i] << "\n\n";
+ }
+ std::printf("Registering image #%zu\n", names.size());
+ return 0;
+}
+
+int fakeBrush(int argc, char** argv) {
+ std::vector<std::string> pos;
+ auto o = options(argc, argv, 1, pos);
+ if (pos.empty() || !fs::is_directory(pos[0])) {
+ std::printf("Error: no dataset at %s\n", pos.empty() ? "" : pos[0].c_str());
+ return 1;
+ }
+ const fs::path ds = pos[0];
+ if (!fs::exists(ds / "transforms.json") && !fs::exists(ds / "sparse" / "0")) {
+ std::printf("Error: %s has neither transforms.json nor sparse/0\n", ds.string().c_str());
+ return 1;
+ }
+ const int total = std::stoi(o.count("total-steps") ? o["total-steps"] : "30000");
+ const int every = std::max(1, std::stoi(o.count("export-every") ? o["export-every"] : "5000"));
+ const int evalEvery = o.count("eval-every") ? std::stoi(o["eval-every"]) : 0;
+ const fs::path out = o.count("export-path") ? o["export-path"] : ".";
+ std::string name = o.count("export-name") ? o["export-name"] : "export_{iter}.ply";
+ const int ms = envInt("FAKE_BRUSH_MS_PER_EXPORT", 20), failAt = envInt("FAKE_BRUSH_FAIL_AT", -1);
+ fs::create_directories(out);
+ std::printf("[2026-10-26T00:00:00Z INFO brush_process::train_stream] Start of training stream\n");
+ std::fflush(stdout);
+ std::mt19937 rng(42);
+ std::normal_distribution<float> normal(0.0f, 1.0f);
+ for (int step = every; step <= total + every - 1; step += every) {
+ const int iter = std::min(step, total);
+ std::this_thread::sleep_for(std::chrono::milliseconds(ms));
+ if (failAt >= 0 && iter >= failAt) {
+ std::printf("thread 'main' panicked at crates/brush-train/src/train.rs:1:1:\nout of memory\n");
+ std::fflush(stdout);
+ return 101;
+ }
+ const int count = std::min(3000, 300 + iter / 2);
+ std::printf("[2026-10-26T00:00:00Z INFO brush_cli] Refine iter %d, %d splats.\n", iter, count);
+ if (evalEvery > 0 && (iter % evalEvery == 0 || iter == total))
+ std::printf("[2026-10-26T00:00:00Z INFO brush_cli] Eval iter %d: PSNR %.2f, ssim %.3f\n", iter,
+ 15.0 + 15.0 * iter / total, 0.5 + 0.4 * iter / total);
+ std::fflush(stdout);
+ sf::SplatCloud cloud;
+ cloud.resize(size_t(count), 0);
+ const float noise = 0.3f * (1.0f - float(iter) / float(total)) + 0.01f;
+ for (int i = 0; i < count; ++i) {
+ glm::vec3 d(normal(rng), normal(rng), normal(rng));
+ d = glm::normalize(d);
+ cloud.positions[size_t(i)] = glm::vec3(1, 2, 3) + 0.5f * d + noise * glm::vec3(normal(rng), normal(rng), normal(rng));
+ cloud.rotations[size_t(i)] = glm::quat(1, 0, 0, 0);
+ cloud.scales[size_t(i)] = glm::vec3(0.02f);
+ cloud.opacities[size_t(i)] = 0.9f;
+ cloud.setColor(size_t(i), glm::vec3(0.5f + 0.5f * d.x, 0.5f + 0.5f * d.y, 0.5f + 0.5f * d.z));
+ }
+ std::string file = name;
+ if (const size_t p = file.find("{iter}"); p != std::string::npos) file.replace(p, 6, std::to_string(iter));
+ std::string err;
+ const fs::path tmp = out / (file + ".part");
+ if (!sf::saveSplatPly(tmp.string(), cloud, err)) {
+ std::printf("Error: %s\n", err.c_str());
+ return 1;
+ }
+ fs::rename(tmp, out / file);
+ if (iter == total) break;
+ }
+ std::printf("Training took 1s\n");
+ return 0;
+}
+
+} // namespace
+
+int main(int argc, char** argv) {
+ for (int i = 1; i < argc; ++i)
+ if (std::string(argv[i]) == "-i") return fakeFfmpeg(argc, argv);
+ if (argc > 1 && std::string(argv[1]) == "automatic_reconstructor") return fakeColmap(argc, argv);
+ return fakeBrush(argc, argv);
+}