Free, open-source 3D sculpting for Linux, macOS and Windows: dense meshes, remeshing, painting, built-in AI (image/text to 3D), Send to Unreal, game-ready LODs, Gaussian splats and OpenUSD.
C++ 3d3d-sculptingcppdear-imguidigital-sculptingfbxgame-developmentgaussian-splattinggltfimage-to-3dmcpopenglopenusdsculptingtext-to-3dunreal-engine
// SculptForge - open source digital sculpting
// SPDX-License-Identifier: GPL-3.0-or-later
#include "core/SplatCapture.h"
#include "core/Json.h"
#include <glm/gtc/quaternion.hpp>
#include <glm/gtc/matrix_transform.hpp>
#include <algorithm>
#include <cctype>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <filesystem>
#include <fstream>
#include <sstream>
namespace sf::capture {
namespace fs = std::filesystem;
namespace {
fs::path u8path(const std::string& s) { return fs::path(std::u8string(s.begin(), s.end())); }
std::string pathString(const fs::path& p) {
const auto s = p.u8string();
return std::string(s.begin(), s.end());
}
std::string lowerExt(const fs::path& p) {
std::string e = p.extension().string();
std::transform(e.begin(), e.end(), e.begin(), [](unsigned char c) { return char(std::tolower(c)); });
return e;
}
bool readText(const std::string& path, std::string& out) {
std::ifstream f(u8path(path), std::ios::binary);
if (!f) return false;
std::stringstream ss;
ss << f.rdbuf();
out = ss.str();
return true;
}
// A 4x4 matrix from JSON: nested rows [[..],[..],[..],[..]] or 16 numbers, row-major.
bool readMatrix(const Json& j, glm::mat4& m) {
float v[16];
if (j.isArray() && j.size() == 4) {
for (int r = 0; r < 4; ++r) {
const Json& row = j[size_t(r)];
if (!row.isArray() || row.size() != 4) return false;
for (int c = 0; c < 4; ++c) v[r * 4 + c] = float(row[size_t(c)].asNumber());
}
} else if (j.isArray() && j.size() == 16) {
for (int i = 0; i < 16; ++i) v[i] = float(j[size_t(i)].asNumber());
} else {
return false;
}
for (int r = 0; r < 4; ++r)
for (int c = 0; c < 4; ++c) m[c][r] = v[r * 4 + c]; // glm is column-major
return true;
}
Json matrixRows(const glm::mat4& m) {
Json rows = Json::array();
for (int r = 0; r < 4; ++r) {
Json row = Json::array();
for (int c = 0; c < 4; ++c) row.push(double(m[c][r]));
rows.push(std::move(row));
}
return rows;
}
void readIntrinsics(const Json& f, CameraView& v) {
v.fx = float(f["fl_x"].asNumber(v.fx));
v.fy = float(f["fl_y"].asNumber(v.fx));
v.cx = float(f["cx"].asNumber(v.cx));
v.cy = float(f["cy"].asNumber(v.cy));
v.width = int(f["w"].asNumber(v.width));
v.height = int(f["h"].asNumber(v.height));
if (v.cx <= 0 && v.width > 0) v.cx = float(v.width) * 0.5f; // principal point: the image centre by default
if (v.cy <= 0 && v.height > 0) v.cy = float(v.height) * 0.5f;
}
// COLMAP pose (world-to-camera rotation as a quaternion and translation, OpenCV axes) to an OpenGL camToWorld.
glm::mat4 colmapToCamToWorld(double qw, double qx, double qy, double qz, double tx, double ty, double tz) {
const glm::mat3 r = glm::mat3_cast(glm::quat(float(qw), float(qx), float(qy), float(qz)));
const glm::mat3 rt = glm::transpose(r);
const glm::vec3 centre = -(rt * glm::vec3(float(tx), float(ty), float(tz)));
glm::mat4 m(1.0f);
m[0] = glm::vec4(rt[0], 0.0f);
m[1] = glm::vec4(-rt[1], 0.0f); // OpenCV y down -> OpenGL y up
m[2] = glm::vec4(-rt[2], 0.0f); // OpenCV looks down +z -> OpenGL -z
m[3] = glm::vec4(centre, 1.0f);
return m;
}
bool readColmapImagesBin(const std::string& path, std::vector<CameraView>& views) {
std::ifstream f(u8path(path), std::ios::binary);
if (!f) return false;
auto rd = [&](void* p, size_t n) { return bool(f.read(static_cast<char*>(p), std::streamsize(n))); };
uint64_t count = 0;
if (!rd(&count, 8) || count > 10'000'000) return false;
for (uint64_t i = 0; i < count; ++i) {
uint32_t id = 0, camera = 0;
double q[4], t[3];
if (!rd(&id, 4) || !rd(q, sizeof q) || !rd(t, sizeof t) || !rd(&camera, 4)) return false;
std::string name;
for (char c; f.get(c) && c != '\0';) name += c;
uint64_t points = 0;
if (!rd(&points, 8) || points > 100'000'000) return false;
f.seekg(std::streamoff(points * 24), std::ios::cur); // x, y (double) + point3D id (int64)
if (!f) return false;
CameraView v;
v.file = "images/" + name;
v.camToWorld = colmapToCamToWorld(q[0], q[1], q[2], q[3], t[0], t[1], t[2]);
views.push_back(std::move(v));
}
return true;
}
bool readColmapImagesTxt(const std::string& path, std::vector<CameraView>& views) {
std::ifstream f(u8path(path));
if (!f) return false;
std::string line;
bool poseLine = true;
while (std::getline(f, line)) {
if (line.empty() || line[0] == '#') continue;
if (poseLine) {
std::istringstream ss(line);
long id = 0, camera = 0;
double q[4], t[3];
std::string name;
if (!(ss >> id >> q[0] >> q[1] >> q[2] >> q[3] >> t[0] >> t[1] >> t[2] >> camera >> name)) return false;
CameraView v;
v.file = "images/" + name;
v.camToWorld = colmapToCamToWorld(q[0], q[1], q[2], q[3], t[0], t[1], t[2]);
views.push_back(std::move(v));
}
poseLine = !poseLine; // every pose line is followed by its 2D points line
}
return true;
}
} // namespace
double sharpness(const Image& image) {
if (image.empty()) return 0.0;
const Image small = image.downscaled(512);
const int w = small.width, h = small.height;
if (w < 3 || h < 3) return 0.0;
std::vector<float> lum(size_t(w) * size_t(h));
for (size_t i = 0; i < lum.size(); ++i) {
const uint8_t* p = &small.rgba[i * 4];
lum[i] = (0.299f * p[0] + 0.587f * p[1] + 0.114f * p[2]) / 255.0f;
}
double sum = 0.0, sum2 = 0.0;
size_t n = 0;
for (int y = 1; y < h - 1; ++y)
for (int x = 1; x < w - 1; ++x) {
const size_t i = size_t(y) * size_t(w) + size_t(x);
const double lap = double(lum[i - 1]) + lum[i + 1] + lum[i - size_t(w)] + lum[i + size_t(w)] - 4.0 * lum[i];
sum += lap;
sum2 += lap * lap;
++n;
}
const double mean = sum / double(n);
return std::max(0.0, sum2 / double(n) - mean * mean) * 1000.0;
}
std::vector<size_t> selectFrames(const std::vector<double>& scores, size_t count) {
std::vector<size_t> out;
const size_t n = scores.size();
if (count == 0 || n == 0) return out;
if (count >= n) {
for (size_t i = 0; i < n; ++i) out.push_back(i);
return out;
}
for (size_t k = 0; k < count; ++k) {
const size_t begin = k * n / count, end = std::max(begin + 1, (k + 1) * n / count);
size_t best = begin;
for (size_t i = begin + 1; i < end; ++i)
if (scores[i] > scores[best]) best = i;
out.push_back(best);
}
return out;
}
bool writeTransformsJson(const std::string& path, const std::vector<CameraView>& views, std::string& error) {
Json root = Json::object();
root.set("camera_model", "PINHOLE");
Json frames = Json::array();
for (const CameraView& v : views) {
Json f = Json::object();
f.set("file_path", v.file);
f.set("transform_matrix", matrixRows(v.camToWorld));
if (v.fx > 0) {
f.set("fl_x", double(v.fx));
f.set("fl_y", double(v.fy > 0 ? v.fy : v.fx));
}
if (v.width > 0 && v.height > 0) {
f.set("cx", double(v.cx > 0 ? v.cx : v.width * 0.5f));
f.set("cy", double(v.cy > 0 ? v.cy : v.height * 0.5f));
f.set("w", v.width);
f.set("h", v.height);
}
frames.push(std::move(f));
}
root.set("frames", std::move(frames));
std::ofstream out(u8path(path), std::ios::binary);
if (!out) {
error = "Cannot write " + path;
return false;
}
out << root.dump(true);
if (!out) {
error = "Cannot write " + path;
return false;
}
return true;
}
bool readTransformsJson(const std::string& path, std::vector<CameraView>& views, std::string& error) {
std::string text;
if (!readText(path, text)) {
error = "Cannot read " + path;
return false;
}
Json root;
if (!Json::parse(text, root, error)) return false;
CameraView shared;
readIntrinsics(root, shared);
for (const Json& f : root["frames"].items()) {
CameraView v = shared;
v.file = f["file_path"].isString() ? f["file_path"].asString() : std::string();
if (!readMatrix(f["transform_matrix"], v.camToWorld)) {
error = "A frame in " + path + " has no 4x4 transform_matrix";
return false;
}
readIntrinsics(f, v);
views.push_back(std::move(v));
}
return true;
}
bool readColmapImages(const std::string& sparseDir, std::vector<CameraView>& views, std::string& error) {
const fs::path dir = u8path(sparseDir);
std::error_code ec;
if (fs::exists(dir / "images.bin", ec)) {
if (readColmapImagesBin(pathString(dir / "images.bin"), views)) return true;
error = "Cannot read " + pathString(dir / "images.bin");
return false;
}
if (fs::exists(dir / "images.txt", ec)) {
if (readColmapImagesTxt(pathString(dir / "images.txt"), views)) return true;
error = "Cannot read " + pathString(dir / "images.txt");
return false;
}
error = "No COLMAP images.bin or images.txt in " + sparseDir;
return false;
}
glm::mat4 orbitFraming(const std::vector<CameraView>& views) {
if (views.size() < 2) return glm::mat4(1.0f);
// The point closest to every viewing ray: sum (I - d d^T) p = sum (I - d d^T) c.
glm::dmat3 a(0.0);
glm::dvec3 b(0.0), up(0.0), mean(0.0);
for (const CameraView& v : views) {
const glm::dvec3 c(v.camToWorld[3]);
const glm::dvec3 d = glm::normalize(-glm::dvec3(v.camToWorld[2]));
const glm::dmat3 p = glm::dmat3(1.0) - glm::outerProduct(d, d);
a += p;
b += p * c;
up += glm::dvec3(v.camToWorld[1]);
mean += c;
}
mean /= double(views.size());
glm::dvec3 target = mean;
if (std::abs(glm::determinant(a)) > 1e-6 * double(views.size())) target = glm::inverse(a) * b;
// Cameras all looking the same way (a straight walk past something) give no
// sensible crossing: fall back to the middle of the cameras.
if (!std::isfinite(target.x) || !std::isfinite(target.y) || !std::isfinite(target.z)) target = mean;
std::vector<double> dist;
for (const CameraView& v : views) dist.push_back(glm::length(glm::dvec3(v.camToWorld[3]) - target));
std::nth_element(dist.begin(), dist.begin() + std::ptrdiff_t(dist.size() / 2), dist.end());
const double median = dist[dist.size() / 2];
const double scale = median > 1e-9 ? 3.0 / median : 1.0;
glm::quat turn(1.0f, 0.0f, 0.0f, 0.0f);
if (glm::length(up) > 1e-9) {
const glm::vec3 u = glm::normalize(glm::vec3(up)), y(0.0f, 1.0f, 0.0f);
const float cosA = glm::clamp(glm::dot(u, y), -1.0f, 1.0f);
glm::vec3 axis = glm::cross(u, y);
if (glm::length(axis) < 1e-6f) axis = cosA > 0.0f ? y : glm::vec3(1.0f, 0.0f, 0.0f); // aligned or opposite
turn = glm::angleAxis(std::acos(cosA), glm::normalize(axis));
}
glm::mat4 m = glm::scale(glm::mat4(1.0f), glm::vec3(float(scale)));
m = m * glm::mat4_cast(turn);
m = glm::translate(m, -glm::vec3(target));
return m;
}
bool readCaptureJson(const std::string& path, CaptureInfo& info, std::string& error) {
std::string text;
if (!readText(path, text)) {
error = "Cannot read " + path;
return false;
}
Json root;
if (!Json::parse(text, root, error)) return false;
if (!root.isObject()) {
error = path + " is not a JSON object";
return false;
}
if (root["kind"].isString()) info.kind = root["kind"].asString();
if (root["device"].isString()) info.device = root["device"].asString();
for (const Json& f : root["files"].items()) {
if (f.isString()) info.files.push_back(f.asString());
else if (f["name"].isString()) info.files.push_back(f["name"].asString()); // {"name":..,"t":..}
}
CameraView shared;
readIntrinsics(root, shared);
for (const Json& f : root["frames"].items()) {
CameraView v = shared;
const Json& name = f["file"].isString() ? f["file"] : f["file_path"];
if (!name.isString() || !readMatrix(f["transform_matrix"], v.camToWorld)) continue;
v.file = name.asString();
readIntrinsics(f, v);
info.poses.push_back(std::move(v));
}
return true;
}
bool isVideoFile(const std::string& path) {
const std::string e = lowerExt(u8path(path));
return e == ".mp4" || e == ".mov" || e == ".m4v" || e == ".webm" || e == ".mkv" || e == ".avi";
}
std::vector<std::string> listImages(const std::string& dir) {
std::vector<std::string> out;
std::error_code ec;
for (fs::directory_iterator it(u8path(dir), ec), end; !ec && it != end; it.increment(ec)) {
if (!it->is_regular_file(ec)) continue;
const std::string e = lowerExt(it->path());
if (e == ".jpg" || e == ".jpeg" || e == ".png") out.push_back(pathString(it->path()));
}
std::sort(out.begin(), out.end());
return out;
}
std::string findVideo(const std::string& dir) {
std::vector<std::string> found;
std::error_code ec;
for (fs::directory_iterator it(u8path(dir), ec), end; !ec && it != end; it.increment(ec))
if (it->is_regular_file(ec) && isVideoFile(pathString(it->path()))) found.push_back(pathString(it->path()));
std::sort(found.begin(), found.end());
return found.empty() ? std::string() : found.front();
}
SourceKind classifySource(const std::string& path) {
std::error_code ec;
const fs::path p = u8path(path);
if (path.empty() || !fs::exists(p, ec)) return SourceKind::Missing;
if (fs::is_regular_file(p, ec)) return isVideoFile(path) ? SourceKind::Video : SourceKind::Unsupported;
if (!fs::is_directory(p, ec)) return SourceKind::Unsupported;
if (fs::exists(p / "transforms.json", ec)) return SourceKind::NerfstudioDataset;
if (fs::is_directory(p / "images", ec) && fs::is_directory(p / "sparse" / "0", ec)) return SourceKind::ColmapDataset;
if (!listImages(path).empty()) return SourceKind::Images;
if (fs::is_directory(p / "images", ec) && !listImages(pathString(p / "images")).empty()) return SourceKind::Images;
if (!findVideo(path).empty()) return SourceKind::VideoFolder;
return SourceKind::Unsupported;
}
const char* sourceKindName(SourceKind kind) {
switch (kind) {
case SourceKind::Missing: return "missing";
case SourceKind::Video: return "video";
case SourceKind::Images: return "photos";
case SourceKind::VideoFolder: return "video";
case SourceKind::NerfstudioDataset: return "nerfstudio dataset";
case SourceKind::ColmapDataset: return "COLMAP dataset";
case SourceKind::Unsupported: break;
}
return "unsupported";
}
const std::vector<TrainPreset>& trainPresets() {
static const std::vector<TrainPreset> presets = {
{"fast", "Fast preview", 5000, 1024, 1, 250, 600'000, 80},
{"quality", "Quality", 30000, 1920, 3, 1000, 3'000'000, 200},
};
return presets;
}
const TrainPreset& trainPreset(const std::string& id) {
for (const TrainPreset& p : trainPresets())
if (id == p.id) return p;
return trainPresets().front();
}
std::vector<std::string> brushArguments(const std::string& dataset, const std::string& exportDir,
const TrainPreset& preset, bool evaluate) {
std::vector<std::string> a = {dataset,
"--total-steps", std::to_string(preset.steps),
"--max-resolution", std::to_string(preset.maxResolution),
"--sh-degree", std::to_string(preset.shDegree),
"--max-splats", std::to_string(preset.maxSplats),
"--export-every", std::to_string(preset.exportEvery),
"--export-path", exportDir,
"--export-name", "export_{iter}.ply"};
if (evaluate) { // hold back every 8th view to measure PSNR on
a.insert(a.end(), {"--eval-split-every", "8", "--eval-every", std::to_string(preset.exportEvery * 2)});
}
return a;
}
bool parseTrainerLog(const std::string& line, TrainerLog& out) {
out = TrainerLog();
if (const size_t p = line.find("Refine iter "); p != std::string::npos) {
int iter = -1, splats = -1;
if (std::sscanf(line.c_str() + p, "Refine iter %d, %d splats", &iter, &splats) >= 1) {
out.refineIter = iter;
out.splats = splats;
return true;
}
}
if (const size_t p = line.find("Eval iter "); p != std::string::npos) {
int iter = -1;
double psnr = -1.0, ssim = -1.0;
if (std::sscanf(line.c_str() + p, "Eval iter %d: PSNR %lf, ssim %lf", &iter, &psnr, &ssim) >= 2) {
out.evalIter = iter;
out.psnr = psnr;
out.ssim = ssim;
return true;
}
}
if (line.find("panicked at") != std::string::npos || line.rfind("Error:", 0) == 0 ||
line.find(" ERROR ") != std::string::npos) {
out.error = true;
return true;
}
return false;
}
int checkpointStep(const std::string& fileName) {
const fs::path p = u8path(fileName);
if (lowerExt(p) != ".ply") return -1;
const std::string stem = pathString(p.stem());
size_t end = stem.size(), begin = end;
while (begin > 0 && std::isdigit(static_cast<unsigned char>(stem[begin - 1]))) --begin;
if (begin == end || end - begin > 9) return -1;
return std::stoi(stem.substr(begin, end - begin));
}
std::string latestCheckpoint(const std::string& dir, int& step) {
step = -1;
std::string best;
std::error_code ec;
for (fs::directory_iterator it(u8path(dir), ec), end; !ec && it != end; it.increment(ec)) {
if (!it->is_regular_file(ec)) continue;
const std::string name = pathString(it->path().filename());
const int s = checkpointStep(name);
if (s > step) {
step = s;
best = pathString(it->path());
}
}
return best;
}
const char* psnrQuality(double psnr) {
if (psnr <= 0.0) return "";
if (psnr < 18.0) return "rough";
if (psnr < 22.0) return "taking shape";
if (psnr < 26.0) return "good";
if (psnr < 30.0) return "sharp";
return "very sharp";
}
} // namespace sf::capture