Free, open-source 3D sculpting for Linux, macOS and Windows: dense meshes, remeshing, painting, built-in AI (image/text to 3D), Send to Unreal, game-ready LODs, Gaussian splats and OpenUSD.
C++ 3d3d-sculptingcppdear-imguidigital-sculptingfbxgame-developmentgaussian-splattinggltfimage-to-3dmcpopenglopenusdsculptingtext-to-3dunreal-engine
Project open: the BVH builds on its own thread while the levels and topology load, big blobs are read in parallel, loaded levels drop their generated quad pairs; Divide's limit follows the computer's memory (100M+ on 30 GB); benchmark reports peak memory
islandmed6ca1a0dc58fb
diff --git a/src/core/Benchmark.cpp b/src/core/Benchmark.cppindex b5d7892f..f4089d69 100644--- a/src/core/Benchmark.cpp+++ b/src/core/Benchmark.cpp@@ -48,6 +48,22 @@ size_t residentBytes() {return 0;}+size_t resetPeakResidentBytes() {+#if defined(__linux__)+ std::ofstream("/proc/self/clear_refs") << "5"; // resets the peak (VmHWM) to the current RSS+#endif+ return residentBytes();+}++size_t peakResidentBytes() {+#if defined(__linux__)+ std::ifstream f("/proc/self/status");+ for (std::string line; std::getline(f, line);)+ if (line.rfind("VmHWM:", 0) == 0) return size_t(std::atoll(line.c_str() + 6)) * 1024;+#endif+ return 0;+}+MemoryUse documentMemory(const Document& doc) {MemoryUse u;for (int i = 0; i < doc.subtoolCount(); ++i) {@@ -288,7 +304,7 @@ std::string runBenchmark(const BenchOptions& o, const BenchGpu* gpu) {const size_t target = size_t(sizeM * 1e6);log("Building a " + millions(double(target)) + " triangle sphere...");Document doc;- const size_t rss0 = residentBytes();+ const size_t rss0 = resetPeakResidentBytes();auto t = Clock::now();doc.mesh = makeBenchSphere(target);const double divideMs = msSince(t);@@ -300,7 +316,7 @@ std::string runBenchmark(const BenchOptions& o, const BenchGpu* gpu) {double uploadAll = 0;if (gpu && gpu->upload) uploadAll = gpu->upload(m);const size_t meshBytes = m.memoryBytes(), bvhBytes = doc.bvh.memoryBytes(), stackBytes = subdivBytes(m);- const size_t rss = residentBytes();+ const size_t rss = residentBytes(), setupPeak = peakResidentBytes();UploadOrder order;order.build(m);@@ -316,7 +332,9 @@ std::string runBenchmark(const BenchOptions& o, const BenchGpu* gpu) {<< " B/triangle |\n";md << "| Subdivision levels (stack) | " << fmt("%.0f MB", double(stackBytes) / 1e6) << " = "<< fmt("%.0f", double(stackBytes) / nt) << " B/triangle |\n";- if (rss) md << "| Process memory (RSS) | " << fmt("%.0f MB", double(rss) / 1e6) << " (+" << fmt("%.0f", double(rss - std::min(rss, rss0)) / 1e6) << " MB) |\n";+ if (rss) md << "| Process memory (RSS) | " << fmt("%.0f MB", double(rss) / 1e6) << " (+" << fmt("%.0f", double(rss - std::min(rss, rss0)) / 1e6) << " MB = "+ << fmt("%.0f", double(rss - std::min(rss, rss0)) / nt) << " B/triangle) |\n";+ if (setupPeak > rss0) md << "| Peak while dividing (RSS) | " << fmt("%.0f MB", double(setupPeak - rss0) / 1e6) << " above the start |\n";md << "\n";md << "| Brush | verts/dab | uploaded verts (ranges) | kernel | normals | BVH refit | pick | " << (gpu ? "GPU upload | " : "")@@ -368,12 +386,17 @@ std::string runBenchmark(const BenchOptions& o, const BenchGpu* gpu) {const double saveMs = msSince(t);const double fileMb = saved ? double(fs::file_size(path, ec)) / 1e6 : 0.0;Document loaded;+ const size_t beforeOpen = resetPeakResidentBytes();t = Clock::now();const bool opened = saved && loadProject(path, loaded, err);const double openMs = msSince(t);+ const size_t openPeak = peakResidentBytes(), afterOpen = residentBytes();fs::remove(path, ec);md << "| Save project (" << fmt("%.0f MB", fileMb) << ") | " << (saved ? fmt("%.0f", saveMs) : "failed: " + err) << " |\n";md << "| Open project | " << (opened ? fmt("%.0f", openMs) : "failed: " + err) << " |\n";+ if (opened && openPeak > beforeOpen)+ md << "| Open project: memory (RSS) | +" << fmt("%.0f MB", double(afterOpen - std::min(afterOpen, beforeOpen)) / 1e6) << ", peak +"+ << fmt("%.0f MB", double(openPeak - beforeOpen) / 1e6) << " |\n";}if (gpu && gpu->orbitFrameMs) {log(" orbit");diff --git a/src/core/Benchmark.h b/src/core/Benchmark.hindex c607ad2c..3dfe0183 100644--- a/src/core/Benchmark.h+++ b/src/core/Benchmark.h@@ -61,6 +61,9 @@ Mesh makeBenchSphere(size_t triangles);// Resident memory of this process in bytes (0 where unknown).size_t residentBytes();+// The most resident memory since the last reset (Linux: VmHWM; 0 where unknown); the reset returns residentBytes().+size_t peakResidentBytes();+size_t resetPeakResidentBytes();// What the scene holds in memory, summed over SubTools (Preferences > Performance, perf_info).struct MemoryUse {diff --git a/src/core/Document.cpp b/src/core/Document.cppindex b2f2459c..182f51e7 100644--- a/src/core/Document.cpp+++ b/src/core/Document.cpp@@ -15,10 +15,11 @@ Document::Document() {subtools.back().name = "Sculpt";}-void Document::setMesh(Mesh m, std::string name) {+void Document::setMesh(Mesh m, std::string name, BVH* built) {mesh = std::move(m);mesh.topologyDirty = true;- bvh.build(mesh);+ if (built) bvh = std::move(*built);+ else bvh.build(mesh);undo.clear();activeLayer = -1;bvhDirty = false;@@ -67,13 +68,14 @@ void Document::selectSubtool(int i) {syncBvh();}-int Document::addSubtool(Mesh m, std::string name, bool select, int at) {+int Document::addSubtool(Mesh m, std::string name, bool select, int at, BVH* built) {SubTool s;s.id = nextSubtoolId_++;s.name = uniqueSubtoolName(name.empty() ? "SubTool" : name);s.mesh = std::move(m);s.mesh.topologyDirty = true;- s.bvh.build(s.mesh);+ if (built) s.bvh = std::move(*built);+ else s.bvh.build(s.mesh);const int index = at < 0 || at > subtoolCount() ? subtoolCount() : at;subtools.insert(subtools.begin() + index, std::move(s));if (index <= activeSubtool) ++activeSubtool;diff --git a/src/core/Document.h b/src/core/Document.hindex 144279f8..cb98db3d 100644--- a/src/core/Document.h+++ b/src/core/Document.h@@ -97,8 +97,9 @@ struct Document {const Mesh& subtoolMesh(int i) const { return i == activeSubtool ? mesh : subtools[size_t(i)].mesh; }// Makes SubTool `i` the one tools and commands edit.void selectSubtool(int i);- // Inserts a SubTool at `at` (-1 = end of the list), made active when `select`; returns its index.- int addSubtool(Mesh m, std::string name, bool select = true, int at = -1);+ // Inserts a SubTool at `at` (-1 = end of the list), made active when `select`; returns its index. `built`:+ // the mesh's BVH, built already (moved from; null = built here).+ int addSubtool(Mesh m, std::string name, bool select = true, int at = -1, BVH* built = nullptr);// Replaces the active SubTool's mesh outside undo (its history is cleared), e.g. after a merge.void replaceActiveMesh(Mesh m);// Removes SubTool `i` (never the last one); the active one moves to a neighbour if needed.@@ -112,8 +113,8 @@ struct Document {// Bounds of every visible SubTool.AABB visibleBounds() const;- // Replaces the whole scene with one SubTool holding `m` and clears the edit history.- void setMesh(Mesh m, std::string name = "Sculpt");+ // Replaces the whole scene with one SubTool holding `m` and clears the edit history (`built`: as addSubtool).+ void setMesh(Mesh m, std::string name = "Sculpt", BVH* built = nullptr);// Replaces the mesh as an undoable topology change. A result without the old mesh's subdivision levels (a// remesh, decimate, ...) leaves a `notice` saying the levels were removed; undo brings them back.void replaceMeshUndoable(Mesh m, std::string label);diff --git a/src/core/Multires.cpp b/src/core/Multires.cppindex ef1b58c4..9af9c940 100644--- a/src/core/Multires.cpp+++ b/src/core/Multires.cpp@@ -15,6 +15,18 @@#include <thread>#include <unordered_map>+#if defined(_WIN32)+#ifndef NOMINMAX+#define NOMINMAX+#endif+#ifndef WIN32_LEAN_AND_MEAN+#define WIN32_LEAN_AND_MEAN+#endif+#include <windows.h>+#else+#include <unistd.h>+#endif+namespace sf {namespace {@@ -1532,7 +1544,21 @@ bool subdivSetLevel(Mesh& mesh, int level, std::string& error, BVH* bvh) {return true;}+size_t subdivMaxTriangles() {+ size_t memory = 0;+#if defined(_WIN32)+ MEMORYSTATUSEX status{};+ status.dwLength = sizeof status;+ if (GlobalMemoryStatusEx(&status)) memory = size_t(status.ullTotalPhys);+#elif defined(_SC_PHYS_PAGES)+ const long pages = sysconf(_SC_PHYS_PAGES), page = sysconf(_SC_PAGESIZE);+ if (pages > 0 && page > 0) memory = size_t(pages) * size_t(page);+#endif+ return std::clamp<size_t>(memory / 10 * 6 / kDivideBytesPerTriangle, 50'000'000, 1'000'000'000);+}+bool subdivDivide(Mesh& mesh, bool smooth, std::string& error, BVH* bvh, size_t maxTriangles) {+ if (maxTriangles == 0) maxTriangles = subdivMaxTriangles();if (mesh.triangleCount() == 0) {error = "Nothing to divide";return false;@@ -1552,7 +1578,8 @@ bool subdivDivide(Mesh& mesh, bool smooth, std::string& error, BVH* bvh, size_tif (cc) projected = mesh.quadCount() * 8 + (mesh.triangleCount() - 2 * mesh.quadCount()) * 6;else projected = mesh.triangleCount() * 4;if (projected > maxTriangles) {- error = "Dividing would make " + std::to_string(projected) + " triangles (the limit is " + std::to_string(maxTriangles) + ")";+ error = "Dividing would make " + std::to_string(projected) + " triangles (the limit for this computer's memory is " ++ std::to_string(maxTriangles) + ")";return false;}MultiresStack& s = ensureStack(mesh);@@ -1669,13 +1696,15 @@ bool attachSubdivStack(Mesh& mesh, MultiresStack stack, std::string& error) {}for (size_t j = 0; j < n; ++j) {SubdivLevel& l = stack.levels[j];- if (l.data && generatedLevel(l) && !l.data->triangles.empty()) { // they follow from the topology- if (l.data.use_count() > 1) l.data = std::make_shared<Mesh>(*l.data);- std::vector<glm::uvec3>().swap(l.data->triangles);+ if (l.data && generatedLevel(l)) { // the triangles and the generated quad pairs follow from the topologybool generatedPairs = l.topo->catmullClark && l.data->quadPartners.size() == l.topo->fineTriangles;for (size_t t = 0; t < l.data->quadPartners.size() && generatedPairs; ++t)generatedPairs = l.data->quadPartners[t] == uint32_t(t ^ 1);- if (generatedPairs) std::vector<uint32_t>().swap(l.data->quadPartners);+ if (!l.data->triangles.empty() || generatedPairs) {+ if (l.data.use_count() > 1) l.data = std::make_shared<Mesh>(*l.data);+ std::vector<glm::uvec3>().swap(l.data->triangles);+ if (generatedPairs) std::vector<uint32_t>().swap(l.data->quadPartners);+ }}if (!l.data || (j > 0 && (!l.topo || l.topo->fineVertices != l.data->vertexCount() ||(!l.topo->generated && l.topo->fineTriangles != l.data->triangleCount()))) ||diff --git a/src/core/Multires.h b/src/core/Multires.hindex c62cda11..a7dee720 100644--- a/src/core/Multires.h+++ b/src/core/Multires.h@@ -176,7 +176,12 @@ void noteSubtoolLevelsDropped(Document& doc, int index);// mesh (rebuilt, or taken from the level cache); otherwise the caller rebuilds it.//// Adds a level on top (stepping to the highest level first) and makes it current. `smooth` = Smt.-bool subdivDivide(Mesh& mesh, bool smooth, std::string& error, BVH* bvh = nullptr, size_t maxTriangles = 50'000'000);+// `maxTriangles` 0 = subdivMaxTriangles().+bool subdivDivide(Mesh& mesh, bool smooth, std::string& error, BVH* bvh = nullptr, size_t maxTriangles = 0);+// The most triangles Divide makes on this computer: what fits in about 60% of its memory at kDivideBytesPerTriangle+// (mesh, levels, BVH, GPU buffers and the peak while dividing), between 50 million and 1 billion (32-bit indices).+constexpr size_t kDivideBytesPerTriangle = 110;+size_t subdivMaxTriangles();// Makes `level` current, with topology and normals.bool subdivSetLevel(Mesh& mesh, int level, std::string& error, BVH* bvh = nullptr);// Deletes the levels below / above the current one (the current level becomes the base / the top).diff --git a/src/core/Project.cpp b/src/core/Project.cppindex 1b133859..f369c476 100644--- a/src/core/Project.cpp+++ b/src/core/Project.cpp@@ -5,16 +5,28 @@#include "core/Multires.h"#include "core/Texture.h"+#include "core/ThreadPool.h"#include "io/Zip.h"#include <algorithm>+#include <atomic>#include <bit>#include <cctype>#include <cstring>+#include <deque>#include <filesystem>#include <fstream>#include <map>#include <system_error>+#include <thread>++#if defined(__unix__) || defined(__APPLE__)+#include <fcntl.h>+#include <unistd.h>+#define SF_PREAD 1+#else+#define SF_PREAD 0+#endif#ifndef SCULPTFORGE_VERSION#define SCULPTFORGE_VERSION "unknown"@@ -136,6 +148,8 @@ class Reader {public:std::map<std::string, ZipFileEntry> files;std::string error;+ // Big blobs are read on the shared thread pool, except while a BVH builds on another thread (it gets the pool).+ bool parallelReads = true;bool open(const std::string& path, std::string& err) {std::vector<ZipFileEntry> index;@@ -146,8 +160,16 @@ public:err = "Cannot open " + path;return false;}+#if SF_PREAD+ fd_ = ::open(path.c_str(), O_RDONLY | O_CLOEXEC);+#endifreturn true;}+ ~Reader() {+#if SF_PREAD+ if (fd_ >= 0) ::close(fd_);+#endif+ }// The whole entry `name` (false if missing or unreadable).bool entry(const std::string& name, std::vector<uint8_t>& out) {auto it = files.find(name);@@ -189,9 +211,34 @@ public:private:std::ifstream file_;+#if SF_PREAD+ int fd_ = -1;+#endifbool readAt(uint64_t offset, size_t n, void* out) {if (n == 0) return true;+#if SF_PREAD+ // Big arrays in 8 MB pieces on every core (the copies and the page faults of the fresh array are the cost).+ constexpr size_t kPiece = size_t(8) << 20;+ if (fd_ >= 0 && n >= 2 * kPiece && parallelReads) {+ std::atomic<bool> ok = true;+ poolFor((n + kPiece - 1) / kPiece, [&](size_t b, size_t e) {+ for (size_t i = b; i < e; ++i) {+ size_t done = i * kPiece;+ const size_t end = std::min(n, done + kPiece);+ while (done < end) {+ const ssize_t got = ::pread(fd_, static_cast<char*>(out) + done, end - done, off_t(offset + done));+ if (got <= 0) {+ ok = false;+ break;+ }+ done += size_t(got);+ }+ }+ }, 1, 1);+ return ok;+ }+#endiffile_.clear();file_.seekg(std::streamoff(offset));return bool(file_.read(static_cast<char*>(out), std::streamsize(n)));@@ -670,6 +717,25 @@ struct LoadedSubtool {bool visible = true;int activeLayer = -1;Mesh mesh;+ // A big mesh's BVH is built on its own thread once the positions and triangles are read (they don't change+ // after that), while the subdivision levels, topology and normals load.+ BVH bvh;+ std::thread bvhBuilder;+ LoadedSubtool() = default;+ LoadedSubtool(const LoadedSubtool&) = delete;+ ~LoadedSubtool() { finishBvh(); }+ void startBvh() {+ if (mesh.triangleCount() >= 1'000'000) bvhBuilder = std::thread([this] { bvh.build(mesh); });+ }+ // The BVH to hand over (null: build it on the spot).+ BVH* finishBvh() {+ if (bvhBuilder.joinable()) {+ bvhBuilder.join();+ built = true;+ }+ return built ? &bvh : nullptr;+ }+ bool built = false;};constexpr size_t kMaxProjectSubtools = 100000;@@ -793,14 +859,13 @@ bool loadProject(const std::string& path, Document& doc, std::string& error) {}// SubTools: version 1 has one mesh at the top level, version 2 a "subtools" list.- std::vector<LoadedSubtool> subtools;+ std::deque<LoadedSubtool> subtools; // (stable addresses: BVH threads use them)int activeSubtool = 0;if (version < 2) {- LoadedSubtool s;+ LoadedSubtool& s = subtools.emplace_back();s.name = "Sculpt";if (!readMesh(r, manifest["mesh"], manifest["layers"], s.mesh, error)) return false;s.activeLayer = int(manifest["active_layer"].asNumber(-1));- subtools.push_back(std::move(s));} else {const Json& list = manifest["subtools"];if (!list.isArray() || list.size() == 0 || list.size() > kMaxProjectSubtools) {@@ -808,18 +873,23 @@ bool loadProject(const std::string& path, Document& doc, std::string& error) {return false;}for (const Json& sj : list.items()) {- LoadedSubtool s;+ LoadedSubtool& s = subtools.emplace_back();s.name = sj["name"].asString();if (s.name.empty()) s.name = "SubTool";s.visible = sj["visible"].asBool(true);s.gozName = sj["goz_name"].asString();if (!readMesh(r, sj["mesh"], sj["layers"], s.mesh, error, false, false)) return false;+ // The BVH builds (on the pool) while the levels and topology load (off it), then the normals.+ s.startBvh();+ r.parallelReads = !s.bvhBuilder.joinable();if (!readPaintLayers(r, sj["paint_layers"], s.mesh, error)) return false;if (!readTexture(r, sj["texture"], s.mesh, error)) return false;if (!readSubdiv(r, sj["subdiv"], s.mesh, error, version < 5)) return false;- finishMesh(s.mesh);+ s.mesh.rebuildTopology();+ s.finishBvh();+ r.parallelReads = true;+ s.mesh.recomputeAllNormals();s.activeLayer = int(sj["active_layer"].asNumber(-1));- subtools.push_back(std::move(s));}const double active = manifest["active_subtool"].asNumber(0);activeSubtool = active >= 0 && active < double(subtools.size()) ? int(active) : 0;@@ -853,12 +923,14 @@ bool loadProject(const std::string& path, Document& doc, std::string& error) {// Everything parsed: replace the scene. SubTool 0 is checked out first; the others are added// behind it, given their layer and GoZ name, and the saved active SubTool is selected last.auto validLayer = [](int layer, const Mesh& m) { return layer >= 0 && layer < int(m.layers.size()) ? layer : -1; };- doc.setMesh(std::move(subtools[0].mesh), subtools[0].name);+ BVH* built = subtools[0].finishBvh();+ doc.setMesh(std::move(subtools[0].mesh), subtools[0].name, built);doc.activeLayer = validLayer(subtools[0].activeLayer, doc.mesh);doc.gozName = subtools[0].gozName;doc.subtools[0].visible = subtools[0].visible;for (size_t i = 1; i < subtools.size(); ++i) {- const int at = doc.addSubtool(std::move(subtools[i].mesh), subtools[i].name, false);+ BVH* builtI = subtools[i].finishBvh();+ const int at = doc.addSubtool(std::move(subtools[i].mesh), subtools[i].name, false, -1, builtI);SubTool& s = doc.subtools[size_t(at)];s.visible = subtools[i].visible;s.activeLayer = validLayer(subtools[i].activeLayer, s.mesh);diff --git a/tests/test_mesh_memory.cpp b/tests/test_mesh_memory.cppindex b88b3ffd..e1d0c00d 100644--- a/tests/test_mesh_memory.cpp+++ b/tests/test_mesh_memory.cpp@@ -156,6 +156,19 @@ TEST(divided_levels_store_little) {CHECK(m.implicitAdjacency());CHECK(stack < 32.0); // was 74CHECK(mesh < 40.0); // was 62+ // Saved and opened again, the levels take as little.+ {+ Document d = makeDoc(m);+ const std::string path = (std::filesystem::temp_directory_path() / "sf_mesh_memory_bench.sfproj").string();+ std::string err;+ CHECK(saveProject(path, d, err));+ Document e;+ CHECK(loadProject(path, e, err));+ CHECK(subdivBytes(e.mesh) == subdivBytes(d.mesh));+ CHECK(e.mesh.memoryBytes() == d.mesh.memoryBytes());+ CHECK(e.mesh.implicitAdjacency());+ std::filesystem::remove(path);+ }// Every generated level stores no triangles; the top keeps its positions but no detail.for (int i = 1; i < subdivLevelCount(m); ++i) {CHECK(subdivLevel(m, i).topo->generated);