Skip to content

Free, open-source 3D sculpting for Linux, macOS and Windows: dense meshes, remeshing, painting, built-in AI (image/text to 3D), Send to Unreal, game-ready LODs, Gaussian splats and OpenUSD.

Project open: the BVH builds on its own thread while the levels and topology load, big blobs are read in parallel, loaded levels drop their generated quad pairs; Divide's limit follows the computer's memory (100M+ on 30 GB); benchmark reports peak memory

islandmed6ca1a0dc58fb
diff --git a/src/core/Benchmark.cpp b/src/core/Benchmark.cpp
index b5d7892f..f4089d69 100644
--- a/src/core/Benchmark.cpp
+++ b/src/core/Benchmark.cpp
@@ -48,6 +48,22 @@ size_t residentBytes() {
return 0;
}
+size_t resetPeakResidentBytes() {
+#if defined(__linux__)
+ std::ofstream("/proc/self/clear_refs") << "5"; // resets the peak (VmHWM) to the current RSS
+#endif
+ return residentBytes();
+}
+
+size_t peakResidentBytes() {
+#if defined(__linux__)
+ std::ifstream f("/proc/self/status");
+ for (std::string line; std::getline(f, line);)
+ if (line.rfind("VmHWM:", 0) == 0) return size_t(std::atoll(line.c_str() + 6)) * 1024;
+#endif
+ return 0;
+}
+
MemoryUse documentMemory(const Document& doc) {
MemoryUse u;
for (int i = 0; i < doc.subtoolCount(); ++i) {
@@ -288,7 +304,7 @@ std::string runBenchmark(const BenchOptions& o, const BenchGpu* gpu) {
const size_t target = size_t(sizeM * 1e6);
log("Building a " + millions(double(target)) + " triangle sphere...");
Document doc;
- const size_t rss0 = residentBytes();
+ const size_t rss0 = resetPeakResidentBytes();
auto t = Clock::now();
doc.mesh = makeBenchSphere(target);
const double divideMs = msSince(t);
@@ -300,7 +316,7 @@ std::string runBenchmark(const BenchOptions& o, const BenchGpu* gpu) {
double uploadAll = 0;
if (gpu && gpu->upload) uploadAll = gpu->upload(m);
const size_t meshBytes = m.memoryBytes(), bvhBytes = doc.bvh.memoryBytes(), stackBytes = subdivBytes(m);
- const size_t rss = residentBytes();
+ const size_t rss = residentBytes(), setupPeak = peakResidentBytes();
UploadOrder order;
order.build(m);
@@ -316,7 +332,9 @@ std::string runBenchmark(const BenchOptions& o, const BenchGpu* gpu) {
<< " B/triangle |\n";
md << "| Subdivision levels (stack) | " << fmt("%.0f MB", double(stackBytes) / 1e6) << " = "
<< fmt("%.0f", double(stackBytes) / nt) << " B/triangle |\n";
- if (rss) md << "| Process memory (RSS) | " << fmt("%.0f MB", double(rss) / 1e6) << " (+" << fmt("%.0f", double(rss - std::min(rss, rss0)) / 1e6) << " MB) |\n";
+ if (rss) md << "| Process memory (RSS) | " << fmt("%.0f MB", double(rss) / 1e6) << " (+" << fmt("%.0f", double(rss - std::min(rss, rss0)) / 1e6) << " MB = "
+ << fmt("%.0f", double(rss - std::min(rss, rss0)) / nt) << " B/triangle) |\n";
+ if (setupPeak > rss0) md << "| Peak while dividing (RSS) | " << fmt("%.0f MB", double(setupPeak - rss0) / 1e6) << " above the start |\n";
md << "\n";
md << "| Brush | verts/dab | uploaded verts (ranges) | kernel | normals | BVH refit | pick | " << (gpu ? "GPU upload | " : "")
@@ -368,12 +386,17 @@ std::string runBenchmark(const BenchOptions& o, const BenchGpu* gpu) {
const double saveMs = msSince(t);
const double fileMb = saved ? double(fs::file_size(path, ec)) / 1e6 : 0.0;
Document loaded;
+ const size_t beforeOpen = resetPeakResidentBytes();
t = Clock::now();
const bool opened = saved && loadProject(path, loaded, err);
const double openMs = msSince(t);
+ const size_t openPeak = peakResidentBytes(), afterOpen = residentBytes();
fs::remove(path, ec);
md << "| Save project (" << fmt("%.0f MB", fileMb) << ") | " << (saved ? fmt("%.0f", saveMs) : "failed: " + err) << " |\n";
md << "| Open project | " << (opened ? fmt("%.0f", openMs) : "failed: " + err) << " |\n";
+ if (opened && openPeak > beforeOpen)
+ md << "| Open project: memory (RSS) | +" << fmt("%.0f MB", double(afterOpen - std::min(afterOpen, beforeOpen)) / 1e6) << ", peak +"
+ << fmt("%.0f MB", double(openPeak - beforeOpen) / 1e6) << " |\n";
}
if (gpu && gpu->orbitFrameMs) {
log(" orbit");
diff --git a/src/core/Benchmark.h b/src/core/Benchmark.h
index c607ad2c..3dfe0183 100644
--- a/src/core/Benchmark.h
+++ b/src/core/Benchmark.h
@@ -61,6 +61,9 @@ Mesh makeBenchSphere(size_t triangles);
// Resident memory of this process in bytes (0 where unknown).
size_t residentBytes();
+// The most resident memory since the last reset (Linux: VmHWM; 0 where unknown); the reset returns residentBytes().
+size_t peakResidentBytes();
+size_t resetPeakResidentBytes();
// What the scene holds in memory, summed over SubTools (Preferences > Performance, perf_info).
struct MemoryUse {
diff --git a/src/core/Document.cpp b/src/core/Document.cpp
index b2f2459c..182f51e7 100644
--- a/src/core/Document.cpp
+++ b/src/core/Document.cpp
@@ -15,10 +15,11 @@ Document::Document() {
subtools.back().name = "Sculpt";
}
-void Document::setMesh(Mesh m, std::string name) {
+void Document::setMesh(Mesh m, std::string name, BVH* built) {
mesh = std::move(m);
mesh.topologyDirty = true;
- bvh.build(mesh);
+ if (built) bvh = std::move(*built);
+ else bvh.build(mesh);
undo.clear();
activeLayer = -1;
bvhDirty = false;
@@ -67,13 +68,14 @@ void Document::selectSubtool(int i) {
syncBvh();
}
-int Document::addSubtool(Mesh m, std::string name, bool select, int at) {
+int Document::addSubtool(Mesh m, std::string name, bool select, int at, BVH* built) {
SubTool s;
s.id = nextSubtoolId_++;
s.name = uniqueSubtoolName(name.empty() ? "SubTool" : name);
s.mesh = std::move(m);
s.mesh.topologyDirty = true;
- s.bvh.build(s.mesh);
+ if (built) s.bvh = std::move(*built);
+ else s.bvh.build(s.mesh);
const int index = at < 0 || at > subtoolCount() ? subtoolCount() : at;
subtools.insert(subtools.begin() + index, std::move(s));
if (index <= activeSubtool) ++activeSubtool;
diff --git a/src/core/Document.h b/src/core/Document.h
index 144279f8..cb98db3d 100644
--- a/src/core/Document.h
+++ b/src/core/Document.h
@@ -97,8 +97,9 @@ struct Document {
const Mesh& subtoolMesh(int i) const { return i == activeSubtool ? mesh : subtools[size_t(i)].mesh; }
// Makes SubTool `i` the one tools and commands edit.
void selectSubtool(int i);
- // Inserts a SubTool at `at` (-1 = end of the list), made active when `select`; returns its index.
- int addSubtool(Mesh m, std::string name, bool select = true, int at = -1);
+ // Inserts a SubTool at `at` (-1 = end of the list), made active when `select`; returns its index. `built`:
+ // the mesh's BVH, built already (moved from; null = built here).
+ int addSubtool(Mesh m, std::string name, bool select = true, int at = -1, BVH* built = nullptr);
// Replaces the active SubTool's mesh outside undo (its history is cleared), e.g. after a merge.
void replaceActiveMesh(Mesh m);
// Removes SubTool `i` (never the last one); the active one moves to a neighbour if needed.
@@ -112,8 +113,8 @@ struct Document {
// Bounds of every visible SubTool.
AABB visibleBounds() const;
- // Replaces the whole scene with one SubTool holding `m` and clears the edit history.
- void setMesh(Mesh m, std::string name = "Sculpt");
+ // Replaces the whole scene with one SubTool holding `m` and clears the edit history (`built`: as addSubtool).
+ void setMesh(Mesh m, std::string name = "Sculpt", BVH* built = nullptr);
// Replaces the mesh as an undoable topology change. A result without the old mesh's subdivision levels (a
// remesh, decimate, ...) leaves a `notice` saying the levels were removed; undo brings them back.
void replaceMeshUndoable(Mesh m, std::string label);
diff --git a/src/core/Multires.cpp b/src/core/Multires.cpp
index ef1b58c4..9af9c940 100644
--- a/src/core/Multires.cpp
+++ b/src/core/Multires.cpp
@@ -15,6 +15,18 @@
#include <thread>
#include <unordered_map>
+#if defined(_WIN32)
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#include <windows.h>
+#else
+#include <unistd.h>
+#endif
+
namespace sf {
namespace {
@@ -1532,7 +1544,21 @@ bool subdivSetLevel(Mesh& mesh, int level, std::string& error, BVH* bvh) {
return true;
}
+size_t subdivMaxTriangles() {
+ size_t memory = 0;
+#if defined(_WIN32)
+ MEMORYSTATUSEX status{};
+ status.dwLength = sizeof status;
+ if (GlobalMemoryStatusEx(&status)) memory = size_t(status.ullTotalPhys);
+#elif defined(_SC_PHYS_PAGES)
+ const long pages = sysconf(_SC_PHYS_PAGES), page = sysconf(_SC_PAGESIZE);
+ if (pages > 0 && page > 0) memory = size_t(pages) * size_t(page);
+#endif
+ return std::clamp<size_t>(memory / 10 * 6 / kDivideBytesPerTriangle, 50'000'000, 1'000'000'000);
+}
+
bool subdivDivide(Mesh& mesh, bool smooth, std::string& error, BVH* bvh, size_t maxTriangles) {
+ if (maxTriangles == 0) maxTriangles = subdivMaxTriangles();
if (mesh.triangleCount() == 0) {
error = "Nothing to divide";
return false;
@@ -1552,7 +1578,8 @@ bool subdivDivide(Mesh& mesh, bool smooth, std::string& error, BVH* bvh, size_t
if (cc) projected = mesh.quadCount() * 8 + (mesh.triangleCount() - 2 * mesh.quadCount()) * 6;
else projected = mesh.triangleCount() * 4;
if (projected > maxTriangles) {
- error = "Dividing would make " + std::to_string(projected) + " triangles (the limit is " + std::to_string(maxTriangles) + ")";
+ error = "Dividing would make " + std::to_string(projected) + " triangles (the limit for this computer's memory is " +
+ std::to_string(maxTriangles) + ")";
return false;
}
MultiresStack& s = ensureStack(mesh);
@@ -1669,13 +1696,15 @@ bool attachSubdivStack(Mesh& mesh, MultiresStack stack, std::string& error) {
}
for (size_t j = 0; j < n; ++j) {
SubdivLevel& l = stack.levels[j];
- if (l.data && generatedLevel(l) && !l.data->triangles.empty()) { // they follow from the topology
- if (l.data.use_count() > 1) l.data = std::make_shared<Mesh>(*l.data);
- std::vector<glm::uvec3>().swap(l.data->triangles);
+ if (l.data && generatedLevel(l)) { // the triangles and the generated quad pairs follow from the topology
bool generatedPairs = l.topo->catmullClark && l.data->quadPartners.size() == l.topo->fineTriangles;
for (size_t t = 0; t < l.data->quadPartners.size() && generatedPairs; ++t)
generatedPairs = l.data->quadPartners[t] == uint32_t(t ^ 1);
- if (generatedPairs) std::vector<uint32_t>().swap(l.data->quadPartners);
+ if (!l.data->triangles.empty() || generatedPairs) {
+ if (l.data.use_count() > 1) l.data = std::make_shared<Mesh>(*l.data);
+ std::vector<glm::uvec3>().swap(l.data->triangles);
+ if (generatedPairs) std::vector<uint32_t>().swap(l.data->quadPartners);
+ }
}
if (!l.data || (j > 0 && (!l.topo || l.topo->fineVertices != l.data->vertexCount() ||
(!l.topo->generated && l.topo->fineTriangles != l.data->triangleCount()))) ||
diff --git a/src/core/Multires.h b/src/core/Multires.h
index c62cda11..a7dee720 100644
--- a/src/core/Multires.h
+++ b/src/core/Multires.h
@@ -176,7 +176,12 @@ void noteSubtoolLevelsDropped(Document& doc, int index);
// mesh (rebuilt, or taken from the level cache); otherwise the caller rebuilds it.
//
// Adds a level on top (stepping to the highest level first) and makes it current. `smooth` = Smt.
-bool subdivDivide(Mesh& mesh, bool smooth, std::string& error, BVH* bvh = nullptr, size_t maxTriangles = 50'000'000);
+// `maxTriangles` 0 = subdivMaxTriangles().
+bool subdivDivide(Mesh& mesh, bool smooth, std::string& error, BVH* bvh = nullptr, size_t maxTriangles = 0);
+// The most triangles Divide makes on this computer: what fits in about 60% of its memory at kDivideBytesPerTriangle
+// (mesh, levels, BVH, GPU buffers and the peak while dividing), between 50 million and 1 billion (32-bit indices).
+constexpr size_t kDivideBytesPerTriangle = 110;
+size_t subdivMaxTriangles();
// Makes `level` current, with topology and normals.
bool subdivSetLevel(Mesh& mesh, int level, std::string& error, BVH* bvh = nullptr);
// Deletes the levels below / above the current one (the current level becomes the base / the top).
diff --git a/src/core/Project.cpp b/src/core/Project.cpp
index 1b133859..f369c476 100644
--- a/src/core/Project.cpp
+++ b/src/core/Project.cpp
@@ -5,16 +5,28 @@
#include "core/Multires.h"
#include "core/Texture.h"
+#include "core/ThreadPool.h"
#include "io/Zip.h"
#include <algorithm>
+#include <atomic>
#include <bit>
#include <cctype>
#include <cstring>
+#include <deque>
#include <filesystem>
#include <fstream>
#include <map>
#include <system_error>
+#include <thread>
+
+#if defined(__unix__) || defined(__APPLE__)
+#include <fcntl.h>
+#include <unistd.h>
+#define SF_PREAD 1
+#else
+#define SF_PREAD 0
+#endif
#ifndef SCULPTFORGE_VERSION
#define SCULPTFORGE_VERSION "unknown"
@@ -136,6 +148,8 @@ class Reader {
public:
std::map<std::string, ZipFileEntry> files;
std::string error;
+ // Big blobs are read on the shared thread pool, except while a BVH builds on another thread (it gets the pool).
+ bool parallelReads = true;
bool open(const std::string& path, std::string& err) {
std::vector<ZipFileEntry> index;
@@ -146,8 +160,16 @@ public:
err = "Cannot open " + path;
return false;
}
+#if SF_PREAD
+ fd_ = ::open(path.c_str(), O_RDONLY | O_CLOEXEC);
+#endif
return true;
}
+ ~Reader() {
+#if SF_PREAD
+ if (fd_ >= 0) ::close(fd_);
+#endif
+ }
// The whole entry `name` (false if missing or unreadable).
bool entry(const std::string& name, std::vector<uint8_t>& out) {
auto it = files.find(name);
@@ -189,9 +211,34 @@ public:
private:
std::ifstream file_;
+#if SF_PREAD
+ int fd_ = -1;
+#endif
bool readAt(uint64_t offset, size_t n, void* out) {
if (n == 0) return true;
+#if SF_PREAD
+ // Big arrays in 8 MB pieces on every core (the copies and the page faults of the fresh array are the cost).
+ constexpr size_t kPiece = size_t(8) << 20;
+ if (fd_ >= 0 && n >= 2 * kPiece && parallelReads) {
+ std::atomic<bool> ok = true;
+ poolFor((n + kPiece - 1) / kPiece, [&](size_t b, size_t e) {
+ for (size_t i = b; i < e; ++i) {
+ size_t done = i * kPiece;
+ const size_t end = std::min(n, done + kPiece);
+ while (done < end) {
+ const ssize_t got = ::pread(fd_, static_cast<char*>(out) + done, end - done, off_t(offset + done));
+ if (got <= 0) {
+ ok = false;
+ break;
+ }
+ done += size_t(got);
+ }
+ }
+ }, 1, 1);
+ return ok;
+ }
+#endif
file_.clear();
file_.seekg(std::streamoff(offset));
return bool(file_.read(static_cast<char*>(out), std::streamsize(n)));
@@ -670,6 +717,25 @@ struct LoadedSubtool {
bool visible = true;
int activeLayer = -1;
Mesh mesh;
+ // A big mesh's BVH is built on its own thread once the positions and triangles are read (they don't change
+ // after that), while the subdivision levels, topology and normals load.
+ BVH bvh;
+ std::thread bvhBuilder;
+ LoadedSubtool() = default;
+ LoadedSubtool(const LoadedSubtool&) = delete;
+ ~LoadedSubtool() { finishBvh(); }
+ void startBvh() {
+ if (mesh.triangleCount() >= 1'000'000) bvhBuilder = std::thread([this] { bvh.build(mesh); });
+ }
+ // The BVH to hand over (null: build it on the spot).
+ BVH* finishBvh() {
+ if (bvhBuilder.joinable()) {
+ bvhBuilder.join();
+ built = true;
+ }
+ return built ? &bvh : nullptr;
+ }
+ bool built = false;
};
constexpr size_t kMaxProjectSubtools = 100000;
@@ -793,14 +859,13 @@ bool loadProject(const std::string& path, Document& doc, std::string& error) {
}
// SubTools: version 1 has one mesh at the top level, version 2 a "subtools" list.
- std::vector<LoadedSubtool> subtools;
+ std::deque<LoadedSubtool> subtools; // (stable addresses: BVH threads use them)
int activeSubtool = 0;
if (version < 2) {
- LoadedSubtool s;
+ LoadedSubtool& s = subtools.emplace_back();
s.name = "Sculpt";
if (!readMesh(r, manifest["mesh"], manifest["layers"], s.mesh, error)) return false;
s.activeLayer = int(manifest["active_layer"].asNumber(-1));
- subtools.push_back(std::move(s));
} else {
const Json& list = manifest["subtools"];
if (!list.isArray() || list.size() == 0 || list.size() > kMaxProjectSubtools) {
@@ -808,18 +873,23 @@ bool loadProject(const std::string& path, Document& doc, std::string& error) {
return false;
}
for (const Json& sj : list.items()) {
- LoadedSubtool s;
+ LoadedSubtool& s = subtools.emplace_back();
s.name = sj["name"].asString();
if (s.name.empty()) s.name = "SubTool";
s.visible = sj["visible"].asBool(true);
s.gozName = sj["goz_name"].asString();
if (!readMesh(r, sj["mesh"], sj["layers"], s.mesh, error, false, false)) return false;
+ // The BVH builds (on the pool) while the levels and topology load (off it), then the normals.
+ s.startBvh();
+ r.parallelReads = !s.bvhBuilder.joinable();
if (!readPaintLayers(r, sj["paint_layers"], s.mesh, error)) return false;
if (!readTexture(r, sj["texture"], s.mesh, error)) return false;
if (!readSubdiv(r, sj["subdiv"], s.mesh, error, version < 5)) return false;
- finishMesh(s.mesh);
+ s.mesh.rebuildTopology();
+ s.finishBvh();
+ r.parallelReads = true;
+ s.mesh.recomputeAllNormals();
s.activeLayer = int(sj["active_layer"].asNumber(-1));
- subtools.push_back(std::move(s));
}
const double active = manifest["active_subtool"].asNumber(0);
activeSubtool = active >= 0 && active < double(subtools.size()) ? int(active) : 0;
@@ -853,12 +923,14 @@ bool loadProject(const std::string& path, Document& doc, std::string& error) {
// Everything parsed: replace the scene. SubTool 0 is checked out first; the others are added
// behind it, given their layer and GoZ name, and the saved active SubTool is selected last.
auto validLayer = [](int layer, const Mesh& m) { return layer >= 0 && layer < int(m.layers.size()) ? layer : -1; };
- doc.setMesh(std::move(subtools[0].mesh), subtools[0].name);
+ BVH* built = subtools[0].finishBvh();
+ doc.setMesh(std::move(subtools[0].mesh), subtools[0].name, built);
doc.activeLayer = validLayer(subtools[0].activeLayer, doc.mesh);
doc.gozName = subtools[0].gozName;
doc.subtools[0].visible = subtools[0].visible;
for (size_t i = 1; i < subtools.size(); ++i) {
- const int at = doc.addSubtool(std::move(subtools[i].mesh), subtools[i].name, false);
+ BVH* builtI = subtools[i].finishBvh();
+ const int at = doc.addSubtool(std::move(subtools[i].mesh), subtools[i].name, false, -1, builtI);
SubTool& s = doc.subtools[size_t(at)];
s.visible = subtools[i].visible;
s.activeLayer = validLayer(subtools[i].activeLayer, s.mesh);
diff --git a/tests/test_mesh_memory.cpp b/tests/test_mesh_memory.cpp
index b88b3ffd..e1d0c00d 100644
--- a/tests/test_mesh_memory.cpp
+++ b/tests/test_mesh_memory.cpp
@@ -156,6 +156,19 @@ TEST(divided_levels_store_little) {
CHECK(m.implicitAdjacency());
CHECK(stack < 32.0); // was 74
CHECK(mesh < 40.0); // was 62
+ // Saved and opened again, the levels take as little.
+ {
+ Document d = makeDoc(m);
+ const std::string path = (std::filesystem::temp_directory_path() / "sf_mesh_memory_bench.sfproj").string();
+ std::string err;
+ CHECK(saveProject(path, d, err));
+ Document e;
+ CHECK(loadProject(path, e, err));
+ CHECK(subdivBytes(e.mesh) == subdivBytes(d.mesh));
+ CHECK(e.mesh.memoryBytes() == d.mesh.memoryBytes());
+ CHECK(e.mesh.implicitAdjacency());
+ std::filesystem::remove(path);
+ }
// Every generated level stores no triangles; the top keeps its positions but no detail.
for (int i = 1; i < subdivLevelCount(m); ++i) {
CHECK(subdivLevel(m, i).topo->generated);