Free, open-source 3D sculpting for Linux, macOS and Windows: dense meshes, remeshing, painting, built-in AI (image/text to 3D), Send to Unreal, game-ready LODs, Gaussian splats and OpenUSD.
C++ 3d3d-sculptingcppdear-imguidigital-sculptingfbxgame-developmentgaussian-splattinggltfimage-to-3dmcpopenglopenusdsculptingtext-to-3dunreal-engine
// SculptForge - open source digital sculpting
// SPDX-License-Identifier: GPL-3.0-or-later
#include "core/ImageTo3D.h"
#include "core/Primitives.h"
#include "core/Sculpt.h"
#include <algorithm>
#include <cmath>
#include <unordered_map>
namespace sf {
namespace {
float heightAt(const Image& img, float u, float v, bool invert) {
float h = img.luminance(u, v);
if (invert) h = 1.0f - h;
return h * img.alpha(u, v);
}
} // namespace
Mesh makeImageRelief(const Image& image, const ReliefOptions& o) {
Mesh m;
if (image.empty()) return m;
const int res = std::clamp(o.resolution, 8, 2048);
const float aspect = image.aspect();
const int nx = aspect >= 1.0f ? res : std::max(2, int(std::lround(res * aspect)));
const int ny = aspect >= 1.0f ? std::max(2, int(std::lround(res / aspect))) : res;
const float width = 2.0f, height = 2.0f / aspect;
const float depth = o.depth * width;
m.positions.reserve(size_t(nx + 1) * size_t(ny + 1));
for (int j = 0; j <= ny; ++j) {
for (int i = 0; i <= nx; ++i) {
float u = float(i) / float(nx), v = float(j) / float(ny);
m.positions.emplace_back((u - 0.5f) * width, (0.5f - v) * height, heightAt(image, u, v, o.invert) * depth);
}
}
for (int j = 0; j < ny; ++j) {
for (int i = 0; i < nx; ++i) {
uint32_t a = uint32_t(j * (nx + 1) + i), b = a + 1, c = a + uint32_t(nx + 1), d = c + 1;
m.triangles.push_back({a, c, b});
m.triangles.push_back({b, c, d});
}
}
m.rebuildTopology();
m.recomputeAllNormals();
return m;
}
size_t projectImage(Mesh& mesh, int layer, const Image& image, const glm::mat4& viewProj, float viewAspect,
const glm::vec3& cameraPos, const ProjectOptions& o) {
if (image.empty() || mesh.vertexCount() == 0) return 0;
// Fit the image inside the view ("contain"), scaled by coverage.
const float ia = image.aspect();
float sx = 1.0f, sy = 1.0f;
if (ia > viewAspect) sy = viewAspect / ia;
else sx = ia / viewAspect;
sx *= o.coverage;
sy *= o.coverage;
auto& target = mesh.layerTarget(layer);
const float invWeight = 1.0f / mesh.layerTargetWeight(layer);
const bool layered = !mesh.layers.empty();
std::vector<uint32_t> moved;
for (uint32_t v = 0; v < mesh.vertexCount(); ++v) {
const glm::vec3 p = mesh.positions[v];
glm::vec4 clip = viewProj * glm::vec4(p, 1.0f);
if (clip.w <= 0.0f) continue;
glm::vec2 ndc = glm::vec2(clip) / clip.w;
float u = (ndc.x / sx + 1.0f) * 0.5f;
float t = 1.0f - (ndc.y / sy + 1.0f) * 0.5f;
if (u < 0.0f || u > 1.0f || t < 0.0f || t > 1.0f) continue;
float facing = glm::dot(mesh.normals[v], glm::normalize(cameraPos - p));
float f = std::clamp((facing - 0.05f) / 0.25f, 0.0f, 1.0f); // fade at grazing angles
float h = heightAt(image, u, t, o.invert) * f * vertexInfluence(mesh, v); // respects mask/hidden like brushes
if (h <= 0.0f) continue;
glm::vec3 d = mesh.normals[v] * (o.depth * h);
mesh.positions[v] += d;
if (layered) target[v] += d * invWeight;
moved.push_back(v);
}
mesh.updateNormals(moved);
return moved.size();
}
Mesh inflateSilhouette(const Image& image, const InflateOptions& o, std::string* error) {
auto fail = [&](const char* msg) {
if (error) *error = msg;
return Mesh{};
};
if (image.empty()) return fail("No image");
const int res = std::clamp(o.resolution, 16, 512);
const float aspect = image.aspect();
const int gw = aspect >= 1.0f ? res : std::max(4, int(std::lround(res * aspect)));
const int gh = aspect >= 1.0f ? std::max(4, int(std::lround(res / aspect))) : res;
// Grid with a one-cell empty border so every shape has an outside.
const int W = gw + 2, H = gh + 2;
auto idx = [W](int x, int y) { return size_t(y) * size_t(W) + size_t(x); };
// 1. Mask: opaque pixels, or pixels that differ from the border colour.
std::vector<uint8_t> inside(size_t(W) * size_t(H), 0);
const bool useAlpha = image.hasTransparency();
float border = 0.0f;
if (!useAlpha) {
int n = 0;
for (int k = 0; k < 64; ++k) {
float s = float(k) / 63.0f;
border += image.luminance(s, 0.0f) + image.luminance(s, 1.0f) + image.luminance(0.0f, s) +
image.luminance(1.0f, s);
n += 4;
}
border /= float(n);
}
for (int y = 0; y < gh; ++y) {
for (int x = 0; x < gw; ++x) {
float u = (float(x) + 0.5f) / float(gw), v = (float(y) + 0.5f) / float(gh);
bool in = useAlpha ? image.alpha(u, v) > 0.5f : std::abs(image.luminance(u, v) - border) > 0.2f;
inside[idx(x + 1, y + 1)] = in;
}
}
// 2. Drop specks: keep connected components at least 5% the size of the largest.
std::vector<int> label(inside.size(), -1);
std::vector<size_t> sizes;
std::vector<size_t> stack;
for (size_t start = 0; start < inside.size(); ++start) {
if (!inside[start] || label[start] >= 0) continue;
int l = int(sizes.size());
size_t count = 0;
stack.assign(1, start);
label[start] = l;
while (!stack.empty()) {
size_t c = stack.back();
stack.pop_back();
++count;
int cx = int(c % size_t(W)), cy = int(c / size_t(W));
const int nx[4] = {cx + 1, cx - 1, cx, cx}, ny[4] = {cy, cy, cy + 1, cy - 1};
for (int k = 0; k < 4; ++k) {
size_t n = idx(nx[k], ny[k]);
if (inside[n] && label[n] < 0) {
label[n] = l;
stack.push_back(n);
}
}
}
sizes.push_back(count);
}
if (sizes.empty()) return fail("No shape found - use an image with a transparent background or a plain backdrop");
const size_t largest = *std::max_element(sizes.begin(), sizes.end());
for (size_t c = 0; c < inside.size(); ++c)
if (inside[c] && sizes[size_t(label[c])] < std::max<size_t>(4, largest / 20)) inside[c] = 0;
// 3. Quads between four inside cells form the sheet. Iteratively remove
// quads that would make the closed surface degenerate: quads whose four
// corners are all on the rim (front and back would coincide) and
// "bow-tie" corners where two quads touch only diagonally.
std::vector<uint8_t> quad(inside.size(), 0); // quad anchored at its top-left cell
for (int y = 1; y < H - 2; ++y)
for (int x = 1; x < W - 2; ++x)
quad[idx(x, y)] = inside[idx(x, y)] && inside[idx(x + 1, y)] && inside[idx(x, y + 1)] &&
inside[idx(x + 1, y + 1)];
auto q = [&](int x, int y) { return x >= 0 && y >= 0 && x < W && y < H && quad[idx(x, y)]; };
auto incident = [&](int x, int y) { return int(q(x - 1, y - 1)) + q(x, y - 1) + q(x - 1, y) + q(x, y); };
std::vector<uint8_t> rim(inside.size(), 0);
for (bool changed = true; changed;) {
changed = false;
for (int y = 1; y < H - 1; ++y)
for (int x = 1; x < W - 1; ++x) {
// Bow-tie: only the two diagonal quads around this corner exist.
bool d1 = q(x - 1, y - 1) && q(x, y) && !q(x, y - 1) && !q(x - 1, y);
bool d2 = q(x, y - 1) && q(x - 1, y) && !q(x - 1, y - 1) && !q(x, y);
if (d1) { quad[idx(x, y)] = 0; changed = true; }
if (d2) { quad[idx(x - 1, y)] = 0; changed = true; }
}
for (int y = 1; y < H - 1; ++y)
for (int x = 1; x < W - 1; ++x) {
int n = incident(x, y);
rim[idx(x, y)] = n > 0 && n < 4;
}
for (int y = 1; y < H - 2; ++y)
for (int x = 1; x < W - 2; ++x)
if (quad[idx(x, y)] && rim[idx(x, y)] && rim[idx(x + 1, y)] && rim[idx(x, y + 1)] &&
rim[idx(x + 1, y + 1)]) {
quad[idx(x, y)] = 0;
changed = true;
}
}
// 4. Distance from each sheet vertex to the rim.
std::vector<glm::vec2> contour;
for (int y = 1; y < H - 1; ++y)
for (int x = 1; x < W - 1; ++x)
if (rim[idx(x, y)]) contour.emplace_back(float(x), float(y));
std::vector<float> dist(inside.size(), 0.0f);
float maxDist = 0.0f;
for (int y = 1; y < H - 1; ++y) {
for (int x = 1; x < W - 1; ++x) {
if (rim[idx(x, y)] || incident(x, y) == 0) continue;
float best = 1e30f;
for (const auto& c : contour) {
float dx = c.x - float(x), dy = c.y - float(y);
best = std::min(best, dx * dx + dy * dy);
}
dist[idx(x, y)] = std::sqrt(best);
maxDist = std::max(maxDist, dist[idx(x, y)]);
}
}
if (maxDist <= 0.0f) return fail("The shape is too thin to inflate - try a bolder silhouette");
// 4b. Heights from a circular profile, then relaxed so the creases along
// the shape's medial axis round off (rim stays pinned at zero).
auto profile = [&](float d) { return std::sqrt(std::max(0.0f, 2.0f * maxDist * d - d * d)); };
std::vector<float> height(inside.size(), 0.0f);
for (size_t c = 0; c < inside.size(); ++c) height[c] = dist[c] > 0.0f ? profile(dist[c]) : 0.0f;
{
std::vector<float> next = height;
const int iterations = std::max(4, int(maxDist * 0.6f));
for (int it = 0; it < iterations; ++it) {
for (int y = 1; y < H - 1; ++y)
for (int x = 1; x < W - 1; ++x) {
size_t c = idx(x, y);
if (dist[c] <= 0.0f) continue;
float sum = 0.0f;
int n = 0;
const int nx[4] = {x + 1, x - 1, x, x}, ny[4] = {y, y, y + 1, y - 1};
for (int k = 0; k < 4; ++k) {
size_t nc = idx(nx[k], ny[k]);
if (dist[nc] > 0.0f || rim[nc]) {
sum += height[nc];
++n;
}
}
// Blend toward the neighbour average but never below the
// profile's rounded rim, so edges keep their fullness.
if (n) next[c] = std::max(0.5f * height[c] + 0.5f * sum / float(n), profile(std::min(dist[c], 2.0f)));
}
height.swap(next);
}
}
// 5. Front sheet over the quads. A circular cross-section profile gives
// a rounded, "balloon" body: h = sqrt(2Rd - d^2).
struct SheetVertex {
glm::vec2 xy; // grid units
float h;
bool rim;
float d = 0.0f; // distance to the rim in cells
};
std::vector<SheetVertex> sheet;
std::vector<glm::uvec3> tris;
constexpr uint32_t kNone = ~0u;
std::vector<uint32_t> slot(inside.size(), kNone);
auto vertexFor = [&](int x, int y) {
size_t c = idx(x, y);
if (slot[c] == kNone) {
slot[c] = uint32_t(sheet.size());
sheet.push_back({glm::vec2(float(x), float(y)), rim[c] ? 0.0f : height[c], rim[c] != 0, dist[c]});
}
return slot[c];
};
for (int y = 1; y < H - 2; ++y) {
for (int x = 1; x < W - 2; ++x) {
if (!quad[idx(x, y)]) continue;
uint32_t a = vertexFor(x, y), b = vertexFor(x + 1, y), c = vertexFor(x, y + 1), d = vertexFor(x + 1, y + 1);
tris.push_back({a, c, b});
tris.push_back({b, c, d});
}
}
if (tris.empty()) return fail("The shape is too thin to inflate - try a bolder silhouette");
// 6. Split "chords": interior edges whose ends are both on the rim. Front
// and back would otherwise share them and pinch the surface.
{
std::unordered_map<uint64_t, int> edgeUse;
auto key = [](uint32_t a, uint32_t b) { return a < b ? (uint64_t(a) << 32 | b) : (uint64_t(b) << 32 | a); };
for (const auto& t : tris)
for (int k = 0; k < 3; ++k) ++edgeUse[key(t[k], t[(k + 1) % 3])];
std::vector<std::pair<uint32_t, uint32_t>> chords;
for (const auto& [k, n] : edgeUse) {
uint32_t a = uint32_t(k >> 32), b = uint32_t(k & 0xffffffffu);
if (n == 2 && sheet[a].rim && sheet[b].rim) chords.emplace_back(a, b);
}
for (const auto& [a, b] : chords) {
const uint32_t mid = uint32_t(sheet.size());
sheet.push_back({(sheet[a].xy + sheet[b].xy) * 0.5f, profile(0.5f), false, 0.5f});
const size_t count = tris.size();
for (size_t i = 0; i < count; ++i) {
glm::uvec3 t = tris[i];
for (int k = 0; k < 3; ++k) {
uint32_t p = t[k], qv = t[(k + 1) % 3], o = t[(k + 2) % 3];
if ((p == a && qv == b) || (p == b && qv == a)) {
tris[i] = {p, mid, o};
tris.push_back({mid, qv, o});
break;
}
}
}
}
}
// 7. Mirror into a back sheet that shares the rim, giving a closed surface.
const float cell = 2.0f / float(std::max(gw, gh));
Mesh m;
std::vector<uint32_t> backIndex(sheet.size());
for (const auto& v : sheet)
m.positions.emplace_back((v.xy.x - float(W) * 0.5f) * cell, (float(H) * 0.5f - v.xy.y) * cell,
v.h * cell * o.thickness);
for (uint32_t i = 0; i < sheet.size(); ++i) {
if (sheet[i].rim) {
backIndex[i] = i;
} else {
backIndex[i] = uint32_t(m.positions.size());
glm::vec3 p = m.positions[i];
m.positions.emplace_back(p.x, p.y, -p.z);
}
}
for (const auto& t : tris) {
m.triangles.push_back(t);
m.triangles.push_back({backIndex[t.x], backIndex[t.z], backIndex[t.y]});
}
m.rebuildTopology();
{
// Rim and near-rim flags per mesh vertex (back copies share the front's flags).
std::vector<uint8_t> isRim(m.vertexCount(), 0), nearRim(m.vertexCount(), 0);
for (uint32_t i = 0; i < sheet.size(); ++i) {
isRim[i] = sheet[i].rim;
nearRim[i] = nearRim[backIndex[i]] = !sheet[i].rim && sheet[i].d <= 4.0f;
}
// Straighten the pixel-stepped outline along itself.
std::vector<glm::vec3> next = m.positions;
for (int it = 0; it < 12; ++it) {
for (uint32_t v = 0; v < m.vertexCount(); ++v) {
if (!isRim[v]) continue;
glm::vec3 sum(0.0f);
int n = 0;
for (uint32_t u : m.neighbors(v))
if (isRim[u]) sum += m.positions[u], ++n;
if (n >= 2) next[v] = glm::mix(m.positions[v], sum / float(n), 0.5f);
}
m.positions = next;
}
// Even out the steep band around the rim, rim included, so the seam rounds off.
for (int it = 0; it < 20; ++it) {
for (uint32_t v = 0; v < m.vertexCount(); ++v) {
if (!nearRim[v] && !(isRim[v] && it % 2)) continue;
glm::vec3 sum(0.0f);
auto ring = m.neighbors(v);
for (uint32_t u : ring) sum += m.positions[u];
next[v] = glm::mix(m.positions[v], sum / float(ring.size()), 0.5f);
}
m.positions = next;
}
}
taubinSmooth(m, 12);
for (int i = 0; i < std::clamp(o.subdivisions, 0, 3); ++i) m = loopSubdivide(m);
m.rebuildTopology();
m.recomputeAllNormals();
return m;
}
} // namespace sf