Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(render): a skinned model can say what it is made of

esenginee3867494b046
diff --git a/desktop/public/scenes/mesh-skin-mat.esscene b/desktop/public/scenes/mesh-skin-mat.esscene
new file mode 100644
index 000000000..1d2f1c24d
--- /dev/null
+++ b/desktop/public/scenes/mesh-skin-mat.esscene
@@ -0,0 +1,114 @@
+{
+ "version": 1,
+ "name": "MeshSkinMaterial",
+ "entities": [
+ {
+ "id": 0,
+ "name": "Camera",
+ "parent": null,
+ "children": [],
+ "components": [
+ {
+ "type": "Transform",
+ "data": {
+ "position": {
+ "x": 60,
+ "y": 20,
+ "z": 10
+ }
+ }
+ },
+ {
+ "type": "Camera",
+ "data": {
+ "projectionType": 1,
+ "orthoSize": 100,
+ "isActive": true,
+ "priority": 0
+ }
+ }
+ ],
+ "visible": true
+ },
+ {
+ "id": 1,
+ "name": "Bone0",
+ "parent": null,
+ "children": [],
+ "components": [
+ {
+ "type": "Transform",
+ "data": {
+ "position": {
+ "x": 0,
+ "y": 0,
+ "z": 0
+ }
+ }
+ }
+ ],
+ "visible": true
+ },
+ {
+ "id": 2,
+ "name": "Bone1",
+ "parent": null,
+ "children": [],
+ "components": [
+ {
+ "type": "Transform",
+ "data": {
+ "position": {
+ "x": 60,
+ "y": 60,
+ "z": 0
+ }
+ }
+ }
+ ],
+ "visible": true
+ },
+ {
+ "id": 3,
+ "name": "Body",
+ "parent": null,
+ "children": [],
+ "components": [
+ {
+ "type": "Transform",
+ "data": {
+ "position": {
+ "x": 0,
+ "y": 0,
+ "z": 0
+ }
+ }
+ },
+ {
+ "type": "Mesh2D",
+ "data": {
+ "mesh": "scenes/skinned-arm.esmesh",
+ "color": {
+ "r": 1,
+ "g": 1,
+ "b": 1,
+ "a": 1
+ },
+ "enabled": true,
+ "material": "/scenes/mesh-skin-mat/skinmat.esmaterial"
+ }
+ },
+ {
+ "type": "MeshSkin",
+ "data": {
+ "joints": [
+ 1,
+ 2
+ ]
+ }
+ }
+ ],
+ "visible": true
+ }
+ ]
+}
diff --git a/desktop/public/scenes/mesh-skin-mat/skinmat.esmaterial b/desktop/public/scenes/mesh-skin-mat/skinmat.esmaterial
new file mode 100644
index 000000000..b40f12f6a
--- /dev/null
+++ b/desktop/public/scenes/mesh-skin-mat/skinmat.esmaterial
@@ -0,0 +1,10 @@
+{
+ "version": "1.0",
+ "type": "material",
+ "shader": "skinmat.esshader",
+ "blendMode": 0,
+ "depthTest": false,
+ "properties": {
+ "u_tint": { "r": 0, "g": 0, "b": 1, "a": 1 }
+ }
+}
diff --git a/desktop/public/scenes/mesh-skin-mat/skinmat.esshader b/desktop/public/scenes/mesh-skin-mat/skinmat.esshader
new file mode 100644
index 000000000..4908ce3bb
--- /dev/null
+++ b/desktop/public/scenes/mesh-skin-mat/skinmat.esshader
@@ -0,0 +1,24 @@
+#pragma shader "SkinnedMaterial"
+#pragma version 300 es
+#pragma domain Unlit2D
+#pragma param u_tint color default(1,1,1,1)
+
+#pragma fragment
+precision mediump float;
+
+in vec4 v_color;
+
+out vec4 fragColor;
+
+// Reads the per-object tint the vertex stage folded in, so a skinned draw that
+// reached this material through the wrong vertex source shows up as black.
+void main() {
+ fragColor = vec4(v_color.rgb * u_tint.rgb, 1.0);
+}
+#pragma end
+
+#pragma fragment wgsl
+@fragment fn fs_main(v : VSOut) -> @location(0) vec4f {
+ return vec4f(v.v_color.rgb * mc.u_tint.rgb, 1.0);
+}
+#pragma end
diff --git a/src/esengine/renderer/plugins/MeshPlugin.cpp b/src/esengine/renderer/plugins/MeshPlugin.cpp
index 5b59cb501..948cbcb0f 100644
--- a/src/esengine/renderer/plugins/MeshPlugin.cpp
+++ b/src/esengine/renderer/plugins/MeshPlugin.cpp
@@ -273,9 +273,9 @@ void MeshPlugin::collect(RenderCollectContext& collect_ctx) {
// source and would draw this at its local origin, so ask for the
// one compiled for THIS source; a failed variant falls back.
u32 materialProgram = 0;
- if (key.materialId != 0 && ctx.materials && !skinned) {
+ if (key.materialId != 0 && ctx.materials) {
materialProgram = ctx.materials->meshProgram(key.materialId, ctx.resources,
- resident->hasNormals);
+ resident->hasNormals, skinned);
if (materialProgram == 0) {
if (!warned_material_) {
warned_material_ = true;
diff --git a/src/esengine/renderer/store/MaterialStore.cpp b/src/esengine/renderer/store/MaterialStore.cpp
index 98b53b42a..9b27e371c 100644
--- a/src/esengine/renderer/store/MaterialStore.cpp
+++ b/src/esengine/renderer/store/MaterialStore.cpp
@@ -45,12 +45,14 @@ void MaterialStore::recreateGpuResources() {
}
u32 MaterialStore::meshProgram(u32 materialId, resource::ResourceManager& resources,
- bool withNormals) const {
+ bool withNormals, bool skinned) const {
const MaterialRecord* rec = find(materialId);
if (!rec || !rec->shaderRef.isValid()) return 0;
// Keyed by shader AND vertex shape: a layout may not declare an attribute its
- // shader ignores, so geometry with normals needs its own program.
- const u64 key = static_cast<u64>(rec->shaderRef.id()) | (withNormals ? (1ull << 32) : 0ull);
+ // shader ignores, so geometry with normals — or posed by bones — needs its own.
+ const u64 key = static_cast<u64>(rec->shaderRef.id())
+ | (withNormals ? (1ull << 32) : 0ull)
+ | (skinned ? (1ull << 33) : 0ull);
auto cached = mesh_programs_.find(key);
if (cached != mesh_programs_.end()) return cached->second;
@@ -68,6 +70,7 @@ u32 MaterialStore::meshProgram(u32 materialId, resource::ResourceManager& resour
std::vector<std::string> features = resource::ShaderParser::splitFeatures(src->second.features);
features.push_back("MESH");
if (withNormals) features.push_back("MESH_NORMALS");
+ if (skinned) features.push_back("SKINNED");
const auto target = resources.preferredShaderTarget();
const std::string vert = resource::ShaderParser::assembleStage(
parsed, resource::ShaderStage::Vertex, "", features, target);
diff --git a/src/esengine/renderer/store/MaterialStore.hpp b/src/esengine/renderer/store/MaterialStore.hpp
index 1f91d5ede..71aaa24bf 100644
--- a/src/esengine/renderer/store/MaterialStore.hpp
+++ b/src/esengine/renderer/store/MaterialStore.hpp
@@ -169,10 +169,11 @@ public:
/// The program that draws @p materialId on GPU-RESIDENT geometry, compiled on
/// first use and cached. @p withNormals selects the variant for geometry that
- /// carries them. 0 when the material's source was never kept or the variant
- /// fails — the caller then falls back rather than drawing it wrong.
+ /// carries them, @p skinned the one posed by bones instead of a per-object
+ /// matrix. 0 when the material's source was never kept or the variant fails —
+ /// the caller then falls back rather than drawing it wrong.
u32 meshProgram(u32 materialId, resource::ResourceManager& resources,
- bool withNormals = false) const;
+ bool withNormals = false, bool skinned = false) const;
/// Registers (or replaces) a shader's MaterialConstants layout — called when a shader
/// authored with #pragma param is compiled, so materials on it can pack their uniforms.
diff --git a/src/esengine/resource/ShaderParser.cpp b/src/esengine/resource/ShaderParser.cpp
index 1b9171947..6ba3265f2 100644
--- a/src/esengine/resource/ShaderParser.cpp
+++ b/src/esengine/resource/ShaderParser.cpp
@@ -123,6 +123,11 @@ std::string wgslCanonicalVSOut(bool lit) {
std::string canonical2DVertexStageWGSL(bool lit) {
std::string src = wgslCanonicalVSOut(lit);
src +=
+ "\n"
+ "#ifdef SKINNED\n"
+ "struct SkinConstants { bones : array<mat4x4f, 64> };\n"
+ "@group(0) @binding(5) var<uniform> skin : SkinConstants;\n"
+ "#endif\n"
"\n"
"struct VSIn {\n"
" @location(0) a_position : vec3f,\n"
@@ -131,6 +136,11 @@ std::string canonical2DVertexStageWGSL(bool lit) {
"#ifdef MESH_NORMALS\n"
" @location(3) a_normal : vec3f,\n"
"#endif\n"
+ "#ifdef SKINNED\n"
+ " @location(5) a_joints : vec4u,\n"
+ " @location(6) a_weights : vec4f,\n"
+ " @location(12) a_instTint : vec4f,\n"
+ "#else\n"
"#ifdef MESH\n"
" @location(8) a_model0 : vec4f,\n"
" @location(9) a_model1 : vec4f,\n"
@@ -143,11 +153,21 @@ std::string canonical2DVertexStageWGSL(bool lit) {
" @location(14) a_nrm1 : vec3f,\n"
" @location(15) a_nrm2 : vec3f,\n"
"#endif\n"
+ "#endif\n"
"};\n"
"\n"
"@vertex fn vs_main(v : VSIn) -> VSOut {\n"
" var out : VSOut;\n"
- "#ifdef MESH\n"
+ // Bones are already world-space, so a skinned mesh's own transform is not
+ // read — which is what glTF requires of one.
+ "#ifdef SKINNED\n"
+ " let pose = v.a_weights.x * skin.bones[v.a_joints.x]\n"
+ " + v.a_weights.y * skin.bones[v.a_joints.y]\n"
+ " + v.a_weights.z * skin.bones[v.a_joints.z]\n"
+ " + v.a_weights.w * skin.bones[v.a_joints.w];\n"
+ " let world = pose * vec4f(v.a_position, 1.0);\n"
+ " out.v_color = v.a_color * v.a_instTint;\n"
+ "#elif defined(MESH)\n"
" let world = mat4x4f(v.a_model0, v.a_model1, v.a_model2, v.a_model3) * vec4f(v.a_position, 1.0);\n"
" out.v_color = v.a_color * v.a_instTint;\n"
"#else\n"
@@ -157,7 +177,11 @@ std::string canonical2DVertexStageWGSL(bool lit) {
" out.pos = frame.projection * world;\n"
" out.v_texCoord = v.a_texCoord;\n"
"#ifdef MESH_NORMALS\n"
+ "#ifdef SKINNED\n"
+ " out.v_worldNormal = mat3x3f(pose[0].xyz, pose[1].xyz, pose[2].xyz) * v.a_normal;\n"
+ "#else\n"
" out.v_worldNormal = mat3x3f(v.a_nrm0, v.a_nrm1, v.a_nrm2) * v.a_normal;\n"
+ "#endif\n"
" out.v_worldXYZ = world.xyz;\n"
"#endif\n";
if (lit) {
@@ -214,11 +238,9 @@ std::string canonicalPPVertexStage() {
"}\n";
}
-// Canonical 2D vertex stage for fragment-only .esshaders: the batch path bakes the
-// world transform into the vertices, so all 2D shaders share this pass-through.
-// The engine's vertex stage, in its two VERTEX SOURCES: the batch stream arrives
-// in world space, a resident mesh in local space with a per-object transform. So
-// MESH is how ONE material serves both rather than a second shader existing.
+// Canonical 2D vertex stage for fragment-only .esshaders, in its three VERTEX
+// SOURCES: batch (already world space), resident mesh (local + a per-object
+// transform), skinned (posed by bones). One material serves all three.
std::string canonical2DVertexStage(bool lit) {
std::string src =
"layout(location = 0) in vec3 a_position;\n"
@@ -227,6 +249,15 @@ std::string canonical2DVertexStage(bool lit) {
"#ifdef MESH_NORMALS\n"
"layout(location = 3) in vec3 a_normal;\n"
"#endif\n"
+ "#ifdef SKINNED\n"
+ "layout(location = 5) in uvec4 a_joints;\n"
+ "layout(location = 6) in vec4 a_weights;\n"
+ "layout(location = 12) in vec4 a_instTint;\n"
+ // This draw's pose, rewritten immediately before it (SkinConstants, binding 5).
+ "layout(std140) uniform SkinConstants {\n"
+ " mat4 u_bones[64];\n"
+ "};\n"
+ "#else\n"
"#ifdef MESH\n"
"layout(location = 8) in vec4 a_model0;\n"
"layout(location = 9) in vec4 a_model1;\n"
@@ -239,6 +270,7 @@ std::string canonical2DVertexStage(bool lit) {
"layout(location = 14) in vec3 a_nrm1;\n"
"layout(location = 15) in vec3 a_nrm2;\n"
"#endif\n"
+ "#endif\n"
"\n"
"out vec4 v_color;\n"
"out vec2 v_texCoord;\n"
@@ -252,7 +284,16 @@ std::string canonical2DVertexStage(bool lit) {
src +=
"\n"
"void main() {\n"
- "#ifdef MESH\n"
+ // Bones are already world-space, so a skinned mesh's own transform is not
+ // read — which is what glTF requires of one.
+ "#ifdef SKINNED\n"
+ " mat4 skin = a_weights.x * u_bones[a_joints.x]\n"
+ " + a_weights.y * u_bones[a_joints.y]\n"
+ " + a_weights.z * u_bones[a_joints.z]\n"
+ " + a_weights.w * u_bones[a_joints.w];\n"
+ " vec4 world = skin * vec4(a_position, 1.0);\n"
+ " v_color = a_color * a_instTint;\n"
+ "#elif defined(MESH)\n"
" vec4 world = mat4(a_model0, a_model1, a_model2, a_model3) * vec4(a_position, 1.0);\n"
" v_color = a_color * a_instTint;\n"
"#else\n"
@@ -262,7 +303,11 @@ std::string canonical2DVertexStage(bool lit) {
" gl_Position = u_projection * world;\n"
" v_texCoord = a_texCoord;\n"
"#ifdef MESH_NORMALS\n"
+ "#ifdef SKINNED\n"
+ " v_worldNormal = mat3(skin) * a_normal;\n"
+ "#else\n"
" v_worldNormal = mat3(a_nrm0, a_nrm1, a_nrm2) * a_normal;\n"
+ "#endif\n"
" v_worldXYZ = world.xyz;\n"
"#endif\n";
if (lit) {
diff --git a/tools/renderScenes.mjs b/tools/renderScenes.mjs
index a682662f2..c65db75a6 100644
--- a/tools/renderScenes.mjs
+++ b/tools/renderScenes.mjs
@@ -48,6 +48,10 @@ export const SCENES = [
// the half bound to it rises by exactly that and the half bound to bone 0 does
// not: the elbow is where the geometry says, not where a whole-mesh move puts it.
{ id: "mesh-skin", tier: "pr", webgpu: true, env: { ESTELLA_VERIFY_SCENE: "/scenes/mesh-skin.esscene", ESTELLA_VERIFY_W: "256", ESTELLA_VERIFY_H: "256", ESTELLA_VERIFY_STEPS: "2", ESTELLA_VERIFY_EXPECT: "[{\"x\":0.35,\"y\":0.6,\"rgb\":[0,255,0],\"tol\":30},{\"x\":0.75,\"y\":0.35,\"rgb\":[0,255,0],\"tol\":30},{\"x\":0.75,\"y\":0.60,\"rgb\":[0,0,0],\"tol\":20}]" } },
+ // The same arm, drawn by a MATERIAL — a program is compiled per vertex source
+ // and there was no skinned one, so a skinned mesh fell back to the engine's
+ // shader and ignored what it was made of (sabotage: the probes come back white).
+ { id: "mesh-skin-material", tier: "pr", webgpu: true, env: { ESTELLA_VERIFY_SCENE: "/scenes/mesh-skin-mat.esscene", ESTELLA_VERIFY_W: "256", ESTELLA_VERIFY_H: "256", ESTELLA_VERIFY_STEPS: "2", ESTELLA_VERIFY_EXPECT: "[{\"x\":0.35,\"y\":0.6,\"rgb\":[0,0,255],\"tol\":30},{\"x\":0.75,\"y\":0.35,\"rgb\":[0,0,255],\"tol\":30},{\"x\":0.75,\"y\":0.60,\"rgb\":[0,0,0],\"tol\":20}]" } },
// A glTF animation, through the shipped importer. The scene puts the quad at
// x=150 and the clip runs -120 to 120 over a second, so 30 fixed steps is its
// midpoint: the model is at the origin and where the SCENE put it is empty.