Code-first video & image, rendered natively on WebGPU.
Load OBJ and FBX textures; assign FBX materials to their meshes
Claude604a62ec3d40
diff --git a/packages/framefields/src/extended-models-render.test.ts b/packages/framefields/src/extended-models-render.test.tsindex ef71fbc2..060d8692 100644--- a/packages/framefields/src/extended-models-render.test.ts+++ b/packages/framefields/src/extended-models-render.test.ts@@ -2,7 +2,7 @@ import fs from "node:fs/promises";import os from "node:os";import path from "node:path";import { Signal } from "@framefields/core";-import { describe, expect, it } from "vitest";+import { describe, expect, it, vi } from "vitest";import { Composition, Layer, Layer3D, LayerAnimation, Light } from "./index.js";import { computeMse, decodePngPixels, getPixel } from "./test-utils/pixels.js";@@ -529,3 +529,191 @@ end_headerawait fs.rm(tmpDir, { recursive: true, force: true });});});++describe("Model textures (OBJ / FBX)", () => {+ /** A 64×64 PNG: top half red, bottom half blue (image space). */+ const redOverBluePng = async (file: string) => {+ const { Canvas } = await import("skia-canvas");+ const c = new Canvas(64, 64);+ const ctx = c.getContext("2d");+ ctx.fillStyle = "rgb(255,0,0)";+ ctx.fillRect(0, 0, 64, 32);+ ctx.fillStyle = "rgb(0,0,255)";+ ctx.fillRect(0, 32, 64, 32);+ await fs.writeFile(file, await c.toBuffer("png"));+ };++ /** Renders a model layer unlit in a 400×300 frame. */+ const render = async (model: ReturnType<typeof Layer.obj>) => {+ const comp = new Composition({+ width: 400,+ height: 300,+ fps: 30,+ durationMs: 1000,+ backgroundColor: "#000000",+ });+ comp.add(+ Layer.camera({+ id: "cam3d",+ x: 200,+ y: 150,+ z: -600,+ targetX: 200,+ targetY: 150,+ targetZ: 0,+ }),+ );+ comp.add(model);+ return decodePngPixels(await comp.renderFrame({ frame: 0 }));+ };++ /** The texture's top (red) shows at the top of the quad, its bottom (blue) below. */+ const expectRedOverBlue = (img: Awaited<ReturnType<typeof render>>) => {+ const top = getPixel(img, 200, 115);+ const bottom = getPixel(img, 200, 185);+ expect(top.r).toBeGreaterThan(150);+ expect(top.b).toBeLessThan(80);+ expect(bottom.b).toBeGreaterThan(150);+ expect(bottom.r).toBeLessThan(80);+ };++ it("draws an OBJ's map_Kd texture from its .mtl", async () => {+ const tmpDir = await fs.mkdtemp(+ path.join(os.tmpdir(), "framefields-obj-tex-"),+ );+ await fs.mkdir(path.join(tmpDir, "maps"));+ await redOverBluePng(path.join(tmpDir, "maps", "paint.png"));+ await fs.writeFile(+ path.join(tmpDir, "quad.mtl"),+ "newmtl Paint\nKd 1 1 1\nmap_Kd maps/paint.png\n",+ );+ // Y up, as OBJ files are authored: vt (0, 0) is the image's bottom-left.+ await fs.writeFile(+ path.join(tmpDir, "quad.obj"),+ `mtllib quad.mtl+v -100 -100 0+v 100 -100 0+v 100 100 0+v -100 100 0+vt 0 0+vt 1 0+vt 1 1+vt 0 1+usemtl Paint+f 1/1 2/2 3/3 4/4+`,+ );++ const img = await render(+ Layer.obj(path.join(tmpDir, "quad.obj"), {+ id: "obj-tex",+ x: 200,+ y: 150,+ z: 0,+ // The world is y-down; turn the y-up model upright.+ rotateX: 180,+ material: "unlit",+ }),+ );+ expectRedOverBlue(img);+ await fs.rm(tmpDir, { recursive: true, force: true });+ });++ it("still renders when a texture cannot be decoded", async () => {+ const tmpDir = await fs.mkdtemp(+ path.join(os.tmpdir(), "framefields-bad-tex-"),+ );+ await fs.writeFile(+ path.join(tmpDir, "paint.tga"),+ Buffer.from([1, 2, 3, 4]),+ );+ await fs.writeFile(+ path.join(tmpDir, "quad.mtl"),+ "newmtl Paint\nKd 0 1 0\nmap_Kd paint.tga\n",+ );+ await fs.writeFile(+ path.join(tmpDir, "quad.obj"),+ "mtllib quad.mtl\nv -100 -100 0\nv 100 -100 0\nv 100 100 0\nv -100 100 0\nusemtl Paint\nf 1 2 3 4\n",+ );+ const warn = vi.spyOn(console, "warn").mockImplementation(() => {});+ const img = await render(+ Layer.obj(path.join(tmpDir, "quad.obj"), {+ id: "obj-bad-tex",+ x: 200,+ y: 150,+ z: 0,+ material: "unlit",+ }),+ );+ // Drawn untextured in its Kd color.+ const center = getPixel(img, 200, 150);+ expect(center.g).toBeGreaterThan(150);+ expect(warn).toHaveBeenCalledWith(+ expect.stringContaining("Could not decode texture"),+ expect.anything(),+ );+ warn.mockRestore();+ await fs.rm(tmpDir, { recursive: true, force: true });+ });++ it("draws an FBX material's diffuse texture", async () => {+ const tmpDir = await fs.mkdtemp(+ path.join(os.tmpdir(), "framefields-fbx-tex-"),+ );+ await redOverBluePng(path.join(tmpDir, "paint.png"));+ await fs.writeFile(+ path.join(tmpDir, "quad.fbx"),+ `; FBX 7.4.0 project file+Objects: {+ Geometry: 1001, "Geometry::Quad", "Mesh" {+ Vertices: *12 {+ a: -100,-100,0, 100,-100,0, 100,100,0, -100,100,0+ }+ PolygonVertexIndex: *4 {+ a: 0,1,2,-4+ }+ LayerElementUV: 0 {+ Version: 101+ Name: "UVMap"+ MappingInformationType: "ByControlPoint"+ ReferenceInformationType: "Direct"+ UV: *8 {+ a: 0,0, 1,0, 1,1, 0,1+ }+ }+ }+ Model: 3001, "Model::Quad", "Mesh" {+ }+ Material: 2001, "Material::Paint", "" {+ Properties70: {+ P: "DiffuseColor", "Color", "", "A",1,1,1+ }+ }+ Texture: 4001, "Texture::PaintDiffuse", "" {+ RelativeFilename: "paint.png"+ }+}++Connections: {+ C: "OO", 1001, 3001+ C: "OO", 2001, 3001+ C: "OP", 4001, 2001, "DiffuseColor"+}+`,+ );++ const img = await render(+ Layer.fbx(path.join(tmpDir, "quad.fbx"), {+ id: "fbx-tex",+ x: 200,+ y: 150,+ z: 0,+ // The world is y-down; turn the y-up model upright.+ rotateX: 180,+ material: "unlit",+ }),+ );+ expectRedOverBlue(img);+ await fs.rm(tmpDir, { recursive: true, force: true });+ });+});diff --git a/packages/webgpu-renderers/src/loaders/fbx-loader.ts b/packages/webgpu-renderers/src/loaders/fbx-loader.tsindex e4537a02..369fca30 100644--- a/packages/webgpu-renderers/src/loaders/fbx-loader.ts+++ b/packages/webgpu-renderers/src/loaders/fbx-loader.ts@@ -1,6 +1,12 @@import fs from "node:fs";+import path from "node:path";import { inflateSync } from "node:zlib";import { type Vec3, Vector3Math } from "../math3d/index.js";+import {+ readTextureFile,+ textureMime,+ warnMissingTexture,+} from "./texture-files.js";import type {AnimationClip3D,LoadModelOptions,@@ -37,8 +43,7 @@ export class FbxLoader {}public static isBinary(buffer: ArrayBuffer | Uint8Array): boolean {- const view =- buffer instanceof Uint8Array ? buffer : new Uint8Array(buffer);+ const view = buffer instanceof Uint8Array ? buffer : new Uint8Array(buffer);if (view.length < 27) return false;const header = "Kaydara FBX Binary \0\x1a\0";for (let i = 0; i < header.length; i++) {@@ -302,6 +307,91 @@ export class FbxLoader {return root;}+ /** Material properties a texture can feed, by the FBX (or Maya / 3ds Max) property name. */+ private static textureSlot(prop: string): "diffuse" | "normal" | undefined {+ const p = prop.toLowerCase();+ if (+ p === "diffusecolor" ||+ p === "diffuse" ||+ p.endsWith("|basecolor") ||+ p.endsWith("|base_color") ||+ p === "basecolor"+ )+ return "diffuse";+ if (+ p === "normalmap" ||+ p.endsWith("|normalcamera") ||+ p.endsWith("|bump_map")+ )+ return "normal";+ return undefined;+ }++ /**+ * Fills a material's diffuse / normal map from its connected Texture+ * objects: the bytes embedded in the Video's `Content`, else the file+ * named by `RelativeFilename` / `FileName`, looked up next to the model.+ */+ private static attachTextures(+ material: Material3D,+ materialId: number | string,+ objects: Map<number | string, { name: string; node: FBXNode }>,+ connections: Array<{+ childId: number | string;+ parentId: number | string;+ propName?: string;+ }>,+ basePath: string | undefined,+ ): void {+ const stringChild = (node: FBXNode | undefined, name: string) => {+ const value = node?.children.find((c) => c.name === name)?.props[0];+ return typeof value === "string" && value ? value : undefined;+ };+ for (const c of connections) {+ if (c.parentId !== materialId || !c.propName) continue;+ const slot = FbxLoader.textureSlot(c.propName);+ const texture = objects.get(c.childId);+ if (!slot || texture?.node.name !== "Texture") continue;+ if (+ slot === "diffuse"+ ? material.diffuseTextureBuffer+ : material.normalTextureBuffer+ )+ continue;++ const video = connections+ .filter((v) => v.parentId === c.childId && !v.propName)+ .map((v) => objects.get(v.childId)?.node)+ .find((n) => n?.name === "Video");+ const ref =+ stringChild(texture.node, "RelativeFilename") ??+ stringChild(texture.node, "FileName") ??+ stringChild(video, "RelativeFilename") ??+ stringChild(video, "Filename");++ let bytes: Uint8Array | undefined;+ const content = video?.children.find((n) => n.name === "Content")+ ?.props[0];+ if (content instanceof Uint8Array && content.length > 0) bytes = content;+ else if (ref) bytes = readTextureFile(ref, basePath)?.bytes;++ if (!bytes) {+ if (ref) warnMissingTexture("FbxLoader", material.name, ref);+ continue;+ }+ if (slot === "diffuse") {+ material.diffuseTextureBuffer = bytes;+ material.diffuseTextureMime = textureMime(ref ?? "");+ if (ref) material.diffuseTexture = ref;+ // FBX UVs commonly tile; textures repeat by default in FBX tools.+ material.textureWrap = "repeat";+ } else {+ material.normalTextureBuffer = bytes;+ if (ref) material.normalTexture = ref;+ }+ }+ }+/*** Builds ready-to-render Model3DData from parsed FBX node tree.*/@@ -351,11 +441,32 @@ export class FbxLoader {}}+ // Geometry -> Model -> Material: a mesh uses its model's first material.+ const materialNameForGeometry = (geometryId: number | string) => {+ for (const c of connections) {+ // Some exporters connect the material to the geometry itself.+ const mat =+ c.parentId === geometryId ? objects.get(c.childId) : undefined;+ if (mat?.node.name === "Material" && !c.propName) return mat.name;+ }+ for (const g of connections) {+ if (g.childId !== geometryId || g.propName) continue;+ for (const m of connections) {+ if (m.parentId !== g.parentId || m.propName) continue;+ const mat = objects.get(m.childId);+ if (mat?.node.name === "Material") return mat.name;+ }+ }+ return undefined;+ };+// 1. Parse Geometry Meshesconst meshes: Mesh3DData[] = [];const materials: Record<string, Material3D> = {};- function extractArrayProp(node: FBXNode | undefined): number[] | Float32Array | Float64Array | Int32Array {+ function extractArrayProp(+ node: FBXNode | undefined,+ ): number[] | Float32Array | Float64Array | Int32Array {if (!node) return [];if (node.props[0] instanceof Float32Array ||@@ -369,14 +480,23 @@ export class FbxLoader {if (aChild && aChild.props.length > 0) {return aChild.props as number[];}- if (Array.isArray(node.props) && node.props.length > 0 && typeof node.props[0] === "number") {+ if (+ Array.isArray(node.props) &&+ node.props.length > 0 &&+ typeof node.props[0] === "number"+ ) {return node.props as number[];}return [];}for (const [id, entry] of objects) {- if (entry.node.name === "Geometry" && (entry.type === "Mesh" || entry.node.props[2] === "Mesh" || entry.type === "")) {+ if (+ entry.node.name === "Geometry" &&+ (entry.type === "Mesh" ||+ entry.node.props[2] === "Mesh" ||+ entry.type === "")+ ) {const geomNode = entry.node;const verticesNode = geomNode.children.find(@@ -403,13 +523,10 @@ export class FbxLoader {const rawNormals = extractArrayProp(normalsNode);const uvsNode = uvLayer?.children.find((c) => c.name === "UV");- const uvIndexNode = uvLayer?.children.find(- (c) => c.name === "UVIndex",- );+ const uvIndexNode = uvLayer?.children.find((c) => c.name === "UVIndex");const rawUvs = extractArrayProp(uvsNode);const rawUvIndices = extractArrayProp(uvIndexNode);-// Triangulate polygons (indices with negative termination)const triangles: Array<[number, number, number]> = [];const triVertexIndexSequence: number[] = [];@@ -596,7 +713,7 @@ export class FbxLoader {: new Uint16Array(outIndices),jointIndices,jointWeights,- materialName: entry.name,+ materialName: materialNameForGeometry(id) ?? entry.name,});}}@@ -620,7 +737,10 @@ export class FbxLoader {const g = Number(p.props[5] ?? 1);const b = Number(p.props[6] ?? 1);diffuseColor = [r, g, b, 1.0];- } else if (pName === "Shininess" || pName === "ShininessExponent") {+ } else if (+ pName === "Shininess" ||+ pName === "ShininessExponent"+ ) {shininess = Number(p.props[4] ?? 32);} else if (pName === "Opacity") {opacity = Number(p.props[4] ?? 1);@@ -630,12 +750,20 @@ export class FbxLoader {}}- materials[entry.name] = {+ const material: Material3D = {name: entry.name,diffuseColor,shininess,opacity,};+ FbxLoader.attachTextures(+ material,+ entry.id,+ objects,+ connections,+ options.basePath,+ );+ materials[entry.name] = material;}}@@ -820,7 +948,9 @@ interface FbxCurve {* Classifies a curve node from its OP connection property ("Lcl Translation",* "Lcl Rotation", "Lcl Scaling") or, failing that, its short name ("T", "R", "S").*/-function transformKindFromProperty(name: string | undefined): TransformKind | undefined {+function transformKindFromProperty(+ name: string | undefined,+): TransformKind | undefined {switch (name) {case "Lcl Translation":case "T":@@ -851,9 +981,14 @@ function axisFromProperty(name: string | undefined): number | undefined {}/** Numeric values of a Properties70 `P` entry (they start at props[4]). */-function readProperty70(node: FBXNode | undefined, name: string): number[] | undefined {+function readProperty70(+ node: FBXNode | undefined,+ name: string,+): number[] | undefined {const props70 = node?.children.find((c) => c.name === "Properties70");- const p = props70?.children.find((c) => c.name === "P" && c.props[0] === name);+ const p = props70?.children.find(+ (c) => c.name === "P" && c.props[0] === name,+ );if (!p) return undefined;return p.props.slice(4).map((v) => Number(v));}@@ -867,7 +1002,11 @@ function staticTransform(const fallback = kind === "S" ? 1 : 0;const modelProp = readProperty70(model,- kind === "T" ? "Lcl Translation" : kind === "R" ? "Lcl Rotation" : "Lcl Scaling",+ kind === "T"+ ? "Lcl Translation"+ : kind === "R"+ ? "Lcl Rotation"+ : "Lcl Scaling",);const out: Vec3 = [fallback, fallback, fallback];for (let axis = 0; axis < 3; axis++) {@@ -965,7 +1104,10 @@ function buildTransformTrack(group: {}const q = fbxEulerToQuat(v, order);// Keep neighbours in the same hemisphere so slerp takes the short way.- if (prev && prev[0] * q[0] + prev[1] * q[1] + prev[2] * q[2] + prev[3] * q[3] < 0) {+ if (+ prev &&+ prev[0] * q[0] + prev[1] * q[1] + prev[2] * q[2] + prev[3] * q[3] < 0+ ) {for (let i = 0; i < 4; i++) q[i] = -q[i]!;}prev = q;@@ -996,8 +1138,7 @@ export function loadFBX(if (typeof srcOrData === "string") {const trimmed = srcOrData.trim();const isInline =- trimmed.includes("FBXHeaderExtension") ||- trimmed.includes("Kaydara FBX");+ trimmed.includes("FBXHeaderExtension") || trimmed.includes("Kaydara FBX");if (!isInline && typeof fs !== "undefined" && fs.existsSync) {try {let filePath = srcOrData.startsWith("file://")@@ -1009,10 +1150,11 @@ export function loadFBX(if (fs.existsSync(filePath)) {const buf = fs.readFileSync(filePath);data = new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);+ // Textures are looked up next to the model.+ options = { basePath: path.dirname(filePath), ...options };}} catch (_) {}}}return FbxLoader.parse(data, options);}-diff --git a/packages/webgpu-renderers/src/loaders/obj-loader.ts b/packages/webgpu-renderers/src/loaders/obj-loader.tsindex ee4dba9c..c665a19c 100644--- a/packages/webgpu-renderers/src/loaders/obj-loader.ts+++ b/packages/webgpu-renderers/src/loaders/obj-loader.ts@@ -1,6 +1,7 @@import fs from "node:fs";import path from "node:path";import { type Vec3, Vector3Math } from "../math3d/index.js";+import { readTextureFile, warnMissingTexture } from "./texture-files.js";import type {LoadModelOptions,Material3D,@@ -22,11 +23,9 @@ export class ObjLoader {const options: LoadModelOptions =typeof mtlTextOrOptions === "object" && mtlTextOrOptions !== null? mtlTextOrOptions- : maybeOptions ?? {};+ : (maybeOptions ?? {});const mtlText =- typeof mtlTextOrOptions === "string"- ? mtlTextOrOptions- : options.mtlText;+ typeof mtlTextOrOptions === "string" ? mtlTextOrOptions : options.mtlText;const rawPositions: Vec3[] = [];const rawNormals: Vec3[] = [];@@ -52,7 +51,10 @@ export class ObjLoader {const materials: Record<string, Material3D> = {};if (mtlText) {- Object.assign(materials, ObjLoader.parseMtl(mtlText));+ Object.assign(+ materials,+ ObjLoader.loadTextures(ObjLoader.parseMtl(mtlText), options.basePath),+ );}const lines = objText.split(/\r?\n/);@@ -71,7 +73,14 @@ export class ObjLoader {const resolvedMtlPath = path.resolve(options.basePath, mtlFileName);if (fs.existsSync(resolvedMtlPath)) {const content = fs.readFileSync(resolvedMtlPath, "utf-8");- Object.assign(materials, ObjLoader.parseMtl(content));+ // Texture paths are relative to the .mtl file.+ Object.assign(+ materials,+ ObjLoader.loadTextures(+ ObjLoader.parseMtl(content),+ path.dirname(resolvedMtlPath),+ ),+ );}} catch (_) {}}@@ -171,12 +180,12 @@ export class ObjLoader {maxZ = 1;}- let centerX = (minX + maxX) * 0.5;- let centerY = (minY + maxY) * 0.5;- let centerZ = (minZ + maxZ) * 0.5;- let sizeX = Math.max(1e-4, maxX - minX);- let sizeY = Math.max(1e-4, maxY - minY);- let sizeZ = Math.max(1e-4, maxZ - minZ);+ const centerX = (minX + maxX) * 0.5;+ const centerY = (minY + maxY) * 0.5;+ const centerZ = (minZ + maxZ) * 0.5;+ const sizeX = Math.max(1e-4, maxX - minX);+ const sizeY = Math.max(1e-4, maxY - minY);+ const sizeZ = Math.max(1e-4, maxZ - minZ);const maxDim = Math.max(sizeX, sizeY, sizeZ);let scaleMultiplier = 1.0;@@ -324,8 +333,7 @@ export class ObjLoader {const finalCenterX = (finalMinX + finalMaxX) * 0.5;const finalCenterY = (finalMinY + finalMaxY) * 0.5;const finalCenterZ = (finalMinZ + finalMaxZ) * 0.5;- const sphereRadius =- Math.hypot(finalSizeX, finalSizeY, finalSizeZ) * 0.5;+ const sphereRadius = Math.hypot(finalSizeX, finalSizeY, finalSizeZ) * 0.5;return {meshes,@@ -373,7 +381,12 @@ export class ObjLoader {const r = Number.parseFloat(parts[1] ?? "1");const g = Number.parseFloat(parts[2] ?? "1");const b = Number.parseFloat(parts[3] ?? "1");- currentMat.diffuseColor = [r, g, b, currentMat.diffuseColor?.[3] ?? 1.0];+ currentMat.diffuseColor = [+ r,+ g,+ b,+ currentMat.diffuseColor?.[3] ?? 1.0,+ ];} else if (cmd === "ks") {// Specular color: Ks r g bconst r = Number.parseFloat(parts[1] ?? "1");@@ -399,14 +412,76 @@ export class ObjLoader {}} else if (cmd === "map_kd") {// Diffuse texture map- currentMat.diffuseTexture = parts.slice(1).join(" ");- } else if (cmd === "map_bump" || cmd === "bump") {- // Normal / bump map- currentMat.normalTexture = parts.slice(1).join(" ");+ currentMat.diffuseTexture = ObjLoader.mapFileName(parts);+ // MTL maps repeat unless `-clamp on`.+ const clamp = parts.findIndex((p) => p.toLowerCase() === "-clamp");+ currentMat.textureWrap =+ clamp > 0 && parts[clamp + 1]?.toLowerCase() === "on"+ ? "clamp"+ : "repeat";+ } else if (cmd === "norm" || cmd === "map_kn") {+ // Tangent-space normal map+ currentMat.normalTexture = ObjLoader.mapFileName(parts);+ }+ // map_Bump / bump are height maps, which the renderer does not use.+ }+ }++ return materials;+ }++ /** Option arities of MTL map statements (`-o`, `-s`, `-t` take 1–3 numbers). */+ private static readonly MAP_OPTION_ARGS: Record<string, number> = {+ "-blendu": 1,+ "-blendv": 1,+ "-cc": 1,+ "-clamp": 1,+ "-texres": 1,+ "-bm": 1,+ "-boost": 1,+ "-imfchan": 1,+ "-type": 1,+ "-mm": 2,+ };++ /** The file name of a map statement (`map_Kd -s 1 1 1 wood grain.png`), past its options. */+ private static mapFileName(parts: string[]): string {+ let i = 1;+ while (i < parts.length && parts[i]?.startsWith("-")) {+ const option = parts[i]!.toLowerCase();+ i++;+ if (option === "-o" || option === "-s" || option === "-t") {+ for (let n = 0; n < 3 && i < parts.length; n++) {+ if (Number.isNaN(Number(parts[i]))) break;+ i++;}+ } else {+ i += ObjLoader.MAP_OPTION_ARGS[option] ?? 0;}}+ return parts.slice(i).join(" ");+ }+ /** Reads the diffuse and normal maps the materials name, relative to `dir`. */+ private static loadTextures(+ materials: Record<string, Material3D>,+ dir: string | undefined,+ ): Record<string, Material3D> {+ if (!dir) return materials;+ for (const mat of Object.values(materials)) {+ if (mat.diffuseTexture && !mat.diffuseTextureBuffer) {+ const file = readTextureFile(mat.diffuseTexture, dir);+ if (file) {+ mat.diffuseTextureBuffer = file.bytes;+ mat.diffuseTextureMime = file.mime;+ } else warnMissingTexture("ObjLoader", mat.name, mat.diffuseTexture);+ }+ if (mat.normalTexture && !mat.normalTextureBuffer) {+ const file = readTextureFile(mat.normalTexture, dir);+ if (file) mat.normalTextureBuffer = file.bytes;+ else warnMissingTexture("ObjLoader", mat.name, mat.normalTexture);+ }+ }return materials;}}@@ -464,4 +539,3 @@ export function loadOBJ(}return ObjLoader.parse(text, mtlText, options);}-diff --git a/packages/webgpu-renderers/src/loaders/texture-files.test.ts b/packages/webgpu-renderers/src/loaders/texture-files.test.tsnew file mode 100644index 00000000..370f287d--- /dev/null+++ b/packages/webgpu-renderers/src/loaders/texture-files.test.ts@@ -0,0 +1,143 @@+import fs from "node:fs";+import os from "node:os";+import path from "node:path";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { FbxLoader, loadFBX } from "./fbx-loader.js";+import { loadOBJ, ObjLoader } from "./obj-loader.js";+import { readTextureFile } from "./texture-files.js";++const PNG_BYTES = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 1, 2, 3, 4]);++const QUAD_OBJ = `mtllib quad.mtl+v -1 -1 0+v 1 -1 0+v 1 1 0+v -1 1 0+vt 0 0+vt 1 0+vt 1 1+vt 0 1+usemtl Wood+f 1/1 2/2 3/3 4/4+`;++const fbxWithTexture = (fileName: string, content = "") => `+; FBX 7.4.0 project file+Objects: {+ Geometry: 1001, "Geometry::Quad", "Mesh" {+ Vertices: *12 {+ a: -10,-10,0, 10,-10,0, 10,10,0, -10,10,0+ }+ PolygonVertexIndex: *4 {+ a: 0,1,2,-4+ }+ }+ Model: 3001, "Model::QuadModel", "Mesh" {+ }+ Material: 2001, "Material::Painted", "" {+ Properties70: {+ P: "DiffuseColor", "Color", "", "A",1,1,1+ }+ }+ Texture: 4001, "Texture::PaintedDiffuse", "" {+ FileName: "C:\\\\Users\\\\artist\\\\textures\\\\${fileName}"+ RelativeFilename: "textures\\\\${fileName}"+ }${content}+}++Connections: {+ C: "OO", 1001, 3001+ C: "OO", 2001, 3001+ C: "OP", 4001, 2001, "DiffuseColor"+}+`;++describe("model textures", () => {+ let dir: string;+ beforeEach(() => {+ dir = fs.mkdtempSync(path.join(os.tmpdir(), "framefields-textures-"));+ });+ afterEach(() => {+ fs.rmSync(dir, { recursive: true, force: true });+ vi.restoreAllMocks();+ });++ it("finds a texture by its file name when the written path is absolute elsewhere", () => {+ fs.mkdirSync(path.join(dir, "textures"));+ fs.writeFileSync(path.join(dir, "textures", "wood.png"), PNG_BYTES);+ const file = readTextureFile("C:\\Users\\artist\\maps\\wood.png", dir);+ expect(file?.bytes).toEqual(PNG_BYTES);+ expect(file?.mime).toBe("image/png");+ });++ it("loads OBJ map_Kd and norm textures relative to the .mtl file, past map options", () => {+ fs.mkdirSync(path.join(dir, "maps"));+ fs.writeFileSync(path.join(dir, "maps", "wood grain.png"), PNG_BYTES);+ fs.writeFileSync(path.join(dir, "maps", "wood_n.png"), PNG_BYTES);+ fs.writeFileSync(+ path.join(dir, "quad.mtl"),+ `newmtl Wood+Kd 1 1 1+map_Kd -s 2 2 1 -bm 1.0 maps/wood grain.png+norm maps/wood_n.png+map_Bump maps/height.png+`,+ );+ fs.writeFileSync(path.join(dir, "quad.obj"), QUAD_OBJ);++ const model = loadOBJ(path.join(dir, "quad.obj"));+ const wood = model.materials.Wood!;+ expect(wood.diffuseTexture).toBe("maps/wood grain.png");+ expect(wood.diffuseTextureBuffer).toEqual(PNG_BYTES);+ expect(wood.textureWrap).toBe("repeat");+ expect(wood.normalTextureBuffer).toEqual(PNG_BYTES);+ expect(model.meshes[0]?.materialName).toBe("Wood");+ });++ it("parses -clamp on as clamped addressing", () => {+ const mats = ObjLoader.parseMtl("newmtl A\nmap_Kd -clamp on a.png\n");+ expect(mats.A?.diffuseTexture).toBe("a.png");+ expect(mats.A?.textureWrap).toBe("clamp");+ });++ it("warns when an OBJ texture is missing", () => {+ const warn = vi.spyOn(console, "warn").mockImplementation(() => {});+ fs.writeFileSync(+ path.join(dir, "quad.mtl"),+ "newmtl Wood\nmap_Kd gone.png\n",+ );+ fs.writeFileSync(path.join(dir, "quad.obj"), QUAD_OBJ);+ const model = loadOBJ(path.join(dir, "quad.obj"));+ expect(model.materials.Wood?.diffuseTextureBuffer).toBeUndefined();+ expect(warn).toHaveBeenCalledWith(expect.stringContaining("gone.png"));+ });++ it("assigns FBX meshes their model's material and loads its diffuse texture", () => {+ fs.mkdirSync(path.join(dir, "textures"));+ fs.writeFileSync(path.join(dir, "textures", "paint.png"), PNG_BYTES);+ fs.writeFileSync(path.join(dir, "quad.fbx"), fbxWithTexture("paint.png"));++ const model = loadFBX(path.join(dir, "quad.fbx"));+ expect(model.meshes[0]?.materialName).toBe("Painted");+ const mat = model.materials.Painted!;+ expect(mat.diffuseTextureBuffer).toEqual(PNG_BYTES);+ expect(mat.textureWrap).toBe("repeat");+ });++ it("prefers FBX textures embedded in a Video's Content", () => {+ const embedded = new Uint8Array([9, 8, 7]);+ const root = FbxLoader.parseAscii(fbxWithTexture("missing.png"));+ const objects = root.children.find((c) => c.name === "Objects")!;+ objects.children.push({+ name: "Video",+ props: [5001, "Video::PaintedDiffuse", "Clip"],+ children: [{ name: "Content", props: [embedded], children: [] }],+ });+ root.children+ .find((c) => c.name === "Connections")!+ .children.push({ name: "C", props: ["OO", 5001, 4001], children: [] });++ const model = FbxLoader.buildModelFromTree(root, { basePath: dir });+ expect(model.materials.Painted?.diffuseTextureBuffer).toEqual(embedded);+ });+});diff --git a/packages/webgpu-renderers/src/loaders/texture-files.ts b/packages/webgpu-renderers/src/loaders/texture-files.tsnew file mode 100644index 00000000..ab871ed7--- /dev/null+++ b/packages/webgpu-renderers/src/loaders/texture-files.ts@@ -0,0 +1,80 @@+import fs from "node:fs";+import path from "node:path";++const MIME_BY_EXT: Record<string, string> = {+ ".png": "image/png",+ ".jpg": "image/jpeg",+ ".jpeg": "image/jpeg",+ ".webp": "image/webp",+ ".gif": "image/gif",+ ".tif": "image/tiff",+ ".tiff": "image/tiff",+ ".bmp": "image/bmp",+ ".tga": "image/x-tga",+};++export interface TextureFile {+ bytes: Uint8Array;+ mime: string;+}++/** MIME type of an image file name, by extension (PNG when unknown). */+export function textureMime(name: string): string {+ return MIME_BY_EXT[path.extname(name).toLowerCase()] ?? "image/png";+}++/**+ * Reads a texture a model file refers to. Exporters often write absolute+ * paths from the author's machine (`C:\Users\…\wood.png`), so after the+ * path as written (relative to `basePath`), the file name is also tried in+ * `basePath`, its `textures/` folder and any `*.fbm` folder next to it.+ */+export function readTextureFile(+ ref: string,+ basePath: string | undefined,+): TextureFile | undefined {+ if (!ref || typeof fs === "undefined" || !fs.existsSync) return undefined;+ let cleaned = ref.trim().replace(/^file:\/\//, "");+ try {+ cleaned = decodeURIComponent(cleaned);+ } catch (_) {}+ const normalized = cleaned.replace(/\\/g, "/");+ const fileName = normalized.split("/").pop() ?? normalized;++ const candidates: string[] = [];+ if (path.isAbsolute(normalized)) candidates.push(normalized);+ if (basePath) {+ candidates.push(path.resolve(basePath, normalized));+ candidates.push(path.join(basePath, fileName));+ candidates.push(path.join(basePath, "textures", fileName));+ try {+ for (const entry of fs.readdirSync(basePath)) {+ if (entry.toLowerCase().endsWith(".fbm"))+ candidates.push(path.join(basePath, entry, fileName));+ }+ } catch (_) {}+ }++ for (const file of candidates) {+ try {+ if (fs.existsSync(file) && fs.statSync(file).isFile()) {+ return {+ bytes: new Uint8Array(fs.readFileSync(file)),+ mime: textureMime(file),+ };+ }+ } catch (_) {}+ }+ return undefined;+}++/** Warns that a model's texture could not be found (the mesh renders untextured). */+export function warnMissingTexture(+ loader: string,+ material: string,+ ref: string,+): void {+ console.warn(+ `[${loader}] Texture "${ref}" for material "${material}" was not found; the material renders without it.`,+ );+}diff --git a/packages/webgpu-renderers/src/renderer3d/renderer3d.ts b/packages/webgpu-renderers/src/renderer3d/renderer3d.tsindex 094f75f7..abe7a318 100644--- a/packages/webgpu-renderers/src/renderer3d/renderer3d.ts+++ b/packages/webgpu-renderers/src/renderer3d/renderer3d.ts@@ -1413,7 +1413,18 @@ export class Renderer3D {const source = Buffer.from(encoded instanceof Uint8Array ? encoded : new Uint8Array(encoded),);- const { width, height } = await sharp(source).metadata();+ let width: number | undefined;+ let height: number | undefined;+ try {+ ({ width, height } = await sharp(source).metadata());+ } catch (err) {+ // TGA, PSD and other formats sharp cannot decode: draw untextured.+ console.warn(+ `[Renderer3D] Could not decode texture "${label}"; drawing without it:`,+ err instanceof Error ? err.message : err,+ );+ return undefined;+ }if (!width || !height) return undefined;const mipLevelCount = Math.floor(Math.log2(Math.max(width, height))) + 1;const texture = this.device.createTexture({