Skip to content

Code-first video & image, rendered natively on WebGPU.

glTF: texture transforms, UV sets, nearest sampling; fail clearly on unsupported data

Claude7a8884431e2d
diff --git a/nodes/node-compositor/src/renderers/webgpu-renderer.ts b/nodes/node-compositor/src/renderers/webgpu-renderer.ts
index e74b26a4..03a72296 100644
--- a/nodes/node-compositor/src/renderers/webgpu-renderer.ts
+++ b/nodes/node-compositor/src/renderers/webgpu-renderer.ts
@@ -1956,6 +1956,7 @@ export const CompositorWebGPURenderer: WebGPUNodeRenderer = async (args) => {
shadeColor: mat?.shadeColor,
alphaCutoff: mat?.alphaCutoff,
textureWrap: mat?.textureWrap,
+ textureFilter: mat?.textureFilter,
shininess:
target?.shininess ?? lop.shininess ?? mat?.shininess ?? 32,
roughness: target?.roughness ?? lop.roughness ?? mat?.roughness,
diff --git a/packages/framefields/src/extended-models-render.test.ts b/packages/framefields/src/extended-models-render.test.ts
index 29249174..1c84c5e0 100644
--- a/packages/framefields/src/extended-models-render.test.ts
+++ b/packages/framefields/src/extended-models-render.test.ts
@@ -657,6 +657,86 @@ f 1/1 2/2 3/3 4/4
await fs.rm(tmpDir, { recursive: true, force: true });
});
+ it("keeps texels sharp with a glTF NEAREST sampler", async () => {
+ const tmpDir = await fs.mkdtemp(
+ path.join(os.tmpdir(), "framefields-nearest-"),
+ );
+ const { Canvas } = await import("skia-canvas");
+ const c = new Canvas(2, 2);
+ const ctx = c.getContext("2d");
+ ctx.fillStyle = "rgb(255,0,0)";
+ ctx.fillRect(0, 0, 1, 2);
+ ctx.fillStyle = "rgb(0,0,255)";
+ ctx.fillRect(1, 0, 1, 2);
+ const png = await c.toBuffer("png");
+
+ const pos = new Float32Array([
+ -100, -100, 0, 100, -100, 0, -100, 100, 0, 100, 100, 0,
+ ]);
+ const uv = new Float32Array([0, 0, 1, 0, 0, 1, 1, 1]);
+ const idx = new Uint16Array([0, 2, 1, 1, 2, 3]);
+ const bin = Buffer.concat([pos, uv, idx].map((a) => Buffer.from(a.buffer)));
+ const file = path.join(tmpDir, "quad.gltf");
+ await fs.writeFile(
+ file,
+ JSON.stringify({
+ asset: { version: "2.0" },
+ buffers: [
+ {
+ byteLength: bin.length,
+ uri: `data:application/octet-stream;base64,${bin.toString("base64")}`,
+ },
+ ],
+ bufferViews: [
+ { buffer: 0, byteOffset: 0, byteLength: 48 },
+ { buffer: 0, byteOffset: 48, byteLength: 32 },
+ { buffer: 0, byteOffset: 80, byteLength: 12 },
+ ],
+ accessors: [
+ { bufferView: 0, componentType: 5126, count: 4, type: "VEC3" },
+ { bufferView: 1, componentType: 5126, count: 4, type: "VEC2" },
+ { bufferView: 2, componentType: 5123, count: 6, type: "SCALAR" },
+ ],
+ meshes: [
+ {
+ primitives: [
+ {
+ attributes: { POSITION: 0, TEXCOORD_0: 1 },
+ indices: 2,
+ material: 0,
+ },
+ ],
+ },
+ ],
+ images: [{ uri: `data:image/png;base64,${png.toString("base64")}` }],
+ samplers: [{ magFilter: 9728, wrapS: 33071, wrapT: 33071 }],
+ textures: [{ source: 0, sampler: 0 }],
+ materials: [
+ { pbrMetallicRoughness: { baseColorTexture: { index: 0 } } },
+ ],
+ }),
+ );
+
+ const img = await render(
+ Layer.gltf(file, {
+ id: "gltf-nearest",
+ x: 200,
+ y: 150,
+ z: 0,
+ material: "unlit",
+ }),
+ );
+ // Just either side of the texel edge (the quad's middle): pure colors, no blend.
+ const a = getPixel(img, 192, 150);
+ const b = getPixel(img, 208, 150);
+ const [red, blue] = a.r > a.b ? [a, b] : [b, a];
+ expect(red.r).toBeGreaterThan(235);
+ expect(red.b).toBeLessThan(20);
+ expect(blue.b).toBeGreaterThan(235);
+ expect(blue.r).toBeLessThan(20);
+ await fs.rm(tmpDir, { recursive: true, force: true });
+ });
+
it("draws an FBX material's diffuse texture", async () => {
const tmpDir = await fs.mkdtemp(
path.join(os.tmpdir(), "framefields-fbx-tex-"),
diff --git a/packages/webgpu-renderers/src/loaders/gltf-features.test.ts b/packages/webgpu-renderers/src/loaders/gltf-features.test.ts
new file mode 100644
index 00000000..92796a60
--- /dev/null
+++ b/packages/webgpu-renderers/src/loaders/gltf-features.test.ts
@@ -0,0 +1,154 @@
+import fs from "node:fs";
+import path from "node:path";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { GltfLoader, loadGLTF } from "./gltf-loader.js";
+
+const IMAGE = "data:image/png;base64,iVBORw0KGgo=";
+
+/** A one-quad glTF with two UV sets (TEXCOORD_1 = TEXCOORD_0 + 10) and the given material. */
+function quadGltf(
+ material: Record<string, unknown>,
+ extra: Record<string, unknown> = {},
+): Record<string, unknown> {
+ const pos = new Float32Array([0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0]);
+ const uv0 = new Float32Array([0, 0, 1, 0, 0, 1, 1, 1]);
+ const uv1 = uv0.map((v) => v + 10);
+ const bin = Buffer.concat([pos, uv0, uv1].map((a) => Buffer.from(a.buffer)));
+ return {
+ asset: { version: "2.0" },
+ buffers: [
+ {
+ byteLength: bin.length,
+ uri: `data:application/octet-stream;base64,${bin.toString("base64")}`,
+ },
+ ],
+ bufferViews: [
+ { buffer: 0, byteOffset: 0, byteLength: 48 },
+ { buffer: 0, byteOffset: 48, byteLength: 32 },
+ { buffer: 0, byteOffset: 80, byteLength: 32 },
+ ],
+ accessors: [
+ { bufferView: 0, componentType: 5126, count: 4, type: "VEC3" },
+ { bufferView: 1, componentType: 5126, count: 4, type: "VEC2" },
+ { bufferView: 2, componentType: 5126, count: 4, type: "VEC2" },
+ ],
+ meshes: [
+ {
+ primitives: [
+ {
+ attributes: { POSITION: 0, TEXCOORD_0: 1, TEXCOORD_1: 2 },
+ material: 0,
+ },
+ ],
+ },
+ ],
+ images: [{ uri: IMAGE }, { uri: "data:image/webp;base64,UklGRg==" }],
+ textures: [{ source: 0, sampler: 0 }],
+ samplers: [{}],
+ materials: [{ name: "M", ...material }],
+ ...extra,
+ };
+}
+
+const parse = (doc: Record<string, unknown>) =>
+ GltfLoader.parse(JSON.stringify(doc));
+
+describe("glTF texture features", () => {
+ afterEach(() => vi.restoreAllMocks());
+
+ it("bakes KHR_texture_transform into the mesh UVs", () => {
+ const model = parse(
+ quadGltf({
+ pbrMetallicRoughness: {
+ baseColorTexture: {
+ index: 0,
+ extensions: {
+ KHR_texture_transform: {
+ offset: [0.5, 0.25],
+ scale: [2, 4],
+ rotation: Math.PI / 2,
+ },
+ },
+ },
+ },
+ }),
+ );
+ const uvs = Array.from(model.meshes[0]!.uvs);
+ // uv (1, 0): scaled to (2, 0), rotated to (0, -2), offset to (0.5, -1.75).
+ expect(uvs[2]).toBeCloseTo(0.5, 5);
+ expect(uvs[3]).toBeCloseTo(-1.75, 5);
+ // uv (0, 1): scaled to (0, 4), rotated to (4, 0), offset to (4.5, 0.25).
+ expect(uvs[4]).toBeCloseTo(4.5, 5);
+ expect(uvs[5]).toBeCloseTo(0.25, 5);
+ });
+
+ it("draws with the base color's UV set and skips maps on another set", () => {
+ const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+ const model = parse(
+ quadGltf({
+ pbrMetallicRoughness: { baseColorTexture: { index: 0, texCoord: 1 } },
+ occlusionTexture: { index: 0 },
+ }),
+ );
+ expect(Array.from(model.meshes[0]!.uvs.slice(0, 4))).toEqual([
+ 10, 10, 11, 10,
+ ]);
+ expect(model.materials.M?.diffuseTextureBuffer).toBeDefined();
+ expect(model.materials.M?.occlusionTextureBuffer).toBeUndefined();
+ expect(warn).toHaveBeenCalledWith(
+ expect.stringContaining('occlusion map of material "M" uses UV set 0'),
+ );
+ });
+
+ it("reads the sampler's NEAREST magnification and clamp wrap", () => {
+ const doc = quadGltf({
+ pbrMetallicRoughness: { baseColorTexture: { index: 0 } },
+ });
+ doc.samplers = [{ magFilter: 9728, wrapS: 33071 }];
+ const mat = parse(doc).materials.M!;
+ expect(mat.textureFilter).toBe("nearest");
+ expect(mat.textureWrap).toBe("clamp");
+ });
+
+ it("prefers EXT_texture_webp and skips KTX2 / Basis textures", () => {
+ const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+ const doc = quadGltf({
+ pbrMetallicRoughness: { baseColorTexture: { index: 0 } },
+ normalTexture: { index: 1 },
+ });
+ doc.textures = [
+ { source: 0, extensions: { EXT_texture_webp: { source: 1 } } },
+ { extensions: { KHR_texture_basisu: { source: 0 } } },
+ ];
+ const mat = parse(doc).materials.M!;
+ expect(mat.diffuseTextureMime).toBe("image/webp");
+ expect(mat.normalTextureBuffer).toBeUndefined();
+ expect(warn).toHaveBeenCalledWith(expect.stringContaining("KTX2 / Basis"));
+ });
+
+ it("fails clearly on required mesh compression", () => {
+ expect(() =>
+ parse(
+ quadGltf({}, { extensionsRequired: ["KHR_draco_mesh_compression"] }),
+ ),
+ ).toThrow(/KHR_draco_mesh_compression.*cannot decode/);
+ expect(() =>
+ parse(quadGltf({}, { extensionsRequired: ["EXT_meshopt_compression"] })),
+ ).toThrow(/meshopt/);
+ });
+
+ it("loads a model from a bare relative path", () => {
+ const dir = fs.mkdtempSync(path.join(process.cwd(), "tmp-gltf-rel-"));
+ try {
+ fs.writeFileSync(
+ path.join(dir, "quad.gltf"),
+ JSON.stringify(quadGltf({})),
+ );
+ // No "./" prefix, as in Layer.glb("models/robot.glb").
+ const bare = `${path.basename(dir)}/quad.gltf`;
+ expect(loadGLTF(bare).meshes).toHaveLength(1);
+ } finally {
+ fs.rmSync(dir, { recursive: true, force: true });
+ }
+ });
+});
diff --git a/packages/webgpu-renderers/src/loaders/gltf-loader.ts b/packages/webgpu-renderers/src/loaders/gltf-loader.ts
index bbc71d11..c86ebc0f 100644
--- a/packages/webgpu-renderers/src/loaders/gltf-loader.ts
+++ b/packages/webgpu-renderers/src/loaders/gltf-loader.ts
@@ -2,6 +2,7 @@ import fs from "node:fs";
import path from "node:path";
import { type Mat4, Matrix4Math, type Vec3 } from "../math3d/index.js";
import { MAX_JOINTS, poseNodes } from "./pose.js";
+import { textureMime } from "./texture-files.js";
import type {
AnimationClip3D,
LoadModelOptions,
@@ -97,6 +98,7 @@ interface GltfPrimitive {
indices?: number;
material?: number;
mode?: number; // 4 = TRIANGLES
+ extensions?: { KHR_draco_mesh_compression?: unknown };
}
interface GltfMesh {
@@ -104,18 +106,31 @@ interface GltfMesh {
primitives: GltfPrimitive[];
}
+interface GltfTextureTransform {
+ offset?: [number, number];
+ rotation?: number;
+ scale?: [number, number];
+ texCoord?: number;
+}
+
+interface GltfTextureInfo {
+ index: number;
+ texCoord?: number;
+ extensions?: { KHR_texture_transform?: GltfTextureTransform };
+}
+
interface GltfMaterial {
name?: string;
pbrMetallicRoughness?: {
baseColorFactor?: [number, number, number, number];
metallicFactor?: number;
roughnessFactor?: number;
- baseColorTexture?: { index: number; texCoord?: number };
- metallicRoughnessTexture?: { index: number; texCoord?: number };
+ baseColorTexture?: GltfTextureInfo;
+ metallicRoughnessTexture?: GltfTextureInfo;
};
- normalTexture?: { index: number; texCoord?: number; scale?: number };
- occlusionTexture?: { index: number; texCoord?: number; strength?: number };
- emissiveTexture?: { index: number; texCoord?: number };
+ normalTexture?: GltfTextureInfo & { scale?: number };
+ occlusionTexture?: GltfTextureInfo & { strength?: number };
+ emissiveTexture?: GltfTextureInfo;
emissiveFactor?: [number, number, number];
doubleSided?: boolean;
alphaMode?: "OPAQUE" | "MASK" | "BLEND";
@@ -137,9 +152,15 @@ interface GltfImage {
interface GltfTexture {
source?: number;
sampler?: number;
+ extensions?: {
+ EXT_texture_webp?: { source?: number };
+ KHR_texture_basisu?: { source?: number };
+ };
}
interface GltfSampler {
+ magFilter?: number;
+ minFilter?: number;
wrapS?: number;
wrapT?: number;
}
@@ -185,6 +206,8 @@ interface GltfAnimation {
interface GltfDocument {
asset?: { version: string };
+ extensionsUsed?: string[];
+ extensionsRequired?: string[];
scene?: number;
scenes?: Array<{ name?: string; nodes?: number[] }>;
nodes?: GltfNode[];
@@ -200,6 +223,21 @@ interface GltfDocument {
buffers?: GltfBuffer[];
}
+/** Mesh compression the loader cannot decode: a file that requires it fails with a clear error. */
+const UNSUPPORTED_REQUIRED_EXTENSIONS: Record<string, string> = {
+ KHR_draco_mesh_compression: "Draco mesh compression",
+ EXT_meshopt_compression: "meshopt compression",
+};
+
+/**
+ * The UV set and transform a material's textures share: the mesh's UVs are
+ * read from that set with the transform baked in.
+ */
+interface UvPlan {
+ texCoord: number;
+ transform?: GltfTextureTransform;
+}
+
const GLB_MAGIC = 0x46546c67; // "glTF"
const CHUNK_TYPE_JSON = 0x4e4f534a; // "JSON"
const CHUNK_TYPE_BIN = 0x004e4942; // "BIN\0"
@@ -282,6 +320,14 @@ export class GltfLoader {
}
}
+ for (const ext of doc.extensionsRequired ?? []) {
+ const what = UNSUPPORTED_REQUIRED_EXTENSIONS[ext];
+ if (what)
+ throw new Error(
+ `glTF file requires ${ext} (${what}), which framefields cannot decode. Re-export the model without mesh compression (in Blender's glTF exporter, turn off Compression).`,
+ );
+ }
+
return GltfLoader.buildModel(doc, binaryBuffers, options, basePath);
}
@@ -403,14 +449,84 @@ export class GltfLoader {
const file = path.isAbsolute(image.uri)
? image.uri
: path.join(basePath, decodeURIComponent(image.uri));
- if (!fs.existsSync(file)) return undefined;
- const ext = path.extname(file).toLowerCase();
+ if (!fs.existsSync(file)) {
+ GltfLoader.warn(
+ `Image "${image.uri}" was not found next to the model; textures using it are skipped.`,
+ );
+ return undefined;
+ }
return {
bytes: new Uint8Array(fs.readFileSync(file)),
- mime: image.mimeType ?? (ext === ".png" ? "image/png" : "image/jpeg"),
+ mime: image.mimeType ?? textureMime(file),
};
}
+ private static warn(message: string): void {
+ console.warn(`[GltfLoader] ${message}`);
+ }
+
+ /** The UV set of a texture reference (KHR_texture_transform may override it). */
+ private static texCoordOf(info: GltfTextureInfo): number {
+ return (
+ info.extensions?.KHR_texture_transform?.texCoord ?? info.texCoord ?? 0
+ );
+ }
+
+ /** The UV set and transform of the material's base color map (else its first map). */
+ private static uvPlan(mat: GltfMaterial | undefined): UvPlan {
+ const first = [
+ mat?.pbrMetallicRoughness?.baseColorTexture,
+ mat?.normalTexture,
+ mat?.pbrMetallicRoughness?.metallicRoughnessTexture,
+ mat?.occlusionTexture,
+ mat?.emissiveTexture,
+ ].find((t) => t !== undefined);
+ return first
+ ? {
+ texCoord: GltfLoader.texCoordOf(first),
+ transform: first.extensions?.KHR_texture_transform,
+ }
+ : { texCoord: 0 };
+ }
+
+ /**
+ * A texture's decodable image: EXT_texture_webp's source when present.
+ * KTX2 / Basis (KHR_texture_basisu) and other formats the renderer
+ * cannot decode are skipped with a warning.
+ */
+ private static textureImage(
+ doc: GltfDocument,
+ binaryBuffers: Uint8Array[],
+ textureIndex: number,
+ basePath: string | undefined,
+ label: string,
+ ): { bytes: Uint8Array; mime: string; sampler?: GltfSampler } | undefined {
+ const texture = doc.textures?.[textureIndex];
+ if (!texture) return undefined;
+ const source =
+ texture.extensions?.EXT_texture_webp?.source ?? texture.source;
+ if (source === undefined) {
+ if (texture.extensions?.KHR_texture_basisu)
+ GltfLoader.warn(
+ `${label} is a KTX2 / Basis texture, which framefields cannot decode; it is skipped (use PNG, JPEG or WebP).`,
+ );
+ return undefined;
+ }
+ const image = GltfLoader.imageBytes(doc, binaryBuffers, source, basePath);
+ if (!image) return undefined;
+ if (!SUPPORTED_IMAGE_MIMES.has(image.mime)) {
+ GltfLoader.warn(
+ `${label} is ${image.mime}, which framefields cannot decode; it is skipped (use PNG, JPEG or WebP).`,
+ );
+ return undefined;
+ }
+ const sampler =
+ texture.sampler !== undefined
+ ? doc.samplers?.[texture.sampler]
+ : undefined;
+ return { ...image, sampler };
+ }
+
private static buildMaterial(
doc: GltfDocument,
binaryBuffers: Uint8Array[],
@@ -449,51 +565,62 @@ export class GltfLoader {
} else if (mat.extensions?.KHR_materials_unlit) {
material.shading = "unlit";
}
- const texRef = pbr?.baseColorTexture;
- const texture = texRef ? doc.textures?.[texRef.index] : undefined;
- if (texture?.source !== undefined) {
- const image = GltfLoader.imageBytes(
+ // Every map is sampled with the mesh's one UV set (see uvPlan).
+ const plan = GltfLoader.uvPlan(mat);
+ const image = (info: GltfTextureInfo | undefined, slot: string) => {
+ if (!info) return undefined;
+ const label = `The ${slot} map of material "${name}"`;
+ const set = GltfLoader.texCoordOf(info);
+ if (set !== plan.texCoord) {
+ GltfLoader.warn(
+ `${label} uses UV set ${set}, but the mesh draws with UV set ${plan.texCoord}; the map is skipped.`,
+ );
+ return undefined;
+ }
+ const t = info.extensions?.KHR_texture_transform;
+ if (JSON.stringify(t ?? null) !== JSON.stringify(plan.transform ?? null))
+ GltfLoader.warn(
+ `${label} has a different KHR_texture_transform than the mesh's first map; it is drawn with that one's.`,
+ );
+ return GltfLoader.textureImage(
doc,
binaryBuffers,
- texture.source,
+ info.index,
basePath,
+ label,
);
- if (image) {
- material.diffuseTextureBuffer = image.bytes;
- material.diffuseTextureMime = image.mime;
- const wrap =
- texture.sampler !== undefined
- ? doc.samplers?.[texture.sampler]?.wrapS
- : undefined;
- material.textureWrap =
- wrap === 33071 ? "clamp" : wrap === 33648 ? "mirror" : "repeat";
- }
- }
- const mapBytes = (ref?: { index: number }) => {
- const tex = ref ? doc.textures?.[ref.index] : undefined;
- return tex?.source !== undefined
- ? GltfLoader.imageBytes(doc, binaryBuffers, tex.source, basePath)?.bytes
- : undefined;
};
- const normal = mapBytes(mat.normalTexture);
+ const base = image(pbr?.baseColorTexture, "base color");
+ if (base) {
+ material.diffuseTextureBuffer = base.bytes;
+ material.diffuseTextureMime = base.mime;
+ const wrap = base.sampler?.wrapS;
+ material.textureWrap =
+ wrap === 33071 ? "clamp" : wrap === 33648 ? "mirror" : "repeat";
+ if (base.sampler?.magFilter === 9728) material.textureFilter = "nearest";
+ }
+ const mapBytes = (info: GltfTextureInfo | undefined, slot: string) =>
+ image(info, slot)?.bytes;
+ const normal = mapBytes(mat.normalTexture, "normal");
if (normal) {
material.normalTextureBuffer = normal;
material.normalScale = mat.normalTexture?.scale ?? 1;
}
- const mr = mapBytes(pbr?.metallicRoughnessTexture);
+ const mr = mapBytes(pbr?.metallicRoughnessTexture, "metallic-roughness");
if (mr) {
material.metallicRoughnessTextureBuffer = mr;
// With a map the factors scale it, and glTF's default factor is 1.
material.roughness = pbr?.roughnessFactor ?? 1;
material.metallic = pbr?.metallicFactor ?? 1;
}
- const occlusion = mapBytes(mat.occlusionTexture);
+ const occlusion = mapBytes(mat.occlusionTexture, "occlusion");
if (occlusion) {
material.occlusionTextureBuffer = occlusion;
material.occlusionStrength = mat.occlusionTexture?.strength ?? 1;
}
// glTF's emissiveFactor defaults to 0: a map alone does not glow.
- const emissiveMap = material.emissive && mapBytes(mat.emissiveTexture);
+ const emissiveMap =
+ material.emissive && mapBytes(mat.emissiveTexture, "emissive");
if (emissiveMap) material.emissiveTextureBuffer = emissiveMap;
return material;
}
@@ -591,6 +718,8 @@ export class GltfLoader {
);
});
+ const uvPlans = (doc.materials ?? []).map((mat) => GltfLoader.uvPlan(mat));
+
const nodes = GltfLoader.buildNodes(doc);
const skins = (doc.skins ?? []).map((skin) => ({
joints: skin.joints,
@@ -658,6 +787,17 @@ export class GltfLoader {
const pos = read(prim.attributes.POSITION)!;
const numVerts = pos.count;
+ const plan =
+ prim.material !== undefined ? uvPlans[prim.material] : undefined;
+ const attributes = prim.attributes as Record<
+ string,
+ number | undefined
+ >;
+ const uvs =
+ read(attributes[`TEXCOORD_${plan?.texCoord ?? 0}`])?.data ??
+ read(prim.attributes.TEXCOORD_0)?.data ??
+ new Float32Array(numVerts * 2);
+ if (plan?.transform) applyTextureTransform(uvs, plan.transform);
const colors = read(prim.attributes.COLOR_0);
let rgba: Float32Array | undefined;
if (colors) {
@@ -703,9 +843,7 @@ export class GltfLoader {
normals:
read(prim.attributes.NORMAL)?.data ??
new Float32Array(numVerts * 3),
- uvs:
- read(prim.attributes.TEXCOORD_0)?.data ??
- new Float32Array(numVerts * 2),
+ uvs,
colors: rgba,
jointIndices: skinned
? read(prim.attributes.JOINTS_0)?.data
@@ -798,15 +936,17 @@ export class GltfLoader {
return GltfLoader.parse(srcOrData, options);
}
+ const cleanPath = srcOrData.startsWith("file://")
+ ? srcOrData.replace("file://", "")
+ : srcOrData;
if (
srcOrData.startsWith("/") ||
srcOrData.startsWith("./") ||
srcOrData.startsWith("../") ||
- srcOrData.startsWith("file://")
+ srcOrData.startsWith("file://") ||
+ // A relative path such as "models/robot.glb" (inline glTF JSON starts with "{").
+ (!srcOrData.trimStart().startsWith("{") && fs.existsSync(cleanPath))
) {
- const cleanPath = srcOrData.startsWith("file://")
- ? srcOrData.replace("file://", "")
- : srcOrData;
const buf = fs.readFileSync(cleanPath);
const basePath = path.dirname(cleanPath);
return GltfLoader.parse(buf, options, basePath);
@@ -848,3 +988,31 @@ function linearToSrgb(c: number): number {
const x = Math.min(Math.max(c, 0), 1);
return x <= 0.0031308 ? x * 12.92 : 1.055 * x ** (1 / 2.4) - 0.055;
}
+
+/** The image types the renderer decodes (sharp). */
+const SUPPORTED_IMAGE_MIMES = new Set([
+ "image/png",
+ "image/jpeg",
+ "image/webp",
+ "image/gif",
+ "image/tiff",
+ "image/avif",
+]);
+
+/**
+ * Bakes KHR_texture_transform into UVs in place: uv' = T * R * S * uv
+ * (offset, then rotation counter-clockwise in UV space, then scale).
+ */
+function applyTextureTransform(
+ uvs: Float32Array,
+ { offset = [0, 0], rotation = 0, scale = [1, 1] }: GltfTextureTransform,
+): void {
+ const c = Math.cos(rotation);
+ const s = Math.sin(rotation);
+ for (let i = 0; i < uvs.length; i += 2) {
+ const u = uvs[i]! * scale[0];
+ const v = uvs[i + 1]! * scale[1];
+ uvs[i] = c * u + s * v + offset[0];
+ uvs[i + 1] = -s * u + c * v + offset[1];
+ }
+}
diff --git a/packages/webgpu-renderers/src/loaders/types.ts b/packages/webgpu-renderers/src/loaders/types.ts
index 793c21e2..ffa0bcce 100644
--- a/packages/webgpu-renderers/src/loaders/types.ts
+++ b/packages/webgpu-renderers/src/loaders/types.ts
@@ -39,6 +39,8 @@ export interface Material3D {
diffuseTextureMime?: string;
/** Texture addressing for UVs outside [0, 1] (glTF samplers default to repeat). */
textureWrap?: "repeat" | "clamp" | "mirror";
+ /** Magnification filter (glTF sampler magFilter); "nearest" keeps texels sharp. */
+ textureFilter?: "linear" | "nearest";
normalTexture?: string;
twoSided?: boolean;
/** "mask" discards texels below `alphaCutoff`; "blend" draws after the opaque meshes. */
diff --git a/packages/webgpu-renderers/src/renderer3d/renderer3d.ts b/packages/webgpu-renderers/src/renderer3d/renderer3d.ts
index abe7a318..57a05252 100644
--- a/packages/webgpu-renderers/src/renderer3d/renderer3d.ts
+++ b/packages/webgpu-renderers/src/renderer3d/renderer3d.ts
@@ -175,6 +175,8 @@ export interface DrawMesh3DOpts extends SurfaceParams {
alphaCutoff?: number;
/** Texture addressing outside [0, 1] */
textureWrap?: "repeat" | "clamp" | "mirror";
+ /** Magnification filter: "nearest" keeps texels sharp (pixel art). */
+ textureFilter?: "linear" | "nearest";
/** Blinn-Phong specular shininess exponent */
shininess?: number;
/** Roughness factor [0, 1] */
@@ -1251,6 +1253,7 @@ export class Renderer3D {
shadeColor,
alphaCutoff,
textureWrap = "clamp",
+ textureFilter = "linear",
shininess = 32.0,
roughness,
specularIntensity = 0.5,
@@ -1265,6 +1268,9 @@ export class Renderer3D {
const sampler = this.samplerCache.getSampler(this.device, {
addressModeU: address,
addressModeV: address,
+ magFilter: textureFilter,
+ // Anisotropy requires linear filtering.
+ ...(textureFilter === "nearest" && { maxAnisotropy: 1 }),
});
const glassy =
diff --git a/plugins/framefields/skills/framefields-compose/SKILL.md b/plugins/framefields/skills/framefields-compose/SKILL.md
index 5959c3c7..11838bb4 100644
--- a/plugins/framefields/skills/framefields-compose/SKILL.md
+++ b/plugins/framefields/skills/framefields-compose/SKILL.md
@@ -406,7 +406,8 @@ Layer3D.cylinder({ ..., color: "#111", emissive: "#4da3ff", emissiveIntensity: 3
- **Bloom only takes light well past white** (emissive ≳ 3, hot speculars), so ordinary bright surfaces don't glow. Raise `emissiveIntensity` to make something glow.
- **Shadows:** one `castShadow` directional or spot light per scene. Meshes cast by default (`castShadow: false` opts out); flat layers (planes, cards) cast only with `castShadow: true`. Everything receives.
- **Fog** fades to `fog.color` exactly, so set it to the composition background for a seamless horizon.
-- **Models:** glTF materials bring base color, normal, metallic-roughness, occlusion and emissive maps.
+- **Models:** glTF materials bring base color, normal, metallic-roughness, occlusion and emissive maps; OBJ (`map_Kd`, `norm` in the `.mtl`) and FBX bring diffuse and normal maps, looked up next to the model. Textures must be PNG, JPEG, WebP, GIF, TIFF or AVIF: TGA, PSD, DDS and KTX2 are skipped with a warning (convert them). Draco / meshopt-compressed glTF files fail to load: re-export without mesh compression. Vertex colors (VOX, PLY, OFF, glTF `COLOR_0`) multiply the base color.
+- **Model orientation:** the world is y-down, so a y-up model (most glTF, OBJ and FBX files) needs `rotateX: 180` to stand upright.
- `emissive` / `emissiveIntensity` / `exposure` animate like any prop (`.keys("emissiveIntensity", …)`).
- **Glass:** `material: "glass" | "acrylic" | "frosted"` on any Layer3D primitive, extruded text (`Layer3D.extrudedText`) or model refracts whatever is behind it on screen — the video, 2D layers, particles — with a rainbow fringe (`dispersion`, default 0.04), bend strength `thickness` (px, default 80) and `ior` (1.5). `color` tints the glass (deeper toward the edges); `frosted` blurs by `roughness`. Glass needs no environment: it draws its own studio reflections, and against a dark grade it reads best — skip `Lighting.*` rigs for glass-only scenes (their environment washes glass out). Glass shows only 2D content and opaque 3D drawn before it, not other glass.