Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

14.1 KB
Raw Blame
import { Group, MeshStandardMaterial } from "three"
import { GLTFExporter } from "three/examples/jsm/exporters/GLTFExporter"
import { cloneSkeleton, combine, combineNoAtlas } from "./merge-geometry"
import VRMExporter from "./VRMExporter"
import VRMExporterv0 from "./VRMExporterv0"
import { findChildrenByType } from "./utils"
import { VRMHumanBoneName, VRMExpression, VRMExpressionPresetName, VRMExpressionManager} from "@pixiv/three-vrm";
import { GetMetadataFromAvatar } from "./vrmMetaUtils"


function cloneAvatarModel (model){
  
    const clone = model.clone()
    /*
      NOTE: After avatar clone, the origIndexBuffer/BufferAttribute in userData will lost many infos:
      From: BufferAttribute {isBufferAttribute: true, name: '', array: Uint32Array(21438), itemSize: 1, count: 21438, …}
      To:   Object          {itemSize: 1, type: 'Uint32Array',  array: Array(21438), normalized: false}
      Especailly notics the change of `array` type, and lost of `count` property, will cause errors later.
      So have to reassign `userData.origIndexBuffer` after avatar clone.
    */
    const origIndexBuffers = []
    model.traverse((child) => {
      if (child.userData.origIndexBuffer)
        origIndexBuffers.push(child.userData.origIndexBuffer)
    })
    clone.traverse((child) => {
      if (child.userData.origIndexBuffer)
        child.userData.origIndexBuffer = origIndexBuffers.shift()
    })
    return clone;
}
function getUnopotimizedGLB (model){

    const modelClone = cloneAvatarModel(model)
    let skeleton
    const skinnedMeshes = []

    modelClone.traverse((child) => {
      if (!skeleton && child.isSkinnedMesh) {
        skeleton = cloneSkeleton(child)
      }
      if (child.isSkinnedMesh) {
        child.geometry = child.geometry.clone()
        child.skeleton = skeleton
        skinnedMeshes.push(child)
        if (Array.isArray(child.material)) {
          const materials = child.material
          child.material = new MeshStandardMaterial()
          child.material.map = materials[0].map
        }
        if (child.userData.origIndexBuffer) {
          child.geometry.setIndex(child.userData.origIndexBuffer)
        }
      }
    })

    const unoptimizedGLB = new Group()
    skinnedMeshes.forEach((skinnedMesh) => {
      unoptimizedGLB.add(skinnedMesh)
    })
    unoptimizedGLB.add(skeleton.bones[0])

    return unoptimizedGLB;
}


export async function getGLBBlobData(model, options){
  const {optimized = true} = options;
  const finalModel = await (optimized ? 
     getOptimizedGLB(model, options) :
     getUnopotimizedGLB(model))
  const glb = await parseGLB(finalModel);
  return new Blob([glb], { type: 'model/gltf-binary' });
}

export async function getVRMBlobData(model, avatar, options){
  const finalModel = await getOptimizedGLB(model, options)
  const vrm = await parseVRM(finalModel, avatar, options);
  // save it as glb now
  return new Blob([vrm], { type: 'model/gltf-binary' });
}

/**
 * Downloads a VRM model with specified options.
 *
 * @param {Object} model - The 3D model object.
 * @param {Object} vrmData - The VRM initial loaded data for the model.
 * @param {string} fileName - The name of the file to be downloaded.
 * @param {Object} options - Additional options for the download.
 * @param {Object} options.screenshot - An optional screenshot for the model.
 * @param {number} options.mToonAtlasSize - Atlas size for opaque parts when using MToon material.
 * @param {number} options.mToonAtlasSizeTransp - Atlas size for transparent parts when using MToon material.
 * @param {number} options.stdAtlasSize - Atlas size for opaque parts when using standard materials.
 * @param {number} options.stdAtlasSizeTransp - Atlas size for transparent parts when using standard materials.
 * @param {boolean} options.exportMtoonAtlas - Whether to export the MToon material atlas.
 * @param {boolean} options.exportStdAtlas - Whether to export the standard material atlas.
 * @param {number} options.scale - Scaling factor for the model.
 * @param {boolean} options.isVrm0 - Whether the VRM version is 0 (true) or 1 (false).
 * @param {Object} options.vrmMeta - Additional metadata for the VRM model.
 * @param {boolean} options.createTextureAtlas - Whether to create a texture atlas.
 * @param {boolean} options.optimized - Whether to optimize the VRM model.
 * @param {boolean} options.ktxCompression - Whether to use ktx2 type texture compression.
 */
export async function downloadVRM(model,vrmData,fileName, options){


    const avatar = {_optimized:{vrm:vrmData}}
    downloadVRMWithAvatar(model, avatar, fileName, options)
}

export function downloadVRMWithAvatar(model, avatar, fileName, options){
  return new Promise((resolve) => {
    const downloadFileName = `${
      fileName && fileName !== "" ? fileName : "AvatarCreatorModel"
    }`
    getVRMData(model, avatar, options).then((vrm)=>{
      saveArrayBuffer(vrm, `${downloadFileName}.vrm`)
      resolve();
    })
  });
}

async function getVRMData(model, avatar, options){
  const vrmModel = await getOptimizedGLB(model,avatar, options);
  return parseVRM(vrmModel,avatar,options) 
}

function getOptimizedGLB(model, avatar, options){
  const modelClone = cloneAvatarModel(model)
  // default for now ?
  options.mergeAppliedMorphs = true;
  const { createTextureAtlas = true } = options;
  if (createTextureAtlas){
    return combine(modelClone, avatar, options);
  }
  else{
    console.log("no atlas");
    return combineNoAtlas(modelClone,avatar, options)
  }
}


export async function downloadGLB(model, fileName = "", options){
  const downloadFileName = `${
    fileName && fileName !== "" ? fileName : "AvatarCreatorModel"
  }`

  const {optimized = true} = options;

  const finalModel = optimized ?
    await getOptimizedGLB(model, options):
    getUnopotimizedGLB(model)

  parseGLB(finalModel)
    .then((result) => {
      if (result instanceof ArrayBuffer) {
        saveArrayBuffer(result, `${downloadFileName}.glb`)
      } else {
        const output = JSON.stringify(result, null, 2)
        saveString(output, `${downloadFileName}.gltf`)
      }
    })
}

function parseGLB (glbModel){
  return new Promise((resolve) => {
    const exporter =  new GLTFExporter();
    return exporter.parse(
        glbModel,
        (result) => {
          resolve(result)
        },
        (error) => {
          console.error("Error parsing", error)
        },
        {
          trs: false,
          onlyVisible: false,
          truncateDrawRange: true,
          binary: true,
          forcePowerOfTwoTextures: false,
          maxTextureSize: 1024,
        },
      )
  })
}

/**
 * 
 * @param {Record<string, Record<string,any>>} avatar 
 * @returns  {GroupSpringBones[]}
 */
function getGroupSpringBones (avatar) {

  /**
   * @typedef {Object} GroupSpringBones
   * @property {THREE.Bone[]} bones
   * @property {VRMSpringBoneJointSettings} settings
   * @property {VRMSpringBoneColliderGroup[]} colliderGroups
   * @property {string} name
   * @property {any} center
   */

  /**
   * @type {Object[]}
   */
  const finalSpringBones= [];

  // add non repeating spring bones
  for(const trait in avatar){
    if (avatar[trait]?.vrm?.springBoneManager!= null){
        const joints = avatar[trait].vrm.springBoneManager.joints;
        for (const item of joints) {
          const doesNameExist = finalSpringBones.some(boneData => boneData.name === item.bone.name);
          if (!doesNameExist) {
            finalSpringBones.push({
              name:item.bone.name, 
              settings:item.settings, 
              bone:item.bone, 
              colliderGroups:item.colliderGroups,
              center:item.center
            }); 
          }

        }
    }
  }

  //get only the root bone of the last array
  /**
   * @type {GroupSpringBones[]}
   */
  const groupSpringBones = [];

    // create a group for each root bone
    finalSpringBones.forEach(springBone => {
      const parent = finalSpringBones.find(bone => bone.name == springBone.bone.parent?.name)
      if(parent == null){
        // current spring bone is a root bone
        groupSpringBones.push({
          bones:[springBone],
          settings:springBone.settings,
          center:springBone.center,
          colliderGroups:springBone.colliderGroups,
          name:springBone.bone.name
        });
        return;
      }
    })

    finalSpringBones.map((springBone) => {
      const group = groupSpringBones.find(group => group.bones.find(bone => bone.name == springBone.bone.parent?.name) != null);
      if(group != null){
        group.bones.push({
          name:springBone.name,
          bone:springBone.bone
        });
      }
    })

  return groupSpringBones;
}

function getRootBones (avatar) {
  const finalSpringBones = [];
  //const springBonesData = [];
  
  // add non repeating spring bones
  for(const trait in avatar){
    if (avatar[trait]?.vrm?.springBoneManager!= null){
        const joints = avatar[trait].vrm.springBoneManager.joints;
        for (const item of joints) {
          const doesNameExist = finalSpringBones.some(boneData => boneData.name === item.bone.name);
          if (!doesNameExist) {
            finalSpringBones.push({
              name:item.bone.name, 
              settings:item.settings, 
              bone:item.bone, 
              colliderGroups:item.colliderGroups,
              center:item.center
            }); 
          }

        }
    }
    
  }

  //get only the root bone of the last array
  const rootSpringBones = [];
  finalSpringBones.forEach(springBone => {
    for (const boneName in VRMHumanBoneName) {
      if(springBone.bone.parent.name == VRMHumanBoneName[boneName]){
        rootSpringBones.push(springBone);
        break;
      }
    }
  });
  return rootSpringBones;
}

function getHumanoidByBoneNames(skinnedMesh){
  const humanBones = {}
  skinnedMesh.skeleton.bones.map((bone)=>{
    for (const boneName in VRMHumanBoneName) {
      if (VRMHumanBoneName[boneName] === bone.name){
        humanBones[bone.name] ={node : bone};
        break;
      }
    }
  })
  return humanBones
}

function getAvatarData (avatarModel, vrmMeta, options){
  const skinnedMeshes = findChildrenByType(avatarModel, "SkinnedMesh")
  return{
    humanBones:getHumanoidByBoneNames(skinnedMeshes[0]),
    materials : avatarModel.userData.atlasMaterial,
    meta : getVRMMeta( vrmMeta),
    ...(options.mergeAppliedMorphs?{expressionManager:getRebindedVRMExpressionManager(avatarModel)}:{}),
  }
}

function getVRMMeta( vrmMeta){
  vrmMeta = vrmMeta||{}

  const defaults = {
    authors:["CharacterStudio"],
    metaVersion:"1",
    version:"v1",
    name:"CharacterCreator",
    licenseUrl:"https://vrm.dev/licenses/1.0/",
    commercialUssageName: "personalNonProfit",
    contactInformation: "https://m3org.com/", 
    allowExcessivelyViolentUsage:false,
    allowExcessivelySexualUsage:false,
    allowPoliticalOrReligiousUsage:false,
    allowAntisocialOrHateUsage:false,
    creditNotation:"required",
    allowRedistribution:false,
    modification:"prohibited"
  }

  return { ...defaults, ...vrmMeta };
}

function parseVRM (glbModel, avatar, options){
  const {
    screenshot = null,
    vrmMeta = null,
    scale = 1,
    vrmName = "CharacterCreator"
  } = options

  const metadataMerged = GetMetadataFromAvatar(avatar, vrmMeta, vrmName);



  return new Promise((resolve) => {
    /**
     * Because vrm1 Exporter is broken, always default to vrm0 exporter;
     */
    const isOutputVRM0 = options.outputVRM0 ?? options.isVrm0 ?? true;
    const exporter = isOutputVRM0 ? new VRMExporterv0() :  new VRMExporter()
    const vrmData = {
      ...getVRMBaseData(avatar),
      ...getAvatarData(glbModel, metadataMerged,options),
    }

    let skinnedMesh;
    glbModel.traverse(child => {
      if (child.isSkinnedMesh) skinnedMesh = child;
    })
    const reverseBonesXZ = () => {
      for (let i = 0; i < skinnedMesh.skeleton.bones.length;i++){
        const bone = skinnedMesh.skeleton.bones[i];
        bone.position.x *= -1;
        bone.position.z *= -1;
      }

      skinnedMesh.skeleton.bones.forEach(bone => {
        bone.updateMatrix();
        bone.updateMatrixWorld();
      })
      skinnedMesh.skeleton.calculateInverses();
      skinnedMesh.skeleton.computeBoneTexture();
      skinnedMesh.skeleton.update();
    }
    reverseBonesXZ();
    


    // @TODO: change springBone selection logic for VRM1
    const rootSpringBones = isOutputVRM0?getGroupSpringBones(avatar):getRootBones(avatar);
    // XXX collider bones should be taken from springBone.colliderBones
    // const colliderBones = [];
    
    if(isOutputVRM0){
      // VRM 0.0
      exporter.parse(vrmData, glbModel, screenshot, rootSpringBones, options.ktxCompression, scale, (vrm) => {
        resolve(vrm)
      })
    }else{
      // VRM 1.0 has a different amount of parameters
      exporter.parse(vrmData, glbModel, screenshot, (vrm) => {
        resolve(vrm)
      })
    }
  })
}

function save(blob, filename) {
  const link = document.createElement("a")
  link.style.display = "none"
  document.body.appendChild(link)

  link.href = URL.createObjectURL(blob)
  link.download = filename
  link.click()
}

function saveString(text, filename) {
  save(new Blob([text], { type: "text/plain" }), filename)
}
function saveArrayBuffer(buffer, filename) {
  save(getArrayBuffer(buffer), filename)
}
function getArrayBuffer(buffer) {
  return new Blob([buffer], { type: "application/octet-stream" })
}
function getVRMBaseData(avatar) {
  // to do, merge data from all vrms, not to get only the first one
  for (const prop in avatar) {
    if (avatar[prop].vrm) {
      return avatar[prop].vrm
    }
  }
}


/**
 * Rebinds the BlendShapes to a new VRMExpressionManager. Used before exporting the VRM.
 * @param {THREE.Object3D} avatarModel - The avatar model.
 */
function getRebindedVRMExpressionManager(avatarModel){
  // Get old expression manager or a new default one if it doesnt exist
  /**
   * @type {THREE.VRMExpressionManager|undefined}
   */
  let oldExpressionManager = avatarModel.userData.expressionManagerToClone;
  if(!oldExpressionManager){
      oldExpressionManager = new VRMExpressionManager();
      for(const exp of Object.values(VRMExpressionPresetName)){
        const expression = new VRMExpression(exp)
        oldExpressionManager.registerExpression(expression)
      }
  }

  return oldExpressionManager;
}