Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

7.1 KB
Raw Blame
/*
 * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: Apache-2.0
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

syntax = "proto3";

package nvidia_ace.a2f.v1;


import "nvidia_ace.animation_id.v1.proto";
import "nvidia_ace.status.v1.proto";
import "nvidia_ace.audio.v1.proto";
import "nvidia_ace.emotion_with_timecode.v1.proto";


message AudioStream {
  // The header must be sent as the first message.
  // One or more audio with emotion messages must be sent thereafter.
  // The end of audio will happen when the client closes the connection
  oneof stream_part {
    AudioStreamHeader audio_stream_header = 1;
    nvidia_ace.a2f.v1.AudioWithEmotion audio_with_emotion = 2;
  }
}

// IMPORTANT NOTE: this is an AudioStreamHeader WITH ID
// A similar AudioStreamHeader exist in nvidia_ace.controller.v1.proto
// but that one does NOT contain IDs
message AudioStreamHeader {
  // IDs of the current stream
  nvidia_ace.animation_id.v1.AnimationIds animation_ids = 1;
  
  nvidia_ace.audio.v1.AudioHeader audio_header = 2;

  // Parameters for updating the facial characteristics of an avatar
  // See the documentation for more information
  FaceParameters face_params = 3;

  // Parameters relative to the emotion blending and processing
  // before using it to generate blendshapes
  // See the documentation for more information
  EmotionPostProcessingParameters emotion_post_processing_params = 4;

  // Multipliers and offsets to apply to the generated blendshape values
  BlendShapeParameters blendshape_params = 5;

  // Emotion parameters (live transition time, beginning emotion)
  EmotionParameters emotion_params = 6;
}

message FloatArray { repeated float values = 1; }


message FaceParameters {
  // The following float parameters are available:
  // "lowerFaceSmoothing", "upperFaceSmoothing", "lowerFaceStrength", "upperFaceStrength",
  // "faceMaskLevel", "faceMaskSoftness", "skinStrength", "blinkStrength", "eyelidOpenOffset",
  // "lipOpenOffset", "blinkOffset", "tongueStrength", "tongueHeightOffset", "tongueDepthOffset"
  map<string, float> float_params = 1;
  // With the current Audio2Face Service no integer parameters are available
  map<string, int32> integer_params = 2;
  // With the current Audio2Face Service no FloatArray parameters are available
  map<string, FloatArray> float_array_params = 3;
}

// The following blendshape can be used here as key:
//  "EyeBlinkLeft", "EyeLookDownLeft", "EyeLookInLeft", "EyeLookOutLeft", "EyeLookUpLeft",
//  "EyeSquintLeft", "EyeWideLeft", "EyeBlinkRight", "EyeLookDownRight", "EyeLookInRight",
//  "EyeLookOutRight", "EyeLookUpRight", "EyeSquintRight", "EyeWideRight", "JawForward",
//  "JawLeft", "JawRight", "JawOpen",  "MouthClose", "MouthFunnel", "MouthPucker", "MouthLeft",
//  "MouthRight", "MouthSmileLeft", "MouthSmileRight", "MouthFrownLeft", "MouthFrownRight",
//  "MouthDimpleLeft", "MouthDimpleRight", "MouthStretchLeft", "MouthStretchRight", "MouthRollLower",
//  "MouthRollUpper", "MouthShrugLower", "MouthShrugUpper", "MouthPressLeft", "MouthPressRight",
//  "MouthLowerDownLeft", "MouthLowerDownRight", "MouthUpperUpLeft", "MouthUpperUpRight", "BrowDownLeft",
//  "BrowDownRight", "BrowInnerUp", "BrowOuterUpLeft", "BrowOuterUpRight", "CheekPuff",
//  "CheekSquintLeft", "CheekSquintRight", "NoseSneerLeft", "NoseSneerRight", "TongueOut"
// Note1: some multipliers and offset visual impact are lighter than others.
// "JawOpen", "MouthSmileLeft" and "MouthSmileRight" have stronger visual impact
// Note2: Blendshape values are after applying multipliers and offset are clamped between 0 and 1
// E.g.:
// * inferenced_weight is 0.9
// * multiplier_value is set to 3
// * offset_value is set to -1
// Then the result will be:
// 0.9 * 3 - 1 = 1.7 ===> clamp between 0, 1 ===> adjusted weight is 1
message BlendShapeParameters {
  // When a key is not specified the default value is 1
  map<string, float> bs_weight_multipliers = 1;
  // When a key is not specified the default value is 0
  map<string, float> bs_weight_offsets = 2;
  // Default output bs weight is unclamped. When clamped, range is [0, 1].
  optional bool enable_clamping_bs_weight = 3;
}

message EmotionParameters {
  // Transition time value used for temporal smoothing by A2E SDK
  // Expected value range: 0 < val < inf
  optional float live_transition_time = 1;

  // Beginning emotion used for temporal emotion smoothing
  // This maps the emotion names to the corresponding emotion strength
  // Missing emotion values will be set to 0.0
  // The following emotions can be set:
  // "amazement", "anger", "cheekiness", "disgust", "fear",
  // "grief", "joy", "outofbreath", "pain", "sadness"
  // Emotion values must be set between 0.0 and 1.0
  map<string, float> beginning_emotion = 2;
}

// For more information refer to the documentation
message EmotionPostProcessingParameters {
  // Increases the spread between emotion values by pushing them higher or lower.
  // Default value: 1
  // Min: 0.3
  // Max: 3
  optional float emotion_contrast = 1;

  // Coefficient for smoothing emotions over time
  //  0 means no smoothing at all (can be jittery)
  //  1 means extreme smoothing (emotion values not updated over time)
  // Default value: 0.7
  // Min: 0
  // Max: 1
  optional float live_blend_coef = 2;

  // Activate blending between the preferred emotions (passed as input) and the emotions detected by A2E.
  // Default: True
  optional bool enable_preferred_emotion = 3;

  // Sets the strength of the preferred emotions (passed as input) relative to emotions detected by A2E.
  // 0 means only A2E output will be used for emotion rendering.
  // 1 means only the preferred emotions will be used for emotion rendering.
  // Default value: 0.5
  // Min: 0
  // Max: 1
  optional float preferred_emotion_strength = 4;

  // Sets the strength of generated emotions relative to neutral emotion.
  // This multiplier is applied globally after the mix of emotion is done.
  // If set to 0, emotion will be neutral.
  // If set to 1, the blend of emotion will be fully used. (can be too intense)
  // Default value: 0.6
  // Min: 0
  // Max: 1
  optional float emotion_strength = 5;

  // Sets a firm limit on the quantity of emotion sliders engaged by A2E
  // emotions with highest weight will be prioritized
  // Default value: 3
  // Min: 1
  // Max: 6
  optional int32 max_emotions = 6;
}

message AudioWithEmotion {
  // audio buffer in bytes to interpret depending on the audio header
  bytes audio_buffer = 1;

  // The time codes are relative to the beginning of the audio clip.
  repeated nvidia_ace.emotion_with_timecode.v1.EmotionWithTimeCode emotions = 2;
}
//nvidia_ace.a2f.v1
//v1.1.0