Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
/*
* SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
syntax = "proto3";
package nvidia_ace.a2f.v1;
import "nvidia_ace.animation_id.v1.proto";
import "nvidia_ace.status.v1.proto";
import "nvidia_ace.audio.v1.proto";
import "nvidia_ace.emotion_with_timecode.v1.proto";
message AudioStream {
// The header must be sent as the first message.
// One or more audio with emotion messages must be sent thereafter.
// The end of audio will happen when the client closes the connection
oneof stream_part {
AudioStreamHeader audio_stream_header = 1;
nvidia_ace.a2f.v1.AudioWithEmotion audio_with_emotion = 2;
}
}
// IMPORTANT NOTE: this is an AudioStreamHeader WITH ID
// A similar AudioStreamHeader exist in nvidia_ace.controller.v1.proto
// but that one does NOT contain IDs
message AudioStreamHeader {
// IDs of the current stream
nvidia_ace.animation_id.v1.AnimationIds animation_ids = 1;
nvidia_ace.audio.v1.AudioHeader audio_header = 2;
// Parameters for updating the facial characteristics of an avatar
// See the documentation for more information
FaceParameters face_params = 3;
// Parameters relative to the emotion blending and processing
// before using it to generate blendshapes
// See the documentation for more information
EmotionPostProcessingParameters emotion_post_processing_params = 4;
// Multipliers and offsets to apply to the generated blendshape values
BlendShapeParameters blendshape_params = 5;
// Emotion parameters (live transition time, beginning emotion)
EmotionParameters emotion_params = 6;
}
message FloatArray { repeated float values = 1; }
message FaceParameters {
// The following float parameters are available:
// "lowerFaceSmoothing", "upperFaceSmoothing", "lowerFaceStrength", "upperFaceStrength",
// "faceMaskLevel", "faceMaskSoftness", "skinStrength", "blinkStrength", "eyelidOpenOffset",
// "lipOpenOffset", "blinkOffset", "tongueStrength", "tongueHeightOffset", "tongueDepthOffset"
map<string, float> float_params = 1;
// With the current Audio2Face Service no integer parameters are available
map<string, int32> integer_params = 2;
// With the current Audio2Face Service no FloatArray parameters are available
map<string, FloatArray> float_array_params = 3;
}
// The following blendshape can be used here as key:
// "EyeBlinkLeft", "EyeLookDownLeft", "EyeLookInLeft", "EyeLookOutLeft", "EyeLookUpLeft",
// "EyeSquintLeft", "EyeWideLeft", "EyeBlinkRight", "EyeLookDownRight", "EyeLookInRight",
// "EyeLookOutRight", "EyeLookUpRight", "EyeSquintRight", "EyeWideRight", "JawForward",
// "JawLeft", "JawRight", "JawOpen", "MouthClose", "MouthFunnel", "MouthPucker", "MouthLeft",
// "MouthRight", "MouthSmileLeft", "MouthSmileRight", "MouthFrownLeft", "MouthFrownRight",
// "MouthDimpleLeft", "MouthDimpleRight", "MouthStretchLeft", "MouthStretchRight", "MouthRollLower",
// "MouthRollUpper", "MouthShrugLower", "MouthShrugUpper", "MouthPressLeft", "MouthPressRight",
// "MouthLowerDownLeft", "MouthLowerDownRight", "MouthUpperUpLeft", "MouthUpperUpRight", "BrowDownLeft",
// "BrowDownRight", "BrowInnerUp", "BrowOuterUpLeft", "BrowOuterUpRight", "CheekPuff",
// "CheekSquintLeft", "CheekSquintRight", "NoseSneerLeft", "NoseSneerRight", "TongueOut"
// Note1: some multipliers and offset visual impact are lighter than others.
// "JawOpen", "MouthSmileLeft" and "MouthSmileRight" have stronger visual impact
// Note2: Blendshape values are after applying multipliers and offset are clamped between 0 and 1
// E.g.:
// * inferenced_weight is 0.9
// * multiplier_value is set to 3
// * offset_value is set to -1
// Then the result will be:
// 0.9 * 3 - 1 = 1.7 ===> clamp between 0, 1 ===> adjusted weight is 1
message BlendShapeParameters {
// When a key is not specified the default value is 1
map<string, float> bs_weight_multipliers = 1;
// When a key is not specified the default value is 0
map<string, float> bs_weight_offsets = 2;
// Default output bs weight is unclamped. When clamped, range is [0, 1].
optional bool enable_clamping_bs_weight = 3;
}
message EmotionParameters {
// Transition time value used for temporal smoothing by A2E SDK
// Expected value range: 0 < val < inf
optional float live_transition_time = 1;
// Beginning emotion used for temporal emotion smoothing
// This maps the emotion names to the corresponding emotion strength
// Missing emotion values will be set to 0.0
// The following emotions can be set:
// "amazement", "anger", "cheekiness", "disgust", "fear",
// "grief", "joy", "outofbreath", "pain", "sadness"
// Emotion values must be set between 0.0 and 1.0
map<string, float> beginning_emotion = 2;
}
// For more information refer to the documentation
message EmotionPostProcessingParameters {
// Increases the spread between emotion values by pushing them higher or lower.
// Default value: 1
// Min: 0.3
// Max: 3
optional float emotion_contrast = 1;
// Coefficient for smoothing emotions over time
// 0 means no smoothing at all (can be jittery)
// 1 means extreme smoothing (emotion values not updated over time)
// Default value: 0.7
// Min: 0
// Max: 1
optional float live_blend_coef = 2;
// Activate blending between the preferred emotions (passed as input) and the emotions detected by A2E.
// Default: True
optional bool enable_preferred_emotion = 3;
// Sets the strength of the preferred emotions (passed as input) relative to emotions detected by A2E.
// 0 means only A2E output will be used for emotion rendering.
// 1 means only the preferred emotions will be used for emotion rendering.
// Default value: 0.5
// Min: 0
// Max: 1
optional float preferred_emotion_strength = 4;
// Sets the strength of generated emotions relative to neutral emotion.
// This multiplier is applied globally after the mix of emotion is done.
// If set to 0, emotion will be neutral.
// If set to 1, the blend of emotion will be fully used. (can be too intense)
// Default value: 0.6
// Min: 0
// Max: 1
optional float emotion_strength = 5;
// Sets a firm limit on the quantity of emotion sliders engaged by A2E
// emotions with highest weight will be prioritized
// Default value: 3
// Min: 1
// Max: 6
optional int32 max_emotions = 6;
}
message AudioWithEmotion {
// audio buffer in bytes to interpret depending on the audio header
bytes audio_buffer = 1;
// The time codes are relative to the beginning of the audio clip.
repeated nvidia_ace.emotion_with_timecode.v1.EmotionWithTimeCode emotions = 2;
}
//nvidia_ace.a2f.v1
//v1.1.0