Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
/*
* SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
syntax = "proto3";
package nvidia_ace.controller.v1;
import "nvidia_ace.a2f.v1.proto";
import "nvidia_ace.animation_data.v1.proto";
import "nvidia_ace.audio.v1.proto";
import "nvidia_ace.status.v1.proto";
import "google/protobuf/any.proto";
message AudioStream {
// This is a marker for the end of an audio clip.
message EndOfAudio {}
oneof stream_part {
// The header must be sent as the first message.
AudioStreamHeader audio_stream_header = 1;
// At least one AudioWithEmotion messages must be sent thereafter.
nvidia_ace.a2f.v1.AudioWithEmotion audio_with_emotion = 2;
// The EndOfAudio must be sent last.
EndOfAudio end_of_audio = 3;
}
}
// IMPORTANT NOTE: this is an AudioStreamHeader WITHOUT ID
// A similar AudioStreamHeader exist in nvidia_ace.a2f.v1.proto
// but that one does contain IDs.
message AudioStreamHeader {
// Metadata about the audio being sent to the service.
nvidia_ace.audio.v1.AudioHeader audio_header = 1;
// Parameters for updating the facial characteristics of an avatar.
// See the documentation for more information.
nvidia_ace.a2f.v1.FaceParameters face_params = 2;
// Parameters relative to the emotion blending and processing
// before using it to generate blendshapes.
// See the documentation for more information.
nvidia_ace.a2f.v1.EmotionPostProcessingParameters emotion_post_processing_params = 3;
// Multipliers and offsets to apply to the generated blendshape values.
nvidia_ace.a2f.v1.BlendShapeParameters blendshape_params = 4;
// Emotion parameters (live transition time, beginning emotion)
nvidia_ace.a2f.v1.EmotionParameters emotion_params = 5;
}
enum EventType {
// This event type means that the A2F Microservice is done processing audio,
// However it doesn't mean that you finished receiving all the audio data,
// You will receive a Status message once you are done receiving all the audio
// data. Events are independent of that.
END_OF_A2F_AUDIO_PROCESSING = 0;
}
message Event {
// Type of the event.
EventType event_type = 1;
// Data attached to the event if any.
optional google.protobuf.Any metadata = 2;
}
// IMPORTANT NOTE: this is an AnimationDataStreamHeader WITHOUT ID
// A similar AudioStreamHeader exist in nvidia_ace.animation_data.v1.proto
// but that one does contain IDs.
message AnimationDataStreamHeader {
// Metadata of the audio buffers. This defines the audio clip properties
// at the beginning the streaming process.
optional nvidia_ace.audio.v1.AudioHeader audio_header = 1;
// Metadata containing the blendshape and joints names.
// This defines the names of the blendshapes and joints flowing through a stream.
optional nvidia_ace.animation_data.v1.SkelAnimationHeader
skel_animation_header = 2;
// Time codes indicate the relative progression of an animation data, audio
// clip, etc. The unit is seconds. In addition, we also need an absolute time
// reference shared across services. The start time is stored in time codes
// elapsed since the Unix time epoch. start_time_code_since_epoch = `Unix
// timestamp in seconds`. NTP should be good enough to synchronize clocks
// across nodes. From Wikipedia: NTP can usually maintain time to within tens
// of milliseconds over the public Internet, and can achieve better than one
// millisecond accuracy in local area networks under ideal conditions.
// Alternatively, there is PTP.
double start_time_code_since_epoch = 3;
// A generic metadata field to attach use case specific data (e.g. session id,
// or user id?) map<string, string> metadata = 4; map<string,
// google.protobuf.Any> metadata = 4;
}
message AnimationDataStream {
// The header must be sent as the first message.
// One or more animation data message must be sent.
// The status must be sent last and may be sent in between.
oneof stream_part {
// The header must be sent as the first message.
AnimationDataStreamHeader animation_data_stream_header = 1;
// Then one or more animation data message must be sent.
nvidia_ace.animation_data.v1.AnimationData animation_data = 2;
// The event may be sent in between.
Event event = 3;
// The status must be sent last and may be sent in between.
nvidia_ace.status.v1.Status status = 4;
}
}
//nvidia_ace.controller.v1
//v1.0.0