Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

4.9 KB
Raw Blame
/*
 * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: Apache-2.0
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

syntax = "proto3";

package nvidia_ace.controller.v1;

import "nvidia_ace.a2f.v1.proto";
import "nvidia_ace.animation_data.v1.proto";
import "nvidia_ace.audio.v1.proto";
import "nvidia_ace.status.v1.proto";
import "google/protobuf/any.proto";

message AudioStream {
  // This is a marker for the end of an audio clip.
  message EndOfAudio {}

  oneof stream_part {
    // The header must be sent as the first message.
    AudioStreamHeader audio_stream_header = 1;
    // At least one AudioWithEmotion messages must be sent thereafter.
    nvidia_ace.a2f.v1.AudioWithEmotion audio_with_emotion = 2;
    // The EndOfAudio must be sent last.
    EndOfAudio end_of_audio = 3;
  }
}

// IMPORTANT NOTE: this is an AudioStreamHeader WITHOUT ID
// A similar AudioStreamHeader exist in nvidia_ace.a2f.v1.proto
// but that one does contain IDs.
message AudioStreamHeader {
  // Metadata about the audio being sent to the service.
  nvidia_ace.audio.v1.AudioHeader audio_header = 1;
  // Parameters for updating the facial characteristics of an avatar.
  // See the documentation for more information.
  nvidia_ace.a2f.v1.FaceParameters face_params = 2;
  // Parameters relative to the emotion blending and processing
  // before using it to generate blendshapes.
  // See the documentation for more information.
  nvidia_ace.a2f.v1.EmotionPostProcessingParameters emotion_post_processing_params = 3;
  // Multipliers and offsets to apply to the generated blendshape values.
  nvidia_ace.a2f.v1.BlendShapeParameters blendshape_params = 4;
  // Emotion parameters (live transition time, beginning emotion)
  nvidia_ace.a2f.v1.EmotionParameters emotion_params = 5;
}

enum EventType {
  // This event type means that the A2F Microservice is done processing audio,
  // However it doesn't mean that you finished receiving all the audio data,
  // You will receive a Status message once you are done receiving all the audio
  // data. Events are independent of that.
  END_OF_A2F_AUDIO_PROCESSING = 0;
}

message Event {
  // Type of the event.
  EventType event_type = 1;
  // Data attached to the event if any.
  optional google.protobuf.Any metadata = 2;
}

// IMPORTANT NOTE: this is an AnimationDataStreamHeader WITHOUT ID
// A similar AudioStreamHeader exist in nvidia_ace.animation_data.v1.proto
// but that one does contain IDs.
message AnimationDataStreamHeader {
  // Metadata of the audio buffers. This defines the audio clip properties
  // at the beginning the streaming process.
  optional nvidia_ace.audio.v1.AudioHeader audio_header = 1;
  // Metadata containing the blendshape and joints names.
  // This defines the names of the blendshapes and joints flowing through a stream.
  optional nvidia_ace.animation_data.v1.SkelAnimationHeader
      skel_animation_header = 2;

  // Time codes indicate the relative progression of an animation data, audio
  // clip, etc. The unit is seconds. In addition, we also need an absolute time
  // reference shared across services. The start time is stored in time codes
  // elapsed since the Unix time epoch. start_time_code_since_epoch = `Unix
  // timestamp in seconds`. NTP should be good enough to synchronize clocks
  // across nodes. From Wikipedia: NTP can usually maintain time to within tens
  // of milliseconds over the public Internet, and can achieve better than one
  // millisecond accuracy in local area networks under ideal conditions.
  // Alternatively, there is PTP.
  double start_time_code_since_epoch = 3;

  // A generic metadata field to attach use case specific data (e.g. session id,
  // or user id?) map<string, string> metadata = 4; map<string,
  // google.protobuf.Any> metadata = 4;
}

message AnimationDataStream {
  // The header must be sent as the first message.
  // One or more animation data message must be sent.
  // The status must be sent last and may be sent in between.
  oneof stream_part {
    // The header must be sent as the first message.
    AnimationDataStreamHeader animation_data_stream_header = 1;
    // Then one or more animation data message must be sent.
    nvidia_ace.animation_data.v1.AnimationData animation_data = 2;
    // The event may be sent in between.
    Event event = 3;
    // The status must be sent last and may be sent in between.
    nvidia_ace.status.v1.Status status = 4;
  }
}
//nvidia_ace.controller.v1
//v1.0.0