Skip to content

Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.

8.5 KB
Raw Blame
package volcengine

import (
	"bytes"
	"context"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"krillin-ai/internal/types"
	"krillin-ai/log"
	"krillin-ai/pkg/util"

	"github.com/google/uuid"
	"go.uber.org/zap"
)

var processAudioFile = util.ProcessAudio

const (
	DefaultAsrBaseURL    = "https://openspeech.bytedance.com"
	DefaultAsrResourceID = "volc.seedasr.auc"
	submitPath           = "/api/v3/auc/bigmodel/submit"
	queryPath            = "/api/v3/auc/bigmodel/query"
	statusSuccess        = "20000000"
	statusProcessing     = "20000001"
	statusQueued         = "20000002"
)

type AsrClient struct {
	baseURL      string
	appID        string
	accessToken  string
	resourceID   string
	httpClient   *http.Client
	pollInterval time.Duration
	maxPollTime  time.Duration
}

// NewAsrClient builds a client for the v3 bigmodel file-ASR API. Newer
// Doubao Voice consoles issue a single API Key, which is passed as accessToken
// with an empty appID and sent as X-Api-Key. Older consoles issue an App ID
// plus Access Token pair, sent as X-Api-App-Key and X-Api-Access-Key.
func NewAsrClient(baseURL, appID, accessToken, resourceID, proxy string) *AsrClient {
	baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")
	if baseURL == "" {
		baseURL = DefaultAsrBaseURL
	}
	resourceID = strings.TrimSpace(resourceID)
	if resourceID == "" {
		resourceID = DefaultAsrResourceID
	}
	transport := http.DefaultTransport.(*http.Transport).Clone()
	if proxyURL, err := url.Parse(strings.TrimSpace(proxy)); err == nil && proxyURL.Scheme != "" {
		transport.Proxy = http.ProxyURL(proxyURL)
	}
	return &AsrClient{
		baseURL:     baseURL,
		appID:       strings.TrimSpace(appID),
		accessToken: strings.TrimSpace(accessToken),
		resourceID:  resourceID,
		httpClient: &http.Client{
			Transport: transport,
			Timeout:   60 * time.Second,
		},
		pollInterval: 3 * time.Second,
		maxPollTime:  10 * time.Minute,
	}
}

type asrSubmitRequest struct {
	User    asrUser    `json:"user"`
	Audio   asrAudio   `json:"audio"`
	Request asrOptions `json:"request"`
}

type asrUser struct {
	UID string `json:"uid"`
}

type asrAudio struct {
	Data     string `json:"data"`
	Format   string `json:"format"`
	Language string `json:"language,omitempty"`
}

type asrOptions struct {
	ModelName      string `json:"model_name"`
	EnableITN      bool   `json:"enable_itn"`
	EnablePunc     bool   `json:"enable_punc"`
	ShowUtterances bool   `json:"show_utterances"`
}

type asrQueryResponse struct {
	Result *asrResult `json:"result"`
}

type asrResult struct {
	Text       string         `json:"text"`
	Utterances []asrUtterance `json:"utterances"`
}

type asrUtterance struct {
	Text      string    `json:"text"`
	StartTime int64     `json:"start_time"`
	EndTime   int64     `json:"end_time"`
	Words     []asrWord `json:"words"`
}

type asrWord struct {
	Text      string `json:"text"`
	StartTime int64  `json:"start_time"`
	EndTime   int64  `json:"end_time"`
}

func (c *AsrClient) Transcription(audioFile, language, workDir string) (*types.TranscriptionData, error) {
	language = string(types.NormalizeLanguage(language))
	if language == "zh_cn" || language == "zh_tw" {
		language = "zh"
	}

	_ = workDir
	if c.accessToken == "" {
		return nil, fmt.Errorf("volcengine asr api key or access token is empty")
	}
	processed, err := processAudioFile(audioFile)
	if err != nil {
		return nil, err
	}
	audio, err := os.ReadFile(processed)
	if err != nil {
		return nil, fmt.Errorf("volcengine asr read audio: %w", err)
	}
	if len(audio) == 0 {
		return nil, fmt.Errorf("volcengine asr audio is empty")
	}
	requestID := uuid.NewString()
	body, err := json.Marshal(asrSubmitRequest{
		User: asrUser{UID: "opencreator"},
		Audio: asrAudio{
			Data:     base64.StdEncoding.EncodeToString(audio),
			Format:   audioFormat(processed),
			Language: volcengineLanguage(language),
		},
		Request: asrOptions{
			ModelName:      "bigmodel",
			EnableITN:      true,
			EnablePunc:     true,
			ShowUtterances: true,
		},
	})
	if err != nil {
		return nil, err
	}
	submitStatus, _, err := c.post(c.baseURL+submitPath, requestID, body)
	if err != nil {
		return nil, err
	}
	if submitStatus != "" && submitStatus != statusSuccess && submitStatus != statusQueued && submitStatus != statusProcessing {
		return nil, fmt.Errorf("volcengine asr submit failed: status %s", submitStatus)
	}
	deadline := time.Now().Add(c.maxPollTime)
	for {
		if time.Now().After(deadline) {
			return nil, fmt.Errorf("volcengine asr polling timeout after %s", c.maxPollTime)
		}
		time.Sleep(c.pollInterval)
		status, payload, err := c.post(c.baseURL+queryPath, requestID, []byte("{}"))
		if err != nil {
			return nil, err
		}
		switch status {
		case "", statusQueued, statusProcessing:
			if result := parseAsrResult(payload); result != nil && result.Text != "" {
				return transcriptionFromAsr(result, language), nil
			}
			continue
		case statusSuccess:
			result := parseAsrResult(payload)
			if result == nil || strings.TrimSpace(result.Text) == "" {
				return nil, fmt.Errorf("volcengine asr returned an empty transcript")
			}
			log.GetLogger().Info("volcengine asr transcription succeeded", zap.String("request_id", requestID))
			return transcriptionFromAsr(result, language), nil
		default:
			return nil, fmt.Errorf("volcengine asr query failed: status %s", status)
		}
	}
}

func (c *AsrClient) post(endpoint, requestID string, body []byte) (string, []byte, error) {
	req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, endpoint, bytes.NewReader(body))
	if err != nil {
		return "", nil, err
	}
	req.Header.Set("Content-Type", "application/json")
	if c.appID == "" {
		req.Header.Set("X-Api-Key", c.accessToken)
	} else {
		req.Header.Set("X-Api-App-Key", c.appID)
		req.Header.Set("X-Api-Access-Key", c.accessToken)
	}
	req.Header.Set("X-Api-Resource-Id", c.resourceID)
	req.Header.Set("X-Api-Request-Id", requestID)
	req.Header.Set("X-Api-Sequence", "-1")
	response, err := c.httpClient.Do(req)
	if err != nil {
		return "", nil, fmt.Errorf("volcengine asr request failed: %w", err)
	}
	defer response.Body.Close()
	payload, err := io.ReadAll(io.LimitReader(response.Body, 32<<20))
	if err != nil {
		return "", nil, err
	}
	if response.StatusCode != http.StatusOK {
		return "", payload, fmt.Errorf("volcengine asr returned HTTP %d: %s", response.StatusCode, boundedMessage(payload))
	}
	return response.Header.Get("X-Api-Status-Code"), payload, nil
}

func parseAsrResult(payload []byte) *asrResult {
	var decoded asrQueryResponse
	if err := json.Unmarshal(payload, &decoded); err != nil {
		return nil
	}
	return decoded.Result
}

func transcriptionFromAsr(result *asrResult, language string) *types.TranscriptionData {
	data := &types.TranscriptionData{
		Language: language,
		Text:     strings.TrimSpace(result.Text),
	}
	var words []types.Word
	index := 0
	for _, utterance := range result.Utterances {
		if len(utterance.Words) > 0 {
			for _, word := range utterance.Words {
				text := strings.TrimSpace(word.Text)
				if text == "" {
					continue
				}
				words = append(words, types.Word{
					Num:   index,
					Text:  text,
					Start: float64(word.StartTime) / 1000,
					End:   float64(word.EndTime) / 1000,
				})
				index++
			}
			continue
		}
		text := strings.TrimSpace(utterance.Text)
		if text == "" {
			continue
		}
		words = append(words, types.Word{
			Num:   index,
			Text:  text,
			Start: float64(utterance.StartTime) / 1000,
			End:   float64(utterance.EndTime) / 1000,
		})
		index++
	}
	data.Words = words
	return data
}

func audioFormat(path string) string {
	switch strings.ToLower(filepath.Ext(path)) {
	case ".wav":
		return "wav"
	case ".ogg":
		return "ogg"
	case ".m4a":
		return "m4a"
	default:
		return "mp3"
	}
}

func volcengineLanguage(language string) string {
	switch strings.ToLower(strings.TrimSpace(language)) {
	case "", "auto":
		return ""
	case "zh", "zh_cn", "zh-cn":
		return "zh-CN"
	case "en", "en-us":
		return "en-US"
	case "ja", "ja-jp":
		return "ja-JP"
	case "ko", "ko-kr":
		return "ko-KR"
	case "es", "es-mx":
		return "es-MX"
	case "fr", "fr-fr":
		return "fr-FR"
	case "de", "de-de":
		return "de-DE"
	case "it", "it-it":
		return "it-IT"
	case "pt", "pt-br":
		return "pt-BR"
	case "id", "id-id":
		return "id-ID"
	case "ms", "ms-my":
		return "ms-MY"
	case "th", "th-th":
		return "th-TH"
	case "ru", "ru-ru":
		return "ru-RU"
	case "tr", "tr-tr":
		return "tr-TR"
	default:
		return language
	}
}

func boundedMessage(value []byte) string {
	const limit = 300
	message := strings.TrimSpace(string(value))
	if len(message) <= limit {
		return message
	}
	return message[:limit]
}