Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

10.1 KB
Raw Blame
#!/usr/bin/env python3
# -*- encoding: utf-8 -*-
# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
#  MIT License  (https://opensource.org/licenses/MIT)
# Modified from 3D-Speaker (https://github.com/alibaba-damo-academy/3D-Speaker)

import scipy
import torch
import sklearn
import numpy as np

from sklearn.cluster._kmeans import k_means
from sklearn.cluster import HDBSCAN
from sklearn.preprocessing import normalize


class SpectralCluster:
    r"""A spectral clustering mehtod using unnormalized Laplacian of affinity matrix.
    This implementation is adapted from https://github.com/speechbrain/speechbrain.
    """

    def __init__(self, min_num_spks=1, max_num_spks=15, pval=0.022):
        """Initialize SpectralCluster.
        
            Args:
                min_num_spks: TODO.
                max_num_spks: TODO.
                pval: TODO.
            """
        self.min_num_spks = min_num_spks
        self.max_num_spks = max_num_spks
        self.pval = pval

    def __call__(self, X, oracle_num=None):
        # Similarity matrix computation
        """Internal: call  .
        
            Args:
                X: TODO.
                oracle_num: TODO.
            """
        sim_mat = self.get_sim_mat(X)

        # Refining similarity matrix with pval
        prunned_sim_mat = self.p_pruning(sim_mat)

        # Symmetrization
        sym_prund_sim_mat = 0.5 * (prunned_sim_mat + prunned_sim_mat.T)

        # Laplacian calculation
        laplacian = self.get_laplacian(sym_prund_sim_mat)

        # Get Spectral Embeddings
        emb, num_of_spk = self.get_spec_embs(laplacian, oracle_num)

        # Perform clustering
        labels = self.cluster_embs(emb, num_of_spk)

        return labels

    def get_sim_mat(self, X):
        # Cosine similarities
        """Get sim mat.
        
            Args:
                X: TODO.
            """
        M = sklearn.metrics.pairwise.cosine_similarity(X, X)
        return M

    def p_pruning(self, A):
        """P pruning.
        
            Args:
                A: TODO.
            """
        if A.shape[0] * self.pval < 6:
            pval = 6.0 / A.shape[0]
        else:
            pval = self.pval

        n_elems = int((1 - pval) * A.shape[0])

        # For each row in a affinity matrix
        for i in range(A.shape[0]):
            low_indexes = np.argsort(A[i, :])
            low_indexes = low_indexes[0:n_elems]

            # Replace smaller similarity values by 0s
            A[i, low_indexes] = 0
        return A

    def get_laplacian(self, M):
        """Get laplacian.
        
            Args:
                M: TODO.
            """
        M[np.diag_indices(M.shape[0])] = 0
        D = np.sum(np.abs(M), axis=1)
        D = np.diag(D)
        L = D - M
        return L

    def get_spec_embs(self, L, k_oracle=None):
        """Get spec embs.

            Args:
                L: TODO.
                k_oracle: TODO.
            """
        # Only the leading eigenpairs are ever used: the speaker count comes
        # from the gaps between the first ``max_num_spks + 1`` eigenvalues, and
        # the embedding keeps the first ``num_of_spk`` eigenvectors. Producing
        # the full spectrum costs O(N^3) and throws all but a handful away.
        n_eig = self.max_num_spks + 1
        if k_oracle is not None:
            n_eig = max(n_eig, int(k_oracle))
        n_eig = min(n_eig, L.shape[0])

        lambdas, eig_vecs = scipy.linalg.eigh(
            L, subset_by_index=[0, n_eig - 1]
        )

        if k_oracle is not None:
            num_of_spk = k_oracle
        else:
            lambda_gap_list = self.getEigenGaps(
                lambdas[self.min_num_spks - 1 : self.max_num_spks + 1]
            )
            num_of_spk = np.argmax(lambda_gap_list) + self.min_num_spks

        emb = eig_vecs[:, :num_of_spk]
        return emb, num_of_spk

    def cluster_embs(self, emb, k):
        """Cluster embs.
        
            Args:
                emb: TODO.
                k: TODO.
            """
        _, labels, _ = k_means(emb, k)
        return labels

    def getEigenGaps(self, eig_vals):
        """Geteigengaps.
        
            Args:
                eig_vals: TODO.
            """
        eig_vals_gap_list = []
        for i in range(len(eig_vals) - 1):
            gap = float(eig_vals[i + 1]) - float(eig_vals[i])
            eig_vals_gap_list.append(gap)
        return eig_vals_gap_list


class UmapHdbscan:
    r"""
    Reference:
    - Siqi Zheng, Hongbin Suo. Reformulating Speaker Diarization as Community Detection With
      Emphasis On Topological Structure. ICASSP2022
    """

    def __init__(
        self, n_neighbors=20, n_components=60, min_samples=10, min_cluster_size=10, metric="cosine"
    ):
        """Initialize UmapHdbscan.
        
            Args:
                n_neighbors: TODO.
                n_components: TODO.
                min_samples: TODO.
                min_cluster_size: Size/dimension parameter.
                metric: TODO.
            """
        self.n_neighbors = n_neighbors
        self.n_components = n_components
        self.min_samples = min_samples
        self.min_cluster_size = min_cluster_size
        self.metric = metric

    def __call__(self, X):
        """Internal: call  .
        
            Args:
                X: TODO.
            """
        import umap.umap_ as umap

        umap_X = umap.UMAP(
            n_neighbors=self.n_neighbors,
            min_dist=0.0,
            n_components=min(self.n_components, X.shape[0] - 2),
            metric=self.metric,
        ).fit_transform(X)
        labels = HDBSCAN(
            min_samples=self.min_samples,
            min_cluster_size=self.min_cluster_size,
            allow_single_cluster=True,
        ).fit_predict(umap_X)
        return labels


class KMeansCluster:
    r"""Cluster a large set of speaker embeddings into a known number of groups."""

    def __call__(self, X, num_clusters):
        """Cluster L2-normalized embeddings with bounded memory usage."""
        normalized_X = normalize(X)
        _, labels, _ = k_means(
            normalized_X,
            num_clusters,
            random_state=0,
            n_init=10,
        )
        return labels


class ClusterBackend(torch.nn.Module):
    r"""Perfom clustering for input embeddings and output the labels.
    Args:
        model_dir: A model dir.
        model_config: The model config.
    """

    def __init__(self, merge_thr=0.78, max_num_spks=15):
        """Initialize ClusterBackend.
        
            Args:
                merge_thr: TODO.
                max_num_spks: Positive upper bound for automatic speaker counting.
            """
        super().__init__()
        if (
            isinstance(max_num_spks, bool)
            or not isinstance(max_num_spks, (int, np.integer))
            or max_num_spks < 1
        ):
            raise ValueError("max_num_spks must be a positive integer")
        self.model_config = {"merge_thr": merge_thr}
        # self.other_config = kwargs

        self.spectral_cluster = SpectralCluster(max_num_spks=max_num_spks)
        self.umap_hdbscan_cluster = UmapHdbscan()
        self.kmeans_cluster = KMeansCluster()

    def forward(self, X, **params):
        # clustering and return the labels
        """Forward pass for training.
        
            Args:
                X: TODO.
                **params: Additional keyword arguments.
            """
        k = params["oracle_num"] if "oracle_num" in params else None
        assert len(X.shape) == 2, "modelscope error: the shape of input should be [N, C]"
        if X.shape[0] < 20:
            return np.zeros(X.shape[0], dtype="int")
        if X.shape[0] < 2048:
            labels = self.spectral_cluster(X, k)
        elif k is not None:
            labels = self.kmeans_cluster(X, k)
        else:
            labels = self.umap_hdbscan_cluster(X)
            labels = self.merge_to_max_spks(labels, X, self.spectral_cluster.max_num_spks)

        if k is None and "merge_thr" in self.model_config:
            labels = self.merge_by_cos(labels, X, self.model_config["merge_thr"])

        return labels

    def merge_to_max_spks(self, labels, embs, max_num_spks):
        """Merge closest speaker centroids until the cap is met, preserving noise labels."""
        labels = labels.copy()
        embs = np.asarray(embs)
        while True:
            speaker_labels = np.unique(labels[labels >= 0])
            if len(speaker_labels) <= max_num_spks:
                return labels
            centers = np.stack([embs[labels == label].mean(0) for label in speaker_labels])
            affinity = sklearn.metrics.pairwise.cosine_similarity(centers)
            rows, cols = np.triu_indices(len(speaker_labels), k=1)
            closest = np.argmax(affinity[rows, cols])
            keep, remove = speaker_labels[rows[closest]], speaker_labels[cols[closest]]
            labels[labels == remove] = keep
            labels[labels > remove] -= 1

    def merge_by_cos(self, labels, embs, cos_thr):
        # merge the similar speakers by cosine similarity
        """Merge by cos.
        
            Args:
                labels: TODO.
                embs: TODO.
                cos_thr: TODO.
            """
        assert cos_thr > 0 and cos_thr <= 1
        while True:
            spk_num = labels.max() + 1
            if spk_num == 1:
                break
            spk_center = []
            for i in range(spk_num):
                spk_emb = embs[labels == i].mean(0)
                spk_center.append(spk_emb)
            assert len(spk_center) > 0
            spk_center = np.stack(spk_center, axis=0)
            norm_spk_center = spk_center / np.linalg.norm(spk_center, axis=1, keepdims=True)
            affinity = np.matmul(norm_spk_center, norm_spk_center.T)
            affinity = np.triu(affinity, 1)
            spks = np.unravel_index(np.argmax(affinity), affinity.shape)
            if affinity[spks] < cos_thr:
                break
            for i in range(len(labels)):
                if labels[i] == spks[1]:
                    labels[i] = spks[0]
                elif labels[i] > spks[1]:
                    labels[i] -= 1
        return labels