Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

9.7 KB
Raw Blame
"""
ONNX module tests
"""

import os
import shutil
import tempfile
import unittest

from unittest.mock import patch

import numpy as np
import onnx

from onnx import helper, TensorProto
from tokenizers import Tokenizer, models, pre_tokenizers

from txtai.pipeline import HFOnnx
from txtai.util import DownloadError
from txtai.vectors import VectorsFactory
from txtai.vectors.dense.onnx import ONNX


class TestONNX(unittest.TestCase):
    """
    ONNX vectors tests
    """

    @classmethod
    def setUpClass(cls):
        """
        Export a model to ONNX and create an ONNX vectors instance.
        """

        path = "sentence-transformers/paraphrase-MiniLM-L3-v2"

        # Export model to ONNX
        cls.path = os.path.join(tempfile.gettempdir(), "vectors", "model.onnx")
        os.makedirs(os.path.dirname(cls.path), exist_ok=True)
        HFOnnx()(path, "pooling", cls.path)

        cls.model = VectorsFactory.create({"path": cls.path, "tokenizer": path, "gpu": False}, None)

    def testMethod(self):
        """
        Test that an .onnx path resolves to the onnx method
        """

        self.assertEqual(VectorsFactory.method({"path": self.path}), "onnx")
        self.assertEqual(VectorsFactory.method({"path": "model.tflite"}), "litert")

    def testIndex(self):
        """
        Test indexing with ONNX vectors
        """

        ids, dimension, batches, stream = self.model.index([(0, "test", None)])

        self.assertEqual(len(ids), 1)
        self.assertEqual(dimension, 384)
        self.assertEqual(batches, 1)
        self.assertIsNotNone(os.path.exists(stream))

        # Test shape of serialized embeddings
        with open(stream, "rb") as queue:
            self.assertEqual(np.load(queue).shape, (1, 384))

    def testEncodeBatch(self):
        """
        Test that results are stable when a batch spans multiple encode batches
        """

        data = ["dog", "puppy", "quantum chromodynamics", "cat", "kitten"]

        single = self.model.encode(data)

        self.model.encodebatch = 2
        batched = self.model.encode(data)
        self.model.encodebatch = 32

        self.assertEqual(single.shape, (5, 384))

        # Each input sees a different amount of padding depending on how the data is batched,
        # so compare direction rather than exact values
        single = single / np.linalg.norm(single, axis=1, keepdims=True)
        batched = batched / np.linalg.norm(batched, axis=1, keepdims=True)
        self.assertTrue(np.all(np.sum(single * batched, axis=1) > 0.99))

    def testSimilarity(self):
        """
        Test that pooled embeddings carry semantics
        """

        embeddings = self.model.encode(["dog", "puppy", "quantum chromodynamics"])
        embeddings = embeddings / np.linalg.norm(embeddings, axis=1, keepdims=True)

        self.assertGreater(float(embeddings[0] @ embeddings[1]), float(embeddings[0] @ embeddings[2]))


class TestONNXModels(unittest.TestCase):
    """
    ONNX vectors tests covering model shapes and configuration a single exported model doesn't reach.
    These build small ONNX graphs locally, no model download required.
    """

    @classmethod
    def setUpClass(cls):
        """
        Create a working directory and a small tokenizer.
        """

        cls.directory = os.path.join(tempfile.gettempdir(), "onnxvectors")
        os.makedirs(cls.directory, exist_ok=True)

        vocab = {"[PAD]": 0, "[UNK]": 1, "dog": 2, "puppy": 3, "quantum": 4, "physics": 5}
        tokenizer = Tokenizer(models.WordLevel(vocab=vocab, unk_token="[UNK]"))
        tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()

        cls.tokenizer = os.path.join(cls.directory, "tokenizer.json")
        tokenizer.save(cls.tokenizer)

        cls.vocab = len(vocab)

    def build(self, name, pooled=False, types=False):
        """
        Builds a small ONNX model that maps input ids to hidden states.

        Args:
            name: model file name
            pooled: True to output a single vector per input, False for token level output
            types: True to declare a token_type_ids input

        Returns:
            model path
        """

        table = np.eye(self.vocab, 4, dtype=np.float32)
        initializers = [helper.make_tensor("table", TensorProto.FLOAT, table.shape, table.flatten())]

        inputs = [
            helper.make_tensor_value_info("input_ids", TensorProto.INT64, ["batch", "sequence"]),
            helper.make_tensor_value_info("attention_mask", TensorProto.INT64, ["batch", "sequence"]),
        ]
        if types:
            inputs.append(helper.make_tensor_value_info("token_type_ids", TensorProto.INT64, ["batch", "sequence"]))

        nodes = [helper.make_node("Gather", ["table", "input_ids"], ["hidden"], axis=0)]
        if pooled:
            nodes.append(helper.make_node("ReduceMean", ["hidden"], ["output"], axes=[1], keepdims=0))
            outputs = [helper.make_tensor_value_info("output", TensorProto.FLOAT, ["batch", 4])]
        else:
            nodes.append(helper.make_node("Identity", ["hidden"], ["output"]))
            outputs = [helper.make_tensor_value_info("output", TensorProto.FLOAT, ["batch", "sequence", 4])]

        graph = helper.make_graph(nodes, "model", inputs, outputs, initializer=initializers)

        # Pin the opset, the default tracks the installed onnx package and can run ahead of onnxruntime
        model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 14)])
        model.ir_version = 9

        path = os.path.join(self.directory, name)
        onnx.save(model, path)

        return path

    def model(self, path, **config):
        """
        Creates an ONNX vectors instance.

        Args:
            path: model path
            config: additional configuration

        Returns:
            ONNX
        """

        return ONNX({"path": path, "tokenizer": self.tokenizer, "gpu": False, **config}, None, None)

    def testTokenOutput(self):
        """
        Test that token level output is pooled into a single vector per input
        """

        model = self.model(self.build("tokens.onnx"))
        self.assertEqual(model.encode(["dog puppy", "quantum"]).shape, (2, 4))

    def testPooledOutput(self):
        """
        Test that output which is already pooled passes through
        """

        model = self.model(self.build("pooled.onnx", pooled=True))
        self.assertEqual(model.encode(["dog puppy"]).shape, (1, 4))

    def testTokenTypeIds(self):
        """
        Test that token_type_ids is set only when the model declares it
        """

        self.assertEqual(self.model(self.build("types.onnx", types=True)).encode(["dog", "quantum physics"]).shape, (2, 4))
        self.assertEqual(self.model(self.build("notypes.onnx")).encode(["dog", "quantum physics"]).shape, (2, 4))

    def testPadding(self):
        """
        Test that padding is excluded when pooling
        """

        model = self.model(self.build("padding.onnx"))

        # An input padded to a longer sequence must produce the same vector as that input alone
        self.assertTrue(np.allclose(model.encode(["dog"])[0], model.encode(["dog", "quantum physics"])[0], atol=1e-6))

    def testMaxLength(self):
        """
        Test that maxlength truncates long inputs
        """

        model = self.model(self.build("maxlength.onnx"), maxlength=1)
        self.assertTrue(np.allclose(model.encode(["dog quantum physics"])[0], model.encode(["dog"])[0], atol=1e-6))

    def testTokenizerPath(self):
        """
        Test that a tokenizer stored alongside the model is found when one isn't configured
        """

        directory = os.path.join(self.directory, "sibling")
        os.makedirs(directory, exist_ok=True)
        shutil.copy(self.tokenizer, os.path.join(directory, "tokenizer.json"))

        path = self.build(os.path.join("sibling", "model.onnx"))
        model = ONNX({"path": path, "gpu": False}, None, None)

        self.assertEqual(model.encode(["dog"]).shape, (1, 4))

    def testTokenizerMissing(self):
        """
        Test that a local model without a tokenizer raises DownloadError
        """

        os.makedirs(os.path.join(self.directory, "missing-tokenizer"), exist_ok=True)
        path = self.build(os.path.join("missing-tokenizer", "model.onnx"))

        with self.assertRaises(DownloadError):
            ONNX({"path": path, "gpu": False}, None, None)

    @patch("huggingface_hub.hf_hub_download")
    def testTokenizerRepo(self, download):
        """
        Test that a tokenizer at the root of a HF Hub repo is found for a model stored in a subdirectory
        """

        files = {"org/repo/onnx/model.onnx": self.build("repo.onnx"), "org/repo/tokenizer.json": self.tokenizer}

        def filedownload(**kwargs):
            path = f"{kwargs['repo_id']}/{kwargs['filename']}"
            if path not in files:
                raise FileNotFoundError

            return files[path]

        download.side_effect = filedownload

        model = ONNX({"path": "org/repo/onnx/model.onnx", "gpu": False}, None, None)
        self.assertEqual(model.encode(["dog"]).shape, (1, 4))

    def testProviders(self):
        """
        Test provider selection
        """

        model = self.model(self.build("providers.onnx"))

        with patch("txtai.vectors.dense.onnx.ort.get_available_providers") as providers:
            providers.return_value = ["CUDAExecutionProvider", "CPUExecutionProvider"]

            model.config["gpu"] = True
            self.assertEqual(model.providers(), ["CUDAExecutionProvider", "CPUExecutionProvider"])

            model.config["gpu"] = False
            self.assertEqual(model.providers(), ["CPUExecutionProvider"])

            providers.return_value = ["CPUExecutionProvider"]
            model.config["gpu"] = True
            self.assertEqual(model.providers(), ["CPUExecutionProvider"])