Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

feat(dspy): Nvidia TensorRT LLM integration

Anindyadeepb75dd7314da2
diff --git a/dsp/modules/tensorrt_llm.py b/dsp/modules/tensorrt_llm.py
new file mode 100644
index 000000000..29a75328f
--- /dev/null
+++ b/dsp/modules/tensorrt_llm.py
@@ -0,0 +1,229 @@
+from pathlib import Path
+from typing import Any, Optional, Union
+
+from dsp.modules.lm import LM
+
+## Utility functions to load models
+
+
+def load_tensorrt_model(
+ engine_dir: Union[str, Path],
+ use_py_session: Optional[bool] = False,
+ **kwargs,
+) -> tuple[Any, dict]:
+ import tensorrt_llm
+ from tensorrt_llm.runtime import ModelRunner, ModelRunnerCpp
+
+ runtime_rank = tensorrt_llm.mpi_rank()
+ runner_cls = ModelRunner if use_py_session else ModelRunnerCpp
+ runner_kwargs = {
+ "engine_dir": engine_dir,
+ "lora_dir": kwargs.get("lora_dir", None),
+ "rank": runtime_rank,
+ "lora_ckpt_source": kwargs.get("lora_ckpt_source", "hf"),
+ }
+
+ if not use_py_session:
+ engine_cpp_kwargs = {}
+ defaults = {
+ "max_batch_size": 1,
+ "max_input_len": 1024,
+ "max_output_len": 1024,
+ "max_beam_width": 1,
+ "max_attention_window_size": None,
+ "sink_token_length": None,
+ }
+
+ for key, value in defaults.items():
+ engine_cpp_kwargs[key] = kwargs.get(key, value)
+ runner_kwargs.update(**engine_cpp_kwargs)
+
+ runner = runner_cls.from_dir(**runner_kwargs)
+ return runner, runner_kwargs
+
+
+def tokenize(prompt: Union[list[dict], str], tokenizer: Any, **kwargs) -> list[int]:
+ defaults = {
+ "add_special_tokens": False,
+ "max_input_length": 1024,
+ "model_name": None,
+ "model_version": None,
+ }
+ if not isinstance(prompt, str):
+ prompt = tokenizer.apply_chat_template(prompt, tokenize=False)
+
+ input_ids = [
+ tokenizer.encode(
+ prompt,
+ add_special_tokens=kwargs.get("add_special_tokens", defaults["add_special_tokens"]),
+ truncation=True,
+ max_length=kwargs.get("max_input_length", defaults["max_input_length"]),
+ ),
+ ]
+ if (
+ kwargs.get("model_name", defaults["model_name"]) == "ChatGLMForCausalLM"
+ and kwargs.get("model_version", defaults["model_version"]) == "glm"
+ ):
+ input_ids.append(tokenizer.stop_token_id)
+ return input_ids
+
+
+class TensorRTModel(LM):
+ """TensorRT integration for dspy LM."""
+
+ def __init__(self, model_name_or_path: str, engine_dir: str, **engine_kwargs: dict) -> None:
+ """Initialize the TensorRTModel.
+
+ Args:
+ model_name_or_path (str): The Huggingface ID or the path where tokenizer files exist.
+ engine_dir (str): The folder where the TensorRT .engine file exists.
+ **engine_kwargs (Optional[dict]): Additional engine loading keyword arguments.
+
+ Keyword Args:
+ use_py_session (bool, optional): Whether to use a Python session or not. Defaults to False.
+ lora_dir (str): The directory of LoRA adapter weights.
+ lora_task_uids (list[str]): list of LoRA task UIDs; use -1 to disable the LoRA module.
+ lora_ckpt_source (str): The source of the LoRA checkpoint.
+
+ If use_py_session is set to False, the following kwargs are supported:
+ max_batch_size (int, optional): The maximum batch size. Defaults to 1.
+ max_input_len (int, optional): The maximum input context length. Defaults to 1024.
+ max_output_len (int, optional): The maximum output context length. Defaults to 1024.
+ max_beam_width (int, optional): The maximum beam width, similar to `n` in OpenAI API. Defaults to 1.
+ max_attention_window_size (int, optional): The attention window size that controls the
+ sliding window attention / cyclic KV cache behavior. Defaults to None.
+ sink_token_length (int, optional): The sink token length. Defaults to 1.
+ """
+ # Implementation here
+ self.model_name_or_path, self.engine_dir = model_name_or_path, engine_dir
+ super().__init__(model=self.model_name_or_path)
+ try:
+ import tensorrt_llm
+ except ImportError as exc:
+ raise ModuleNotFoundError(
+ "You need to install tensorrt-llm to use TensorRTModel",
+ ) from exc
+
+ try:
+ from transformers import AutoTokenizer
+ except ImportError as exc:
+ raise ModuleNotFoundError(
+ "You need to install torch and transformers",
+ ) from exc
+
+ # Configure tokenizer
+ self.tokenizer = AutoTokenizer.from_pretrained(
+ self.model_name_or_path,
+ legacy=False,
+ padding_side="left",
+ truncation_side="left",
+ trust_remote_code=True,
+ use_fast=True,
+ )
+
+ self.pad_id = (
+ self.tokenizer.eos_token_id if self.tokenizer.pad_token_id is None else self.tokenizer.pad_token_id
+ )
+ self.end_id = self.tokenizer.eos_token_id
+
+ # Configure TensorRT
+ self.runtime_rank = tensorrt_llm.mpi_rank()
+ self.runner, self._runner_kwargs = load_tensorrt_model(engine_dir=self.engine_dir, **engine_kwargs)
+ self.history = []
+
+ def _generate(self, prompt: Union[list[dict[str, str]], str], **kwargs) -> list[str]:
+ import torch
+
+ input_ids = tokenize(prompt=prompt, tokenizer=self.tokenizer, **kwargs)
+ input_ids = torch.tensor(input_ids, dtype=torch.int32)
+
+ run_kwargs = {}
+ defaults = {
+ "max_new_tokens": 1024,
+ "max_attention_window_size": None,
+ "sink_token_length": None,
+ "end_id": self.end_id,
+ "pad_id": self.pad_id,
+ "temperature": 1.0,
+ "top_k": 1,
+ "top_p": 0.0,
+ "num_beams": 1,
+ "length_penalty": 1.0,
+ "early_stopping": 1,
+ "repetition_penalty": 1.0,
+ "presence_penalty": 0.0,
+ "frequency_penalty": 0.0,
+ "stop_words_list": None,
+ "bad_words_list": None,
+ "streaming": False,
+ "return_dict": True,
+ "output_log_probs": False,
+ "output_cum_log_probs": False,
+ "output_sequence_lengths": True,
+ }
+
+ for k, v in defaults.items():
+ run_kwargs[k] = kwargs.get(k, v)
+
+ with torch.no_grad():
+ outputs = self.runner.generate(input_ids, **run_kwargs)
+ input_lengths = [x.size(0) for x in input_ids]
+
+ output_ids, sequence_lengths = outputs["output_ids"], outputs["sequence_lengths"]
+
+ # In case of current version of dspy it will always stay as 1
+ batch_size, num_beams, _ = output_ids.size()
+ batch_idx, beams = 0, []
+
+ for beam in range(num_beams):
+ output_begin = input_lengths[batch_idx]
+ output_end = sequence_lengths[batch_idx][beam]
+ outputs = output_ids[batch_idx][beam][output_begin:output_end].tolist()
+ output_text = self.tokenizer.decode(outputs)
+ beams.append(output_text)
+
+ return beams, run_kwargs
+
+ def basic_request(self, prompt, **kwargs) -> list[str]:
+ raw_kwargs = kwargs
+ response, all_kwargs = self._generate(prompt, **kwargs)
+ history = {
+ "prompt": prompt,
+ "response": response,
+ "raw_kwargs": raw_kwargs,
+ "kwargs": all_kwargs,
+ }
+ self.history.append(history)
+ return response
+
+ def __call__(
+ self,
+ prompt: Union[list[dict[str, str]], str],
+ **kwargs,
+ ):
+ """TensorRTLLM generate method in dspy.
+
+ Args:
+ prompt (Union[list[dict[str, str]], str]): The prompt to pass. If prompt is not string
+ then it will assume that chat mode / instruct mode is triggered.
+ **kwargs (Optional[dict]): Optional keyword arguments.
+
+ Additional Parameters:
+ max_new_tokens (int): The maximum number of tokens to output. Defaults to 1024
+ max_attention_window_size (int) Defaults to None
+ sink_token_length (int): Defaults to None
+ end_id (int): The end of sequence of ID of tokenize, defaults to tokenizer's default
+ end id
+ pad_id (int): The pd sequence of ID of tokenize, defaults to tokenizer's default end id
+ temperature (float): The temperature to control probabilistic behaviour in generation
+ Defaults to 1.0
+ top_k (int): Defaults to 1
+ top_p (float): Defaults to 1
+ num_beams: (int): The number of responses to generate. Defaults to 1
+ length_penalty (float): Defaults to 1.0
+ repetition_penalty (float): Defaults to 1.0
+ presence_penalty (float): Defaults to 0.0
+ frequency_penalty (float): Defaults to 0.0
+ early_stopping (int): Use this only when num_beams > 1, Defaults to 1
+ """
+ return self.request(prompt, **kwargs)