DSPy: The framework for programming—not prompting—language models
Python
fix(dspy): fix return type in _generate
Anindyadeep6f9c1f37d2dc
diff --git a/dsp/modules/tensorrt_llm.py b/dsp/modules/tensorrt_llm.pyindex 006e643ff..f675f2303 100644--- a/dsp/modules/tensorrt_llm.py+++ b/dsp/modules/tensorrt_llm.py@@ -132,7 +132,7 @@ class TensorRTModel(LM):self.runner, self._runner_kwargs = load_tensorrt_model(engine_dir=self.engine_dir, **engine_kwargs)self.history: list[dict[str, Any]] = []- def _generate(self, prompt: Union[list[dict[str, str]], str], **kwargs) -> list[str]:+ def _generate(self, prompt: Union[list[dict[str, str]], str], **kwargs: dict) -> tuple[list[str], dict]:import torchinput_ids = tokenize(prompt=prompt, tokenizer=self.tokenizer, **kwargs)@@ -173,7 +173,7 @@ class TensorRTModel(LM):output_ids, sequence_lengths = outputs["output_ids"], outputs["sequence_lengths"]# In case of current version of dspy it will always stay as 1- batch_size, num_beams, _ = output_ids.size()+ _, num_beams, _ = output_ids.size()batch_idx, beams = 0, []for beam in range(num_beams):@@ -185,7 +185,7 @@ class TensorRTModel(LM):return beams, run_kwargs- def basic_request(self, prompt, **kwargs) -> list[str]:+ def basic_request(self, prompt, **kwargs: dict) -> list[str]:raw_kwargs = kwargsresponse, all_kwargs = self._generate(prompt, **kwargs)history = {