DSPy: The framework for programming—not prompting—language models
Python
added client class for MLC llama
Arnav Singhvid6b93dd296dd
diff --git a/dsp/modules/hf_client.py b/dsp/modules/hf_client.pyindex af4cb495f..8a674ed6f 100644--- a/dsp/modules/hf_client.py+++ b/dsp/modules/hf_client.py@@ -27,6 +27,7 @@ from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory, cache_turn# from dsp.modules.adapter import TurboAdapter, DavinciAdapter, LlamaAdapter+from mlc_chat import ChatModuleclass HFClientTGI(HFModel):def __init__(self, model, port, url="http://future-hgx-1", **kwargs):@@ -92,3 +93,23 @@ class HFClientTGI(HFModel):@CacheMemory.cachedef send_hftgi_request_v00(arg, **kwargs):return requests.post(arg, **kwargs)++class ChatModuleClient(HFModel):+ def __init__(self, model, model_path):+ super().__init__(model=model, is_client=True)+ self.cm = ChatModule(model=model, lib_path=model_path)++ def _generate(self, prompt, **kwargs):+ output = self.cm.generate(+ prompt=prompt,+ )+ try:+ completions = [{"text": output}]+ response = {+ "prompt": prompt,+ "choices": completions+ }+ return response+ except Exception as e:+ print("Failed to parse output:", response.text)+ raise Exception("Received invalid output")