Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

added client class for MLC llama

Arnav Singhvid6b93dd296dd
diff --git a/dsp/modules/hf_client.py b/dsp/modules/hf_client.py
index af4cb495f..8a674ed6f 100644
--- a/dsp/modules/hf_client.py
+++ b/dsp/modules/hf_client.py
@@ -27,6 +27,7 @@ from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory, cache_turn
# from dsp.modules.adapter import TurboAdapter, DavinciAdapter, LlamaAdapter
+from mlc_chat import ChatModule
class HFClientTGI(HFModel):
def __init__(self, model, port, url="http://future-hgx-1", **kwargs):
@@ -92,3 +93,23 @@ class HFClientTGI(HFModel):
@CacheMemory.cache
def send_hftgi_request_v00(arg, **kwargs):
return requests.post(arg, **kwargs)
+
+class ChatModuleClient(HFModel):
+ def __init__(self, model, model_path):
+ super().__init__(model=model, is_client=True)
+ self.cm = ChatModule(model=model, lib_path=model_path)
+
+ def _generate(self, prompt, **kwargs):
+ output = self.cm.generate(
+ prompt=prompt,
+ )
+ try:
+ completions = [{"text": output}]
+ response = {
+ "prompt": prompt,
+ "choices": completions
+ }
+ return response
+ except Exception as e:
+ print("Failed to parse output:", response.text)
+ raise Exception("Received invalid output")