DSPy: The framework for programming—not prompting—language models
Python
Merge branch 'main' into local_model_loading_TGI
Omar Khattab23995244d6e8
diff --cc dsp/modules/hf_client.pyindex 8554ca007,bd2061e7a..cd7dc027e--- a/dsp/modules/hf_client.py+++ b/dsp/modules/hf_client.py@@@ -88,54 -99,47 +104,94 @@@ def send_hftgi_request_v01_wrapped(argdef send_hftgi_request_v00(arg, **kwargs):return requests.post(arg, **kwargs)+class HFServerTGI:+ def __init__(self, user_dir):+ self.model_weights_dir = os.path.abspath(os.path.join(os.getcwd(), "text-generation-inference", user_dir))+ if not os.path.exists(self.model_weights_dir):+ os.makedirs(self.model_weights_dir)++ def close_server(self, port):+ process = subprocess.Popen(['docker', 'ps'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)+ stdout, _ = process.communicate()+ print(stdout)+ if stdout:+ container_ids = stdout.decode().strip().split('\n')+ container_ids = container_ids[1:]+ for container_id in container_ids:+ match = re.search(r'^([a-zA-Z0-9]+)', container_id)+ if match:+ container_id = match.group(1)+ port_mapping = subprocess.check_output(['docker', 'port', container_id]).decode().strip()+ if f'0.0.0.0:{port}' in port_mapping:+ subprocess.run(['docker', 'stop', container_id])++ def run_server(self, port, model_name=None, model_path=None, env_variable=None, gpus="all", num_shard=1, max_input_length=4000, max_total_tokens=4096, max_best_of=100):+ self.close_server(port)+ if model_path:+ model_file_name = os.path.basename(model_path)+ link_path = os.path.join(self.model_weights_dir, model_file_name)+ shutil.copytree(model_path, link_path)+ model_name = os.path.sep + os.path.basename(self.model_weights_dir) + os.path.sep + os.path.basename(model_path)+ docker_command = f'docker run --gpus {gpus} --shm-size 1g -p {port}:80 -v {self.model_weights_dir}:{os.path.sep + os.path.basename(self.model_weights_dir)} -e {env_variable} ghcr.io/huggingface/text-generation-inference:1.1.0 --model-id {model_name} --num-shard {num_shard} --max-input-length {max_input_length} --max-total-tokens {max_total_tokens} --max-best-of {max_best_of}'+ print(f"Connect Command: {docker_command}")+ docker_process = subprocess.Popen(docker_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)+ connected = False+ output = []+ while True:+ line = docker_process.stdout.readline()+ if not line:+ break+ output.append(line.strip())+ if 'Connected' in line:+ connected = True+ break+ if not connected:+ print("Could not connect to server. Error log:")+ for line in output:+ print(line)+ docker_process.terminate()+ docker_process.wait()+ class Anyscale(HFModel):+ def __init__(self, model, **kwargs):+ super().__init__(model=model, is_client=True)+ self.session = requests.Session()+ self.api_base = os.getenv("OPENAI_API_BASE")+ self.token = os.getenv("OPENAI_API_KEY")+ self.model = model+ self.kwargs = {+ "temperature": 0.0,+ "n": 1,+ **kwargs+ }++ def _generate(self, prompt, **kwargs):+ url = f"{self.api_base}/chat/completions"+ kwargs = {**self.kwargs, **kwargs}++ temperature = kwargs.get("temperature")+ messages = [{"role": "system", "content": "You are a helpful assistant. You must continue the user text directly without *any* additional interjections."}, {"role": "user", "content": prompt}]++ body = {+ "model": self.model,+ "messages": messages,+ "temperature": temperature,+ "max_tokens": 150+ }++ headers = {"Authorization": f"Bearer {self.token}"}++ try:+ with self.session.post(url, headers=headers, json=body) as resp:+ resp_json = resp.json()+ completions = [resp_json.get('choices', [])[0].get('message', {}).get('content', "")]+ response = {"prompt": prompt, "choices": [{"text": c} for c in completions]}+ return response+ except Exception as e:+ print(f"Failed to parse JSON response: {e}")+ raise Exception("Received invalid JSON response from server")++class ChatModuleClient(HFModel):def __init__(self, model, model_path):super().__init__(model=model, is_client=True)diff --cc dspy/__init__.pyindex b7964cb35,7ecbb1444..558163d13--- a/dspy/__init__.py+++ b/dspy/__init__.py@@@ -18,6 -17,7 +18,8 @@@ settings = dsp.settingOpenAI = dsp.GPT3ColBERTv2 = dsp.ColBERTv2Pyserini = dsp.PyseriniRetriever++HFClientTGI = dsp.HFClientTGI- HFServerTGI = HFServerTGI- ChatModuleClient = ChatModuleClient+++ Anyscale = dsp.Anyscale-ChatModuleClient = ChatModuleClient+ HFModel = dsp.HFModel