Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Merge branch 'main' into local_model_loading_TGI

Omar Khattab23995244d6e8
diff --cc dsp/modules/hf_client.py
index 8554ca007,bd2061e7a..cd7dc027e
--- a/dsp/modules/hf_client.py
+++ b/dsp/modules/hf_client.py
@@@ -88,54 -99,47 +104,94 @@@ def send_hftgi_request_v01_wrapped(arg
def send_hftgi_request_v00(arg, **kwargs):
return requests.post(arg, **kwargs)
+class HFServerTGI:
+ def __init__(self, user_dir):
+ self.model_weights_dir = os.path.abspath(os.path.join(os.getcwd(), "text-generation-inference", user_dir))
+ if not os.path.exists(self.model_weights_dir):
+ os.makedirs(self.model_weights_dir)
+
+ def close_server(self, port):
+ process = subprocess.Popen(['docker', 'ps'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, _ = process.communicate()
+ print(stdout)
+ if stdout:
+ container_ids = stdout.decode().strip().split('\n')
+ container_ids = container_ids[1:]
+ for container_id in container_ids:
+ match = re.search(r'^([a-zA-Z0-9]+)', container_id)
+ if match:
+ container_id = match.group(1)
+ port_mapping = subprocess.check_output(['docker', 'port', container_id]).decode().strip()
+ if f'0.0.0.0:{port}' in port_mapping:
+ subprocess.run(['docker', 'stop', container_id])
+
+ def run_server(self, port, model_name=None, model_path=None, env_variable=None, gpus="all", num_shard=1, max_input_length=4000, max_total_tokens=4096, max_best_of=100):
+ self.close_server(port)
+ if model_path:
+ model_file_name = os.path.basename(model_path)
+ link_path = os.path.join(self.model_weights_dir, model_file_name)
+ shutil.copytree(model_path, link_path)
+ model_name = os.path.sep + os.path.basename(self.model_weights_dir) + os.path.sep + os.path.basename(model_path)
+ docker_command = f'docker run --gpus {gpus} --shm-size 1g -p {port}:80 -v {self.model_weights_dir}:{os.path.sep + os.path.basename(self.model_weights_dir)} -e {env_variable} ghcr.io/huggingface/text-generation-inference:1.1.0 --model-id {model_name} --num-shard {num_shard} --max-input-length {max_input_length} --max-total-tokens {max_total_tokens} --max-best-of {max_best_of}'
+ print(f"Connect Command: {docker_command}")
+ docker_process = subprocess.Popen(docker_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+ connected = False
+ output = []
+ while True:
+ line = docker_process.stdout.readline()
+ if not line:
+ break
+ output.append(line.strip())
+ if 'Connected' in line:
+ connected = True
+ break
+ if not connected:
+ print("Could not connect to server. Error log:")
+ for line in output:
+ print(line)
+ docker_process.terminate()
+ docker_process.wait()
+ class Anyscale(HFModel):
+ def __init__(self, model, **kwargs):
+ super().__init__(model=model, is_client=True)
+ self.session = requests.Session()
+ self.api_base = os.getenv("OPENAI_API_BASE")
+ self.token = os.getenv("OPENAI_API_KEY")
+ self.model = model
+ self.kwargs = {
+ "temperature": 0.0,
+ "n": 1,
+ **kwargs
+ }
+
+ def _generate(self, prompt, **kwargs):
+ url = f"{self.api_base}/chat/completions"
+ kwargs = {**self.kwargs, **kwargs}
+
+ temperature = kwargs.get("temperature")
+ messages = [{"role": "system", "content": "You are a helpful assistant. You must continue the user text directly without *any* additional interjections."}, {"role": "user", "content": prompt}]
+
+ body = {
+ "model": self.model,
+ "messages": messages,
+ "temperature": temperature,
+ "max_tokens": 150
+ }
+
+ headers = {"Authorization": f"Bearer {self.token}"}
+
+ try:
+ with self.session.post(url, headers=headers, json=body) as resp:
+ resp_json = resp.json()
+ completions = [resp_json.get('choices', [])[0].get('message', {}).get('content', "")]
+ response = {"prompt": prompt, "choices": [{"text": c} for c in completions]}
+ return response
+ except Exception as e:
+ print(f"Failed to parse JSON response: {e}")
+ raise Exception("Received invalid JSON response from server")
+
+
class ChatModuleClient(HFModel):
def __init__(self, model, model_path):
super().__init__(model=model, is_client=True)
diff --cc dspy/__init__.py
index b7964cb35,7ecbb1444..558163d13
--- a/dspy/__init__.py
+++ b/dspy/__init__.py
@@@ -18,6 -17,7 +18,8 @@@ settings = dsp.setting
OpenAI = dsp.GPT3
ColBERTv2 = dsp.ColBERTv2
Pyserini = dsp.PyseriniRetriever
++
HFClientTGI = dsp.HFClientTGI
- HFServerTGI = HFServerTGI
- ChatModuleClient = ChatModuleClient
++
+ Anyscale = dsp.Anyscale
-ChatModuleClient = ChatModuleClient
+ HFModel = dsp.HFModel