DSPy: The framework for programming—not prompting—language models
Python
Add code for supporting arbitrary Hugging Face models
santhnm2feaec3342f81
diff --git a/dsp/modules/__init__.py b/dsp/modules/__init__.pyindex b2b5f3049..03171fc84 100644--- a/dsp/modules/__init__.py+++ b/dsp/modules/__init__.py@@ -1,3 +1,4 @@from .cache_utils import *from .gpt3 import *-from .colbertv2 import *\ No newline at end of file+from .hf import HFModel+from .colbertv2 import *diff --git a/dsp/modules/gpt3.py b/dsp/modules/gpt3.pyindex e9a08a598..c379e63d5 100644--- a/dsp/modules/gpt3.py+++ b/dsp/modules/gpt3.py@@ -3,6 +3,7 @@ import openai.errorimport backoffimport functools+from dsp.modules.lm import LMfrom dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory, cache_turn_on@@ -13,14 +14,12 @@ def backoff_hdlr(details):"{kwargs}".format(**details))-class GPT3:+class GPT3(LM):def __init__(self, model='text-davinci-002', api_key=None):+ super().__init__(model)if api_key:openai.api_key = api_key- self.kwargs = {'model': model, 'temperature': 0.0, 'max_tokens': 150, 'top_p': 1,- 'frequency_penalty': 0, 'presence_penalty': 0, 'n': 1}-self.history = []def basic_request(self, prompt, **kwargs):@@ -40,38 +39,7 @@ class GPT3:max_time=1000,on_backoff=backoff_hdlr)def request(self, prompt, **kwargs):- return self.basic_request(prompt, **kwargs)-- def print_green(self, text, end='\n'):- print("\x1b[32m" + str(text) + "\x1b[0m", end=end)-- def print_red(self, text, end='\n'):- print("\x1b[31m" + str(text) + "\x1b[0m", end=end)-- def inspect_history(self, n=1):- last_prompt = None- printed = []-- for x in reversed(self.history[-100:]):- prompt = x['prompt']-- if prompt != last_prompt:- printed.append((prompt, x['response']['choices']))-- last_prompt = prompt-- if len(printed) >= n:- break-- for prompt, choices in reversed(printed):- print('\n\n\n')- print(prompt, end='')- self.print_green(choices[0]['text'], end='')-- if len(choices) > 1:- self.print_red(f" \t (and {len(choices)-1} other completions)", end='')- print('\n\n\n')-+ return super().request(prompt, **kwargs)def __call__(self, prompt, only_completed=True, return_sorted=False, **kwargs):assert only_completed, "for now"diff --git a/dsp/modules/hf.py b/dsp/modules/hf.pynew file mode 100644index 000000000..b6518cdf0--- /dev/null+++ b/dsp/modules/hf.py@@ -0,0 +1,83 @@+from tokenizers import AddedToken+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer++from dsp.modules.lm import LM+++def openai_to_hf(**kwargs):+ hf_kwargs = {}+ for k, v in kwargs.items():+ if k == "n":+ hf_kwargs["num_return_sequences"] = v+ elif k == "frequency_penalty":+ hf_kwargs["repetition_penalty"] = 1.0 - v+ elif k == "presence_penalty":+ hf_kwargs["diversity_penalty"] = v+ elif k == "max_tokens":+ hf_kwargs["max_new_tokens"] = v+ elif k == "model":+ pass+ else:+ hf_kwargs[k] = v++ return hf_kwargs+++class HFModel(LM):+ def __init__(self, model, checkpoint=None, is_client=False):+ super().__init__(model)++ self.is_client = is_client+ if not self.is_client:+ self.model = AutoModelForSeq2SeqLM.from_pretrained(model if checkpoint is None else checkpoint).to("cuda")+ self.tokenizer = AutoTokenizer.from_pretrained(model)+ self.tokenizer.add_tokens(AddedToken("\n", normalized=False))++ self.history = []++ def basic_request(self, prompt, **kwargs):+ raw_kwargs = kwargs+ kwargs = {**self.kwargs, **kwargs}+ response = self._generate(prompt, **kwargs)++ history = {+ "prompt": prompt,+ "response": response,+ "kwargs": kwargs,+ "raw_kwargs": raw_kwargs,+ }+ self.history.append(history)++ return response++ def _generate(self, prompt, **kwargs):+ assert not self.is_client+ # TODO: Add caching+ kwargs = {**openai_to_hf(**self.kwargs), **openai_to_hf(**kwargs)}+ inputs = self.tokenizer(prompt, return_tensors="pt").to("cuda")+ outputs = self.model.generate(**inputs, **kwargs)+ completions = [+ {"text": c}+ for c in self.tokenizer.batch_decode(outputs, skip_special_tokens=True)+ ]+ response = {+ "prompt": prompt,+ "choices": completions,+ }+ return response++ def __call__(self, prompt, only_completed=True, return_sorted=False, **kwargs):+ assert only_completed, "for now"+ assert return_sorted is False, "for now"++ if kwargs.get("n", 1) > 1:+ kwargs["num_beams"] = max(5, kwargs["n"])++ response = self.request(prompt, **kwargs)+ return [c["text"] for c in response["choices"]]+++if __name__ == "__main__":+ model = HFModel(model="google/flan-t5-base")+ response = model("Who was the first man to walk on the moon?\nFinal answer: ")+ print(response)diff --git a/dsp/modules/hf_client.py b/dsp/modules/hf_client.pynew file mode 100644index 000000000..5e42e1387--- /dev/null+++ b/dsp/modules/hf_client.py@@ -0,0 +1,18 @@+import functools+import requests++from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory+from dsp.modules.hf import HFModel+from dsp.utils import dotdict+++class HFModelClient(HFModel):+ def __init__(self, port, model, url="http://0.0.0.0"):+ super().__init__(model=model, is_client=True)+ self.url = f"{url}:{port}"+ self.headers = {"Content-Type": "application/json; charset=utf-8"}++ def _generate(self, prompt, **kwargs):+ payload = {"prompt": prompt, **kwargs}+ response = requests.post(self.url, json=payload, headers=self.headers)+ return response.json()diff --git a/dsp/modules/hf_server.py b/dsp/modules/hf_server.pynew file mode 100644index 000000000..23dc997a6--- /dev/null+++ b/dsp/modules/hf_server.py@@ -0,0 +1,75 @@+# To Run:+# python -m dsp.modules.hf_server --port 4242 --model "google/flan-t5-base"++# To Query:+# curl -d '{"query":"What is the answer to life, the universe, and everything?"}' \+# -X POST "http://0.0.0.0:4242" \+# -H 'Content-Type: application/json'+++from functools import lru_cache+import argparse+import time+import random+import os+import sys+import uvicorn+import warnings++from fastapi import FastAPI+from pydantic import BaseModel+from argparse import ArgumentParser+from starlette.middleware.cors import CORSMiddleware++from dsp.modules.hf import HFModel+++class Query(BaseModel):+ prompt: str+++warnings.filterwarnings("ignore")++app = FastAPI()+app.add_middleware(+ CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"]+)++parser = argparse.ArgumentParser("Server for Hugging Face models")+parser.add_argument("--port", type=int, required=True, help="Server port")+parser.add_argument("--model", type=str, required=True, help="Hugging Face model")+args = parser.parse_args()+# TODO: Convert this to a log message+print(f"#> Loading the language model {args.model}")+lm = HFModel(args.model)+++@lru_cache(maxsize=None)+def generate(prompt, **kwargs):+ global lm+ generateStart = time.time()+ # TODO: Convert this to a log message+ print(f'#> Prompt: "{prompt}"')+ response = lm._generate(prompt, **kwargs)+ # TODO: Convert this to a log message+ print(f'#> Response: "{response}"')+ latency = (time.time() - generateStart) * 1000.0+ response["latency"] = latency+ return response++[email protected]("/")+async def generate_post(query: Query):+ # TODO: Handle kwargs++ return generate(query.prompt)+++if __name__ == "__main__":+ uvicorn.run(+ app,+ host="0.0.0.0",+ port=args.port,+ reload=False,+ log_level="info",+ ) # can make reload=True laterdiff --git a/dsp/modules/lm.py b/dsp/modules/lm.pynew file mode 100644index 000000000..0e9ff3348--- /dev/null+++ b/dsp/modules/lm.py@@ -0,0 +1,58 @@+from abc import ABC, abstractmethod+++class LM(ABC):+ def __init__(self, model):++ self.kwargs = {+ "model": model,+ "temperature": 0.0,+ "max_tokens": 150,+ "top_p": 1,+ "frequency_penalty": 0,+ "presence_penalty": 0,+ "n": 1,+ }++ self.history = []++ @abstractmethod+ def basic_request(self, prompt, **kwargs):+ pass++ def request(self, prompt, **kwargs):+ return self.basic_request(prompt, **kwargs)++ def print_green(self, text, end="\n"):+ print("\x1b[32m" + str(text) + "\x1b[0m", end=end)++ def print_red(self, text, end="\n"):+ print("\x1b[31m" + str(text) + "\x1b[0m", end=end)++ def inspect_history(self, n=1):+ last_prompt = None+ printed = []++ for x in reversed(self.history[-100:]):+ prompt = x["prompt"]++ if prompt != last_prompt:+ printed.append((prompt, x["response"]["choices"]))++ last_prompt = prompt++ if len(printed) >= n:+ break++ for prompt, choices in reversed(printed):+ print("\n\n\n")+ print(prompt, end="")+ self.print_green(choices[0]["text"], end="")++ if len(choices) > 1:+ self.print_red(f" \t (and {len(choices)-1} other completions)", end="")+ print("\n\n\n")++ @abstractmethod+ def __call__(self, prompt, only_completed=True, return_sorted=False, **kwargs):+ pass