Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Add code for supporting arbitrary Hugging Face models

santhnm2feaec3342f81
diff --git a/dsp/modules/__init__.py b/dsp/modules/__init__.py
index b2b5f3049..03171fc84 100644
--- a/dsp/modules/__init__.py
+++ b/dsp/modules/__init__.py
@@ -1,3 +1,4 @@
from .cache_utils import *
from .gpt3 import *
-from .colbertv2 import *
\ No newline at end of file
+from .hf import HFModel
+from .colbertv2 import *
diff --git a/dsp/modules/gpt3.py b/dsp/modules/gpt3.py
index e9a08a598..c379e63d5 100644
--- a/dsp/modules/gpt3.py
+++ b/dsp/modules/gpt3.py
@@ -3,6 +3,7 @@ import openai.error
import backoff
import functools
+from dsp.modules.lm import LM
from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory, cache_turn_on
@@ -13,14 +14,12 @@ def backoff_hdlr(details):
"{kwargs}".format(**details))
-class GPT3:
+class GPT3(LM):
def __init__(self, model='text-davinci-002', api_key=None):
+ super().__init__(model)
if api_key:
openai.api_key = api_key
- self.kwargs = {'model': model, 'temperature': 0.0, 'max_tokens': 150, 'top_p': 1,
- 'frequency_penalty': 0, 'presence_penalty': 0, 'n': 1}
-
self.history = []
def basic_request(self, prompt, **kwargs):
@@ -40,38 +39,7 @@ class GPT3:
max_time=1000,
on_backoff=backoff_hdlr)
def request(self, prompt, **kwargs):
- return self.basic_request(prompt, **kwargs)
-
- def print_green(self, text, end='\n'):
- print("\x1b[32m" + str(text) + "\x1b[0m", end=end)
-
- def print_red(self, text, end='\n'):
- print("\x1b[31m" + str(text) + "\x1b[0m", end=end)
-
- def inspect_history(self, n=1):
- last_prompt = None
- printed = []
-
- for x in reversed(self.history[-100:]):
- prompt = x['prompt']
-
- if prompt != last_prompt:
- printed.append((prompt, x['response']['choices']))
-
- last_prompt = prompt
-
- if len(printed) >= n:
- break
-
- for prompt, choices in reversed(printed):
- print('\n\n\n')
- print(prompt, end='')
- self.print_green(choices[0]['text'], end='')
-
- if len(choices) > 1:
- self.print_red(f" \t (and {len(choices)-1} other completions)", end='')
- print('\n\n\n')
-
+ return super().request(prompt, **kwargs)
def __call__(self, prompt, only_completed=True, return_sorted=False, **kwargs):
assert only_completed, "for now"
diff --git a/dsp/modules/hf.py b/dsp/modules/hf.py
new file mode 100644
index 000000000..b6518cdf0
--- /dev/null
+++ b/dsp/modules/hf.py
@@ -0,0 +1,83 @@
+from tokenizers import AddedToken
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+
+from dsp.modules.lm import LM
+
+
+def openai_to_hf(**kwargs):
+ hf_kwargs = {}
+ for k, v in kwargs.items():
+ if k == "n":
+ hf_kwargs["num_return_sequences"] = v
+ elif k == "frequency_penalty":
+ hf_kwargs["repetition_penalty"] = 1.0 - v
+ elif k == "presence_penalty":
+ hf_kwargs["diversity_penalty"] = v
+ elif k == "max_tokens":
+ hf_kwargs["max_new_tokens"] = v
+ elif k == "model":
+ pass
+ else:
+ hf_kwargs[k] = v
+
+ return hf_kwargs
+
+
+class HFModel(LM):
+ def __init__(self, model, checkpoint=None, is_client=False):
+ super().__init__(model)
+
+ self.is_client = is_client
+ if not self.is_client:
+ self.model = AutoModelForSeq2SeqLM.from_pretrained(model if checkpoint is None else checkpoint).to("cuda")
+ self.tokenizer = AutoTokenizer.from_pretrained(model)
+ self.tokenizer.add_tokens(AddedToken("\n", normalized=False))
+
+ self.history = []
+
+ def basic_request(self, prompt, **kwargs):
+ raw_kwargs = kwargs
+ kwargs = {**self.kwargs, **kwargs}
+ response = self._generate(prompt, **kwargs)
+
+ history = {
+ "prompt": prompt,
+ "response": response,
+ "kwargs": kwargs,
+ "raw_kwargs": raw_kwargs,
+ }
+ self.history.append(history)
+
+ return response
+
+ def _generate(self, prompt, **kwargs):
+ assert not self.is_client
+ # TODO: Add caching
+ kwargs = {**openai_to_hf(**self.kwargs), **openai_to_hf(**kwargs)}
+ inputs = self.tokenizer(prompt, return_tensors="pt").to("cuda")
+ outputs = self.model.generate(**inputs, **kwargs)
+ completions = [
+ {"text": c}
+ for c in self.tokenizer.batch_decode(outputs, skip_special_tokens=True)
+ ]
+ response = {
+ "prompt": prompt,
+ "choices": completions,
+ }
+ return response
+
+ def __call__(self, prompt, only_completed=True, return_sorted=False, **kwargs):
+ assert only_completed, "for now"
+ assert return_sorted is False, "for now"
+
+ if kwargs.get("n", 1) > 1:
+ kwargs["num_beams"] = max(5, kwargs["n"])
+
+ response = self.request(prompt, **kwargs)
+ return [c["text"] for c in response["choices"]]
+
+
+if __name__ == "__main__":
+ model = HFModel(model="google/flan-t5-base")
+ response = model("Who was the first man to walk on the moon?\nFinal answer: ")
+ print(response)
diff --git a/dsp/modules/hf_client.py b/dsp/modules/hf_client.py
new file mode 100644
index 000000000..5e42e1387
--- /dev/null
+++ b/dsp/modules/hf_client.py
@@ -0,0 +1,18 @@
+import functools
+import requests
+
+from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory
+from dsp.modules.hf import HFModel
+from dsp.utils import dotdict
+
+
+class HFModelClient(HFModel):
+ def __init__(self, port, model, url="http://0.0.0.0"):
+ super().__init__(model=model, is_client=True)
+ self.url = f"{url}:{port}"
+ self.headers = {"Content-Type": "application/json; charset=utf-8"}
+
+ def _generate(self, prompt, **kwargs):
+ payload = {"prompt": prompt, **kwargs}
+ response = requests.post(self.url, json=payload, headers=self.headers)
+ return response.json()
diff --git a/dsp/modules/hf_server.py b/dsp/modules/hf_server.py
new file mode 100644
index 000000000..23dc997a6
--- /dev/null
+++ b/dsp/modules/hf_server.py
@@ -0,0 +1,75 @@
+# To Run:
+# python -m dsp.modules.hf_server --port 4242 --model "google/flan-t5-base"
+
+# To Query:
+# curl -d '{"query":"What is the answer to life, the universe, and everything?"}' \
+# -X POST "http://0.0.0.0:4242" \
+# -H 'Content-Type: application/json'
+
+
+from functools import lru_cache
+import argparse
+import time
+import random
+import os
+import sys
+import uvicorn
+import warnings
+
+from fastapi import FastAPI
+from pydantic import BaseModel
+from argparse import ArgumentParser
+from starlette.middleware.cors import CORSMiddleware
+
+from dsp.modules.hf import HFModel
+
+
+class Query(BaseModel):
+ prompt: str
+
+
+warnings.filterwarnings("ignore")
+
+app = FastAPI()
+app.add_middleware(
+ CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"]
+)
+
+parser = argparse.ArgumentParser("Server for Hugging Face models")
+parser.add_argument("--port", type=int, required=True, help="Server port")
+parser.add_argument("--model", type=str, required=True, help="Hugging Face model")
+args = parser.parse_args()
+# TODO: Convert this to a log message
+print(f"#> Loading the language model {args.model}")
+lm = HFModel(args.model)
+
+
+@lru_cache(maxsize=None)
+def generate(prompt, **kwargs):
+ global lm
+ generateStart = time.time()
+ # TODO: Convert this to a log message
+ print(f'#> Prompt: "{prompt}"')
+ response = lm._generate(prompt, **kwargs)
+ # TODO: Convert this to a log message
+ print(f'#> Response: "{response}"')
+ latency = (time.time() - generateStart) * 1000.0
+ response["latency"] = latency
+ return response
+
+
+async def generate_post(query: Query):
+ # TODO: Handle kwargs
+
+ return generate(query.prompt)
+
+
+if __name__ == "__main__":
+ uvicorn.run(
+ app,
+ host="0.0.0.0",
+ port=args.port,
+ reload=False,
+ log_level="info",
+ ) # can make reload=True later
diff --git a/dsp/modules/lm.py b/dsp/modules/lm.py
new file mode 100644
index 000000000..0e9ff3348
--- /dev/null
+++ b/dsp/modules/lm.py
@@ -0,0 +1,58 @@
+from abc import ABC, abstractmethod
+
+
+class LM(ABC):
+ def __init__(self, model):
+
+ self.kwargs = {
+ "model": model,
+ "temperature": 0.0,
+ "max_tokens": 150,
+ "top_p": 1,
+ "frequency_penalty": 0,
+ "presence_penalty": 0,
+ "n": 1,
+ }
+
+ self.history = []
+
+ @abstractmethod
+ def basic_request(self, prompt, **kwargs):
+ pass
+
+ def request(self, prompt, **kwargs):
+ return self.basic_request(prompt, **kwargs)
+
+ def print_green(self, text, end="\n"):
+ print("\x1b[32m" + str(text) + "\x1b[0m", end=end)
+
+ def print_red(self, text, end="\n"):
+ print("\x1b[31m" + str(text) + "\x1b[0m", end=end)
+
+ def inspect_history(self, n=1):
+ last_prompt = None
+ printed = []
+
+ for x in reversed(self.history[-100:]):
+ prompt = x["prompt"]
+
+ if prompt != last_prompt:
+ printed.append((prompt, x["response"]["choices"]))
+
+ last_prompt = prompt
+
+ if len(printed) >= n:
+ break
+
+ for prompt, choices in reversed(printed):
+ print("\n\n\n")
+ print(prompt, end="")
+ self.print_green(choices[0]["text"], end="")
+
+ if len(choices) > 1:
+ self.print_red(f" \t (and {len(choices)-1} other completions)", end="")
+ print("\n\n\n")
+
+ @abstractmethod
+ def __call__(self, prompt, only_completed=True, return_sorted=False, **kwargs):
+ pass