DSPy: The framework for programming—not prompting—language models
Python
feat: added new AzureOpenAI class
KCaverly712eeb09537f
diff --git a/dsp/modules/__init__.py b/dsp/modules/__init__.pyindex 1fb45bdac..c1c6875eb 100644--- a/dsp/modules/__init__.py+++ b/dsp/modules/__init__.py@@ -1,4 +1,5 @@from .cache_utils import *+from .azure_openai import *from .gpt3 import *from .hf import HFModelfrom .colbertv2 import ColBERTv2diff --git a/dsp/modules/azure_openai.py b/dsp/modules/azure_openai.pynew file mode 100644index 000000000..65b73186c--- /dev/null+++ b/dsp/modules/azure_openai.py@@ -0,0 +1,301 @@+import logging++# Configure logging+logging.basicConfig(+ level=logging.INFO,+ format="%(message)s",+ handlers=[logging.FileHandler("openai_usage.log")],+)++import functools+import json+from typing import Any, Literal, Optional, cast++import backoff+import openai++import dsp+from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory, cache_turn_on+from dsp.modules.lm import LM++try:+ OPENAI_LEGACY = int(openai.version.__version__[0]) == 0+except Exception:+ OPENAI_LEGACY = True++try:+ import openai.error+ from openai.openai_object import OpenAIObject++ ERRORS = (+ openai.error.RateLimitError,+ openai.error.ServiceUnavailableError,+ openai.error.APIError,+ )+except Exception:+ ERRORS = (openai.RateLimitError, openai.APIError)+ OpenAIObject = dict+++def backoff_hdlr(details):+ """Handler from https://pypi.org/project/backoff/"""+ print(+ "Backing off {wait:0.1f} seconds after {tries} tries "+ "calling function {target} with kwargs "+ "{kwargs}".format(**details),+ )+++class AzureOpenAI(LM):+ """Wrapper around Azure's API for OpenAI.++ Args:+ model (str, optional): OpenAI or Azure supported LLM model to use. Defaults to "text-davinci-002".+ api_key (Optional[str], optional): API provider Authentication token. use Defaults to None.+ model_type (Literal["chat", "text"], optional): The type of model that was specified. Mainly to decide the optimal prompting strategy. Defaults to "text".+ **kwargs: Additional arguments to pass to the API provider.+ """++ def __init__(+ self,+ model: str = "gpt-3.5-turbo-instruct",+ api_key: Optional[str] = None,+ api_base: Optional[str] = None,+ model_type: Literal["chat", "text"] = None,+ **kwargs,+ ):+ super().__init__(model)+ self.provider = "openai"++ # Assert that all variables are available+ assert (+ "engine" in kwargs or "deployment_id" in kwargs+ ), "Must specify engine or deployment_id for Azure API instead of model."+ assert "api_version" in kwargs, "Must specify api_version for Azure API"+ assert api_base is not None, "Must specify api_base for Azure API"++ # Define Client+ if OPENAI_LEGACY:+ openai.api_base = api_base+ openai.api_key = api_key+ openai.api_type = "azure"+ openai.api_version = kwargs["api_version"]++ self.client = None++ else:+ client = openai.AzureOpenAI(+ azure_endpoint=api_base,+ api_key=api_key,+ api_version=kwargs["api_version"],+ )++ self.client = client++ # Define model type+ default_model_type = (+ "chat"+ if ("gpt-3.5" in model or "turbo" in model or "gpt-4" in model)+ and ("instruct" not in model)+ else "text"+ )+ self.model_type = model_type if model_type else default_model_type++ if not OPENAI_LEGACY and "model" not in kwargs:+ if "deployment_id" in kwargs:+ kwargs["model"] = kwargs["deployment_id"]+ del kwargs["deployment_id"]++ if "api_version" in kwargs:+ del kwargs["api_version"]++ if "model" not in kwargs:+ kwargs["model"] = model++ self.kwargs = {+ "temperature": 0.0,+ "max_tokens": 150,+ "top_p": 1,+ "frequency_penalty": 0,+ "presence_penalty": 0,+ "n": 1,+ **kwargs,+ } # TODO: add kwargs above for </s>++ self.history: list[dict[str, Any]] = []++ def _openai_client(self):+ if OPENAI_LEGACY:+ return openai++ return self.client++ def log_usage(self, response):+ """Log the total tokens from the OpenAI API response."""+ usage_data = response.get("usage")+ if usage_data:+ total_tokens = usage_data.get("total_tokens")+ logging.info(f"{total_tokens}")++ def basic_request(self, prompt: str, **kwargs):+ raw_kwargs = kwargs++ kwargs = {**self.kwargs, **kwargs}+ if self.model_type == "chat":+ # caching mechanism requires hashable kwargs+ kwargs["messages"] = [{"role": "user", "content": prompt}]+ kwargs = {"stringify_request": json.dumps(kwargs)}+ response = chat_request(self.client, **kwargs)++ else:+ kwargs["prompt"] = prompt+ response = completions_request(self.client, **kwargs)++ history = {+ "prompt": prompt,+ "response": response,+ "kwargs": kwargs,+ "raw_kwargs": raw_kwargs,+ }+ self.history.append(history)++ return response++ @backoff.on_exception(+ backoff.expo,+ ERRORS,+ max_time=1000,+ on_backoff=backoff_hdlr,+ )+ def request(self, prompt: str, **kwargs):+ """Handles retrieval of GPT-3 completions whilst handling rate limiting and caching."""+ if "model_type" in kwargs:+ del kwargs["model_type"]++ return self.basic_request(prompt, **kwargs)++ def _get_choice_text(self, choice: dict[str, Any]) -> str:+ if self.model_type == "chat":+ return choice["message"]["content"]+ return choice["text"]++ def __call__(+ self,+ prompt: str,+ only_completed: bool = True,+ return_sorted: bool = False,+ **kwargs,+ ) -> list[dict[str, Any]]:+ """Retrieves completions from OpenAI Model.++ Args:+ prompt (str): prompt to send to GPT-3+ only_completed (bool, optional): return only completed responses and ignores completion due to length. Defaults to True.+ return_sorted (bool, optional): sort the completion choices using the returned probabilities. Defaults to False.++ Returns:+ list[dict[str, Any]]: list of completion choices+ """++ assert only_completed, "for now"+ assert return_sorted is False, "for now"++ response = self.request(prompt, **kwargs)++ if dsp.settings.log_openai_usage:+ self.log_usage(response)++ choices = response["choices"]++ completed_choices = [c for c in choices if c["finish_reason"] != "length"]++ if only_completed and len(completed_choices):+ choices = completed_choices++ completions = [self._get_choice_text(c) for c in choices]+ if return_sorted and kwargs.get("n", 1) > 1:+ scored_completions = []++ for c in choices:+ tokens, logprobs = (+ c["logprobs"]["tokens"],+ c["logprobs"]["token_logprobs"],+ )++ if "<|endoftext|>" in tokens:+ index = tokens.index("<|endoftext|>") + 1+ tokens, logprobs = tokens[:index], logprobs[:index]++ avglog = sum(logprobs) / len(logprobs)+ scored_completions.append((avglog, self._get_choice_text(c)))++ scored_completions = sorted(scored_completions, reverse=True)+ completions = [c for _, c in scored_completions]++ return completions+++def cached_gpt3_request_v2(**kwargs):+ return openai.Completion.create(**kwargs)++[email protected]_cache(maxsize=None if cache_turn_on else 0)+def cached_gpt3_request_v2_wrapped(**kwargs):+ return cached_gpt3_request_v2(**kwargs)+++def _cached_gpt3_turbo_request_v2(**kwargs) -> OpenAIObject:+ if "stringify_request" in kwargs:+ kwargs = json.loads(kwargs["stringify_request"])+ return cast(OpenAIObject, openai.ChatCompletion.create(**kwargs))++[email protected]_cache(maxsize=None if cache_turn_on else 0)+def _cached_gpt3_turbo_request_v2_wrapped(**kwargs) -> OpenAIObject:+ return _cached_gpt3_turbo_request_v2(**kwargs)+++def v1_chat_request(client, **kwargs):+ @functools.lru_cache(maxsize=None if cache_turn_on else 0)+ @NotebookCacheMemory.cache+ def v1_cached_gpt3_turbo_request_v2_wrapped(**kwargs):+ @CacheMemory.cache+ def v1_cached_gpt3_turbo_request_v2(**kwargs):+ if "stringify_request" in kwargs:+ kwargs = json.loads(kwargs["stringify_request"])+ return client.chat.completion.create(**kwargs)++ return v1_cached_gpt3_turbo_request_v2(**kwargs)++ return v1_cached_gpt3_turbo_request_v2_wrapped(**kwargs).model_dump()+++def v1_completions_request(client, **kwargs):+ @functools.lru_cache(maxsize=None if cache_turn_on else 0)+ @NotebookCacheMemory.cache+ def v1_cached_gpt3_request_v2_wrapped(**kwargs):+ @CacheMemory.cache+ def v1_cached_gpt3_request_v2(**kwargs):+ return client.completions.create(**kwargs)++ return v1_cached_gpt3_request_v2(**kwargs)++ return v1_cached_gpt3_request_v2_wrapped(**kwargs).model_dump()+++def chat_request(client, **kwargs):+ if OPENAI_LEGACY:+ return _cached_gpt3_turbo_request_v2_wrapped(**kwargs)++ return v1_chat_request(client, **kwargs)+++def completions_request(client, **kwargs):+ if OPENAI_LEGACY:+ return cached_gpt3_request_v2_wrapped(**kwargs)++ return v1_completions_request(client, **kwargs)diff --git a/dsp/modules/gpt3.py b/dsp/modules/gpt3.pyindex f7293bdeb..e19ca40f5 100644--- a/dsp/modules/gpt3.py+++ b/dsp/modules/gpt3.py@@ -1,4 +1,5 @@import logging+from logging.handlers import RotatingFileHandler# Configure logginglogging.basicConfig(@@ -11,21 +12,21 @@ import functoolsimport jsonfrom typing import Any, Literal, Optional, cast+import dspimport backoffimport openai-import dspfrom dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory, cache_turn_onfrom dsp.modules.lm import LMtry:- OPENAI_LEGACY = int(openai.__version__[0]) == 0+ OPENAI_LEGACY = int(openai.version.__version__[0]) == 0except Exception:OPENAI_LEGACY = Truetry:- import openai.errorfrom openai.openai_object import OpenAIObject+ import openai.errorERRORS = (openai.error.RateLimitError,@@ -42,17 +43,17 @@ def backoff_hdlr(details):print("Backing off {wait:0.1f} seconds after {tries} tries ""calling function {target} with kwargs "- "{kwargs}".format(**details),+ "{kwargs}".format(**details))class GPT3(LM):- """Wrapper around OpenAI's GPT API. Supports both the OpenAI and Azure APIs.+ """Wrapper around OpenAI's GPT API.Args:- model (str, optional): OpenAI or Azure supported LLM model to use. Defaults to "text-davinci-002".+ model (str, optional): OpenAI supported LLM model to use. Defaults to "text-davinci-002".api_key (Optional[str], optional): API provider Authentication token. use Defaults to None.- api_provider (Literal["openai", "azure"], optional): The API provider to use. Defaults to "openai".+ api_provider (Literal["openai"], optional): The API provider to use. Defaults to "openai".model_type (Literal["chat", "text"], optional): The type of model that was specified. Mainly to decide the optimal prompting strategy. Defaults to "text".**kwargs: Additional arguments to pass to the API provider."""@@ -61,47 +62,19 @@ class GPT3(LM):self,model: str = "gpt-3.5-turbo-instruct",api_key: Optional[str] = None,- api_provider: Literal["openai", "azure"] = "openai",+ api_provider: Literal["openai"] = "openai",api_base: Optional[str] = None,model_type: Literal["chat", "text"] = None,**kwargs,):super().__init__(model)self.provider = "openai"+ openai.api_type = api_provider- # Assert that all variables are available- assert api_key is not None, "Must provide api_key for OpenAI/Azure API"- if api_provider == "azure":- assert (- "engine" in kwargs or "deployment_id" in kwargs- ), "Must specify engine or deployment_id for Azure API instead of model."- assert "api_version" in kwargs, "Must specify api_version for Azure API"- assert api_base is not None, "Must specify api_base for Azure API"-- # Define Client- if OPENAI_LEGACY:- openai.api_base = api_base- openai.api_key = api_key- openai.api_type = api_provider-- if api_provider == "azure":- openai.api_version = kwargs["api_version"]-- self.client = None+ assert (+ api_provider != "azure"+ ), "Azure functionality with base OpenAI has been deprecated, please use dspy.AzureOpenAI instead."- else:- if api_provider == "azure":- client = openai.AzureOpenAI(- azure_endpoint=api_base,- api_key=api_key,- api_version=kwargs["api_version"],- )- else:- client = openai.OpenAI(api_key=api_key, base_url=api_base)-- self.client = client-- # Define model typedefault_model_type = ("chat"if ("gpt-3.5" in model or "turbo" in model or "gpt-4" in model)@@ -110,16 +83,14 @@ class GPT3(LM):)self.model_type = model_type if model_type else default_model_type- if not OPENAI_LEGACY and api_provider == "azure" and "model" not in kwargs:- if "deployment_id" in kwargs:- kwargs["model"] = kwargs["deployment_id"]- del kwargs["deployment_id"]-- if "api_version" in kwargs:- del kwargs["api_version"]+ if api_key:+ openai.api_key = api_key- if "model" not in kwargs:- kwargs["model"] = model+ if api_base:+ if OPENAI_LEGACY:+ openai.api_base = api_base+ else:+ openai.base_url = api_baseself.kwargs = {"temperature": 0.0,@@ -131,13 +102,11 @@ class GPT3(LM):**kwargs,} # TODO: add kwargs above for </s>+ self.kwargs["model"] = modelself.history: list[dict[str, Any]] = []def _openai_client(self):- if OPENAI_LEGACY:- return openai-- return self.client+ return openaidef log_usage(self, response):"""Log the total tokens from the OpenAI API response."""@@ -154,11 +123,11 @@ class GPT3(LM):# caching mechanism requires hashable kwargskwargs["messages"] = [{"role": "user", "content": prompt}]kwargs = {"stringify_request": json.dumps(kwargs)}- response = chat_request(self.client, **kwargs)+ response = chat_request(**kwargs)else:kwargs["prompt"] = prompt- response = completions_request(self.client, **kwargs)+ response = completions_request(**kwargs)history = {"prompt": prompt,@@ -274,43 +243,39 @@ def _cached_gpt3_turbo_request_v2_wrapped(**kwargs) -> OpenAIObject:return _cached_gpt3_turbo_request_v2(**kwargs)-def v1_chat_request(client, **kwargs):- @functools.lru_cache(maxsize=None if cache_turn_on else 0)- @NotebookCacheMemory.cache- def v1_cached_gpt3_turbo_request_v2_wrapped(**kwargs):- @CacheMemory.cache- def v1_cached_gpt3_turbo_request_v2(**kwargs):- if "stringify_request" in kwargs:- kwargs = json.loads(kwargs["stringify_request"])- return client.chat.completion.create(**kwargs)+def v1_cached_gpt3_request_v2(**kwargs):+ return openai.completions.create(**kwargs)- return v1_cached_gpt3_turbo_request_v2(**kwargs)- return v1_cached_gpt3_turbo_request_v2_wrapped(**kwargs).model_dump()[email protected]_cache(maxsize=None if cache_turn_on else 0)+def v1_cached_gpt3_request_v2_wrapped(**kwargs):+ return v1_cached_gpt3_request_v2(**kwargs)-def v1_completions_request(client, **kwargs):- @functools.lru_cache(maxsize=None if cache_turn_on else 0)- @NotebookCacheMemory.cache- def v1_cached_gpt3_request_v2_wrapped(**kwargs):- @CacheMemory.cache- def v1_cached_gpt3_request_v2(**kwargs):- return client.completions.create(**kwargs)+def v1_cached_gpt3_turbo_request_v2(**kwargs):+ if "stringify_request" in kwargs:+ kwargs = json.loads(kwargs["stringify_request"])+ return openai.chat.completions.create(**kwargs)- return v1_cached_gpt3_request_v2(**kwargs)- return v1_cached_gpt3_request_v2_wrapped(**kwargs).model_dump()[email protected]_cache(maxsize=None if cache_turn_on else 0)+def v1_cached_gpt3_turbo_request_v2_wrapped(**kwargs):+ return v1_cached_gpt3_turbo_request_v2(**kwargs)-def chat_request(client, **kwargs):+def chat_request(**kwargs):if OPENAI_LEGACY:return _cached_gpt3_turbo_request_v2_wrapped(**kwargs)- return v1_chat_request(client, **kwargs)+ return v1_cached_gpt3_turbo_request_v2_wrapped(**kwargs).model_dump()-def completions_request(client, **kwargs):+def completions_request(**kwargs):if OPENAI_LEGACY:return cached_gpt3_request_v2_wrapped(**kwargs)- return v1_completions_request(client, **kwargs)+ return v1_cached_gpt3_request_v2_wrapped(**kwargs).model_dump()diff --git a/dspy/__init__.py b/dspy/__init__.pyindex c30295518..afd3c5756 100644--- a/dspy/__init__.py+++ b/dspy/__init__.py@@ -16,6 +16,7 @@ import dspsettings = dsp.settings+AzureOpenAI = dsp.AzureOpenAIOpenAI = dsp.GPT3Cohere = dsp.CohereColBERTv2 = dsp.ColBERTv2