The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
community: add new parameter default_headers (#28700)
Lu Peng50afa7c4e7fc
diff --git a/docs/docs/integrations/chat/oci_data_science.ipynb b/docs/docs/integrations/chat/oci_data_science.ipynbindex 6196d2025c..61b0a41ae1 100644--- a/docs/docs/integrations/chat/oci_data_science.ipynb+++ b/docs/docs/integrations/chat/oci_data_science.ipynb@@ -137,7 +137,7 @@},{"cell_type": "code",- "execution_count": 1,+ "execution_count": null,"metadata": {},"outputs": [],"source": [@@ -156,6 +156,10 @@" \"temperature\": 0.2,\n"," \"max_tokens\": 512,\n"," }, # other model params...\n",+ " default_headers={\n",+ " \"route\": \"/v1/chat/completions\",\n",+ " # other request headers ...\n",+ " },\n",")"]},diff --git a/libs/community/langchain_community/chat_models/oci_data_science.py b/libs/community/langchain_community/chat_models/oci_data_science.pyindex cdb181df89..8ac571adbd 100644--- a/libs/community/langchain_community/chat_models/oci_data_science.py+++ b/libs/community/langchain_community/chat_models/oci_data_science.py@@ -47,6 +47,7 @@ from langchain_community.llms.oci_data_science_model_deployment_endpoint import)logger = logging.getLogger(__name__)+DEFAULT_INFERENCE_ENDPOINT_CHAT = "/v1/chat/completions"def _is_pydantic_class(obj: Any) -> bool:@@ -56,6 +57,13 @@ def _is_pydantic_class(obj: Any) -> bool:class ChatOCIModelDeployment(BaseChatModel, BaseOCIModelDeployment):"""OCI Data Science Model Deployment chat model integration.+ Prerequisite+ The OCI Model Deployment plugins are installable only on+ python version 3.9 and above. If you're working inside the notebook,+ try installing the python 3.10 based conda pack and running the+ following setup.++Setup:Install ``oracle-ads`` and ``langchain-openai``.@@ -90,6 +98,8 @@ class ChatOCIModelDeployment(BaseChatModel, BaseOCIModelDeployment):Key init args — client params:auth: dictADS auth dictionary for OCI authentication.+ default_headers: Optional[Dict]+ The headers to be added to the Model Deployment request.Instantiate:.. code-block:: python@@ -98,7 +108,7 @@ class ChatOCIModelDeployment(BaseChatModel, BaseOCIModelDeployment):chat = ChatOCIModelDeployment(endpoint="https://modeldeployment.<region>.oci.customer-oci.com/<ocid>/predict",- model="odsc-llm",+ model="odsc-llm", # this is the default model name if deployed with AQUAstreaming=True,max_retries=3,model_kwargs={@@ -106,6 +116,10 @@ class ChatOCIModelDeployment(BaseChatModel, BaseOCIModelDeployment):"temperature": 0.2,# other model parameters ...},+ default_headers={+ "route": "/v1/chat/completions",+ # other request headers ...+ },)Invocation:@@ -288,6 +302,25 @@ class ChatOCIModelDeployment(BaseChatModel, BaseOCIModelDeployment):"stream": self.streaming,}+ def _headers(+ self, is_async: Optional[bool] = False, body: Optional[dict] = None+ ) -> Dict:+ """Construct and return the headers for a request.++ Args:+ is_async (bool, optional): Indicates if the request is asynchronous.+ Defaults to `False`.+ body (optional): The request body to be included in the headers if+ the request is asynchronous.++ Returns:+ Dict: A dictionary containing the appropriate headers for the request.+ """+ return {+ "route": DEFAULT_INFERENCE_ENDPOINT_CHAT,+ **super()._headers(is_async=is_async, body=body),+ }+def _generate(self,messages: List[BaseMessage],@@ -701,7 +734,7 @@ class ChatOCIModelDeployment(BaseChatModel, BaseOCIModelDeployment):for choice in choices:message = _convert_dict_to_message(choice["message"])- generation_info = dict(finish_reason=choice.get("finish_reason"))+ generation_info = {"finish_reason": choice.get("finish_reason")}if "logprobs" in choice:generation_info["logprobs"] = choice["logprobs"]diff --git a/libs/community/langchain_community/llms/oci_data_science_model_deployment_endpoint.py b/libs/community/langchain_community/llms/oci_data_science_model_deployment_endpoint.pyindex c998fbc0ec..e18a1a0847 100644--- a/libs/community/langchain_community/llms/oci_data_science_model_deployment_endpoint.py+++ b/libs/community/langchain_community/llms/oci_data_science_model_deployment_endpoint.py@@ -32,6 +32,7 @@ from pydantic import Field, model_validatorfrom langchain_community.utilities.requests import Requestslogger = logging.getLogger(__name__)+DEFAULT_INFERENCE_ENDPOINT = "/v1/completions"DEFAULT_TIME_OUT = 300@@ -81,6 +82,9 @@ class BaseOCIModelDeployment(Serializable):max_retries: int = 3"""Maximum number of retries to make when generating."""+ default_headers: Optional[Dict[str, Any]] = None+ """The headers to be added to the Model Deployment request."""+@model_validator(mode="before")@classmethoddef validate_environment(cls, values: Dict) -> Dict:@@ -120,12 +124,12 @@ class BaseOCIModelDeployment(Serializable):Returns:Dict: A dictionary containing the appropriate headers for the request."""+ headers = self.default_headers or {}if is_async:signer = self.auth["signer"]_req = requests.Request("POST", self.endpoint, json=body)req = _req.prepare()req = signer(req)- headers = {}for key, value in req.headers.items():headers[key] = value@@ -135,7 +139,7 @@ class BaseOCIModelDeployment(Serializable):)return headers- return (+ headers.update({"Content-Type": DEFAULT_CONTENT_TYPE_JSON,"enable-streaming": "true",@@ -147,6 +151,8 @@ class BaseOCIModelDeployment(Serializable):})+ return headers+def completion_with_retry(self, run_manager: Optional[CallbackManagerForLLMRun] = None, **kwargs: Any) -> Any:@@ -383,6 +389,10 @@ class OCIModelDeploymentLLM(BaseLLM, BaseOCIModelDeployment):model="odsc-llm",streaming=True,model_kwargs={"frequency_penalty": 1.0},+ headers={+ "route": "/v1/completions",+ # other request headers ...+ })llm.invoke("tell me a joke.")@@ -426,7 +436,7 @@ class OCIModelDeploymentLLM(BaseLLM, BaseOCIModelDeployment):temperature: float = 0.2"""A non-negative float that tunes the degree of randomness in generation."""- k: int = -1+ k: int = 50"""Number of most likely tokens to consider at each step."""p: float = 0.75@@ -472,6 +482,25 @@ class OCIModelDeploymentLLM(BaseLLM, BaseOCIModelDeployment):**self._default_params,}+ def _headers(+ self, is_async: Optional[bool] = False, body: Optional[dict] = None+ ) -> Dict:+ """Construct and return the headers for a request.++ Args:+ is_async (bool, optional): Indicates if the request is asynchronous.+ Defaults to `False`.+ body (optional): The request body to be included in the headers if+ the request is asynchronous.++ Returns:+ Dict: A dictionary containing the appropriate headers for the request.+ """+ return {+ "route": DEFAULT_INFERENCE_ENDPOINT,+ **super()._headers(is_async=is_async, body=body),+ }+def _generate(self,prompts: List[str],diff --git a/libs/community/tests/unit_tests/chat_models/test_oci_data_science.py b/libs/community/tests/unit_tests/chat_models/test_oci_data_science.pyindex f385f2ddb0..f5b7dae643 100644--- a/libs/community/tests/unit_tests/chat_models/test_oci_data_science.py+++ b/libs/community/tests/unit_tests/chat_models/test_oci_data_science.py@@ -19,6 +19,7 @@ CONST_MODEL_NAME = "odsc-vllm"CONST_ENDPOINT = "https://oci.endpoint/ocid/predict"CONST_PROMPT = "This is a prompt."CONST_COMPLETION = "This is a completion."+CONST_COMPLETION_ROUTE = "/v1/chat/completions"CONST_COMPLETION_RESPONSE = {"id": "chat-123456789","object": "chat.completion",@@ -120,6 +121,7 @@ def mocked_requests_post(url: str, **kwargs: Any) -> MockResponse:def test_invoke_vllm(*args: Any) -> None:"""Tests invoking vLLM endpoint."""llm = ChatOCIModelDeploymentVLLM(endpoint=CONST_ENDPOINT, model=CONST_MODEL_NAME)+ assert llm._headers().get("route") == CONST_COMPLETION_ROUTEoutput = llm.invoke(CONST_PROMPT)assert isinstance(output, AIMessage)assert output.content == CONST_COMPLETION@@ -132,6 +134,7 @@ def test_invoke_vllm(*args: Any) -> None:def test_invoke_tgi(*args: Any) -> None:"""Tests invoking TGI endpoint using OpenAI Spec."""llm = ChatOCIModelDeploymentTGI(endpoint=CONST_ENDPOINT, model=CONST_MODEL_NAME)+ assert llm._headers().get("route") == CONST_COMPLETION_ROUTEoutput = llm.invoke(CONST_PROMPT)assert isinstance(output, AIMessage)assert output.content == CONST_COMPLETION@@ -146,6 +149,7 @@ def test_stream_vllm(*args: Any) -> None:llm = ChatOCIModelDeploymentVLLM(endpoint=CONST_ENDPOINT, model=CONST_MODEL_NAME, streaming=True)+ assert llm._headers().get("route") == CONST_COMPLETION_ROUTEoutput = Nonecount = 0for chunk in llm.stream(CONST_PROMPT):@@ -184,6 +188,7 @@ async def test_stream_async(*args: Any) -> None:llm = ChatOCIModelDeploymentVLLM(endpoint=CONST_ENDPOINT, model=CONST_MODEL_NAME, streaming=True)+ assert llm._headers().get("route") == CONST_COMPLETION_ROUTEwith mock.patch.object(llm,"_aiter_sse",diff --git a/libs/community/tests/unit_tests/llms/test_oci_model_deployment_endpoint.py b/libs/community/tests/unit_tests/llms/test_oci_model_deployment_endpoint.pyindex c87b05f12a..1055552801 100644--- a/libs/community/tests/unit_tests/llms/test_oci_model_deployment_endpoint.py+++ b/libs/community/tests/unit_tests/llms/test_oci_model_deployment_endpoint.py@@ -18,6 +18,7 @@ CONST_MODEL_NAME = "odsc-vllm"CONST_ENDPOINT = "https://oci.endpoint/ocid/predict"CONST_PROMPT = "This is a prompt."CONST_COMPLETION = "This is a completion."+CONST_COMPLETION_ROUTE = "/v1/completions"CONST_COMPLETION_RESPONSE = {"choices": [{@@ -114,6 +115,7 @@ async def mocked_async_streaming_response(def test_invoke_vllm(*args: Any) -> None:"""Tests invoking vLLM endpoint."""llm = OCIModelDeploymentVLLM(endpoint=CONST_ENDPOINT, model=CONST_MODEL_NAME)+ assert llm._headers().get("route") == CONST_COMPLETION_ROUTEoutput = llm.invoke(CONST_PROMPT)assert output == CONST_COMPLETION@@ -126,6 +128,7 @@ def test_stream_tgi(*args: Any) -> None:llm = OCIModelDeploymentTGI(endpoint=CONST_ENDPOINT, model=CONST_MODEL_NAME, streaming=True)+ assert llm._headers().get("route") == CONST_COMPLETION_ROUTEoutput = ""count = 0for chunk in llm.stream(CONST_PROMPT):@@ -143,6 +146,7 @@ def test_generate_tgi(*args: Any) -> None:llm = OCIModelDeploymentTGI(endpoint=CONST_ENDPOINT, api="/generate", model=CONST_MODEL_NAME)+ assert llm._headers().get("route") == CONST_COMPLETION_ROUTEoutput = llm.invoke(CONST_PROMPT)assert output == CONST_COMPLETION@@ -161,6 +165,7 @@ async def test_stream_async(*args: Any) -> None:llm = OCIModelDeploymentTGI(endpoint=CONST_ENDPOINT, model=CONST_MODEL_NAME, streaming=True)+ assert llm._headers().get("route") == CONST_COMPLETION_ROUTEwith mock.patch.object(llm,"_aiter_sse",