Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: Add SambaNova Cloud API support (#930)

Wendong-Fan0d047e5429d0
diff --git a/camel/configs/__init__.py b/camel/configs/__init__.py
index 59ffda397..a6a061975 100644
--- a/camel/configs/__init__.py
+++ b/camel/configs/__init__.py
@@ -21,8 +21,10 @@ from .ollama_config import OLLAMA_API_PARAMS, OllamaConfig
from .openai_config import OPENAI_API_PARAMS, ChatGPTConfig, OpenSourceConfig
from .reka_config import REKA_API_PARAMS, RekaConfig
from .samba_config import (
+ SAMBA_CLOUD_API_PARAMS,
SAMBA_FAST_API_PARAMS,
SAMBA_VERSE_API_PARAMS,
+ SambaCloudAPIConfig,
SambaFastAPIConfig,
SambaVerseAPIConfig,
)
@@ -57,6 +59,8 @@ __all__ = [
'SAMBA_FAST_API_PARAMS',
'SambaVerseAPIConfig',
'SAMBA_VERSE_API_PARAMS',
+ 'SambaCloudAPIConfig',
+ 'SAMBA_CLOUD_API_PARAMS',
'TogetherAIConfig',
'TOGETHERAI_API_PARAMS',
]
diff --git a/camel/configs/samba_config.py b/camel/configs/samba_config.py
index dbfc8dc38..444172270 100644
--- a/camel/configs/samba_config.py
+++ b/camel/configs/samba_config.py
@@ -13,7 +13,10 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
from __future__ import annotations
-from typing import Any, Dict, Optional, Union
+from typing import Any, Dict, Optional, Sequence, Union
+
+from openai._types import NOT_GIVEN, NotGiven
+from pydantic import Field
from camel.configs.base_config import BaseConfig
@@ -107,3 +110,98 @@ class SambaVerseAPIConfig(BaseConfig):
SAMBA_VERSE_API_PARAMS = {
param for param in SambaVerseAPIConfig().model_fields.keys()
}
+
+
+class SambaCloudAPIConfig(BaseConfig):
+ r"""Defines the parameters for generating chat completions using the
+ OpenAI API.
+
+ Args:
+ temperature (float, optional): Sampling temperature to use, between
+ :obj:`0` and :obj:`2`. Higher values make the output more random,
+ while lower values make it more focused and deterministic.
+ (default: :obj:`0.2`)
+ top_p (float, optional): An alternative to sampling with temperature,
+ called nucleus sampling, where the model considers the results of
+ the tokens with top_p probability mass. So :obj:`0.1` means only
+ the tokens comprising the top 10% probability mass are considered.
+ (default: :obj:`1.0`)
+ n (int, optional): How many chat completion choices to generate for
+ each input message. (default: :obj:`1`)
+ response_format (object, optional): An object specifying the format
+ that the model must output. Compatible with GPT-4 Turbo and all
+ GPT-3.5 Turbo models newer than gpt-3.5-turbo-1106. Setting to
+ {"type": "json_object"} enables JSON mode, which guarantees the
+ message the model generates is valid JSON. Important: when using
+ JSON mode, you must also instruct the model to produce JSON
+ yourself via a system or user message. Without this, the model
+ may generate an unending stream of whitespace until the generation
+ reaches the token limit, resulting in a long-running and seemingly
+ "stuck" request. Also note that the message content may be
+ partially cut off if finish_reason="length", which indicates the
+ generation exceeded max_tokens or the conversation exceeded the
+ max context length.
+ stream (bool, optional): If True, partial message deltas will be sent
+ as data-only server-sent events as they become available.
+ (default: :obj:`False`)
+ stop (str or list, optional): Up to :obj:`4` sequences where the API
+ will stop generating further tokens. (default: :obj:`None`)
+ max_tokens (int, optional): The maximum number of tokens to generate
+ in the chat completion. The total length of input tokens and
+ generated tokens is limited by the model's context length.
+ (default: :obj:`None`)
+ presence_penalty (float, optional): Number between :obj:`-2.0` and
+ :obj:`2.0`. Positive values penalize new tokens based on whether
+ they appear in the text so far, increasing the model's likelihood
+ to talk about new topics. See more information about frequency and
+ presence penalties. (default: :obj:`0.0`)
+ frequency_penalty (float, optional): Number between :obj:`-2.0` and
+ :obj:`2.0`. Positive values penalize new tokens based on their
+ existing frequency in the text so far, decreasing the model's
+ likelihood to repeat the same line verbatim. See more information
+ about frequency and presence penalties. (default: :obj:`0.0`)
+ logit_bias (dict, optional): Modify the likelihood of specified tokens
+ appearing in the completion. Accepts a json object that maps tokens
+ (specified by their token ID in the tokenizer) to an associated
+ bias value from :obj:`-100` to :obj:`100`. Mathematically, the bias
+ is added to the logits generated by the model prior to sampling.
+ The exact effect will vary per model, but values between:obj:` -1`
+ and :obj:`1` should decrease or increase likelihood of selection;
+ values like :obj:`-100` or :obj:`100` should result in a ban or
+ exclusive selection of the relevant token. (default: :obj:`{}`)
+ user (str, optional): A unique identifier representing your end-user,
+ which can help OpenAI to monitor and detect abuse.
+ (default: :obj:`""`)
+ tools (list[OpenAIFunction], optional): A list of tools the model may
+ call. Currently, only functions are supported as a tool. Use this
+ to provide a list of functions the model may generate JSON inputs
+ for. A max of 128 functions are supported.
+ tool_choice (Union[dict[str, str], str], optional): Controls which (if
+ any) tool is called by the model. :obj:`"none"` means the model
+ will not call any tool and instead generates a message.
+ :obj:`"auto"` means the model can pick between generating a
+ message or calling one or more tools. :obj:`"required"` means the
+ model must call one or more tools. Specifying a particular tool
+ via {"type": "function", "function": {"name": "my_function"}}
+ forces the model to call that tool. :obj:`"none"` is the default
+ when no tools are present. :obj:`"auto"` is the default if tools
+ are present.
+ """
+
+ temperature: float = 0.2 # openai default: 1.0
+ top_p: float = 1.0
+ n: int = 1
+ stream: bool = False
+ stop: Union[str, Sequence[str], NotGiven] = NOT_GIVEN
+ max_tokens: Union[int, NotGiven] = NOT_GIVEN
+ presence_penalty: float = 0.0
+ response_format: Union[dict, NotGiven] = NOT_GIVEN
+ frequency_penalty: float = 0.0
+ logit_bias: dict = Field(default_factory=dict)
+ user: str = ""
+ tool_choice: Optional[Union[dict[str, str], str]] = None
+
+
+SAMBA_CLOUD_API_PARAMS = {
+ param for param in SambaCloudAPIConfig().model_fields.keys()
+}
diff --git a/camel/models/ollama_model.py b/camel/models/ollama_model.py
index bdc974026..1f601f315 100644
--- a/camel/models/ollama_model.py
+++ b/camel/models/ollama_model.py
@@ -76,10 +76,10 @@ class OllamaModel:
)
print(
f"Ollama server started on http://localhost:11434/v1 "
- f"for {self.model_type} model"
+ f"for {self.model_type} model."
)
except Exception as e:
- print(f"Failed to start Ollama server: {e}")
+ print(f"Failed to start Ollama server: {e}.")
@property
def token_counter(self) -> BaseTokenCounter:
diff --git a/camel/models/samba_model.py b/camel/models/samba_model.py
index e66b1c8b4..3366b8204 100644
--- a/camel/models/samba_model.py
+++ b/camel/models/samba_model.py
@@ -20,7 +20,11 @@ from typing import Any, Dict, List, Optional, Union
import httpx
from openai import OpenAI, Stream
-from camel.configs import SAMBA_FAST_API_PARAMS, SAMBA_VERSE_API_PARAMS
+from camel.configs import (
+ SAMBA_CLOUD_API_PARAMS,
+ SAMBA_FAST_API_PARAMS,
+ SAMBA_VERSE_API_PARAMS,
+)
from camel.messages import OpenAIMessage
from camel.types import (
ChatCompletion,
@@ -59,9 +63,10 @@ class SambaModel:
SambaNova service. (default: :obj:`None`)
url (Optional[str]): The url to the SambaNova service. Current
support SambaNova Fast API: :obj:`"https://fast-api.snova.ai/
- v1/chat/ completions"` and SambaVerse API: :obj:`"https://
- sambaverse.sambanova.ai/api/predict"`. (default::obj:`"https://
- fast-api.snova.ai/v1/chat/completions"`)
+ v1/chat/ completions"`, SambaVerse API: :obj:`"https://
+ sambaverse.sambanova.ai/api/predict"` and SambaNova Cloud:
+ :obj:`"https://api.sambanova.ai/v1"`
+ (default::obj:`"https://fast-api.snova.ai/v1/chat/completions"`)
token_counter (Optional[BaseTokenCounter]): Token counter to use
for the model. If not provided, `OpenAITokenCounter(ModelType.
GPT_4O_MINI)` will be used.
@@ -76,6 +81,14 @@ class SambaModel:
self.model_config_dict = model_config_dict
self.check_model_config()
+ if self._url == "https://api.sambanova.ai/v1":
+ self._client = OpenAI(
+ timeout=60,
+ max_retries=3,
+ base_url=self._url,
+ api_key=self._api_key,
+ )
+
@property
def token_counter(self) -> BaseTokenCounter:
r"""Initialize the token counter for the model backend.
@@ -111,6 +124,14 @@ class SambaModel:
"input into SambaVerse API."
)
+ elif self._url == "https://api.sambanova.ai/v1":
+ for param in self.model_config_dict:
+ if param not in SAMBA_CLOUD_API_PARAMS:
+ raise ValueError(
+ f"Unexpected argument `{param}` is "
+ "input into SambaCloud API."
+ )
+
else:
raise ValueError(
f"{self._url} is not supported, please check the url to the"
@@ -141,7 +162,7 @@ class SambaModel:
def _run_streaming( # type: ignore[misc]
self, messages: List[OpenAIMessage]
) -> Stream[ChatCompletionChunk]:
- r"""Handles streaming inference with SambaNova FastAPI.
+ r"""Handles streaming inference with SambaNova's API.
Args:
messages (List[OpenAIMessage]): A list of messages representing the
@@ -189,6 +210,15 @@ class SambaModel:
except httpx.HTTPError as e:
raise RuntimeError(f"HTTP request failed: {e!s}")
+ # Handle SambaNova's Cloud API
+ elif self._url == "https://api.sambanova.ai/v1":
+ response = self._client.chat.completions.create(
+ messages=messages,
+ model=self.model_type,
+ **self.model_config_dict,
+ )
+ return response
+
elif self._url == "https://sambaverse.sambanova.ai/api/predict":
raise ValueError(
"https://sambaverse.sambanova.ai/api/predict doesn't support"
@@ -198,7 +228,7 @@ class SambaModel:
def _run_non_streaming(
self, messages: List[OpenAIMessage]
) -> ChatCompletion:
- r"""Handles non-streaming inference with SambaNova FastAPI.
+ r"""Handles non-streaming inference with SambaNova's API.
Args:
messages (List[OpenAIMessage]): A list of messages representing the
@@ -251,6 +281,15 @@ class SambaModel:
except json.JSONDecodeError as e:
raise ValueError(f"Failed to decode JSON response: {e!s}")
+ # Handle SambaNova's Cloud API
+ elif self._url == "https://api.sambanova.ai/v1":
+ response = self._client.chat.completions.create(
+ messages=messages,
+ model=self.model_type,
+ **self.model_config_dict,
+ )
+ return response
+
# Handle SambaNova's Sambaverse API
else:
headers = {
diff --git a/camel/models/vllm_model.py b/camel/models/vllm_model.py
index afe6496a0..0fb32d25f 100644
--- a/camel/models/vllm_model.py
+++ b/camel/models/vllm_model.py
@@ -80,10 +80,10 @@ class VLLMModel:
)
print(
f"vllm server started on http://localhost:8000/v1 "
- f"for {self.model_type} model"
+ f"for {self.model_type} model."
)
except Exception as e:
- print(f"Failed to start vllm server: {e}")
+ print(f"Failed to start vllm server: {e}.")
@property
def token_counter(self) -> BaseTokenCounter:
diff --git a/examples/models/samba_model_example.py b/examples/models/samba_model_example.py
index 42c6855d4..afb8c3ad0 100644
--- a/examples/models/samba_model_example.py
+++ b/examples/models/samba_model_example.py
@@ -12,7 +12,11 @@
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
from camel.agents import ChatAgent
-from camel.configs import SambaFastAPIConfig, SambaVerseAPIConfig
+from camel.configs import (
+ SambaCloudAPIConfig,
+ SambaFastAPIConfig,
+ SambaVerseAPIConfig,
+)
from camel.messages import BaseMessage
from camel.models import ModelFactory
from camel.types import ModelPlatformType
@@ -21,7 +25,7 @@ samba_fast_api_model = ModelFactory.create(
model_platform=ModelPlatformType.SAMBA,
model_type="llama3-70b",
model_config_dict=SambaFastAPIConfig(max_tokens=800).as_dict(),
- api_key="Your Fast API Key",
+ api_key="Your SambaNova Fast API Key",
url="https://fast-api.snova.ai/v1/chat/completions",
)
@@ -33,6 +37,15 @@ sambaverse_api_model = ModelFactory.create(
url="https://sambaverse.sambanova.ai/api/predict",
)
+sambacloud_api_model = ModelFactory.create(
+ model_platform=ModelPlatformType.SAMBA,
+ model_type="Meta-Llama-3.1-405B-Instruct",
+ model_config_dict=SambaCloudAPIConfig(max_tokens=800).as_dict(),
+ api_key="Your SambaNova Cloud API Key",
+ url="https://api.sambanova.ai/v1",
+)
+
+
# Define system message
sys_msg = BaseMessage.make_assistant_message(
role_name="Assistant",
@@ -50,10 +63,15 @@ user_msg = BaseMessage.make_user_message(
camel_agent_fast_api = ChatAgent(
system_message=sys_msg, model=samba_fast_api_model
)
+
camel_agent_sambaverse_api = ChatAgent(
system_message=sys_msg, model=sambaverse_api_model
)
+camel_agent_sambacloud_api = ChatAgent(
+ system_message=sys_msg, model=sambacloud_api_model
+)
+
# Get response information
response = camel_agent_fast_api.step(user_msg)
print(response.msgs[0].content)
@@ -83,3 +101,20 @@ related to autonomous and communicative agents. Let me know how I can be
of assistance.
===============================================================================
'''
+
+# Get response information
+response = camel_agent_sambacloud_api.step(user_msg)
+print(response.msgs[0].content)
+'''
+===============================================================================
+Hello to the CAMEL AI community. It's great to see open-source communities
+like yours working on autonomous and communicative agents, as this field has
+the potential to revolutionize many areas of our lives, from customer service
+to healthcare and beyond.
+
+What specific projects or initiatives is the CAMEL AI community currently
+working on? Are there any exciting developments or breakthroughs that you'd
+like to share? I'm all ears (or rather, all text) and happy to learn more
+about your work!
+===============================================================================
+'''