Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

Add Llama API Support (#9899)

Chirag Modi2dc038b6c08a
diff --git a/autogpt_platform/backend/.env.example b/autogpt_platform/backend/.env.example
index 60d941b839..ebd9d7c6ad 100644
--- a/autogpt_platform/backend/.env.example
+++ b/autogpt_platform/backend/.env.example
@@ -129,6 +129,7 @@ OPENAI_API_KEY=
ANTHROPIC_API_KEY=
GROQ_API_KEY=
OPEN_ROUTER_API_KEY=
+LLAMA_API_KEY=
# Reddit
# Go to https://www.reddit.com/prefs/apps and create a new app
diff --git a/autogpt_platform/backend/backend/blocks/llm.py b/autogpt_platform/backend/backend/blocks/llm.py
index f630937e61..675b77ee5d 100644
--- a/autogpt_platform/backend/backend/blocks/llm.py
+++ b/autogpt_platform/backend/backend/blocks/llm.py
@@ -36,6 +36,7 @@ LLMProviderName = Literal[
ProviderName.OLLAMA,
ProviderName.OPENAI,
ProviderName.OPEN_ROUTER,
+ ProviderName.LLAMA_API,
]
AICredentials = CredentialsMetaInput[LLMProviderName, Literal["api_key"]]
@@ -141,6 +142,11 @@ class LlmModel(str, Enum, metaclass=LlmModelMeta):
GRYPHE_MYTHOMAX_L2_13B = "gryphe/mythomax-l2-13b"
META_LLAMA_4_SCOUT = "meta-llama/llama-4-scout"
META_LLAMA_4_MAVERICK = "meta-llama/llama-4-maverick"
+ # Llama API models
+ LLAMA_API_LLAMA_4_SCOUT = "Llama-4-Scout-17B-16E-Instruct-FP8"
+ LLAMA_API_LLAMA4_MAVERICK = "Llama-4-Maverick-17B-128E-Instruct-FP8"
+ LLAMA_API_LLAMA3_3_8B = "Llama-3.3-8B-Instruct"
+ LLAMA_API_LLAMA3_3_70B = "Llama-3.3-70B-Instruct"
@property
def metadata(self) -> ModelMetadata:
@@ -230,6 +236,11 @@ MODEL_METADATA = {
LlmModel.GRYPHE_MYTHOMAX_L2_13B: ModelMetadata("open_router", 4096, 4096),
LlmModel.META_LLAMA_4_SCOUT: ModelMetadata("open_router", 131072, 131072),
LlmModel.META_LLAMA_4_MAVERICK: ModelMetadata("open_router", 1048576, 1000000),
+ # Llama API models
+ LlmModel.LLAMA_API_LLAMA_4_SCOUT: ModelMetadata("llama_api", 128000, 4028),
+ LlmModel.LLAMA_API_LLAMA4_MAVERICK: ModelMetadata("llama_api", 128000, 4028),
+ LlmModel.LLAMA_API_LLAMA3_3_8B: ModelMetadata("llama_api", 128000, 4028),
+ LlmModel.LLAMA_API_LLAMA3_3_70B: ModelMetadata("llama_api", 128000, 4028),
}
for model in LlmModel:
@@ -507,6 +518,53 @@ def llm_call(
api_key=credentials.api_key.get_secret_value(),
)
+ response = client.chat.completions.create(
+ extra_headers={
+ "HTTP-Referer": "https://agpt.co",
+ "X-Title": "AutoGPT",
+ },
+ model=llm_model.value,
+ messages=prompt, # type: ignore
+ max_tokens=max_tokens,
+ tools=tools_param, # type: ignore
+ )
+
+ # If there's no response, raise an error
+ if not response.choices:
+ if response:
+ raise ValueError(f"OpenRouter error: {response}")
+ else:
+ raise ValueError("No response from OpenRouter.")
+
+ if response.choices[0].message.tool_calls:
+ tool_calls = [
+ ToolContentBlock(
+ id=tool.id,
+ type=tool.type,
+ function=ToolCall(
+ name=tool.function.name, arguments=tool.function.arguments
+ ),
+ )
+ for tool in response.choices[0].message.tool_calls
+ ]
+ else:
+ tool_calls = None
+
+ return LLMResponse(
+ raw_response=response.choices[0].message,
+ prompt=prompt,
+ response=response.choices[0].message.content or "",
+ tool_calls=tool_calls,
+ prompt_tokens=response.usage.prompt_tokens if response.usage else 0,
+ completion_tokens=response.usage.completion_tokens if response.usage else 0,
+ )
+ elif provider == "llama_api":
+ tools_param = tools if tools else openai.NOT_GIVEN
+ client = openai.OpenAI(
+ base_url="https://api.llama.com/compat/v1/",
+ api_key=credentials.api_key.get_secret_value(),
+ )
+
response = client.chat.completions.create(
extra_headers={
"HTTP-Referer": "https://agpt.co",
@@ -524,9 +582,9 @@ def llm_call(
# If there's no response, raise an error
if not response.choices:
if response:
- raise ValueError(f"OpenRouter error: {response}")
+ raise ValueError(f"Llama API error: {response}")
else:
- raise ValueError("No response from OpenRouter.")
+ raise ValueError("No response from Llama API.")
if response.choices[0].message.tool_calls:
tool_calls = [
diff --git a/autogpt_platform/backend/backend/data/block_cost_config.py b/autogpt_platform/backend/backend/data/block_cost_config.py
index 2397ce0a0b..1e15b5f620 100644
--- a/autogpt_platform/backend/backend/data/block_cost_config.py
+++ b/autogpt_platform/backend/backend/data/block_cost_config.py
@@ -26,6 +26,7 @@ from backend.integrations.credentials_store import (
groq_credentials,
ideogram_credentials,
jina_credentials,
+ llama_api_credentials,
open_router_credentials,
openai_credentials,
replicate_credentials,
@@ -81,6 +82,10 @@ MODEL_COST: dict[LlmModel, int] = {
LlmModel.GRYPHE_MYTHOMAX_L2_13B: 1,
LlmModel.META_LLAMA_4_SCOUT: 1,
LlmModel.META_LLAMA_4_MAVERICK: 1,
+ LlmModel.LLAMA_API_LLAMA_4_SCOUT: 1,
+ LlmModel.LLAMA_API_LLAMA4_MAVERICK: 1,
+ LlmModel.LLAMA_API_LLAMA3_3_8B: 1,
+ LlmModel.LLAMA_API_LLAMA3_3_70B: 1,
}
for model in LlmModel:
@@ -153,6 +158,23 @@ LLM_COST = (
for model, cost in MODEL_COST.items()
if MODEL_METADATA[model].provider == "open_router"
]
+ # Llama API Models
+ + [
+ BlockCost(
+ cost_type=BlockCostType.RUN,
+ cost_filter={
+ "model": model,
+ "credentials": {
+ "id": llama_api_credentials.id,
+ "provider": llama_api_credentials.provider,
+ "type": llama_api_credentials.type,
+ },
+ },
+ cost_amount=cost,
+ )
+ for model, cost in MODEL_COST.items()
+ if MODEL_METADATA[model].provider == "llama_api"
+ ]
)
# =============== This is the exhaustive list of cost for each Block =============== #
diff --git a/autogpt_platform/backend/backend/integrations/credentials_store.py b/autogpt_platform/backend/backend/integrations/credentials_store.py
index 152aa7ee39..09849536c6 100644
--- a/autogpt_platform/backend/backend/integrations/credentials_store.py
+++ b/autogpt_platform/backend/backend/integrations/credentials_store.py
@@ -177,6 +177,14 @@ zerobounce_credentials = APIKeyCredentials(
expires_at=None,
)
+llama_api_credentials = APIKeyCredentials(
+ id="d44045af-1c33-4833-9e19-752313214de2",
+ provider="llama_api",
+ api_key=SecretStr(settings.secrets.llama_api_key),
+ title="Use Credits for Llama API",
+ expires_at=None,
+)
+
DEFAULT_CREDENTIALS = [
ollama_credentials,
revid_credentials,
diff --git a/autogpt_platform/backend/backend/integrations/providers.py b/autogpt_platform/backend/backend/integrations/providers.py
index bf409b68d3..bd054641b8 100644
--- a/autogpt_platform/backend/backend/integrations/providers.py
+++ b/autogpt_platform/backend/backend/integrations/providers.py
@@ -20,6 +20,7 @@ class ProviderName(str, Enum):
IDEOGRAM = "ideogram"
JINA = "jina"
LINEAR = "linear"
+ LLAMA_API = "llama_api"
MEDIUM = "medium"
MEM0 = "mem0"
NOTION = "notion"
diff --git a/autogpt_platform/backend/backend/util/settings.py b/autogpt_platform/backend/backend/util/settings.py
index 159e31bb88..3ae35d6794 100644
--- a/autogpt_platform/backend/backend/util/settings.py
+++ b/autogpt_platform/backend/backend/util/settings.py
@@ -388,6 +388,7 @@ class Secrets(UpdateTrackingModel["Secrets"], BaseSettings):
anthropic_api_key: str = Field(default="", description="Anthropic API key")
groq_api_key: str = Field(default="", description="Groq API key")
open_router_api_key: str = Field(default="", description="Open Router API Key")
+ llama_api_key: str = Field(default="", description="Llama API Key")
reddit_client_id: str = Field(default="", description="Reddit client ID")
reddit_client_secret: str = Field(default="", description="Reddit client secret")
diff --git a/autogpt_platform/frontend/src/app/(platform)/profile/(user)/integrations/page.tsx b/autogpt_platform/frontend/src/app/(platform)/profile/(user)/integrations/page.tsx
index e5f1cd7ee9..534c50bce1 100644
--- a/autogpt_platform/frontend/src/app/(platform)/profile/(user)/integrations/page.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/profile/(user)/integrations/page.tsx
@@ -117,6 +117,7 @@ export default function PrivatePage() {
"3bcdbda3-84a3-46af-8fdb-bfd2472298b8", // SmartLead
"63a6e279-2dc2-448e-bf57-85776f7176dc", // ZeroBounce
"9aa1bde0-4947-4a70-a20c-84daa3850d52", // Google Maps
+ "d44045af-1c33-4833-9e19-752313214de2", // Llama API
],
[],
);
diff --git a/autogpt_platform/frontend/src/components/integrations/credentials-input.tsx b/autogpt_platform/frontend/src/components/integrations/credentials-input.tsx
index 3d2246f910..83e337efe9 100644
--- a/autogpt_platform/frontend/src/components/integrations/credentials-input.tsx
+++ b/autogpt_platform/frontend/src/components/integrations/credentials-input.tsx
@@ -76,6 +76,7 @@ export const providerIcons: Record<
openai: fallbackIcon,
openweathermap: fallbackIcon,
open_router: fallbackIcon,
+ llama_api: fallbackIcon,
pinecone: fallbackIcon,
slant3d: fallbackIcon,
screenshotone: fallbackIcon,
diff --git a/autogpt_platform/frontend/src/components/integrations/credentials-provider.tsx b/autogpt_platform/frontend/src/components/integrations/credentials-provider.tsx
index 4f0fe0a60d..d31cda8530 100644
--- a/autogpt_platform/frontend/src/components/integrations/credentials-provider.tsx
+++ b/autogpt_platform/frontend/src/components/integrations/credentials-provider.tsx
@@ -40,6 +40,7 @@ const providerDisplayNames: Record<CredentialsProviderName, string> = {
openai: "OpenAI",
openweathermap: "OpenWeatherMap",
open_router: "Open Router",
+ llama_api: "Llama API",
pinecone: "Pinecone",
screenshotone: "ScreenshotOne",
slant3d: "Slant3D",
diff --git a/autogpt_platform/frontend/src/lib/autogpt-server-api/types.ts b/autogpt_platform/frontend/src/lib/autogpt-server-api/types.ts
index 696d9f3b37..1aec142578 100644
--- a/autogpt_platform/frontend/src/lib/autogpt-server-api/types.ts
+++ b/autogpt_platform/frontend/src/lib/autogpt-server-api/types.ts
@@ -172,6 +172,7 @@ export const PROVIDER_NAMES = {
OPENAI: "openai",
OPENWEATHERMAP: "openweathermap",
OPEN_ROUTER: "open_router",
+ LLAMA_API: "llama_api",
PINECONE: "pinecone",
SCREENSHOTONE: "screenshotone",
SLANT3D: "slant3d",
diff --git a/docs/content/platform/blocks/llm.md b/docs/content/platform/blocks/llm.md
index 02c0b79985..4eafa62bee 100644
--- a/docs/content/platform/blocks/llm.md
+++ b/docs/content/platform/blocks/llm.md
@@ -156,4 +156,18 @@ The block formulates a prompt based on the given focus or source data, sends it
| Error | Any error message if the process fails |
### Possible use case
-Automatically generating a list of key points or action items from a long meeting transcript or summarizing the main topics discussed in a series of documents.
\ No newline at end of file
+Automatically generating a list of key points or action items from a long meeting transcript or summarizing the main topics discussed in a series of documents.
+
+# Providers
+There are severals providers that AutoGPT users can use for running inference with LLM models.
+
+## Llama API
+Llama API is a Meta-hosted API service that helps you integrate Llama models quickly and efficiently. Using OpenAI comptability endpoints, you can easily access the power of Llama models without the need for complex setup or configuration!
+
+Join the [waitlist](https://llama.developer.meta.com?utm_source=partner-autogpt&utm_medium=readme) to get access!
+
+Try the Llama API provider by selecting any of the following LLM Model names from the AI blocks mentioned above:
+- Llama-4-Scout-17B-16E-Instruct-FP8
+- Llama-4-Maverick-17B-128E-Instruct-FP8
+- Llama-3.3-8B-Instruct
+- Llama-3-70B-Instruct