AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
Add Llama API Support (#9899)
Chirag Modi2dc038b6c08a
diff --git a/autogpt_platform/backend/.env.example b/autogpt_platform/backend/.env.exampleindex 60d941b839..ebd9d7c6ad 100644--- a/autogpt_platform/backend/.env.example+++ b/autogpt_platform/backend/.env.example@@ -129,6 +129,7 @@ OPENAI_API_KEY=ANTHROPIC_API_KEY=GROQ_API_KEY=OPEN_ROUTER_API_KEY=+LLAMA_API_KEY=# Go to https://www.reddit.com/prefs/apps and create a new appdiff --git a/autogpt_platform/backend/backend/blocks/llm.py b/autogpt_platform/backend/backend/blocks/llm.pyindex f630937e61..675b77ee5d 100644--- a/autogpt_platform/backend/backend/blocks/llm.py+++ b/autogpt_platform/backend/backend/blocks/llm.py@@ -36,6 +36,7 @@ LLMProviderName = Literal[ProviderName.OLLAMA,ProviderName.OPENAI,ProviderName.OPEN_ROUTER,+ ProviderName.LLAMA_API,]AICredentials = CredentialsMetaInput[LLMProviderName, Literal["api_key"]]@@ -141,6 +142,11 @@ class LlmModel(str, Enum, metaclass=LlmModelMeta):GRYPHE_MYTHOMAX_L2_13B = "gryphe/mythomax-l2-13b"META_LLAMA_4_SCOUT = "meta-llama/llama-4-scout"META_LLAMA_4_MAVERICK = "meta-llama/llama-4-maverick"+ # Llama API models+ LLAMA_API_LLAMA_4_SCOUT = "Llama-4-Scout-17B-16E-Instruct-FP8"+ LLAMA_API_LLAMA4_MAVERICK = "Llama-4-Maverick-17B-128E-Instruct-FP8"+ LLAMA_API_LLAMA3_3_8B = "Llama-3.3-8B-Instruct"+ LLAMA_API_LLAMA3_3_70B = "Llama-3.3-70B-Instruct"@propertydef metadata(self) -> ModelMetadata:@@ -230,6 +236,11 @@ MODEL_METADATA = {LlmModel.GRYPHE_MYTHOMAX_L2_13B: ModelMetadata("open_router", 4096, 4096),LlmModel.META_LLAMA_4_SCOUT: ModelMetadata("open_router", 131072, 131072),LlmModel.META_LLAMA_4_MAVERICK: ModelMetadata("open_router", 1048576, 1000000),+ # Llama API models+ LlmModel.LLAMA_API_LLAMA_4_SCOUT: ModelMetadata("llama_api", 128000, 4028),+ LlmModel.LLAMA_API_LLAMA4_MAVERICK: ModelMetadata("llama_api", 128000, 4028),+ LlmModel.LLAMA_API_LLAMA3_3_8B: ModelMetadata("llama_api", 128000, 4028),+ LlmModel.LLAMA_API_LLAMA3_3_70B: ModelMetadata("llama_api", 128000, 4028),}for model in LlmModel:@@ -507,6 +518,53 @@ def llm_call(api_key=credentials.api_key.get_secret_value(),)+ response = client.chat.completions.create(+ extra_headers={+ "HTTP-Referer": "https://agpt.co",+ "X-Title": "AutoGPT",+ },+ model=llm_model.value,+ messages=prompt, # type: ignore+ max_tokens=max_tokens,+ tools=tools_param, # type: ignore+ )++ # If there's no response, raise an error+ if not response.choices:+ if response:+ raise ValueError(f"OpenRouter error: {response}")+ else:+ raise ValueError("No response from OpenRouter.")++ if response.choices[0].message.tool_calls:+ tool_calls = [+ ToolContentBlock(+ id=tool.id,+ type=tool.type,+ function=ToolCall(+ name=tool.function.name, arguments=tool.function.arguments+ ),+ )+ for tool in response.choices[0].message.tool_calls+ ]+ else:+ tool_calls = None++ return LLMResponse(+ raw_response=response.choices[0].message,+ prompt=prompt,+ response=response.choices[0].message.content or "",+ tool_calls=tool_calls,+ prompt_tokens=response.usage.prompt_tokens if response.usage else 0,+ completion_tokens=response.usage.completion_tokens if response.usage else 0,+ )+ elif provider == "llama_api":+ tools_param = tools if tools else openai.NOT_GIVEN+ client = openai.OpenAI(+ base_url="https://api.llama.com/compat/v1/",+ api_key=credentials.api_key.get_secret_value(),+ )+response = client.chat.completions.create(extra_headers={"HTTP-Referer": "https://agpt.co",@@ -524,9 +582,9 @@ def llm_call(# If there's no response, raise an errorif not response.choices:if response:- raise ValueError(f"OpenRouter error: {response}")+ raise ValueError(f"Llama API error: {response}")else:- raise ValueError("No response from OpenRouter.")+ raise ValueError("No response from Llama API.")if response.choices[0].message.tool_calls:tool_calls = [diff --git a/autogpt_platform/backend/backend/data/block_cost_config.py b/autogpt_platform/backend/backend/data/block_cost_config.pyindex 2397ce0a0b..1e15b5f620 100644--- a/autogpt_platform/backend/backend/data/block_cost_config.py+++ b/autogpt_platform/backend/backend/data/block_cost_config.py@@ -26,6 +26,7 @@ from backend.integrations.credentials_store import (groq_credentials,ideogram_credentials,jina_credentials,+ llama_api_credentials,open_router_credentials,openai_credentials,replicate_credentials,@@ -81,6 +82,10 @@ MODEL_COST: dict[LlmModel, int] = {LlmModel.GRYPHE_MYTHOMAX_L2_13B: 1,LlmModel.META_LLAMA_4_SCOUT: 1,LlmModel.META_LLAMA_4_MAVERICK: 1,+ LlmModel.LLAMA_API_LLAMA_4_SCOUT: 1,+ LlmModel.LLAMA_API_LLAMA4_MAVERICK: 1,+ LlmModel.LLAMA_API_LLAMA3_3_8B: 1,+ LlmModel.LLAMA_API_LLAMA3_3_70B: 1,}for model in LlmModel:@@ -153,6 +158,23 @@ LLM_COST = (for model, cost in MODEL_COST.items()if MODEL_METADATA[model].provider == "open_router"]+ # Llama API Models+ + [+ BlockCost(+ cost_type=BlockCostType.RUN,+ cost_filter={+ "model": model,+ "credentials": {+ "id": llama_api_credentials.id,+ "provider": llama_api_credentials.provider,+ "type": llama_api_credentials.type,+ },+ },+ cost_amount=cost,+ )+ for model, cost in MODEL_COST.items()+ if MODEL_METADATA[model].provider == "llama_api"+ ])# =============== This is the exhaustive list of cost for each Block =============== #diff --git a/autogpt_platform/backend/backend/integrations/credentials_store.py b/autogpt_platform/backend/backend/integrations/credentials_store.pyindex 152aa7ee39..09849536c6 100644--- a/autogpt_platform/backend/backend/integrations/credentials_store.py+++ b/autogpt_platform/backend/backend/integrations/credentials_store.py@@ -177,6 +177,14 @@ zerobounce_credentials = APIKeyCredentials(expires_at=None,)+llama_api_credentials = APIKeyCredentials(+ id="d44045af-1c33-4833-9e19-752313214de2",+ provider="llama_api",+ api_key=SecretStr(settings.secrets.llama_api_key),+ title="Use Credits for Llama API",+ expires_at=None,+)+DEFAULT_CREDENTIALS = [ollama_credentials,revid_credentials,diff --git a/autogpt_platform/backend/backend/integrations/providers.py b/autogpt_platform/backend/backend/integrations/providers.pyindex bf409b68d3..bd054641b8 100644--- a/autogpt_platform/backend/backend/integrations/providers.py+++ b/autogpt_platform/backend/backend/integrations/providers.py@@ -20,6 +20,7 @@ class ProviderName(str, Enum):IDEOGRAM = "ideogram"JINA = "jina"LINEAR = "linear"+ LLAMA_API = "llama_api"MEDIUM = "medium"MEM0 = "mem0"NOTION = "notion"diff --git a/autogpt_platform/backend/backend/util/settings.py b/autogpt_platform/backend/backend/util/settings.pyindex 159e31bb88..3ae35d6794 100644--- a/autogpt_platform/backend/backend/util/settings.py+++ b/autogpt_platform/backend/backend/util/settings.py@@ -388,6 +388,7 @@ class Secrets(UpdateTrackingModel["Secrets"], BaseSettings):anthropic_api_key: str = Field(default="", description="Anthropic API key")groq_api_key: str = Field(default="", description="Groq API key")open_router_api_key: str = Field(default="", description="Open Router API Key")+ llama_api_key: str = Field(default="", description="Llama API Key")reddit_client_id: str = Field(default="", description="Reddit client ID")reddit_client_secret: str = Field(default="", description="Reddit client secret")diff --git a/autogpt_platform/frontend/src/app/(platform)/profile/(user)/integrations/page.tsx b/autogpt_platform/frontend/src/app/(platform)/profile/(user)/integrations/page.tsxindex e5f1cd7ee9..534c50bce1 100644--- a/autogpt_platform/frontend/src/app/(platform)/profile/(user)/integrations/page.tsx+++ b/autogpt_platform/frontend/src/app/(platform)/profile/(user)/integrations/page.tsx@@ -117,6 +117,7 @@ export default function PrivatePage() {"3bcdbda3-84a3-46af-8fdb-bfd2472298b8", // SmartLead"63a6e279-2dc2-448e-bf57-85776f7176dc", // ZeroBounce"9aa1bde0-4947-4a70-a20c-84daa3850d52", // Google Maps+ "d44045af-1c33-4833-9e19-752313214de2", // Llama API],[],);diff --git a/autogpt_platform/frontend/src/components/integrations/credentials-input.tsx b/autogpt_platform/frontend/src/components/integrations/credentials-input.tsxindex 3d2246f910..83e337efe9 100644--- a/autogpt_platform/frontend/src/components/integrations/credentials-input.tsx+++ b/autogpt_platform/frontend/src/components/integrations/credentials-input.tsx@@ -76,6 +76,7 @@ export const providerIcons: Record<openai: fallbackIcon,openweathermap: fallbackIcon,open_router: fallbackIcon,+ llama_api: fallbackIcon,pinecone: fallbackIcon,slant3d: fallbackIcon,screenshotone: fallbackIcon,diff --git a/autogpt_platform/frontend/src/components/integrations/credentials-provider.tsx b/autogpt_platform/frontend/src/components/integrations/credentials-provider.tsxindex 4f0fe0a60d..d31cda8530 100644--- a/autogpt_platform/frontend/src/components/integrations/credentials-provider.tsx+++ b/autogpt_platform/frontend/src/components/integrations/credentials-provider.tsx@@ -40,6 +40,7 @@ const providerDisplayNames: Record<CredentialsProviderName, string> = {openai: "OpenAI",openweathermap: "OpenWeatherMap",open_router: "Open Router",+ llama_api: "Llama API",pinecone: "Pinecone",screenshotone: "ScreenshotOne",slant3d: "Slant3D",diff --git a/autogpt_platform/frontend/src/lib/autogpt-server-api/types.ts b/autogpt_platform/frontend/src/lib/autogpt-server-api/types.tsindex 696d9f3b37..1aec142578 100644--- a/autogpt_platform/frontend/src/lib/autogpt-server-api/types.ts+++ b/autogpt_platform/frontend/src/lib/autogpt-server-api/types.ts@@ -172,6 +172,7 @@ export const PROVIDER_NAMES = {OPENAI: "openai",OPENWEATHERMAP: "openweathermap",OPEN_ROUTER: "open_router",+ LLAMA_API: "llama_api",PINECONE: "pinecone",SCREENSHOTONE: "screenshotone",SLANT3D: "slant3d",diff --git a/docs/content/platform/blocks/llm.md b/docs/content/platform/blocks/llm.mdindex 02c0b79985..4eafa62bee 100644--- a/docs/content/platform/blocks/llm.md+++ b/docs/content/platform/blocks/llm.md@@ -156,4 +156,18 @@ The block formulates a prompt based on the given focus or source data, sends it| Error | Any error message if the process fails |### Possible use case-Automatically generating a list of key points or action items from a long meeting transcript or summarizing the main topics discussed in a series of documents.\ No newline at end of file+Automatically generating a list of key points or action items from a long meeting transcript or summarizing the main topics discussed in a series of documents.++# Providers+There are severals providers that AutoGPT users can use for running inference with LLM models.++## Llama API+Llama API is a Meta-hosted API service that helps you integrate Llama models quickly and efficiently. Using OpenAI comptability endpoints, you can easily access the power of Llama models without the need for complex setup or configuration!++Join the [waitlist](https://llama.developer.meta.com?utm_source=partner-autogpt&utm_medium=readme) to get access!++Try the Llama API provider by selecting any of the following LLM Model names from the AI blocks mentioned above:+- Llama-4-Scout-17B-16E-Instruct-FP8+- Llama-4-Maverick-17B-128E-Instruct-FP8+- Llama-3.3-8B-Instruct+- Llama-3-70B-Instruct