Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(blocks): Add AI/ML API support to LLM blocks (#9996)

Dmitrya259eac9ffb1
diff --git a/autogpt_platform/backend/.env.example b/autogpt_platform/backend/.env.example
index ebd9d7c6ad..4c5830d704 100644
--- a/autogpt_platform/backend/.env.example
+++ b/autogpt_platform/backend/.env.example
@@ -127,6 +127,7 @@ TODOIST_CLIENT_SECRET=
# LLM
OPENAI_API_KEY=
ANTHROPIC_API_KEY=
+AIML_API_KEY=
GROQ_API_KEY=
OPEN_ROUTER_API_KEY=
LLAMA_API_KEY=
diff --git a/autogpt_platform/backend/backend/blocks/llm.py b/autogpt_platform/backend/backend/blocks/llm.py
index f4c16e48ad..a1fedeec52 100644
--- a/autogpt_platform/backend/backend/blocks/llm.py
+++ b/autogpt_platform/backend/backend/blocks/llm.py
@@ -31,6 +31,7 @@ logger = TruncatedLogger(logging.getLogger(__name__), "[LLM-Block]")
fmt = TextFormatter()
LLMProviderName = Literal[
+ ProviderName.AIML_API,
ProviderName.ANTHROPIC,
ProviderName.GROQ,
ProviderName.OLLAMA,
@@ -107,6 +108,12 @@ class LlmModel(str, Enum, metaclass=LlmModelMeta):
CLAUDE_3_5_SONNET = "claude-3-5-sonnet-latest"
CLAUDE_3_5_HAIKU = "claude-3-5-haiku-latest"
CLAUDE_3_HAIKU = "claude-3-haiku-20240307"
+ # AI/ML API models
+ AIML_API_QWEN2_5_72B = "Qwen/Qwen2.5-72B-Instruct-Turbo"
+ AIML_API_LLAMA3_1_70B = "nvidia/llama-3.1-nemotron-70b-instruct"
+ AIML_API_LLAMA3_3_70B = "meta-llama/Llama-3.3-70B-Instruct-Turbo"
+ AIML_API_META_LLAMA_3_1_70B = "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo"
+ AIML_API_LLAMA_3_2_3B = "meta-llama/Llama-3.2-3B-Instruct-Turbo"
# Groq models
GEMMA2_9B = "gemma2-9b-it"
LLAMA3_3_70B = "llama-3.3-70b-versatile"
@@ -204,6 +211,12 @@ MODEL_METADATA = {
LlmModel.CLAUDE_3_HAIKU: ModelMetadata(
"anthropic", 200000, 4096
), # claude-3-haiku-20240307
+ # https://docs.aimlapi.com/api-overview/model-database/text-models
+ LlmModel.AIML_API_QWEN2_5_72B: ModelMetadata("aiml_api", 32000, 8000),
+ LlmModel.AIML_API_LLAMA3_1_70B: ModelMetadata("aiml_api", 128000, 40000),
+ LlmModel.AIML_API_LLAMA3_3_70B: ModelMetadata("aiml_api", 128000, None),
+ LlmModel.AIML_API_META_LLAMA_3_1_70B: ModelMetadata("aiml_api", 131000, 2000),
+ LlmModel.AIML_API_LLAMA_3_2_3B: ModelMetadata("aiml_api", 128000, None),
# https://console.groq.com/docs/models
LlmModel.GEMMA2_9B: ModelMetadata("groq", 8192, None),
LlmModel.LLAMA3_3_70B: ModelMetadata("groq", 128000, 32768),
@@ -616,6 +629,29 @@ def llm_call(
prompt_tokens=response.usage.prompt_tokens if response.usage else 0,
completion_tokens=response.usage.completion_tokens if response.usage else 0,
)
+ elif provider == "aiml_api":
+ client = openai.OpenAI(
+ base_url="https://api.aimlapi.com/v2",
+ api_key=credentials.api_key.get_secret_value(),
+ default_headers={"X-Project": "AutoGPT"},
+ )
+
+ completion = client.chat.completions.create(
+ model=llm_model.value,
+ messages=prompt, # type: ignore
+ max_tokens=max_tokens,
+ )
+
+ return LLMResponse(
+ raw_response=completion.choices[0].message,
+ prompt=prompt,
+ response=completion.choices[0].message.content or "",
+ tool_calls=None,
+ prompt_tokens=completion.usage.prompt_tokens if completion.usage else 0,
+ completion_tokens=(
+ completion.usage.completion_tokens if completion.usage else 0
+ ),
+ )
else:
raise ValueError(f"Unsupported LLM provider: {provider}")
diff --git a/autogpt_platform/backend/backend/data/block_cost_config.py b/autogpt_platform/backend/backend/data/block_cost_config.py
index e099ebfea0..864ed0e989 100644
--- a/autogpt_platform/backend/backend/data/block_cost_config.py
+++ b/autogpt_platform/backend/backend/data/block_cost_config.py
@@ -22,6 +22,7 @@ from backend.blocks.text_to_speech_block import UnrealTextToSpeechBlock
from backend.data.block import Block
from backend.data.cost import BlockCost, BlockCostType
from backend.integrations.credentials_store import (
+ aiml_api_credentials,
anthropic_credentials,
did_credentials,
groq_credentials,
@@ -54,6 +55,11 @@ MODEL_COST: dict[LlmModel, int] = {
LlmModel.CLAUDE_3_5_SONNET: 4,
LlmModel.CLAUDE_3_5_HAIKU: 1, # $0.80 / $4.00
LlmModel.CLAUDE_3_HAIKU: 1,
+ LlmModel.AIML_API_QWEN2_5_72B: 1,
+ LlmModel.AIML_API_LLAMA3_1_70B: 1,
+ LlmModel.AIML_API_LLAMA3_3_70B: 1,
+ LlmModel.AIML_API_META_LLAMA_3_1_70B: 1,
+ LlmModel.AIML_API_LLAMA_3_2_3B: 1,
LlmModel.LLAMA3_8B: 1,
LlmModel.LLAMA3_70B: 1,
LlmModel.MIXTRAL_8X7B: 1,
@@ -178,6 +184,23 @@ LLM_COST = (
for model, cost in MODEL_COST.items()
if MODEL_METADATA[model].provider == "llama_api"
]
+ # AI/ML Api Models
+ + [
+ BlockCost(
+ cost_type=BlockCostType.RUN,
+ cost_filter={
+ "model": model,
+ "credentials": {
+ "id": aiml_api_credentials.id,
+ "provider": aiml_api_credentials.provider,
+ "type": aiml_api_credentials.type,
+ },
+ },
+ cost_amount=cost,
+ )
+ for model, cost in MODEL_COST.items()
+ if MODEL_METADATA[model].provider == "aiml_api"
+ ]
)
# =============== This is the exhaustive list of cost for each Block =============== #
diff --git a/autogpt_platform/backend/backend/integrations/credentials_store.py b/autogpt_platform/backend/backend/integrations/credentials_store.py
index 09849536c6..847b20fa6c 100644
--- a/autogpt_platform/backend/backend/integrations/credentials_store.py
+++ b/autogpt_platform/backend/backend/integrations/credentials_store.py
@@ -60,6 +60,13 @@ openai_credentials = APIKeyCredentials(
title="Use Credits for OpenAI",
expires_at=None,
)
+aiml_api_credentials = APIKeyCredentials(
+ id="aad82a89-9794-4ebb-977f-d736aa5260a3",
+ provider="aiml_api",
+ api_key=SecretStr(settings.secrets.aiml_api_key),
+ title="Use Credits for AI/ML API",
+ expires_at=None,
+)
anthropic_credentials = APIKeyCredentials(
id="24e5d942-d9e3-4798-8151-90143ee55629",
provider="anthropic",
@@ -191,6 +198,7 @@ DEFAULT_CREDENTIALS = [
ideogram_credentials,
replicate_credentials,
openai_credentials,
+ aiml_api_credentials,
anthropic_credentials,
groq_credentials,
did_credentials,
@@ -252,6 +260,8 @@ class IntegrationCredentialsStore:
all_credentials.append(replicate_credentials)
if settings.secrets.openai_api_key:
all_credentials.append(openai_credentials)
+ if settings.secrets.aiml_api_key:
+ all_credentials.append(aiml_api_credentials)
if settings.secrets.anthropic_api_key:
all_credentials.append(anthropic_credentials)
if settings.secrets.did_api_key:
diff --git a/autogpt_platform/backend/backend/integrations/providers.py b/autogpt_platform/backend/backend/integrations/providers.py
index bd054641b8..30444c1ea3 100644
--- a/autogpt_platform/backend/backend/integrations/providers.py
+++ b/autogpt_platform/backend/backend/integrations/providers.py
@@ -3,6 +3,7 @@ from enum import Enum
# --8<-- [start:ProviderName]
class ProviderName(str, Enum):
+ AIML_API = "aiml_api"
ANTHROPIC = "anthropic"
APOLLO = "apollo"
COMPASS = "compass"
diff --git a/autogpt_platform/backend/backend/util/settings.py b/autogpt_platform/backend/backend/util/settings.py
index 3ae35d6794..c349f0c804 100644
--- a/autogpt_platform/backend/backend/util/settings.py
+++ b/autogpt_platform/backend/backend/util/settings.py
@@ -385,6 +385,7 @@ class Secrets(UpdateTrackingModel["Secrets"], BaseSettings):
)
openai_api_key: str = Field(default="", description="OpenAI API key")
+ aiml_api_key: str = Field(default="", description="'AI/ML API' key")
anthropic_api_key: str = Field(default="", description="Anthropic API key")
groq_api_key: str = Field(default="", description="Groq API key")
open_router_api_key: str = Field(default="", description="Open Router API Key")
diff --git a/autogpt_platform/frontend/src/app/(platform)/profile/(user)/integrations/page.tsx b/autogpt_platform/frontend/src/app/(platform)/profile/(user)/integrations/page.tsx
index 80e34a134c..3cceb3c6b3 100644
--- a/autogpt_platform/frontend/src/app/(platform)/profile/(user)/integrations/page.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/profile/(user)/integrations/page.tsx
@@ -103,6 +103,7 @@ export default function UserIntegrationsPage() {
"6b9fc200-4726-4973-86c9-cd526f5ce5db", // Replicate
"53c25cb8-e3ee-465c-a4d1-e75a4c899c2a", // OpenAI
"24e5d942-d9e3-4798-8151-90143ee55629", // Anthropic
+ "aad82a89-9794-4ebb-977f-d736aa5260a3", // AI/ML
"4ec22295-8f97-4dd1-b42b-2c6957a02545", // Groq
"7f7b0654-c36b-4565-8fa7-9a52575dfae2", // D-ID
"7f26de70-ba0d-494e-ba76-238e65e7b45f", // Jina
diff --git a/autogpt_platform/frontend/src/components/integrations/credentials-input.tsx b/autogpt_platform/frontend/src/components/integrations/credentials-input.tsx
index 83e337efe9..690f58de33 100644
--- a/autogpt_platform/frontend/src/components/integrations/credentials-input.tsx
+++ b/autogpt_platform/frontend/src/components/integrations/credentials-input.tsx
@@ -56,6 +56,7 @@ export const providerIcons: Record<
CredentialsProviderName,
React.FC<{ className?: string }>
> = {
+ aiml_api: fallbackIcon,
anthropic: fallbackIcon,
apollo: fallbackIcon,
e2b: fallbackIcon,
diff --git a/autogpt_platform/frontend/src/components/integrations/credentials-provider.tsx b/autogpt_platform/frontend/src/components/integrations/credentials-provider.tsx
index 91bdc40f46..f74fc3c7c5 100644
--- a/autogpt_platform/frontend/src/components/integrations/credentials-provider.tsx
+++ b/autogpt_platform/frontend/src/components/integrations/credentials-provider.tsx
@@ -18,6 +18,7 @@ const CREDENTIALS_PROVIDER_NAMES = Object.values(
// --8<-- [start:CredentialsProviderNames]
const providerDisplayNames: Record<CredentialsProviderName, string> = {
+ aiml_api: "AI/ML",
anthropic: "Anthropic",
apollo: "Apollo",
discord: "Discord",
diff --git a/autogpt_platform/frontend/src/lib/autogpt-server-api/types.ts b/autogpt_platform/frontend/src/lib/autogpt-server-api/types.ts
index 635fa44e2d..873a6c735f 100644
--- a/autogpt_platform/frontend/src/lib/autogpt-server-api/types.ts
+++ b/autogpt_platform/frontend/src/lib/autogpt-server-api/types.ts
@@ -149,6 +149,7 @@ export type Credentials =
// --8<-- [start:BlockIOCredentialsSubSchema]
export const PROVIDER_NAMES = {
+ AIML_API: "aiml_api",
ANTHROPIC: "anthropic",
APOLLO: "apollo",
D_ID: "d_id",
diff --git a/docs/content/index.md b/docs/content/index.md
index 243b56ea41..cd69844364 100644
--- a/docs/content/index.md
+++ b/docs/content/index.md
@@ -66,6 +66,7 @@ The platform comes pre-integrated with cutting-edge LLM providers:
- OpenAI
- Anthropic
+- AI/ML API
- Groq
- Llama