AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(blocks): add Tavily provider blocks (search, extract, crawl, map) (#13499)
Oren Levitin3b2b05e60f60
diff --git a/autogpt_platform/backend/.env.default b/autogpt_platform/backend/.env.defaultindex bf5632af17..d9e00c6e10 100644--- a/autogpt_platform/backend/.env.default+++ b/autogpt_platform/backend/.env.default@@ -249,6 +249,7 @@ EXA_API_KEY=JINA_API_KEY=MEM0_API_KEY=OPENWEATHERMAP_API_KEY=+TAVILY_API_KEY=GOOGLE_MAPS_API_KEY=# Platform Bot Linkingdiff --git a/autogpt_platform/backend/backend/blocks/tavily/__init__.py b/autogpt_platform/backend/backend/blocks/tavily/__init__.pynew file mode 100644index 0000000000..e69de29bb2diff --git a/autogpt_platform/backend/backend/blocks/tavily/_api.py b/autogpt_platform/backend/backend/blocks/tavily/_api.pynew file mode 100644index 0000000000..ae83bc2305--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/tavily/_api.py@@ -0,0 +1,67 @@+from enum import Enum+from typing import Any, Optional++from backend.sdk import BaseModel, SchemaField++# USD cost of one Tavily API credit on the pay-as-you-go plan.+CREDIT_USD = 0.008+++def credits_from_response(response: dict[str, Any], estimate: float) -> float:+ """Actual credits consumed per the API's usage report (include_usage=True),+ falling back to the documented credit schedule when absent."""+ usage = response.get("usage") or {}+ credits = usage.get("credits")+ return credits if isinstance(credits, (int, float)) else estimate+++class TavilySearchDepth(Enum):+ BASIC = "basic"+ ADVANCED = "advanced"+ FAST = "fast"+ ULTRA_FAST = "ultra-fast"+++class TavilyTopic(Enum):+ GENERAL = "general"+ NEWS = "news"+ FINANCE = "finance"+++class TavilyTimeRange(Enum):+ DAY = "day"+ WEEK = "week"+ MONTH = "month"+ YEAR = "year"+++class TavilyExtractDepth(Enum):+ BASIC = "basic"+ ADVANCED = "advanced"+++class TavilyFormat(Enum):+ MARKDOWN = "markdown"+ TEXT = "text"+++class TavilySearchResult(BaseModel):+ """Schema for a single Tavily search result."""++ title: str = SchemaField(description="Title of the result page")+ url: str = SchemaField(description="URL of the result page")+ content: str = SchemaField(+ description="Query-relevant content snippet extracted from the page"+ )+ score: float = SchemaField(description="Relevance score for the query (0-1)")+ raw_content: Optional[str] = SchemaField(+ description="Full page content, present when include_raw_content is enabled",+ default=None,+ )+++class TavilyPageContent(BaseModel):+ """Schema for a page returned by Tavily extract/crawl."""++ url: str = SchemaField(description="URL of the page")+ raw_content: str = SchemaField(description="Extracted page content")diff --git a/autogpt_platform/backend/backend/blocks/tavily/_config.py b/autogpt_platform/backend/backend/blocks/tavily/_config.pynew file mode 100644index 0000000000..8802f20ce1--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/tavily/_config.py@@ -0,0 +1,15 @@+from backend.sdk import BlockCostType, ProviderBuilder++# Tavily bills in API credits (1 credit = $0.008 pay-as-you-go). Each block's+# run() estimates USD spend from the documented credit schedule (basic search+# 1 credit, advanced 2; extract 1 credit per 5 URLs; map 1 credit per 10+# pages) and merge_stats populates NodeExecutionStats.provider_cost before+# billing reconciliation. 1000 platform credits per USD matches the+# convention used by the other search providers (see firecrawl/_config.py).+tavily = (+ ProviderBuilder("tavily")+ .with_description("AI-native web search, extract, crawl and map")+ .with_api_key("TAVILY_API_KEY", "Tavily API Key")+ .with_base_cost(1000, BlockCostType.COST_USD)+ .build()+)diff --git a/autogpt_platform/backend/backend/blocks/tavily/crawl.py b/autogpt_platform/backend/backend/blocks/tavily/crawl.pynew file mode 100644index 0000000000..58ef80aa89--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/tavily/crawl.py@@ -0,0 +1,165 @@+import math+from typing import Any++from tavily import AsyncTavilyClient++from backend.data.model import NodeExecutionStats+from backend.sdk import (+ APIKeyCredentials,+ Block,+ BlockCategory,+ BlockOutput,+ BlockSchemaInput,+ BlockSchemaOutput,+ CredentialsMetaInput,+ SchemaField,+)+from backend.util.exceptions import BlockExecutionError++from ._api import (+ CREDIT_USD,+ TavilyExtractDepth,+ TavilyFormat,+ TavilyPageContent,+ credits_from_response,+)+from ._config import tavily+++class TavilyCrawlBlock(Block):+ class Input(BlockSchemaInput):+ credentials: CredentialsMetaInput = tavily.credentials_field(+ description="The Tavily integration requires an API Key."+ )+ url: str = SchemaField(description="The root URL to start crawling from")+ limit: int = SchemaField(+ description="Maximum number of pages to crawl",+ default=10,+ ge=1,+ advanced=True,+ )+ max_depth: int = SchemaField(+ description="Maximum link depth from the root URL",+ default=1,+ ge=1,+ advanced=True,+ )+ instructions: str | None = SchemaField(+ description="Natural language instructions guiding which pages to crawl (e.g. 'Find all the API reference pages')",+ default=None,+ )+ extract_depth: TavilyExtractDepth = SchemaField(+ description="Depth of the extraction: basic or advanced (retrieves more data, including tables and embedded content)",+ default=TavilyExtractDepth.BASIC,+ advanced=True,+ )+ format: TavilyFormat = SchemaField(+ description="The format of the extracted content",+ default=TavilyFormat.MARKDOWN,+ advanced=True,+ )++ class Output(BlockSchemaOutput):+ results: list[TavilyPageContent] = SchemaField(+ description="List of crawled pages with their content"+ )+ result: TavilyPageContent = SchemaField(description="Single crawled page")+ error: str = SchemaField(+ description="Error message if the crawl failed",+ default="",+ )++ def __init__(self):+ super().__init__(+ id="ad89e82c-cdab-4784-bf54-9666ab0d2952",+ description="Crawls a website with Tavily, following links from the root URL and extracting page content",+ categories={BlockCategory.SEARCH},+ input_schema=self.Input,+ output_schema=self.Output,+ test_input={+ "credentials": tavily.get_test_credentials().model_dump(),+ "url": "https://agpt.co",+ "limit": 2,+ },+ test_credentials=tavily.get_test_credentials(),+ test_output=[+ ("results", lambda x: isinstance(x, list) and len(x) == 2),+ ("result", lambda x: x.url == "https://agpt.co"),+ ("result", lambda x: x.url == "https://agpt.co/blog"),+ ],+ test_mock={+ "_crawl": lambda *args, **kwargs: {+ "base_url": "https://agpt.co",+ "results": [+ {+ "url": "https://agpt.co",+ "raw_content": "AutoGPT is a platform for building AI agents.",+ },+ {+ "url": "https://agpt.co/blog",+ "raw_content": "AutoGPT blog.",+ },+ ],+ "usage": {"credits": 2},+ }+ },+ )++ async def _crawl(self, credentials: APIKeyCredentials, **kwargs) -> dict[str, Any]:+ """Private method to call the Tavily API - can be mocked for testing."""+ client = AsyncTavilyClient(api_key=credentials.api_key.get_secret_value())+ return await client.crawl(**kwargs)++ async def run(+ self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs+ ) -> BlockOutput:+ sdk_kwargs: dict[str, Any] = {+ "url": input_data.url,+ "limit": input_data.limit,+ "max_depth": input_data.max_depth,+ "extract_depth": input_data.extract_depth.value,+ "format": input_data.format.value,+ "include_usage": True,+ }+ if input_data.instructions:+ sdk_kwargs["instructions"] = input_data.instructions++ try:+ response = await self._crawl(credentials, **sdk_kwargs)+ except Exception as e:+ raise BlockExecutionError(+ message=f"Crawl failed: {e}",+ block_name=self.name,+ block_id=self.id,+ ) from e++ results = [+ TavilyPageContent(+ url=r.get("url") or "",+ raw_content=r.get("raw_content") or "",+ )+ for r in response.get("results", [])+ ]++ # Actual credit spend from the API's usage report; falls back to the+ # documented schedule (crawl combines mapping and extraction costs:+ # 1 credit per 10 pages mapped, 2 per 10 with instructions, plus+ # 1 credit per 5 pages extracted, 2 per 5 for advanced extraction).+ pages = len(results)+ map_credits = math.ceil(pages / 10) * (2 if input_data.instructions else 1)+ extract_credits = math.ceil(pages / 5) * (+ 2 if input_data.extract_depth == TavilyExtractDepth.ADVANCED else 1+ )+ credits_used = credits_from_response(+ response, estimate=map_credits + extract_credits+ )+ self.merge_stats(+ NodeExecutionStats(+ provider_cost=credits_used * CREDIT_USD,+ provider_cost_type="cost_usd",+ )+ )++ yield "results", results+ for result in results:+ yield "result", resultdiff --git a/autogpt_platform/backend/backend/blocks/tavily/extract.py b/autogpt_platform/backend/backend/blocks/tavily/extract.pynew file mode 100644index 0000000000..e5ed892872--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/tavily/extract.py@@ -0,0 +1,151 @@+import math+from typing import Any++from tavily import AsyncTavilyClient++from backend.data.model import NodeExecutionStats+from backend.sdk import (+ APIKeyCredentials,+ Block,+ BlockCategory,+ BlockOutput,+ BlockSchemaInput,+ BlockSchemaOutput,+ CredentialsMetaInput,+ SchemaField,+)+from backend.util.exceptions import BlockExecutionError++from ._api import (+ CREDIT_USD,+ TavilyExtractDepth,+ TavilyFormat,+ TavilyPageContent,+ credits_from_response,+)+from ._config import tavily+++class TavilyExtractBlock(Block):+ class Input(BlockSchemaInput):+ credentials: CredentialsMetaInput = tavily.credentials_field(+ description="The Tavily integration requires an API Key."+ )+ urls: list[str] = SchemaField(+ description="The URLs to extract content from (up to 20 per request)",+ max_length=20,+ )+ extract_depth: TavilyExtractDepth = SchemaField(+ description="Depth of the extraction: basic or advanced (retrieves more data, including tables and embedded content)",+ default=TavilyExtractDepth.BASIC,+ advanced=True,+ )+ format: TavilyFormat = SchemaField(+ description="The format of the extracted content",+ default=TavilyFormat.MARKDOWN,+ advanced=True,+ )++ class Output(BlockSchemaOutput):+ results: list[TavilyPageContent] = SchemaField(+ description="List of successfully extracted pages"+ )+ result: TavilyPageContent = SchemaField(description="Single extracted page")+ failed_urls: list[str] = SchemaField(+ description="URLs that could not be extracted"+ )+ error: str = SchemaField(+ description="Error message if the extraction failed",+ default="",+ )++ def __init__(self):+ super().__init__(+ id="d90fc44b-d462-42be-8327-09b75ad06721",+ description="Extracts page content from one or more URLs using Tavily, optimized for LLM consumption",+ categories={BlockCategory.SEARCH},+ input_schema=self.Input,+ output_schema=self.Output,+ test_input={+ "credentials": tavily.get_test_credentials().model_dump(),+ "urls": ["https://agpt.co"],+ },+ test_credentials=tavily.get_test_credentials(),+ test_output=[+ ("results", lambda x: isinstance(x, list) and len(x) == 1),+ ("result", lambda x: x.url == "https://agpt.co"),+ ("failed_urls", []),+ ],+ test_mock={+ "_extract": lambda *args, **kwargs: {+ "results": [+ {+ "url": "https://agpt.co",+ "raw_content": "AutoGPT is a platform for building AI agents.",+ }+ ],+ "failed_results": [],+ "usage": {"credits": 1},+ }+ },+ )++ async def _extract(+ self, credentials: APIKeyCredentials, **kwargs+ ) -> dict[str, Any]:+ """Private method to call the Tavily API - can be mocked for testing."""+ client = AsyncTavilyClient(api_key=credentials.api_key.get_secret_value())+ return await client.extract(**kwargs)++ async def run(+ self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs+ ) -> BlockOutput:+ try:+ response = await self._extract(+ credentials,+ urls=input_data.urls,+ extract_depth=input_data.extract_depth.value,+ format=input_data.format.value,+ include_usage=True,+ )+ except Exception as e:+ raise BlockExecutionError(+ message=f"Extract failed: {e}",+ block_name=self.name,+ block_id=self.id,+ ) from e++ results = [+ TavilyPageContent(+ url=r.get("url") or "",+ raw_content=r.get("raw_content") or "",+ )+ for r in response.get("results", [])+ ]++ # Actual credit spend from the API's usage report; falls back to the+ # documented schedule (1 credit per 5 successfully extracted URLs,+ # 2 per 5 for advanced extraction; failed extractions are free).+ # The usage report is incremental: extract charges at each 5-URL+ # account boundary, so 0 credits on a given request is a valid+ # report, not a missing one.+ credits_per_batch = (+ 2 if input_data.extract_depth == TavilyExtractDepth.ADVANCED else 1+ )+ credits_used = credits_from_response(+ response, estimate=math.ceil(len(results) / 5) * credits_per_batch+ )+ self.merge_stats(+ NodeExecutionStats(+ provider_cost=credits_used * CREDIT_USD,+ provider_cost_type="cost_usd",+ )+ )++ yield "results", results+ for result in results:+ yield "result", result++ yield "failed_urls", [+ f.get("url", "") for f in response.get("failed_results", [])+ ]diff --git a/autogpt_platform/backend/backend/blocks/tavily/map.py b/autogpt_platform/backend/backend/blocks/tavily/map.pynew file mode 100644index 0000000000..9a506ca17d--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/tavily/map.py@@ -0,0 +1,121 @@+import math+from typing import Any++from tavily import AsyncTavilyClient++from backend.data.model import NodeExecutionStats+from backend.sdk import (+ APIKeyCredentials,+ Block,+ BlockCategory,+ BlockOutput,+ BlockSchemaInput,+ BlockSchemaOutput,+ CredentialsMetaInput,+ SchemaField,+)+from backend.util.exceptions import BlockExecutionError++from ._api import CREDIT_USD, credits_from_response+from ._config import tavily+++class TavilyMapBlock(Block):+ class Input(BlockSchemaInput):+ credentials: CredentialsMetaInput = tavily.credentials_field(+ description="The Tavily integration requires an API Key."+ )+ url: str = SchemaField(description="The root URL to map")+ limit: int = SchemaField(+ description="Maximum number of pages to discover",+ default=50,+ ge=1,+ advanced=True,+ )+ max_depth: int = SchemaField(+ description="Maximum link depth from the root URL",+ default=1,+ ge=1,+ advanced=True,+ )+ instructions: str | None = SchemaField(+ description="Natural language instructions guiding which pages to include (e.g. 'Only documentation pages')",+ default=None,+ )++ class Output(BlockSchemaOutput):+ links: list[str] = SchemaField(+ description="List of URLs discovered on the website"+ )+ error: str = SchemaField(+ description="Error message if the map failed",+ default="",+ )++ def __init__(self):+ super().__init__(+ id="ddc65c9a-c4b0-4389-8604-4aba01a3dfe6",+ description="Maps a website's structure with Tavily, discovering its URLs without extracting content",+ categories={BlockCategory.SEARCH},+ input_schema=self.Input,+ output_schema=self.Output,+ test_input={+ "credentials": tavily.get_test_credentials().model_dump(),+ "url": "https://agpt.co",+ },+ test_credentials=tavily.get_test_credentials(),+ test_output=[+ ("links", ["https://agpt.co", "https://agpt.co/blog"]),+ ],+ test_mock={+ "_map": lambda *args, **kwargs: {+ "base_url": "https://agpt.co",+ "results": ["https://agpt.co", "https://agpt.co/blog"],+ "usage": {"credits": 1},+ }+ },+ )++ async def _map(self, credentials: APIKeyCredentials, **kwargs) -> dict[str, Any]:+ """Private method to call the Tavily API - can be mocked for testing."""+ client = AsyncTavilyClient(api_key=credentials.api_key.get_secret_value())+ return await client.map(**kwargs)++ async def run(+ self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs+ ) -> BlockOutput:+ sdk_kwargs: dict[str, Any] = {+ "url": input_data.url,+ "limit": input_data.limit,+ "max_depth": input_data.max_depth,+ "include_usage": True,+ }+ if input_data.instructions:+ sdk_kwargs["instructions"] = input_data.instructions++ try:+ response = await self._map(credentials, **sdk_kwargs)+ except Exception as e:+ raise BlockExecutionError(+ message=f"Map failed: {e}",+ block_name=self.name,+ block_id=self.id,+ ) from e++ links = [str(link) for link in response.get("results", [])]++ # Actual credit spend from the API's usage report; falls back to the+ # documented schedule (1 credit per 10 successfully mapped pages,+ # 2 per 10 when instructions are provided).+ credits_used = credits_from_response(+ response,+ estimate=math.ceil(len(links) / 10) * (2 if input_data.instructions else 1),+ )+ self.merge_stats(+ NodeExecutionStats(+ provider_cost=credits_used * CREDIT_USD,+ provider_cost_type="cost_usd",+ )+ )++ yield "links", linksdiff --git a/autogpt_platform/backend/backend/blocks/tavily/search.py b/autogpt_platform/backend/backend/blocks/tavily/search.pynew file mode 100644index 0000000000..1145f9389c--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/tavily/search.py@@ -0,0 +1,193 @@+from typing import Any++from tavily import AsyncTavilyClient++from backend.data.model import NodeExecutionStats+from backend.sdk import (+ APIKeyCredentials,+ Block,+ BlockCategory,+ BlockOutput,+ BlockSchemaInput,+ BlockSchemaOutput,+ CredentialsMetaInput,+ SchemaField,+)+from backend.util.exceptions import BlockExecutionError++from ._api import (+ CREDIT_USD,+ TavilySearchDepth,+ TavilySearchResult,+ TavilyTimeRange,+ TavilyTopic,+ credits_from_response,+)+from ._config import tavily+++class TavilySearchBlock(Block):+ class Input(BlockSchemaInput):+ credentials: CredentialsMetaInput = tavily.credentials_field(+ description="The Tavily integration requires an API Key."+ )+ query: str = SchemaField(description="The search query")+ topic: TavilyTopic = SchemaField(+ description="Search category: general, news, or finance",+ default=TavilyTopic.GENERAL,+ advanced=True,+ )+ search_depth: TavilySearchDepth = SchemaField(+ description="Depth of the search: basic, fast or ultra-fast (1 credit), or advanced (2 credits)",+ default=TavilySearchDepth.BASIC,+ advanced=True,+ )+ max_results: int = SchemaField(+ description="Maximum number of results to return",+ default=5,+ ge=1,+ le=20,+ advanced=True,+ )+ time_range: TavilyTimeRange | None = SchemaField(+ description="Only include results published within this time range",+ default=None,+ advanced=True,+ )+ include_domains: list[str] = SchemaField(+ description="Domains to include in search", default_factory=list+ )+ exclude_domains: list[str] = SchemaField(+ description="Domains to exclude from search",+ default_factory=list,+ advanced=True,+ )+ include_answer: bool = SchemaField(+ description="Include an LLM-generated answer to the query, based on the search results",+ default=True,+ )+ include_raw_content: bool = SchemaField(+ description="Include the full page content for each result",+ default=False,+ advanced=True,+ )++ class Output(BlockSchemaOutput):+ results: list[TavilySearchResult] = SchemaField(+ description="List of search results"+ )+ result: TavilySearchResult = SchemaField(description="Single search result")+ answer: str = SchemaField(+ description="LLM-generated answer to the query, based on the search results"+ )+ context: str = SchemaField(+ description="A formatted string of the search results ready for LLMs."+ )+ error: str = SchemaField(+ description="Error message if the search failed",+ default="",+ )++ def __init__(self):+ super().__init__(+ id="363bb641-2147-47a0-9aaf-caae26b06dcb",+ description="Searches the web using Tavily's AI-native search API",+ categories={BlockCategory.SEARCH},+ input_schema=self.Input,+ output_schema=self.Output,+ test_input={+ "credentials": tavily.get_test_credentials().model_dump(),+ "query": "What is AutoGPT?",+ "max_results": 1,+ },+ test_credentials=tavily.get_test_credentials(),+ test_output=[+ ("results", lambda x: isinstance(x, list) and len(x) == 1),+ ("result", lambda x: x.url == "https://agpt.co"),+ ("answer", "AutoGPT is a platform for building AI agents."),+ ("context", lambda x: "AutoGPT" in x),+ ],+ test_mock={+ "_search": lambda *args, **kwargs: {+ "query": "What is AutoGPT?",+ "answer": "AutoGPT is a platform for building AI agents.",+ "results": [+ {+ "title": "AutoGPT",+ "url": "https://agpt.co",+ "content": "AutoGPT is a platform for building AI agents.",+ "score": 0.98,+ }+ ],+ "usage": {"credits": 1},+ }+ },+ )++ async def _search(self, credentials: APIKeyCredentials, **kwargs) -> dict[str, Any]:+ """Private method to call the Tavily API - can be mocked for testing."""+ client = AsyncTavilyClient(api_key=credentials.api_key.get_secret_value())+ return await client.search(**kwargs)++ async def run(+ self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs+ ) -> BlockOutput:+ sdk_kwargs: dict[str, Any] = {+ "query": input_data.query,+ "topic": input_data.topic.value,+ "search_depth": input_data.search_depth.value,+ "max_results": input_data.max_results,+ "include_answer": input_data.include_answer,+ "include_raw_content": input_data.include_raw_content,+ "include_usage": True,+ }+ if input_data.time_range:+ sdk_kwargs["time_range"] = input_data.time_range.value+ if input_data.include_domains:+ sdk_kwargs["include_domains"] = input_data.include_domains+ if input_data.exclude_domains:+ sdk_kwargs["exclude_domains"] = input_data.exclude_domains++ try:+ response = await self._search(credentials, **sdk_kwargs)+ except Exception as e:+ raise BlockExecutionError(+ message=f"Search failed: {e}",+ block_name=self.name,+ block_id=self.id,+ ) from e++ # Actual credit spend from the API's usage report; falls back to the+ # documented schedule (basic search 1 credit, advanced 2).+ credits_used = credits_from_response(+ response,+ estimate=2 if input_data.search_depth == TavilySearchDepth.ADVANCED else 1,+ )+ self.merge_stats(+ NodeExecutionStats(+ provider_cost=credits_used * CREDIT_USD,+ provider_cost_type="cost_usd",+ )+ )++ results = [+ TavilySearchResult(+ title=r.get("title", ""),+ url=r.get("url", ""),+ content=r.get("content", ""),+ score=r.get("score", 0.0),+ raw_content=r.get("raw_content"),+ )+ for r in response.get("results", [])+ ]++ yield "results", results+ for result in results:+ yield "result", result++ if response.get("answer"):+ yield "answer", response["answer"]++ yield "context", "\n\n".join(+ f"[{r.title}]({r.url})\n{r.content}" for r in results+ )diff --git a/autogpt_platform/backend/poetry.lock b/autogpt_platform/backend/poetry.lockindex 81fef0640b..37c5778209 100644--- a/autogpt_platform/backend/poetry.lock+++ b/autogpt_platform/backend/poetry.lock@@ -7175,6 +7175,23 @@ files = [requests = {version = ">=2.20", markers = "python_version >= \"3.0\""}typing-extensions = {version = ">=4.5.0", markers = "python_version >= \"3.7\""}+[[package]]+name = "tavily-python"+version = "0.7.27"+description = "Python wrapper for the Tavily API"+optional = false+python-versions = ">=3.8"+groups = ["main"]+files = [+ {file = "tavily_python-0.7.27-py3-none-any.whl", hash = "sha256:e5cb40cc852d108ced8a313379b7098108642eedfbd97f821296a5e1a483e9b9"},+ {file = "tavily_python-0.7.27.tar.gz", hash = "sha256:3fbbee7fc7e252479b264835e6f943b4a81395429c1bd419e8024d11bf2c1831"},+]++[package.dependencies]+httpx = "*"+requests = "*"+tiktoken = ">=0.5.1"+[[package]]name = "tenacity"version = "9.1.4"@@ -8607,4 +8624,4 @@ cffi = ["cffi (>=1.17,<2.0) ; platform_python_implementation != \"PyPy\" and pyt[metadata]lock-version = "2.1"python-versions = ">=3.10,<3.14"-content-hash = "c3a3271f197bce923c6d52e7e1c7f5c6939470dc9f14e9221a60a2c224e8ee70"+content-hash = "d306ca3867ee81ce402c01db09c3b00e588b4e66605226f4c9a78ec91c416371"diff --git a/autogpt_platform/backend/pyproject.toml b/autogpt_platform/backend/pyproject.tomlindex 0831d097c4..f58f48c062 100644--- a/autogpt_platform/backend/pyproject.toml+++ b/autogpt_platform/backend/pyproject.toml@@ -105,6 +105,7 @@ pyarrow = "^23.0.0"sqlparse = "^0.5.5"pillow = ">=9.1.0,<12"pypdfium2 = "^5.8.0"+tavily-python = "^0.7.26"[tool.poetry.group.dev.dependencies]aiohappyeyeballs = "^2.6.1"diff --git a/docs/integrations/README.md b/docs/integrations/README.mdindex f2da8a5e2d..93d43631c6 100644--- a/docs/integrations/README.md+++ b/docs/integrations/README.md@@ -298,6 +298,10 @@ Below is a comprehensive list of all available blocks, categorized by their prim| [Search Organizations](block-integrations/apollo/organization.md#search-organizations) | Search for organizations in Apollo || [Search People](block-integrations/apollo/people.md#search-people) | Search for people in Apollo || [Search The Web](block-integrations/jina/search.md#search-the-web) | This block searches the internet for the given search query |+| [Tavily Crawl](block-integrations/tavily/crawl.md#tavily-crawl) | Crawls a website with Tavily, following links from the root URL and extracting page content |+| [Tavily Extract](block-integrations/tavily/extract.md#tavily-extract) | Extracts page content from one or more URLs using Tavily, optimized for LLM consumption |+| [Tavily Map](block-integrations/tavily/map.md#tavily-map) | Maps a website's structure with Tavily, discovering its URLs without extracting content |+| [Tavily Search](block-integrations/tavily/search.md#tavily-search) | Searches the web using Tavily's AI-native search API || [Validate Emails](block-integrations/zerobounce/validate_emails.md#validate-emails) | Validate emails |## Social Media and Contentdiff --git a/docs/integrations/SUMMARY.md b/docs/integrations/SUMMARY.mdindex e419729215..b3a3a73bb1 100644--- a/docs/integrations/SUMMARY.md+++ b/docs/integrations/SUMMARY.md@@ -114,6 +114,10 @@* [Stagehand Blocks](block-integrations/stagehand/blocks.md)* [System Library Operations](block-integrations/system/library_operations.md)* [System Store Operations](block-integrations/system/store_operations.md)+* [Tavily Crawl](block-integrations/tavily/crawl.md)+* [Tavily Extract](block-integrations/tavily/extract.md)+* [Tavily Map](block-integrations/tavily/map.md)+* [Tavily Search](block-integrations/tavily/search.md)* [Telegram Blocks](block-integrations/telegram/blocks.md)* [Telegram Triggers](block-integrations/telegram/triggers.md)* [Text](block-integrations/text.md)diff --git a/docs/integrations/block-integrations/tavily/crawl.md b/docs/integrations/block-integrations/tavily/crawl.mdnew file mode 100644index 0000000000..f9d37aceed--- /dev/null+++ b/docs/integrations/block-integrations/tavily/crawl.md@@ -0,0 +1,46 @@+# Tavily Crawl+<!-- MANUAL: file_description -->+Blocks for crawling a website and extracting page content with Tavily.+<!-- END MANUAL -->++## Tavily Crawl++### What it is+Crawls a website with Tavily, following links from the root URL and extracting page content++### How it works+<!-- MANUAL: how_it_works -->+Starting from a root URL, the block follows links up to `max_depth` and extracts the content of each page it visits, stopping at `limit` pages. Optional natural-language `instructions` steer which pages to follow (for example, "Find all the API reference pages"), and `extract_depth` and `format` control how much content is captured and whether it comes back as markdown or text.++Each crawled page is returned on `results` (and one at a time on `result`) with its extracted content. A crawl combines mapping and extraction, so its credit cost is the sum of both per Tavily's schedule; actual spend is read from the API's usage report.+<!-- END MANUAL -->++### Inputs++| Input | Description | Type | Required |+|-------|-------------|------|----------|+| url | The root URL to start crawling from | str | Yes |+| limit | Maximum number of pages to crawl | int | No |+| max_depth | Maximum link depth from the root URL | int | No |+| instructions | Natural language instructions guiding which pages to crawl (e.g. 'Find all the API reference pages') | str | No |+| extract_depth | Depth of the extraction: basic or advanced (retrieves more data, including tables and embedded content) | "basic" \| "advanced" | No |+| format | The format of the extracted content | "markdown" \| "text" | No |++### Outputs++| Output | Description | Type |+|--------|-------------|------|+| error | Error message if the crawl failed | str |+| results | List of crawled pages with their content | List[TavilyPageContent] |+| result | Single crawled page | TavilyPageContent |++### Possible use case+<!-- MANUAL: use_case -->+**Documentation Ingestion**: Crawl a docs site and extract every page to build a knowledge base or RAG pipeline.++**Targeted Site Harvesting**: Use `instructions` to gather only pricing, product, or reference pages from a large site.++**Competitive Content Snapshots**: Capture the content of a section of a site in one pass for analysis or change tracking.+<!-- END MANUAL -->++---diff --git a/docs/integrations/block-integrations/tavily/extract.md b/docs/integrations/block-integrations/tavily/extract.mdnew file mode 100644index 0000000000..393940206d--- /dev/null+++ b/docs/integrations/block-integrations/tavily/extract.md@@ -0,0 +1,44 @@+# Tavily Extract+<!-- MANUAL: file_description -->+Blocks for extracting clean page content from URLs with Tavily.+<!-- END MANUAL -->++## Tavily Extract++### What it is+Extracts page content from one or more URLs using Tavily, optimized for LLM consumption++### How it works+<!-- MANUAL: how_it_works -->+The block takes up to 20 URLs per request, fetches each page, and returns cleaned content in `markdown` or `text` format optimized for LLM consumption. Setting `extract_depth` to advanced retrieves more from each page, including tables and embedded content, at a higher credit cost.++Successfully extracted pages are returned on `results` (and one at a time on `result`), while any URLs that could not be fetched are surfaced separately on `failed_urls` — so a few bad URLs never fail the whole batch. Cost is 1 credit per 5 successfully extracted URLs (2 per 5 for advanced); failed extractions are free, and actual spend is read from the API's usage report.+<!-- END MANUAL -->++### Inputs++| Input | Description | Type | Required |+|-------|-------------|------|----------|+| urls | The URLs to extract content from (up to 20 per request) | List[str] | Yes |+| extract_depth | Depth of the extraction: basic or advanced (retrieves more data, including tables and embedded content) | "basic" \| "advanced" | No |+| format | The format of the extracted content | "markdown" \| "text" | No |++### Outputs++| Output | Description | Type |+|--------|-------------|------|+| error | Error message if the extraction failed | str |+| results | List of successfully extracted pages | List[TavilyPageContent] |+| result | Single extracted page | TavilyPageContent |+| failed_urls | URLs that could not be extracted | List[str] |++### Possible use case+<!-- MANUAL: use_case -->+**Content Ingestion**: Turn a list of article or documentation URLs into clean text for summarization or RAG indexing.++**Link Enrichment**: Follow up a search or map result by pulling the full content of the most relevant pages.++**Resilient Batch Scraping**: Extract many URLs at once and route any failures via `failed_urls` for retry without losing the successful pages.+<!-- END MANUAL -->++---diff --git a/docs/integrations/block-integrations/tavily/map.md b/docs/integrations/block-integrations/tavily/map.mdnew file mode 100644index 0000000000..c005662c4a--- /dev/null+++ b/docs/integrations/block-integrations/tavily/map.md@@ -0,0 +1,43 @@+# Tavily Map+<!-- MANUAL: file_description -->+Blocks for discovering a website's URL structure with Tavily.+<!-- END MANUAL -->++## Tavily Map++### What it is+Maps a website's structure with Tavily, discovering its URLs without extracting content++### How it works+<!-- MANUAL: how_it_works -->+The block discovers the URLs reachable from a root URL, following links up to `max_depth` and returning a flat list of `links` — without fetching page content, which makes it faster and cheaper than a full crawl. Use `limit` to cap how many pages are discovered and optional natural-language `instructions` to narrow the results (for example, "Only documentation pages").++Because no content is extracted, mapping is billed at 1 credit per 10 discovered pages (2 per 10 when instructions are provided); actual spend is read from the API's usage report.+<!-- END MANUAL -->++### Inputs++| Input | Description | Type | Required |+|-------|-------------|------|----------|+| url | The root URL to map | str | Yes |+| limit | Maximum number of pages to discover | int | No |+| max_depth | Maximum link depth from the root URL | int | No |+| instructions | Natural language instructions guiding which pages to include (e.g. 'Only documentation pages') | str | No |++### Outputs++| Output | Description | Type |+|--------|-------------|------|+| error | Error message if the map failed | str |+| links | List of URLs discovered on the website | List[str] |++### Possible use case+<!-- MANUAL: use_case -->+**Sitemap Discovery**: Enumerate a site's pages before deciding what to crawl or extract.++**Scoped Crawl Planning**: Map first, filter the links, then feed a subset into the Extract block to control cost.++**Coverage Auditing**: List all reachable URLs under a section to spot orphaned or missing pages.+<!-- END MANUAL -->++---diff --git a/docs/integrations/block-integrations/tavily/search.md b/docs/integrations/block-integrations/tavily/search.mdnew file mode 100644index 0000000000..11cb5f1da6--- /dev/null+++ b/docs/integrations/block-integrations/tavily/search.md@@ -0,0 +1,51 @@+# Tavily Search+<!-- MANUAL: file_description -->+Blocks for searching the web with Tavily's AI-native search API.+<!-- END MANUAL -->++## Tavily Search++### What it is+Searches the web using Tavily's AI-native search API++### How it works+<!-- MANUAL: how_it_works -->+The block sends your query to Tavily's search endpoint and returns ranked web results, each with a relevance score and a query-relevant content snippet. You can scope the search by `topic` (general, news, or finance), `time_range`, and domain include/exclude lists, and trade cost for quality with `search_depth` (basic/fast/ultra-fast at 1 credit, advanced at 2).++Beyond the raw `results` list (also emitted one `result` at a time), the block can return an LLM-generated `answer` synthesized from the results when `include_answer` is enabled, and always emits a `context` string — the results formatted as markdown, ready to feed straight into an LLM block. Actual credit spend is read from the API's usage report and reported to the platform's cost tracking.+<!-- END MANUAL -->++### Inputs++| Input | Description | Type | Required |+|-------|-------------|------|----------|+| query | The search query | str | Yes |+| topic | Search category: general, news, or finance | "general" \| "news" \| "finance" | No |+| search_depth | Depth of the search: basic, fast or ultra-fast (1 credit), or advanced (2 credits) | "basic" \| "advanced" \| "fast" \| "ultra-fast" | No |+| max_results | Maximum number of results to return | int | No |+| time_range | Only include results published within this time range | "day" \| "week" \| "month" \| "year" | No |+| include_domains | Domains to include in search | List[str] | No |+| exclude_domains | Domains to exclude from search | List[str] | No |+| include_answer | Include an LLM-generated answer to the query, based on the search results | bool | No |+| include_raw_content | Include the full page content for each result | bool | No |++### Outputs++| Output | Description | Type |+|--------|-------------|------|+| error | Error message if the search failed | str |+| results | List of search results | List[TavilySearchResult] |+| result | Single search result | TavilySearchResult |+| answer | LLM-generated answer to the query, based on the search results | str |+| context | A formatted string of the search results ready for LLMs. | str |++### Possible use case+<!-- MANUAL: use_case -->+**Research Automation**: Pull current, ranked sources on a topic and feed the `context` output directly into an LLM block for summarization or synthesis.++**Grounded Q&A**: Enable `include_answer` to get a concise, source-backed answer for chatbots or agents that need up-to-date facts.++**News & Market Monitoring**: Set `topic` to news or finance and narrow `time_range` to recent windows to track breaking developments.+<!-- END MANUAL -->++---