Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(block): Improve CreateListBlock to support batching based on token count (#10257)

Zamil Majdy4d0db27d5e9f
diff --git a/autogpt_platform/backend/backend/blocks/basic.py b/autogpt_platform/backend/backend/blocks/basic.py
index fc1d22e534..13d3b4e62f 100644
--- a/autogpt_platform/backend/backend/blocks/basic.py
+++ b/autogpt_platform/backend/backend/blocks/basic.py
@@ -6,6 +6,7 @@ from backend.data.model import SchemaField
from backend.util import json
from backend.util.file import store_media_file
from backend.util.mock import MockObject
+from backend.util.prompt import estimate_token_count_str
from backend.util.type import MediaFileType, convert
@@ -466,6 +467,11 @@ class CreateListBlock(Block):
description="Maximum size of the list. If provided, the list will be yielded in chunks of this size.",
advanced=True,
)
+ max_tokens: int | None = SchemaField(
+ default=None,
+ description="Maximum tokens for the list. If provided, the list will be yielded in chunks that fit within this token limit.",
+ advanced=True,
+ )
class Output(BlockSchema):
list: List[Any] = SchemaField(
@@ -476,7 +482,7 @@ class CreateListBlock(Block):
def __init__(self):
super().__init__(
id="a912d5c7-6e00-4542-b2a9-8034136930e4",
- description="Creates a list with the specified values. Use this when you know all the values you want to add upfront.",
+ description="Creates a list with the specified values. Use this when you know all the values you want to add upfront. This block can also yield the list in batches based on a maximum size or token limit.",
categories={BlockCategory.DATA},
input_schema=CreateListBlock.Input,
output_schema=CreateListBlock.Output,
@@ -501,12 +507,30 @@ class CreateListBlock(Block):
)
async def run(self, input_data: Input, **kwargs) -> BlockOutput:
- try:
- max_size = input_data.max_size or len(input_data.values)
- for i in range(0, len(input_data.values), max_size):
- yield "list", input_data.values[i : i + max_size]
- except Exception as e:
- yield "error", f"Failed to create list: {str(e)}"
+ chunk = []
+ cur_tokens, max_tokens = 0, input_data.max_tokens
+ cur_size, max_size = 0, input_data.max_size
+
+ for value in input_data.values:
+ if max_tokens:
+ tokens = estimate_token_count_str(value)
+ else:
+ tokens = 0
+
+ # Check if adding this value would exceed either limit
+ if (max_tokens and (cur_tokens + tokens > max_tokens)) or (
+ max_size and (cur_size + 1 > max_size)
+ ):
+ yield "list", chunk
+ chunk = [value]
+ cur_size, cur_tokens = 1, tokens
+ else:
+ chunk.append(value)
+ cur_size, cur_tokens = cur_size + 1, cur_tokens + tokens
+
+ # Yield final chunk if any
+ if chunk:
+ yield "list", chunk
class TypeOptions(enum.Enum):
diff --git a/autogpt_platform/backend/backend/util/json.py b/autogpt_platform/backend/backend/util/json.py
index d71b61a246..505a7b27c8 100644
--- a/autogpt_platform/backend/backend/util/json.py
+++ b/autogpt_platform/backend/backend/util/json.py
@@ -14,8 +14,37 @@ def to_dict(data) -> dict:
return jsonable_encoder(data)
-def dumps(data) -> str:
- return json.dumps(to_dict(data))
+def dumps(data: Any, *args: Any, **kwargs: Any) -> str:
+ """
+ Serialize data to JSON string with automatic conversion of Pydantic models and complex types.
+
+ This function converts the input data to a JSON-serializable format using FastAPI's
+ jsonable_encoder before dumping to JSON. It handles Pydantic models, complex types,
+ and ensures proper serialization.
+
+ Parameters
+ ----------
+ data : Any
+ The data to serialize. Can be any type including Pydantic models, dicts, lists, etc.
+ *args : Any
+ Additional positional arguments passed to json.dumps()
+ **kwargs : Any
+ Additional keyword arguments passed to json.dumps() (e.g., indent, separators)
+
+ Returns
+ -------
+ str
+ JSON string representation of the data
+
+ Examples
+ --------
+ >>> dumps({"name": "Alice", "age": 30})
+ '{"name": "Alice", "age": 30}'
+
+ >>> dumps(pydantic_model_instance, indent=2)
+ '{\n "field1": "value1",\n "field2": "value2"\n}'
+ """
+ return json.dumps(to_dict(data), *args, **kwargs)
T = TypeVar("T")
diff --git a/autogpt_platform/backend/backend/util/prompt.py b/autogpt_platform/backend/backend/util/prompt.py
index 50e04e069e..b098e48d6f 100644
--- a/autogpt_platform/backend/backend/util/prompt.py
+++ b/autogpt_platform/backend/backend/util/prompt.py
@@ -1,8 +1,10 @@
-import json
from copy import deepcopy
+from typing import Any
from tiktoken import encoding_for_model
+from backend.util import json
+
# ---------------------------------------------------------------------------#
# INTERNAL UTILITIES #
# ---------------------------------------------------------------------------#
@@ -179,3 +181,26 @@ def estimate_token_count(
"""
enc = encoding_for_model(model) # best-match tokenizer
return sum(_msg_tokens(m, enc) for m in messages)
+
+
+def estimate_token_count_str(
+ text: Any,
+ *,
+ model: str = "gpt-4o",
+) -> int:
+ """
+ Return the true token count of *text* when encoded for *model*.
+
+ Parameters
+ ----------
+ text Input text.
+ model Model name; passed to tiktoken to pick the right
+ tokenizer (gpt-4o → 'o200k_base', others fallback).
+
+ Returns
+ -------
+ int – Token count.
+ """
+ enc = encoding_for_model(model) # best-match tokenizer
+ text = json.dumps(text) if not isinstance(text, str) else text
+ return _tok_len(text, enc)