Skip to content

🌟 The Multi-Agent Framework: First AI Software Company, Towards Natural Language Programming

fix conflicts

seehi77703f12369d
diff --cc .gitignore
index 73f32f75d,841306429..7434aec84
--- a/.gitignore
+++ b/.gitignore
@@@ -190,4 -189,9 +191,10 @@@ cov.xm
*-structure.json
*.dot
.python-version
+tests/data/requirements/*.jpg
+ *.csv
+ metagpt/ext/sela/results/*
+ .chainlit/
+
+ metagpt/ext/aflow/data
+ metagpt/ext/aflow/scripts/optimized
diff --cc config/config2.example.yaml
index f6970dabe,b82468eed..eb556fecf
--- a/config/config2.example.yaml
+++ b/config/config2.example.yaml
@@@ -1,49 -1,25 +1,56 @@@
llm:
- api_type: "openai" # or azure / ollama / open_llm etc. Check LLMType for more options
+ api_type: "openai" # or azure / ollama / groq etc.
base_url: "YOUR_BASE_URL"
api_key: "YOUR_API_KEY"
- model: "gpt-4-turbo-preview" # or gpt-3.5-turbo-1106 / gpt-4-1106-preview
+ model: "gpt-4-turbo" # or gpt-3.5-turbo
proxy: "YOUR_PROXY" # for LLM API requests
# timeout: 600 # Optional. If set to 0, default value is 300.
- pricing_plan: "" # Optional. If invalid, it will be automatically filled in with the value of the `model`.
- # Azure-exclusive pricing plan mappings:
- # - gpt-3.5-turbo 4k: "gpt-3.5-turbo-1106"
- # - gpt-4-turbo: "gpt-4-turbo-preview"
- # - gpt-4-turbo-vision: "gpt-4-vision-preview"
- # - gpt-4 8k: "gpt-4"
- # See for more: https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/
+ # Details: https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/
+ pricing_plan: "" # Optional. Use for Azure LLM when its model name is not the same as OpenAI's
+
+
+ # RAG Embedding.
+ # For backward compatibility, if the embedding is not set and the llm's api_type is either openai or azure, the llm's config will be used.
+ embedding:
+ api_type: "" # openai / azure / gemini / ollama etc. Check EmbeddingType for more options.
+ base_url: ""
+ api_key: ""
+ model: ""
+ api_version: ""
+ embed_batch_size: 100
+ dimensions: # output dimension of embedding model
+# Role's custom configuration
+roles:
+ - role: "ProductManager" # role's className or role's role_id
+ llm:
+ api_type: "openai" # or azure / ollama / open_llm etc. Check LLMType for more options
+ base_url: "YOUR_BASE_URL"
+ api_key: "YOUR_API_KEY"
+ proxy: "YOUR_PROXY" # for LLM API requests
+ model: "gpt-4-turbo-1106"
+ - role: "Architect"
+ llm:
+ api_type: "openai" # or azure / ollama / open_llm etc. Check LLMType for more options
+ base_url: "YOUR_BASE_URL"
+ api_key: "YOUR_API_KEY"
+ proxy: "YOUR_PROXY" # for LLM API requests
+ model: "gpt-35-turbo"
+ - role: "ProjectManager"
+ llm:
+ api_type: "azure"
+ base_url: "YOUR_BASE_URL"
+ api_key: "YOUR_API_KEY"
+ api_version: "YOUR_API_VERSION"
+ model: "gpt-4-1106"
+ - role: "Engineer"
+ llm:
+ api_type: "azure"
+ base_url: "YOUR_BASE_URL"
+ api_key: "YOUR_API_KEY"
+ api_version: "YOUR_API_VERSION"
+ model: "gpt-35-turbo-1106"
+
repair_llm_output: true # when the output is not a valid json, try to repair it
proxy: "YOUR_PROXY" # for tools like requests, playwright, selenium, etc.
@@@ -98,3 -59,27 +105,25 @@@ iflytek_api_key: "YOUR_API_KEY
iflytek_api_secret: "YOUR_API_SECRET"
metagpt_tti_url: "YOUR_MODEL_URL"
+
+ omniparse:
+ api_key: "YOUR_API_KEY"
+ base_url: "YOUR_BASE_URL"
+
+ models:
+ # "YOUR_MODEL_NAME_1 or YOUR_API_TYPE_1": # model: "gpt-4-turbo" # or gpt-3.5-turbo
+ # api_type: "openai" # or azure / ollama / groq etc.
+ # base_url: "YOUR_BASE_URL"
+ # api_key: "YOUR_API_KEY"
+ # proxy: "YOUR_PROXY" # for LLM API requests
+ # # timeout: 600 # Optional. If set to 0, default value is 300.
+ # # Details: https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/
+ # pricing_plan: "" # Optional. Use for Azure LLM when its model name is not the same as OpenAI's
+ # "YOUR_MODEL_NAME_2 or YOUR_API_TYPE_2": # api_type: "openai" # or azure / ollama / groq etc.
+ # api_type: "openai" # or azure / ollama / groq etc.
+ # base_url: "YOUR_BASE_URL"
+ # api_key: "YOUR_API_KEY"
+ # proxy: "YOUR_PROXY" # for LLM API requests
+ # # timeout: 600 # Optional. If set to 0, default value is 300.
+ # # Details: https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/
+ # pricing_plan: "" # Optional. Use for Azure LLM when its model name is not the same as OpenAI's
-
-agentops_api_key: "YOUR_AGENTOPS_API_KEY" # get key from https://app.agentops.ai/settings/projects
diff --cc examples/use_off_the_shelf_agent.py
index 0d224a92b,7ccf87a2b..34d605f1a
--- a/examples/use_off_the_shelf_agent.py
+++ b/examples/use_off_the_shelf_agent.py
@@@ -5,11 -5,9 +5,11 @@@ Author: garylin209
"""
import asyncio
-from metagpt.context import Context
++from metagpt.environment.mgx.mgx_env import MGXEnv
from metagpt.logs import logger
++from metagpt.roles.di.team_leader import TeamLeader
from metagpt.roles.product_manager import ProductManager
- from metagpt.environment.mgx.mgx_env import MGXEnv
+from metagpt.schema import Message
- from metagpt.roles.di.team_leader import TeamLeader
async def main():
diff --cc metagpt/actions/action.py
index 8733947f5,20c052aa9..ba8a94803
--- a/metagpt/actions/action.py
+++ b/metagpt/actions/action.py
@@@ -34,7 -37,26 +36,20 @@@ class Action(SerializationMixin, Contex
prefix: str = "" # aask*时会加上prefix,作为system_message
desc: str = "" # for skill manager
node: ActionNode = Field(default=None, exclude=True)
+ # The model name or API type of LLM of the `models` in the `config2.yaml`;
+ # Using `None` to use the `llm` configuration in the `config2.yaml`.
+ llm_name_or_type: Optional[str] = None
+
+ @model_validator(mode="after")
+ @classmethod
+ def _update_private_llm(cls, data: Any) -> Any:
+ config = ModelsConfig.default().get(data.llm_name_or_type)
+ if config:
+ llm = create_llm_instance(config)
+ llm.cost_manager = data.llm.cost_manager
+ data.llm = llm
+ return data
- @property
- def repo(self) -> ProjectRepo:
- if not self.context.repo:
- self.context.repo = ProjectRepo(self.context.git_repo)
- return self.context.repo
-
@property
def prompt_schema(self):
return self.config.prompt_schema
diff --cc metagpt/actions/action_node.py
index c1de16656,c94349a76..7109f287e
--- a/metagpt/actions/action_node.py
+++ b/metagpt/actions/action_node.py
@@@ -467,11 -481,119 +483,121 @@@ class ActionNode
return self
+ def get_field_name(self):
+ """
+ Get the field name from the Pydantic model associated with this ActionNode.
+ """
+ model_class = self.create_class()
+ fields = model_class.model_fields
+
+ # Assuming there's only one field in the model
+ if len(fields) == 1:
+ return next(iter(fields))
+
+ # If there are multiple fields, we might want to use self.key to find the right one
+ return self.key
+
+ def get_field_names(self):
+ """
+ Get the field names associated with this ActionNode's Pydantic model.
+ """
+ model_class = self.create_class()
+ return model_class.model_fields.keys()
+
+ def get_field_types(self):
+ """
+ Get the field types associated with this ActionNode's Pydantic model.
+ """
+ model_class = self.create_class()
+ return {field_name: field.annotation for field_name, field in model_class.model_fields.items()}
+
+ def xml_compile(self, context):
+ """
+ Compile the prompt to make it easier for the model to understand the xml format.
+ """
+ field_names = self.get_field_names()
+ # Construct the example using the field names
+ examples = []
+ for field_name in field_names:
+ examples.append(f"<{field_name}>content</{field_name}>")
+
+ # Join all examples into a single string
+ example_str = "\n".join(examples)
+ # Add the example to the context
+ context += f"""
+ ### Response format (must be strictly followed): All content must be enclosed in the given XML tags, ensuring each opening <tag> has a corresponding closing </tag>, with no incomplete or self-closing tags allowed.\n
+ {example_str}
+ """
+ return context
+
+ async def code_fill(
+ self, context: str, function_name: Optional[str] = None, timeout: int = USE_CONFIG_TIMEOUT
+ ) -> Dict[str, str]:
+ """
+ Fill CodeBlock Using ``` ```
+ """
+ field_name = self.get_field_name()
+ prompt = context
+ content = await self.llm.aask(prompt, timeout=timeout)
+ extracted_code = sanitize(code=content, entrypoint=function_name)
+ result = {field_name: extracted_code}
+ return result
+
+ async def single_fill(self, context: str, images: Optional[Union[str, list[str]]] = None) -> Dict[str, str]:
+ field_name = self.get_field_name()
+ prompt = context
+ content = await self.llm.aask(prompt, images=images)
+ result = {field_name: content}
+ return result
+
+ async def xml_fill(self, context: str, images: Optional[Union[str, list[str]]] = None) -> Dict[str, Any]:
+ """
+ Fill context with XML tags and convert according to field types, including string, integer, boolean, list and dict types
+ """
+ field_names = self.get_field_names()
+ field_types = self.get_field_types()
+
+ extracted_data: Dict[str, Any] = {}
+ content = await self.llm.aask(context, images=images)
+
+ for field_name in field_names:
+ pattern = rf"<{field_name}>(.*?)</{field_name}>"
+ match = re.search(pattern, content, re.DOTALL)
+ if match:
+ raw_value = match.group(1).strip()
+ field_type = field_types.get(field_name)
+
+ if field_type == str:
+ extracted_data[field_name] = raw_value
+ elif field_type == int:
+ try:
+ extracted_data[field_name] = int(raw_value)
+ except ValueError:
+ extracted_data[field_name] = 0 # 或者其他默认值
+ elif field_type == bool:
+ extracted_data[field_name] = raw_value.lower() in ("true", "yes", "1", "on", "True")
+ elif field_type == list:
+ try:
+ extracted_data[field_name] = eval(raw_value)
+ if not isinstance(extracted_data[field_name], list):
+ raise ValueError
+ except:
+ extracted_data[field_name] = [] # 默认空列表
+ elif field_type == dict:
+ try:
+ extracted_data[field_name] = eval(raw_value)
+ if not isinstance(extracted_data[field_name], dict):
+ raise ValueError
+ except:
+ extracted_data[field_name] = {} # 默认空字典
+
+ return extracted_data
+
+ @exp_cache(serializer=ActionNodeSerializer())
async def fill(
self,
- context,
+ *,
+ req,
llm,
schema="json",
mode="auto",
diff --cc metagpt/actions/project_management_an.py
index 7131a6c99,f53062433..a953feb4c
--- a/metagpt/actions/project_management_an.py
+++ b/metagpt/actions/project_management_an.py
@@@ -9,10 -9,10 +9,10 @@@ from typing import List, Optiona
from metagpt.actions.action_node import ActionNode
- REQUIRED_PYTHON_PACKAGES = ActionNode(
+ REQUIRED_PACKAGES = ActionNode(
key="Required packages",
- expected_type=List[str],
+ expected_type=Optional[List[str]],
- instruction="Provide required third-party packages in requirements.txt format.",
+ instruction="Provide required packages The response language should correspond to the context and requirements.",
example=["flask==1.1.2", "bcrypt==3.2.0"],
)
diff --cc metagpt/actions/research.py
index 99f72b076,5086f10cf..266584485
--- a/metagpt/actions/research.py
+++ b/metagpt/actions/research.py
@@@ -164,11 -159,12 +164,13 @@@ class CollectLinks(Action)
Returns:
A list of ranked URLs.
"""
- max_results = max(num_results * 2, 6)
- results = await self.search_engine.run(query, max_results=max_results, as_string=False)
+ max_results = max_num_results or max(num_results * 2, 6)
+ results = await self._search_urls(query, max_results=max_results)
+ if len(results) == 0:
+ return []
_results = "\n".join(f"{i}: {j}" for i, j in zip(range(max_results), results))
- prompt = COLLECT_AND_RANKURLS_PROMPT.format(topic=topic, query=query, results=_results)
+ time_stamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+ prompt = COLLECT_AND_RANKURLS_PROMPT.format(topic=topic, query=query, results=_results, time_stamp=time_stamp)
logger.debug(prompt)
indices = await self._aask(prompt)
try:
diff --cc metagpt/actions/write_code_review.py
index 6a283f812,c9b494dff..209e4e8ac
--- a/metagpt/actions/write_code_review.py
+++ b/metagpt/actions/write_code_review.py
@@@ -115,9 -118,9 +124,9 @@@ LGT
"""
REWRITE_CODE_TEMPLATE = """
- # Instruction: rewrite code based on the Code Review and Actions
+ # Instruction: rewrite the `{filename}` based on the Code Review and Actions
-## Rewrite Code: CodeBlock. If it still has some bugs, rewrite {filename} with triple quotes. Do your utmost to optimize THIS SINGLE FILE. Return all completed codes and prohibit the return of unfinished codes.
-```Code
+## Rewrite Code: CodeBlock. If it still has some bugs, rewrite {filename} using a Markdown code block, with the filename docstring preceding the code block. Do your utmost to optimize THIS SINGLE FILE. Return all completed codes and prohibit the return of unfinished codes.
+```python
## {filename}
...
```
diff --cc metagpt/config2.py
index fd0cb0948,27b228b33..1942e5ef5
--- a/metagpt/config2.py
+++ b/metagpt/config2.py
@@@ -86,15 -84,8 +89,14 @@@ class Config(CLIParams, YamlModel)
iflytek_api_key: str = ""
azure_tts_subscription_key: str = ""
azure_tts_region: str = ""
+ _extra: dict = dict() # extra config dict
+ # Role's custom configuration
+ roles: Optional[List[RoleCustomConfig]] = None
+
+ # RoleZero's configuration
+ role_zero: RoleZeroConfig = Field(default_factory=RoleZeroConfig)
+
- omniparse: Optional[OmniParseConfig] = None
-
@classmethod
def from_home(cls, path):
"""Load config from ~/.metagpt/config2.yaml"""
@@@ -104,7 -95,7 +106,7 @@@
return Config.from_yaml_file(pathname)
@classmethod
- def default(cls, reload: bool = False, **kwargs):
- def default(cls):
++ def default(cls, reload: bool = False, **kwargs) -> "Config":
"""Load default config
- Priority: env < default_config_paths
- Inside default_config_paths, the latter one overwrites the former one
diff --cc metagpt/configs/llm_config.py
index 57913956c,ef034ca49..9e27d76a6
--- a/metagpt/configs/llm_config.py
+++ b/metagpt/configs/llm_config.py
@@@ -10,8 -11,7 +11,8 @@@ from typing import Optiona
from pydantic import field_validator
+from metagpt.configs.compress_msg_config import CompressType
- from metagpt.const import LLM_API_TIMEOUT
+ from metagpt.const import CONFIG_ROOT, LLM_API_TIMEOUT, METAGPT_ROOT
from metagpt.utils.yaml_model import YamlModel
@@@ -32,9 -35,9 +36,12 @@@ class LLMType(Enum)
MOONSHOT = "moonshot"
MISTRAL = "mistral"
YI = "yi" # lingyiwanwu
+ OPEN_ROUTER = "open_router"
+ DEEPSEEK = "deepseek"
+ SILICONFLOW = "siliconflow"
+ OPENROUTER = "openrouter"
+ BEDROCK = "bedrock"
+ ARK = "ark" # https://www.volcengine.com/docs/82379/1263482#python-sdk
def __missing__(self, key):
return self.OPENAI
@@@ -87,9 -97,9 +101,12 @@@ class LLMConfig(YamlModel)
# Cost Control
calc_usage: bool = True
+ # Compress request messages under token limit
+ compress_type: CompressType = CompressType.NO_COMPRESS
+
+ # For Messages Control
+ use_system_prompt: bool = True
+
@field_validator("api_key")
@classmethod
def check_llm_key(cls, v):
diff --cc metagpt/configs/omniparse_config.py
index ecae78697,39742c8a4..8f38f9f51
--- a/metagpt/configs/omniparse_config.py
+++ b/metagpt/configs/omniparse_config.py
@@@ -2,5 -2,5 +2,6 @@@ from metagpt.utils.yaml_model import Ya
class OmniParseConfig(YamlModel):
- url: str = ""
+ api_key: str = ""
+ base_url: str = ""
+ timeout: int = 600
diff --cc metagpt/environment/werewolf/werewolf_ext_env.py
index d9644eb9b,a8636536b..c5ecb8b34
--- a/metagpt/environment/werewolf/werewolf_ext_env.py
+++ b/metagpt/environment/werewolf/werewolf_ext_env.py
@@@ -9,8 -8,10 +8,10 @@@ from typing import Any, Callable, Optio
from pydantic import ConfigDict, Field
- from metagpt.base.base_env_space import BaseEnvAction, BaseEnvObsParams
++from metagpt.base.base_env_space import BaseEnvObsParams
from metagpt.environment.base_env import ExtEnv, mark_as_readable, mark_as_writeable
-from metagpt.environment.base_env_space import BaseEnvObsParams
+ from metagpt.environment.werewolf.const import STEP_INSTRUCTIONS, RoleState, RoleType
+ from metagpt.environment.werewolf.env_space import EnvAction, EnvActionType
from metagpt.logs import logger
diff --cc metagpt/ext/stanford_town/utils/utils.py
index e09cce8fe,4e81298c9..b4e15f485
--- a/metagpt/ext/stanford_town/utils/utils.py
+++ b/metagpt/ext/stanford_town/utils/utils.py
@@@ -48,8 -48,8 +48,9 @@@ def read_csv_to_list(curr_file: str, he
def get_embedding(text, model: str = "text-embedding-ada-002"):
+ config = Config.default()
text = text.replace("\n", " ")
+ embedding = None
if not text:
text = "this is blank"
for idx in range(3):
diff --cc metagpt/logs.py
index 63c10fa2f,7ab51ff8e..ce2f12e4c
--- a/metagpt/logs.py
+++ b/metagpt/logs.py
@@@ -21,19 -13,8 +21,21 @@@ from pydantic import BaseModel, Fiel
from metagpt.const import METAGPT_ROOT
+LLM_STREAM_QUEUE: ContextVar[asyncio.Queue] = ContextVar("llm-stream")
+
+
+class ToolLogItem(BaseModel):
+ type_: str = Field(alias="type", default="str", description="Data type of `value` field.")
+ name: str
+ value: Any
+
+
+TOOL_LOG_END_MARKER = ToolLogItem(
+ type="str", name="end_marker", value="\x18\x19\x1B\x18"
+) # A special log item to suggest the end of a stream log
+
+ _print_level = "INFO"
+
def define_log_level(print_level="INFO", logfile_level="DEBUG", name: str = None):
"""Adjust the log level to above level"""
@@@ -91,53 -43,6 +96,58 @@@ def set_llm_stream_logfunc(func)
_llm_stream_log = func
+def set_tool_output_logfunc(func):
+ global _tool_output_log
+ _tool_output_log = func
+
+
+async def set_tool_output_logfunc_async(func):
+ # async version
+ global _tool_output_log_async
+ _tool_output_log_async = func
+
+
+def set_human_input_func(func):
+ global _get_human_input
+ _get_human_input = func
+
+
+_llm_stream_log = partial(print, end="")
+
+
+_tool_output_log = (
+ lambda *args, **kwargs: None
+) # a dummy function to avoid errors if set_tool_output_logfunc is not called
+
+
+async def _tool_output_log_async(*args, **kwargs):
+ # async version
+ pass
+
+
+def create_llm_stream_queue():
+ """Creates a new LLM stream queue and sets it in the context variable.
+
+ Returns:
+ The newly created asyncio.Queue instance.
+ """
+ queue = asyncio.Queue()
+ LLM_STREAM_QUEUE.set(queue)
+ return queue
+
+
+def get_llm_stream_queue():
+ """Retrieves the current LLM stream queue from the context variable.
+
+ Returns:
+ The asyncio.Queue instance if set, otherwise None.
+ """
+ return LLM_STREAM_QUEUE.get(None)
+
+
+_get_human_input = input # get human input from console by default
++
++
+ def _llm_stream_log(msg):
+ if _print_level in ["INFO"]:
+ print(msg, end="")
diff --cc metagpt/provider/base_llm.py
index f9111ffe0,a95e8dbd3..ec9e7fcf4
--- a/metagpt/provider/base_llm.py
+++ b/metagpt/provider/base_llm.py
@@@ -153,10 -146,10 +153,12 @@@ class BaseLLM(ABC)
message.append(self._user_msg(msg, images=images))
else:
message.extend(msg)
+ if stream is None:
+ stream = self.config.stream
logger.debug(message)
- rsp = await self.acompletion_text(message, stream=stream, timeout=self.get_timeout(timeout))
+ compressed_message = self.compress_messages(message, compress_type=self.config.compress_type)
+ rsp = await self.acompletion_text(compressed_message, stream=stream, timeout=self.get_timeout(timeout))
+ # rsp = await self.acompletion_text(message, stream=stream, timeout=self.get_timeout(timeout))
return rsp
def _extract_assistant_rsp(self, context):
diff --cc metagpt/provider/openai_api.py
index 8cb503572,bd9c02231..da4278524
--- a/metagpt/provider/openai_api.py
+++ b/metagpt/provider/openai_api.py
@@@ -34,8 -34,8 +34,8 @@@ from metagpt.utils.common import CodePa
from metagpt.utils.cost_manager import CostManager
from metagpt.utils.exceptions import handle_exception
from metagpt.utils.token_counter import (
- count_input_tokens,
+ count_message_tokens,
- count_string_tokens,
+ count_output_tokens,
get_max_completion_tokens,
)
@@@ -48,9 -48,7 +48,10 @@@
LLMType.MOONSHOT,
LLMType.MISTRAL,
LLMType.YI,
+ LLMType.OPEN_ROUTER,
+ LLMType.DEEPSEEK,
+ LLMType.SILICONFLOW,
+ LLMType.OPENROUTER,
]
)
class OpenAILLM(BaseLLM):
@@@ -253,8 -262,8 +265,8 @@@
return usage
try:
- usage.prompt_tokens = count_input_tokens(messages, self.pricing_plan)
+ usage.prompt_tokens = count_message_tokens(messages, self.pricing_plan)
- usage.completion_tokens = count_string_tokens(rsp, self.pricing_plan)
+ usage.completion_tokens = count_output_tokens(rsp, self.pricing_plan)
except Exception as e:
logger.warning(f"usage calculation failed: {e}")
diff --cc metagpt/rag/engines/simple.py
index 8d78fcad7,a03e0149c..61200a295
--- a/metagpt/rag/engines/simple.py
+++ b/metagpt/rag/engines/simple.py
@@@ -38,12 -38,8 +40,13 @@@ from metagpt.rag.factories import
get_retriever,
)
from metagpt.rag.interface import NoEmbedding, RAGObject
+ from metagpt.rag.parsers import OmniParse
-from metagpt.rag.retrievers.base import ModifiableRAGRetriever, PersistableRAGRetriever
+from metagpt.rag.retrievers.base import (
+ DeletableRAGRetriever,
+ ModifiableRAGRetriever,
+ PersistableRAGRetriever,
+ QueryableRAGRetriever,
+)
from metagpt.rag.retrievers.hybrid_retriever import SimpleHybridRetriever
from metagpt.rag.schema import (
BaseIndexConfig,
@@@ -110,7 -106,10 +116,10 @@@ class SimpleEngine(RetrieverQueryEngine
if not input_dir and not input_files:
raise ValueError("Must provide either `input_dir` or `input_files`.")
- documents = SimpleDirectoryReader(input_dir=input_dir, input_files=input_files, fs=fs).load_data()
+ file_extractor = cls._get_file_extractor()
+ documents = SimpleDirectoryReader(
- input_dir=input_dir, input_files=input_files, file_extractor=file_extractor
++ input_dir=input_dir, input_files=input_files, file_extractor=file_extractor, fs=fs
+ ).load_data()
cls._fix_document_metadata(documents)
transformations = transformations or cls._default_transformations()
@@@ -217,65 -216,29 +226,88 @@@
self._persist(str(persist_dir), **kwargs)
+ def count(self) -> int:
+ """Count."""
+ self._ensure_retriever_queryable()
+
+ return self.retriever.query_total_count()
+
+ def clear(self, **kwargs):
+ """Clear."""
+ self._ensure_retriever_deletable()
+
+ return self.retriever.clear(**kwargs)
+
+ def delete_docs(self, input_files: List[Union[str, Path]]):
+ """Delete documents from the index and document store.
+
+ Args:
+ input_files (List[Union[str, Path]]): A list of file paths or file names to be deleted.
+
+ Raises:
+ NotImplementedError: If the method is not implemented.
+ """
+ exists_filenames = set()
+ filenames = {str(i) for i in input_files}
+ for doc_id, info in self.retriever._index.ref_doc_info.items():
+ if info.metadata.get("file_path") in filenames:
+ exists_filenames.add(doc_id)
+
+ for doc_id in exists_filenames:
+ self.retriever._index.delete_ref_doc(doc_id, delete_from_docstore=True)
+
+ @staticmethod
+ def get_obj_nodes(objs: Optional[list[RAGObject]] = None) -> list[ObjectNode]:
+ """Converts a list of RAGObjects to a list of ObjectNodes."""
+
+ return [ObjectNode(text=obj.rag_key(), metadata=ObjectNode.get_obj_metadata(obj)) for obj in objs]
+
+ @classmethod
+ def _from_nodes(
+ cls,
+ nodes: list[BaseNode],
+ transformations: Optional[list[TransformComponent]] = None,
+ embed_model: BaseEmbedding = None,
+ llm: LLM = None,
+ retriever_configs: list[BaseRetrieverConfig] = None,
+ ranker_configs: list[BaseRankerConfig] = None,
+ ) -> "SimpleEngine":
+ embed_model = cls._resolve_embed_model(embed_model, retriever_configs)
+ llm = llm or get_rag_llm()
+
+ retriever = get_retriever(configs=retriever_configs, nodes=nodes, embed_model=embed_model)
+ rankers = get_rankers(configs=ranker_configs, llm=llm) # Default []
+
+ return cls(
+ retriever=retriever,
+ node_postprocessors=rankers,
+ response_synthesizer=get_response_synthesizer(llm=llm),
+ transformations=transformations,
+ )
+
+ @classmethod
+ def _from_nodes(
+ cls,
+ nodes: list[BaseNode],
+ transformations: Optional[list[TransformComponent]] = None,
+ embed_model: BaseEmbedding = None,
+ llm: LLM = None,
+ retriever_configs: list[BaseRetrieverConfig] = None,
+ ranker_configs: list[BaseRankerConfig] = None,
+ ) -> "SimpleEngine":
+ embed_model = cls._resolve_embed_model(embed_model, retriever_configs)
+ llm = llm or get_rag_llm()
+
+ retriever = get_retriever(configs=retriever_configs, nodes=nodes, embed_model=embed_model)
+ rankers = get_rankers(configs=ranker_configs, llm=llm) # Default []
+
+ return cls(
+ retriever=retriever,
+ node_postprocessors=rankers,
+ response_synthesizer=get_response_synthesizer(llm=llm),
+ transformations=transformations,
+ )
+
@classmethod
def _from_index(
cls,
@@@ -354,6 -311,22 +386,26 @@@
def _default_transformations():
return [SentenceSplitter()]
+ @property
+ def filenames(self) -> Set[str]:
+ return self._filenames
++
+ @staticmethod
+ def _get_file_extractor() -> dict[str:BaseReader]:
+ """
+ Get the file extractor.
+ Currently, only PDF use OmniParse. Other document types use the built-in reader from llama_index.
+
+ Returns:
+ dict[file_type: BaseReader]
+ """
+ file_extractor: dict[str:BaseReader] = {}
+ if config.omniparse.base_url:
+ pdf_parser = OmniParse(
+ api_key=config.omniparse.api_key,
+ base_url=config.omniparse.base_url,
+ parse_options=OmniParseOptions(parse_type=OmniParseType.PDF, result_type=ParseResultType.MD),
+ )
+ file_extractor[".pdf"] = pdf_parser
+
+ return file_extractor
diff --cc metagpt/rag/factories/embedding.py
index 19b8b36f6,3613fd228..6212aae25
--- a/metagpt/rag/factories/embedding.py
+++ b/metagpt/rag/factories/embedding.py
@@@ -5,8 -5,11 +5,11 @@@ from typing import Any, Optiona
from llama_index.core.embeddings import BaseEmbedding
from llama_index.embeddings.azure_openai import AzureOpenAIEmbedding
+ from llama_index.embeddings.gemini import GeminiEmbedding
+ from llama_index.embeddings.ollama import OllamaEmbedding
+ from llama_index.embeddings.openai import OpenAIEmbedding
-from metagpt.config2 import config
+from metagpt.config2 import Config
from metagpt.configs.embedding_config import EmbeddingType
from metagpt.configs.llm_config import LLMType
from metagpt.rag.factories.base import GenericFactory
diff --cc metagpt/rag/schema.py
index 4180536a3,1e04a546f..12e635a1a
--- a/metagpt/rag/schema.py
+++ b/metagpt/rag/schema.py
@@@ -70,6 -62,36 +70,37 @@@ class BM25RetrieverConfig(IndexRetrieve
_no_embedding: bool = PrivateAttr(default=True)
+ class MilvusRetrieverConfig(IndexRetrieverConfig):
+ """Config for Milvus-based retrievers."""
+
+ uri: str = Field(default="./milvus_local.db", description="The directory to save data.")
+ collection_name: str = Field(default="metagpt", description="The name of the collection.")
+ token: str = Field(default=None, description="The token for Milvus")
+ metadata: Optional[CollectionMetadata] = Field(
+ default=None, description="Optional metadata to associate with the collection"
+ )
+ dimensions: int = Field(default=0, description="Dimensionality of the vectors for Milvus index construction.")
+
+ _embedding_type_to_dimensions: ClassVar[dict[EmbeddingType, int]] = {
+ EmbeddingType.GEMINI: 768,
+ EmbeddingType.OLLAMA: 4096,
+ }
+
+ @model_validator(mode="after")
+ def check_dimensions(self):
+ if self.dimensions == 0:
++ config = Config.default()
+ self.dimensions = config.embedding.dimensions or self._embedding_type_to_dimensions.get(
+ config.embedding.api_type, 1536
+ )
+ if not config.embedding.dimensions and config.embedding.api_type not in self._embedding_type_to_dimensions:
+ logger.warning(
+ f"You didn't set dimensions in config when using {config.embedding.api_type}, default to 1536"
+ )
+
+ return self
+
+
class ChromaRetrieverConfig(IndexRetrieverConfig):
"""Config for Chroma-based retrievers."""
diff --cc metagpt/utils/__init__.py
index 26042eb0e,74652b74b..be911f9dd
--- a/metagpt/utils/__init__.py
+++ b/metagpt/utils/__init__.py
@@@ -10,8 -10,8 +10,8 @@@ from metagpt.utils.read_document impor
from metagpt.utils.singleton import Singleton
from metagpt.utils.token_counter import (
TOKEN_COSTS,
- count_input_tokens,
+ count_message_tokens,
- count_string_tokens,
+ count_output_tokens,
)
@@@ -19,7 -19,6 +19,8 @@@ __all__ =
"read_docx",
"Singleton",
"TOKEN_COSTS",
- "count_input_tokens",
+ "new_transaction_id",
+ "count_message_tokens",
+ "count_string_tokens",
+ "count_output_tokens",
]
diff --cc metagpt/utils/common.py
index 90f13da23,47f2768cd..cf2aa58ba
--- a/metagpt/utils/common.py
+++ b/metagpt/utils/common.py
@@@ -603,12 -577,34 +603,36 @@@ def write_json_file(json_file: str, dat
if not folder_path.exists():
folder_path.mkdir(parents=True, exist_ok=True)
+ custom_default = partial(to_jsonable_python, fallback=handle_unknown_serialization if use_fallback else None)
+
with open(json_file, "w", encoding=encoding) as fout:
- json.dump(data, fout, ensure_ascii=False, indent=indent, default=to_jsonable_python)
+ json.dump(data, fout, ensure_ascii=False, indent=indent, default=custom_default)
+ def read_jsonl_file(jsonl_file: str, encoding="utf-8") -> list[dict]:
+ if not Path(jsonl_file).exists():
+ raise FileNotFoundError(f"json_file: {jsonl_file} not exist, return []")
+ datas = []
+ with open(jsonl_file, "r", encoding=encoding) as fin:
+ try:
+ for line in fin:
+ data = json.loads(line)
+ datas.append(data)
+ except Exception:
+ raise ValueError(f"read jsonl file: {jsonl_file} failed")
+ return datas
+
+
+ def add_jsonl_file(jsonl_file: str, data: list[dict], encoding: str = None):
+ folder_path = Path(jsonl_file).parent
+ if not folder_path.exists():
+ folder_path.mkdir(parents=True, exist_ok=True)
+
+ with open(jsonl_file, "a", encoding=encoding) as fout:
+ for json_item in data:
+ fout.write(json.dumps(json_item) + "\n")
+
+
def read_csv_to_list(curr_file: str, header=False, strip_trail=True):
"""
Reads in a csv file to a list of list. If header is True, it returns a
@@@ -931,268 -867,19 +958,286 @@@ def get_markdown_codeblock_type(filenam
return mappings.get(mime_type, "text")
+def get_project_srcs_path(workdir: str | Path) -> Path:
+ src_workdir_path = workdir / ".src_workspace"
+ if src_workdir_path.exists():
+ with open(src_workdir_path, "r") as file:
+ src_name = file.read()
+ else:
+ src_name = Path(workdir).name
+ return Path(workdir) / src_name
+
+
+async def init_python_folder(workdir: str | Path):
+ if not workdir:
+ return
+ workdir = Path(workdir)
+ if not workdir.exists():
+ return
+ init_filename = Path(workdir) / "__init__.py"
+ if init_filename.exists():
+ return
+ async with aiofiles.open(init_filename, "a"):
+ os.utime(init_filename, None)
+
+
+def get_markdown_code_block_type(filename: str) -> str:
+ if not filename:
+ return ""
+ ext = Path(filename).suffix
+ types = {
+ ".py": "python",
+ ".js": "javascript",
+ ".java": "java",
+ ".cpp": "cpp",
+ ".c": "c",
+ ".html": "html",
+ ".css": "css",
+ ".xml": "xml",
+ ".json": "json",
+ ".yaml": "yaml",
+ ".md": "markdown",
+ ".sql": "sql",
+ ".rb": "ruby",
+ ".php": "php",
+ ".sh": "bash",
+ ".swift": "swift",
+ ".go": "go",
+ ".rs": "rust",
+ ".pl": "perl",
+ ".asm": "assembly",
+ ".r": "r",
+ ".scss": "scss",
+ ".sass": "sass",
+ ".lua": "lua",
+ ".ts": "typescript",
+ ".tsx": "tsx",
+ ".jsx": "jsx",
+ ".yml": "yaml",
+ ".ini": "ini",
+ ".toml": "toml",
+ ".svg": "xml", # SVG can often be treated as XML
+ # Add more file extensions and corresponding code block types as needed
+ }
+ return types.get(ext, "")
+
+
+def to_markdown_code_block(val: str, type_: str = "") -> str:
+ """
+ Convert a string to a Markdown code block.
+
+ This function takes a string and wraps it in a Markdown code block.
+ If a type is provided, it adds it as a language identifier for syntax highlighting.
+
+ Args:
+ val (str): The string to be converted to a Markdown code block.
+ type_ (str, optional): The language identifier for syntax highlighting.
+ Defaults to an empty string.
+
+ Returns:
+ str: The input string wrapped in a Markdown code block.
+ If the input string is empty, it returns an empty string.
+
+ Examples:
+ >>> to_markdown_code_block("print('Hello, World!')", "python")
+ \n```python\nprint('Hello, World!')\n```\n
+
+ >>> to_markdown_code_block("Some text")
+ \n```\nSome text\n```\n
+ """
+ if not val:
+ return val or ""
+ val = val.replace("```", "\\`\\`\\`")
+ return f"\n```{type_}\n{val}\n```\n"
+
+
+async def save_json_to_markdown(content: str, output_filename: str | Path):
+ """
+ Saves the provided JSON content as a Markdown file.
+
+ This function takes a JSON string, converts it to Markdown format,
+ and writes it to the specified output file.
+
+ Args:
+ content (str): The JSON content to be converted.
+ output_filename (str or Path): The path where the output Markdown file will be saved.
+
+ Returns:
+ None
+
+ Raises:
+ None: Any exceptions are logged and the function returns without raising them.
+
+ Examples:
+ >>> await save_json_to_markdown('{"key": "value"}', Path("/path/to/output.md"))
+ This will save the Markdown converted JSON to the specified file.
+
+ Notes:
+ - This function handles `json.JSONDecodeError` specifically for JSON parsing errors.
+ - Any other exceptions during the process are also logged and handled gracefully.
+ """
+ try:
+ m = json.loads(content)
+ except json.JSONDecodeError as e:
+ logger.warning(f"Failed to decode JSON content: {e}")
+ return
+ except Exception as e:
+ logger.warning(f"An unexpected error occurred: {e}")
+ return
+ await awrite(filename=output_filename, data=json_to_markdown(m))
+
+
+def tool2name(cls, methods: List[str], entry) -> Dict[str, Any]:
+ """
+ Generates a mapping of class methods to a given entry with class name as a prefix.
+
+ Args:
+ cls: The class from which the methods are derived.
+ methods (List[str]): A list of method names as strings.
+ entry (Any): The entry to be mapped to each method.
+
+ Returns:
+ Dict[str, Any]: A dictionary where keys are method names prefixed with the class name and
+ values are the given entry. If the number of methods is less than 2,
+ the dictionary will contain a single entry with the class name as the key.
+
+ Example:
+ >>> class MyClass:
+ >>> pass
+ >>>
+ >>> tool2name(MyClass, ['method1', 'method2'], 'some_entry')
+ {'MyClass.method1': 'some_entry', 'MyClass.method2': 'some_entry'}
+
+ >>> tool2name(MyClass, ['method1'], 'some_entry')
+ {'MyClass': 'some_entry', 'MyClass.method1': 'some_entry'}
+ """
+ class_name = cls.__name__
+ mappings = {f"{class_name}.{i}": entry for i in methods}
+ if len(mappings) < 2:
+ mappings[class_name] = entry
+ return mappings
+
+
+def new_transaction_id(postfix_len=8) -> str:
+ """
+ Generates a new unique transaction ID based on current timestamp and a random UUID.
+
+ Args:
+ postfix_len (int): Length of the random UUID postfix to include in the transaction ID. Default is 8.
+
+ Returns:
+ str: A unique transaction ID composed of timestamp and a random UUID.
+ """
+ return datetime.now().strftime("%Y%m%d%H%M%ST") + uuid.uuid4().hex[0:postfix_len]
+
+
+def log_time(method):
+ """A time-consuming decorator for printing execution duration."""
+
+ def before_call():
+ start_time, cpu_start_time = time.perf_counter(), time.process_time()
+ logger.info(f"[{method.__name__}] started at: " f"{datetime.now().strftime('%Y-%m-%d %H:%m:%S')}")
+ return start_time, cpu_start_time
+
+ def after_call(start_time, cpu_start_time):
+ end_time, cpu_end_time = time.perf_counter(), time.process_time()
+ logger.info(
+ f"[{method.__name__}] ended. "
+ f"Time elapsed: {end_time - start_time:.4} sec, CPU elapsed: {cpu_end_time - cpu_start_time:.4} sec"
+ )
+
+ @functools.wraps(method)
+ def timeit_wrapper(*args, **kwargs):
+ start_time, cpu_start_time = before_call()
+ result = method(*args, **kwargs)
+ after_call(start_time, cpu_start_time)
+ return result
+
+ @functools.wraps(method)
+ async def timeit_wrapper_async(*args, **kwargs):
+ start_time, cpu_start_time = before_call()
+ result = await method(*args, **kwargs)
+ after_call(start_time, cpu_start_time)
+ return result
+
+ return timeit_wrapper_async if iscoroutinefunction(method) else timeit_wrapper
+
+
+async def check_http_endpoint(url: str, timeout: int = 3) -> bool:
+ """
+ Checks the status of an HTTP endpoint.
+
+ Args:
+ url (str): The URL of the HTTP endpoint to check.
+ timeout (int, optional): The timeout in seconds for the HTTP request. Defaults to 3.
+
+ Returns:
+ bool: True if the endpoint is online and responding with a 200 status code, False otherwise.
+ """
+ async with aiohttp.ClientSession() as session:
+ try:
+ async with session.get(url, timeout=timeout) as response:
+ return response.status == 200
+ except Exception as e:
+ print(f"Error accessing the endpoint {url}: {e}")
+ return False
+
+
+def rectify_pathname(path: Union[str, Path], default_filename: str) -> Path:
+ """
+ Rectifies the given path to ensure a valid output file path.
+
+ If the given `path` is a directory, it creates the directory (if it doesn't exist) and appends the `default_filename` to it. If the `path` is a file path, it creates the parent directory (if it doesn't exist) and returns the `path`.
+
+ Args:
+ path (Union[str, Path]): The input path, which can be a string or a `Path` object.
+ default_filename (str): The default filename to use if the `path` is a directory.
+
+ Returns:
+ Path: The rectified output path.
+ """
+ output_pathname = Path(path)
+ if output_pathname.is_dir():
+ output_pathname.mkdir(parents=True, exist_ok=True)
+ output_pathname = output_pathname / default_filename
+ else:
+ output_pathname.parent.mkdir(parents=True, exist_ok=True)
+ return output_pathname
+
+
+def generate_fingerprint(text: str) -> str:
+ """
+ Generate a fingerprint for the given text
+
+ Args:
+ text (str): The text for which the fingerprint needs to be generated
+
+ Returns:
+ str: The fingerprint value of the text
+ """
+ text_bytes = text.encode("utf-8")
+
+ # calculate SHA-256 hash
+ sha256 = hashlib.sha256()
+ sha256.update(text_bytes)
+ fingerprint = sha256.hexdigest()
+
+ return fingerprint
++
++
+ def download_model(file_url: str, target_folder: Path) -> Path:
+ file_name = file_url.split("/")[-1]
+ file_path = target_folder.joinpath(f"{file_name}")
+ if not file_path.exists():
+ file_path.mkdir(parents=True, exist_ok=True)
+ try:
+ response = requests.get(file_url, stream=True)
+ response.raise_for_status() # 检查请求是否成功
+ # 保存文件
+ with open(file_path, "wb") as f:
+ for chunk in response.iter_content(chunk_size=8192):
+ f.write(chunk)
+ logger.info(f"权重文件已下载并保存至 {file_path}")
+ except requests.exceptions.HTTPError as err:
+ logger.info(f"权重文件下载过程中发生错误: {err}")
+ return file_path
diff --cc metagpt/utils/file.py
index a3f612bcc,f62b44eb8..75107c8be
--- a/metagpt/utils/file.py
+++ b/metagpt/utils/file.py
@@@ -75,128 -68,3 +75,128 @@@ class File
content = b"".join(chunks)
logger.debug(f"Successfully read file, the path of file: {file_path}")
return content
+
+ @staticmethod
+ async def is_textual_file(filename: Union[str, Path]) -> bool:
+ """Determines if a given file is a textual file.
+
+ A file is considered a textual file if it is plain text or has a
+ specific set of MIME types associated with textual formats,
+ including PDF and Microsoft Word documents.
+
+ Args:
+ filename (Union[str, Path]): The path to the file to be checked.
+
+ Returns:
+ bool: True if the file is a textual file, False otherwise.
+ """
+ is_text, mime_type = await is_text_file(filename)
+ if is_text:
+ return True
+ if mime_type == "application/pdf":
+ return True
+ if mime_type in {
+ "application/msword",
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ "application/vnd.ms-word.document.macroEnabled.12",
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.template",
+ "application/vnd.ms-word.template.macroEnabled.12",
+ }:
+ return True
+ return False
+
+ @staticmethod
+ async def read_text_file(filename: Union[str, Path]) -> Optional[str]:
+ """Read the whole content of a file. Using absolute paths as the argument for specifying the file location."""
+ is_text, mime_type = await is_text_file(filename)
+ if is_text:
+ return await File._read_text(filename)
+ if mime_type == "application/pdf":
+ return await File._read_pdf(filename)
+ if mime_type in {
+ "application/msword",
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ "application/vnd.ms-word.document.macroEnabled.12",
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.template",
+ "application/vnd.ms-word.template.macroEnabled.12",
+ }:
+ return await File._read_docx(filename)
+ return None
+
+ @staticmethod
+ async def _read_text(path: Union[str, Path]) -> str:
+ return await aread(path)
+
+ @staticmethod
+ async def _read_pdf(path: Union[str, Path]) -> str:
+ result = await File._omniparse_read_file(path)
+ if result:
+ return result
+
+ from llama_index.readers.file import PDFReader
+
+ reader = PDFReader()
+ lines = reader.load_data(file=Path(path))
+ return "\n".join([i.text for i in lines])
+
+ @staticmethod
+ async def _read_docx(path: Union[str, Path]) -> str:
+ result = await File._omniparse_read_file(path)
+ if result:
+ return result
+ return "\n".join(read_docx(str(path)))
+
+ @staticmethod
+ async def _omniparse_read_file(path: Union[str, Path], auto_save_image: bool = False) -> Optional[str]:
+ from metagpt.tools.libs import get_env_default
+ from metagpt.utils.omniparse_client import OmniParseClient
+
+ env_base_url = await get_env_default(key="base_url", app_name="OmniParse", default_value="")
+ env_timeout = await get_env_default(key="timeout", app_name="OmniParse", default_value="")
+ conf_base_url, conf_timeout = await File._read_omniparse_config()
+
+ base_url = env_base_url or conf_base_url
+ if not base_url:
+ return None
+ api_key = await get_env_default(key="api_key", app_name="OmniParse", default_value="")
+ timeout = env_timeout or conf_timeout or 600
+ try:
+ timeout = int(timeout)
+ except ValueError:
+ timeout = 600
+
+ try:
+ if not await check_http_endpoint(url=base_url):
+ logger.warning(f"{base_url}: NOT AVAILABLE")
+ return None
+ client = OmniParseClient(api_key=api_key, base_url=base_url, max_timeout=timeout)
+ file_data = await aread_bin(filename=path)
+ ret = await client.parse_document(file_input=file_data, bytes_filename=str(path))
+ except (ValueError, Exception) as e:
+ logger.exception(f"{path}: {e}")
+ return None
+ if not ret.images or not auto_save_image:
+ return ret.text
+
+ result = [ret.text]
+ img_dir = Path(path).parent / (Path(path).name.replace(".", "_") + "_images")
+ img_dir.mkdir(parents=True, exist_ok=True)
+ for i in ret.images:
+ byte_data = base64.b64decode(i.image)
+ filename = img_dir / i.image_name
+ await awrite_bin(filename=filename, data=byte_data)
+ result.append(f"![{i.image_name}]({str(filename)})")
+ return "\n".join(result)
+
+ @staticmethod
+ async def _read_omniparse_config() -> Tuple[str, int]:
+ config = Config.default()
- if config.omniparse and config.omniparse.url:
- return config.omniparse.url, config.omniparse.timeout
++ if config.omniparse and config.omniparse.base_url:
++ return config.omniparse.base_url, config.omniparse.timeout
+ return "", 0
+
+
+class MemoryFileSystem(_MemoryFileSystem):
+ @classmethod
+ def _strip_protocol(cls, path):
+ return super()._strip_protocol(str(path))
diff --cc metagpt/utils/token_counter.py
index b235ceb7b,1bc6b597d..f710a0e06
--- a/metagpt/utils/token_counter.py
+++ b/metagpt/utils/token_counter.py
@@@ -10,10 -10,15 +10,13 @@@ ref3: https://github.com/Significant-Gr
ref4: https://github.com/hwchase17/langchain/blob/master/langchain/chat_models/openai.py
ref5: https://ai.google.dev/models/gemini
"""
+ import anthropic
import tiktoken
-from openai.types import CompletionUsage
-from openai.types.chat import ChatCompletionChunk
+ from metagpt.logs import logger
-from metagpt.utils.ahttp_client import apost
+
TOKEN_COSTS = {
+ "anthropic/claude-3.5-sonnet": {"prompt": 0.003, "completion": 0.015},
"gpt-3.5-turbo": {"prompt": 0.0015, "completion": 0.002},
"gpt-3.5-turbo-0301": {"prompt": 0.0015, "completion": 0.002},
"gpt-3.5-turbo-0613": {"prompt": 0.0015, "completion": 0.002},
@@@ -53,21 -69,35 +67,43 @@@
"claude-2.0": {"prompt": 0.008, "completion": 0.024},
"claude-2.1": {"prompt": 0.008, "completion": 0.024},
"claude-3-sonnet-20240229": {"prompt": 0.003, "completion": 0.015},
+ "claude-3-5-sonnet": {"prompt": 0.003, "completion": 0.015},
+ "claude-3-5-sonnet-v2": {"prompt": 0.003, "completion": 0.015}, # alias of newer 3.5 sonnet
+ "claude-3-5-sonnet-20240620": {"prompt": 0.003, "completion": 0.015},
"claude-3-opus-20240229": {"prompt": 0.015, "completion": 0.075},
+ "claude-3-haiku-20240307": {"prompt": 0.00025, "completion": 0.00125},
"yi-34b-chat-0205": {"prompt": 0.0003, "completion": 0.0003},
"yi-34b-chat-200k": {"prompt": 0.0017, "completion": 0.0017},
+ "openai/gpt-4": {"prompt": 0.03, "completion": 0.06}, # start, for openrouter
+ "openai/gpt-4-turbo": {"prompt": 0.01, "completion": 0.03},
+ "openai/gpt-4o": {"prompt": 0.005, "completion": 0.015},
+ "openai/gpt-4o-2024-05-13": {"prompt": 0.005, "completion": 0.015},
+ "openai/gpt-4o-mini": {"prompt": 0.00015, "completion": 0.0006},
+ "openai/gpt-4o-mini-2024-07-18": {"prompt": 0.00015, "completion": 0.0006},
- "google/gemini-pro-1.5": {"prompt": 0.0025, "completion": 0.0075},
+ "google/gemini-flash-1.5": {"prompt": 0.00025, "completion": 0.00075},
+ "deepseek/deepseek-coder": {"prompt": 0.00014, "completion": 0.00028},
+ "deepseek/deepseek-chat": {"prompt": 0.00014, "completion": 0.00028}, # end, for openrouter
+ "yi-large": {"prompt": 0.0028, "completion": 0.0028},
+ "microsoft/wizardlm-2-8x22b": {"prompt": 0.00108, "completion": 0.00108}, # for openrouter, start
+ "meta-llama/llama-3-70b-instruct": {"prompt": 0.008, "completion": 0.008},
+ "llama3-70b-8192": {"prompt": 0.0059, "completion": 0.0079},
+ "openai/gpt-3.5-turbo-0125": {"prompt": 0.0005, "completion": 0.0015},
+ "openai/gpt-4-turbo-preview": {"prompt": 0.01, "completion": 0.03},
+ "openai/o1-preview": {"prompt": 0.015, "completion": 0.06},
+ "openai/o1-mini": {"prompt": 0.003, "completion": 0.012},
+ "anthropic/claude-3-opus": {"prompt": 0.015, "completion": 0.075},
- "anthropic/claude-3.5-sonnet": {"prompt": 0.003, "completion": 0.015},
+ "google/gemini-pro-1.5": {"prompt": 0.0025, "completion": 0.0075}, # for openrouter, end
"deepseek-chat": {"prompt": 0.00014, "completion": 0.00028},
"deepseek-coder": {"prompt": 0.00014, "completion": 0.00028},
+ # For ark model https://www.volcengine.com/docs/82379/1099320
+ "doubao-lite-4k-240515": {"prompt": 0.000043, "completion": 0.000086},
+ "doubao-lite-32k-240515": {"prompt": 0.000043, "completion": 0.000086},
+ "doubao-lite-128k-240515": {"prompt": 0.00011, "completion": 0.00014},
+ "doubao-pro-4k-240515": {"prompt": 0.00011, "completion": 0.00029},
+ "doubao-pro-32k-240515": {"prompt": 0.00011, "completion": 0.00029},
+ "doubao-pro-128k-240515": {"prompt": 0.0007, "completion": 0.0013},
+ "llama3-70b-llama3-70b-instruct": {"prompt": 0.0, "completion": 0.0},
+ "llama3-8b-llama3-8b-instruct": {"prompt": 0.0, "completion": 0.0},
}
@@@ -200,27 -272,124 +278,134 @@@ TOKEN_MAX =
"claude-2.1": 200000,
"claude-3-sonnet-20240229": 200000,
"claude-3-opus-20240229": 200000,
+ "claude-3-5-sonnet-20240620": 200000,
+ "claude-3-haiku-20240307": 200000,
"yi-34b-chat-0205": 4000,
"yi-34b-chat-200k": 200000,
+ "openai/gpt-4": 8192, # start, for openrouter
+ "openai/gpt-4-turbo": 128000,
+ "openai/gpt-4o": 128000,
+ "openai/gpt-4o-2024-05-13": 128000,
+ "openai/gpt-4o-mini": 128000,
+ "openai/gpt-4o-mini-2024-07-18": 128000,
- "anthropic/claude-3.5-sonnet": 200000,
- "google/gemini-pro-1.5": 2800000,
+ "google/gemini-flash-1.5": 2800000,
+ "deepseek/deepseek-coder": 128000,
+ "deepseek/deepseek-chat": 128000, # end, for openrouter
+ "deepseek-chat": 128000,
+ "deepseek-coder": 128000,
+ "deepseek-ai/DeepSeek-Coder-V2-Instruct": 32000, # siliconflow
+ "yi-large": 16385,
+ "microsoft/wizardlm-2-8x22b": 65536,
+ "meta-llama/llama-3-70b-instruct": 8192,
+ "llama3-70b-8192": 8192,
+ "openai/gpt-3.5-turbo-0125": 16385,
+ "openai/gpt-4-turbo-preview": 128000,
+ "openai/o1-preview": 128000,
+ "openai/o1-mini": 128000,
+ "anthropic/claude-3-opus": 200000,
+ "anthropic/claude-3.5-sonnet": 200000,
+ "google/gemini-pro-1.5": 4000000,
- "deepseek-chat": 32768,
- "deepseek-coder": 16385,
+ "doubao-lite-4k-240515": 4000,
+ "doubao-lite-32k-240515": 32000,
+ "doubao-lite-128k-240515": 128000,
+ "doubao-pro-4k-240515": 4000,
+ "doubao-pro-32k-240515": 32000,
+ "doubao-pro-128k-240515": 128000,
+ # Qwen https://help.aliyun.com/zh/dashscope/developer-reference/tongyi-qianwen-7b-14b-72b-api-detailes?spm=a2c4g.11186623.0.i20
+ "qwen2.5-72b-instruct": 131072,
+ "qwen2.5-32b-instruct": 131072,
+ "qwen2.5-14b-instruct": 131072,
+ "qwen2.5-7b-instruct": 131072,
+ "qwen2.5-3b-instruct": 32768,
+ "qwen2.5-1.5b-instruct": 32768,
+ "qwen2.5-0.5b-instruct": 32768,
+ "qwen2-57b-a14b-instruct": 32768,
+ "qwen2-72b-instruct": 131072,
+ "qwen2-7b-instruct": 32768,
+ "qwen2-1.5b-instruct": 32768,
+ "qwen2-0.5b-instruct": 32768,
+ "qwen1.5-110b-chat": 32000,
+ "qwen1.5-72b-chat": 32000,
+ "qwen1.5-32b-chat": 32000,
+ "qwen1.5-14b-chat": 8000,
+ "qwen1.5-7b-chat": 32000,
+ "qwen1.5-1.8b-chat": 32000,
+ "qwen1.5-0.5b-chat": 32000,
+ "codeqwen1.5-7b-chat": 64000,
+ "qwen-72b-chat": 32000,
+ "qwen-14b-chat": 8000,
+ "qwen-7b-chat": 32000,
+ "qwen-1.8b-longcontext-chat": 32000,
+ "qwen-1.8b-chat": 8000,
+ }
+
+ # For Amazon Bedrock US region
+ # See https://aws.amazon.com/cn/bedrock/pricing/
+
+ BEDROCK_TOKEN_COSTS = {
+ "amazon.titan-tg1-large": {"prompt": 0.0008, "completion": 0.0008},
+ "amazon.titan-text-express-v1": {"prompt": 0.0008, "completion": 0.0008},
+ "amazon.titan-text-express-v1:0:8k": {"prompt": 0.0008, "completion": 0.0008},
+ "amazon.titan-text-lite-v1:0:4k": {"prompt": 0.0003, "completion": 0.0004},
+ "amazon.titan-text-lite-v1": {"prompt": 0.0003, "completion": 0.0004},
+ "anthropic.claude-instant-v1": {"prompt": 0.0008, "completion": 0.00024},
+ "anthropic.claude-instant-v1:2:100k": {"prompt": 0.0008, "completion": 0.00024},
+ "anthropic.claude-v1": {"prompt": 0.008, "completion": 0.0024},
+ "anthropic.claude-v2": {"prompt": 0.008, "completion": 0.0024},
+ "anthropic.claude-v2:1": {"prompt": 0.008, "completion": 0.0024},
+ "anthropic.claude-v2:0:18k": {"prompt": 0.008, "completion": 0.0024},
+ "anthropic.claude-v2:1:200k": {"prompt": 0.008, "completion": 0.0024},
+ "anthropic.claude-3-sonnet-20240229-v1:0": {"prompt": 0.003, "completion": 0.015},
+ "anthropic.claude-3-sonnet-20240229-v1:0:28k": {"prompt": 0.003, "completion": 0.015},
+ "anthropic.claude-3-sonnet-20240229-v1:0:200k": {"prompt": 0.003, "completion": 0.015},
+ "anthropic.claude-3-5-sonnet-20240620-v1:0": {"prompt": 0.003, "completion": 0.015},
+ "anthropic.claude-3-haiku-20240307-v1:0": {"prompt": 0.00025, "completion": 0.00125},
+ "anthropic.claude-3-haiku-20240307-v1:0:48k": {"prompt": 0.00025, "completion": 0.00125},
+ "anthropic.claude-3-haiku-20240307-v1:0:200k": {"prompt": 0.00025, "completion": 0.00125},
+ # currently (2024-4-29) only available at US West (Oregon) AWS Region.
+ "anthropic.claude-3-opus-20240229-v1:0": {"prompt": 0.015, "completion": 0.075},
+ "cohere.command-text-v14": {"prompt": 0.0015, "completion": 0.0015},
+ "cohere.command-text-v14:7:4k": {"prompt": 0.0015, "completion": 0.0015},
+ "cohere.command-light-text-v14": {"prompt": 0.0003, "completion": 0.0003},
+ "cohere.command-light-text-v14:7:4k": {"prompt": 0.0003, "completion": 0.0003},
+ "meta.llama2-13b-chat-v1:0:4k": {"prompt": 0.00075, "completion": 0.001},
+ "meta.llama2-13b-chat-v1": {"prompt": 0.00075, "completion": 0.001},
+ "meta.llama2-70b-v1": {"prompt": 0.00195, "completion": 0.00256},
+ "meta.llama2-70b-v1:0:4k": {"prompt": 0.00195, "completion": 0.00256},
+ "meta.llama2-70b-chat-v1": {"prompt": 0.00195, "completion": 0.00256},
+ "meta.llama2-70b-chat-v1:0:4k": {"prompt": 0.00195, "completion": 0.00256},
+ "meta.llama3-8b-instruct-v1:0": {"prompt": 0.0004, "completion": 0.0006},
+ "meta.llama3-70b-instruct-v1:0": {"prompt": 0.00265, "completion": 0.0035},
+ "mistral.mistral-7b-instruct-v0:2": {"prompt": 0.00015, "completion": 0.0002},
+ "mistral.mixtral-8x7b-instruct-v0:1": {"prompt": 0.00045, "completion": 0.0007},
+ "mistral.mistral-large-2402-v1:0": {"prompt": 0.008, "completion": 0.024},
+ "ai21.j2-grande-instruct": {"prompt": 0.0125, "completion": 0.0125},
+ "ai21.j2-jumbo-instruct": {"prompt": 0.0188, "completion": 0.0188},
+ "ai21.j2-mid": {"prompt": 0.0125, "completion": 0.0125},
+ "ai21.j2-mid-v1": {"prompt": 0.0125, "completion": 0.0125},
+ "ai21.j2-ultra": {"prompt": 0.0188, "completion": 0.0188},
+ "ai21.j2-ultra-v1": {"prompt": 0.0188, "completion": 0.0188},
+ }
+
+ # https://xinghuo.xfyun.cn/sparkapi?scr=price
+ SPARK_TOKENS = {
+ "general": {"prompt": 0.0, "completion": 0.0}, # Spark-Lite
+ "generalv2": {"prompt": 0.0188, "completion": 0.0188}, # Spark V2.0
+ "generalv3": {"prompt": 0.0035, "completion": 0.0035}, # Spark Pro
+ "generalv3.5": {"prompt": 0.0035, "completion": 0.0035}, # Spark3.5 Max
}
-def count_input_tokens(messages, model="gpt-3.5-turbo-0125"):
+def count_message_tokens(messages, model="gpt-3.5-turbo-0125"):
"""Return the number of tokens used by a list of messages."""
+ if "claude" in model:
+ # rough estimation for models newer than claude-2.1
+ vo = anthropic.Client()
+ num_tokens = 0
+ for message in messages:
+ for key, value in message.items():
+ num_tokens += vo.count_tokens(str(value))
+ return num_tokens
try:
encoding = tiktoken.encoding_for_model(model)
except KeyError:
@@@ -254,11 -430,11 +446,11 @@@
tokens_per_message = 4 # every message follows <|start|>{role/name}\n{content}<|end|>\n
tokens_per_name = -1 # if there's a name, the role is omitted
elif "gpt-3.5-turbo" == model:
- print("Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0125.")
+ logger.info("Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0125.")
- return count_input_tokens(messages, model="gpt-3.5-turbo-0125")
+ return count_message_tokens(messages, model="gpt-3.5-turbo-0125")
elif "gpt-4" == model:
- print("Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.")
+ logger.info("Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.")
- return count_input_tokens(messages, model="gpt-4-0613")
+ return count_message_tokens(messages, model="gpt-4-0613")
elif "open-llm-model" == model:
"""
For self-hosted open_llm api, they include lots of different models. The message tokens calculation is
diff --cc requirements.txt
index 5aa404e7e,51b3d786b..67aabf5dc
--- a/requirements.txt
+++ b/requirements.txt
@@@ -66,15 -71,17 +71,21 @@@ anytre
ipywidgets==8.1.1
Pillow
imap_tools==1.5.0 # Used by metagpt/tools/libs/email_login.py
- qianfan==0.3.2
- dashscope==1.14.1
- rank-bm25==0.2.2 # for tool recommendation
- gymnasium==0.29.1
+pylint~=3.0.3
+pygithub~=2.3
+htmlmin
+fsspec
+grep-ast~=0.3.3 # linter
- tree-sitter~=0.21.3 # linter
+unidiff==0.7.5 # used at metagpt/tools/libs/cr.py
+ qianfan~=0.4.4
+ dashscope~=1.19.3
+ rank-bm25==0.2.2 # for tool recommendation
+ jieba==0.42.1 # for tool recommendation
+ volcengine-python-sdk[ark]~=1.0.94 # Solution for installation error in Windows: https://github.com/volcengine/volcengine-python-sdk/issues/5
-# llama-index-vector-stores-elasticsearch~=0.2.5 # Used by `metagpt/memory/longterm_memory.py`
-# llama-index-vector-stores-chroma~=0.1.10 # Used by `metagpt/memory/longterm_memory.py`
+ gymnasium==0.29.1
+ boto3~=1.34.69
+ spark_ai_python~=0.3.30
+ agentops
+ tree_sitter~=0.23.2
+ tree_sitter_python~=0.23.2
httpx==0.27.2
diff --cc tests/metagpt/rag/engines/test_simple.py
index e0a174ed2,a10fcbe63..877a789a5
--- a/tests/metagpt/rag/engines/test_simple.py
+++ b/tests/metagpt/rag/engines/test_simple.py
@@@ -75,7 -83,9 +83,9 @@@ class TestSimpleEngine
)
# Assert
- mock_simple_directory_reader.assert_called_once_with(input_dir=input_dir, input_files=input_files, fs=None)
+ mock_simple_directory_reader.assert_called_once_with(
- input_dir=input_dir, input_files=input_files, file_extractor=file_extractor
++ input_dir=input_dir, input_files=input_files, file_extractor=file_extractor, fs=None
+ )
mock_get_retriever.assert_called_once()
mock_get_rankers.assert_called_once()
mock_get_response_synthesizer.assert_called_once_with(llm=llm)
diff --cc tests/metagpt/roles/test_product_manager.py
index 2ec6c9baa,143eef2f2..77df9af40
--- a/tests/metagpt/roles/test_product_manager.py
+++ b/tests/metagpt/roles/test_product_manager.py
@@@ -14,8 -15,7 +14,8 @@@ from metagpt.context import Contex
from metagpt.logs import logger
from metagpt.roles import ProductManager
from metagpt.utils.common import any_to_str
- from tests.metagpt.roles.mock import MockMessages
+from metagpt.utils.git_repository import GitRepository
+ from tests.metagpt.roles.mock import MockMessages
@pytest.mark.asyncio
diff --cc tests/metagpt/utils/test_token_counter.py
index acb99d717,5c8d5e15f..b6eefd61d
--- a/tests/metagpt/utils/test_token_counter.py
+++ b/tests/metagpt/utils/test_token_counter.py
@@@ -7,7 -7,7 +7,7 @@@
"""
import pytest
- from metagpt.utils.token_counter import count_message_tokens, count_string_tokens
-from metagpt.utils.token_counter import count_input_tokens, count_output_tokens
++from metagpt.utils.token_counter import count_message_tokens, count_output_tokens
def test_count_message_tokens():