🌟 The Multi-Agent Framework: First AI Software Company, Towards Natural Language Programming
fix conflicts
seehi77703f12369d
diff --cc .gitignoreindex 73f32f75d,841306429..7434aec84--- a/.gitignore+++ b/.gitignore@@@ -190,4 -189,9 +191,10 @@@ cov.xm*-structure.json*.dot.python-version+tests/data/requirements/*.jpg+ *.csv+ metagpt/ext/sela/results/*+ .chainlit/++ metagpt/ext/aflow/data+ metagpt/ext/aflow/scripts/optimizeddiff --cc config/config2.example.yamlindex f6970dabe,b82468eed..eb556fecf--- a/config/config2.example.yaml+++ b/config/config2.example.yaml@@@ -1,49 -1,25 +1,56 @@@llm:- api_type: "openai" # or azure / ollama / open_llm etc. Check LLMType for more options+ api_type: "openai" # or azure / ollama / groq etc.base_url: "YOUR_BASE_URL"api_key: "YOUR_API_KEY"- model: "gpt-4-turbo-preview" # or gpt-3.5-turbo-1106 / gpt-4-1106-preview+ model: "gpt-4-turbo" # or gpt-3.5-turboproxy: "YOUR_PROXY" # for LLM API requests# timeout: 600 # Optional. If set to 0, default value is 300.- pricing_plan: "" # Optional. If invalid, it will be automatically filled in with the value of the `model`.- # Azure-exclusive pricing plan mappings:- # - gpt-3.5-turbo 4k: "gpt-3.5-turbo-1106"- # - gpt-4-turbo: "gpt-4-turbo-preview"- # - gpt-4-turbo-vision: "gpt-4-vision-preview"- # - gpt-4 8k: "gpt-4"- # See for more: https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/+ # Details: https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/+ pricing_plan: "" # Optional. Use for Azure LLM when its model name is not the same as OpenAI's+++ # RAG Embedding.+ # For backward compatibility, if the embedding is not set and the llm's api_type is either openai or azure, the llm's config will be used.+ embedding:+ api_type: "" # openai / azure / gemini / ollama etc. Check EmbeddingType for more options.+ base_url: ""+ api_key: ""+ model: ""+ api_version: ""+ embed_batch_size: 100+ dimensions: # output dimension of embedding model+# Role's custom configuration+roles:+ - role: "ProductManager" # role's className or role's role_id+ llm:+ api_type: "openai" # or azure / ollama / open_llm etc. Check LLMType for more options+ base_url: "YOUR_BASE_URL"+ api_key: "YOUR_API_KEY"+ proxy: "YOUR_PROXY" # for LLM API requests+ model: "gpt-4-turbo-1106"+ - role: "Architect"+ llm:+ api_type: "openai" # or azure / ollama / open_llm etc. Check LLMType for more options+ base_url: "YOUR_BASE_URL"+ api_key: "YOUR_API_KEY"+ proxy: "YOUR_PROXY" # for LLM API requests+ model: "gpt-35-turbo"+ - role: "ProjectManager"+ llm:+ api_type: "azure"+ base_url: "YOUR_BASE_URL"+ api_key: "YOUR_API_KEY"+ api_version: "YOUR_API_VERSION"+ model: "gpt-4-1106"+ - role: "Engineer"+ llm:+ api_type: "azure"+ base_url: "YOUR_BASE_URL"+ api_key: "YOUR_API_KEY"+ api_version: "YOUR_API_VERSION"+ model: "gpt-35-turbo-1106"+repair_llm_output: true # when the output is not a valid json, try to repair itproxy: "YOUR_PROXY" # for tools like requests, playwright, selenium, etc.@@@ -98,3 -59,27 +105,25 @@@ iflytek_api_key: "YOUR_API_KEYiflytek_api_secret: "YOUR_API_SECRET"metagpt_tti_url: "YOUR_MODEL_URL"++ omniparse:+ api_key: "YOUR_API_KEY"+ base_url: "YOUR_BASE_URL"++ models:+ # "YOUR_MODEL_NAME_1 or YOUR_API_TYPE_1": # model: "gpt-4-turbo" # or gpt-3.5-turbo+ # api_type: "openai" # or azure / ollama / groq etc.+ # base_url: "YOUR_BASE_URL"+ # api_key: "YOUR_API_KEY"+ # proxy: "YOUR_PROXY" # for LLM API requests+ # # timeout: 600 # Optional. If set to 0, default value is 300.+ # # Details: https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/+ # pricing_plan: "" # Optional. Use for Azure LLM when its model name is not the same as OpenAI's+ # "YOUR_MODEL_NAME_2 or YOUR_API_TYPE_2": # api_type: "openai" # or azure / ollama / groq etc.+ # api_type: "openai" # or azure / ollama / groq etc.+ # base_url: "YOUR_BASE_URL"+ # api_key: "YOUR_API_KEY"+ # proxy: "YOUR_PROXY" # for LLM API requests+ # # timeout: 600 # Optional. If set to 0, default value is 300.+ # # Details: https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/+ # pricing_plan: "" # Optional. Use for Azure LLM when its model name is not the same as OpenAI's--agentops_api_key: "YOUR_AGENTOPS_API_KEY" # get key from https://app.agentops.ai/settings/projectsdiff --cc examples/use_off_the_shelf_agent.pyindex 0d224a92b,7ccf87a2b..34d605f1a--- a/examples/use_off_the_shelf_agent.py+++ b/examples/use_off_the_shelf_agent.py@@@ -5,11 -5,9 +5,11 @@@ Author: garylin209"""import asyncio-from metagpt.context import Context++from metagpt.environment.mgx.mgx_env import MGXEnvfrom metagpt.logs import logger++from metagpt.roles.di.team_leader import TeamLeaderfrom metagpt.roles.product_manager import ProductManager- from metagpt.environment.mgx.mgx_env import MGXEnv+from metagpt.schema import Message- from metagpt.roles.di.team_leader import TeamLeaderasync def main():diff --cc metagpt/actions/action.pyindex 8733947f5,20c052aa9..ba8a94803--- a/metagpt/actions/action.py+++ b/metagpt/actions/action.py@@@ -34,7 -37,26 +36,20 @@@ class Action(SerializationMixin, Contexprefix: str = "" # aask*时会加上prefix,作为system_messagedesc: str = "" # for skill managernode: ActionNode = Field(default=None, exclude=True)+ # The model name or API type of LLM of the `models` in the `config2.yaml`;+ # Using `None` to use the `llm` configuration in the `config2.yaml`.+ llm_name_or_type: Optional[str] = None++ @model_validator(mode="after")+ @classmethod+ def _update_private_llm(cls, data: Any) -> Any:+ config = ModelsConfig.default().get(data.llm_name_or_type)+ if config:+ llm = create_llm_instance(config)+ llm.cost_manager = data.llm.cost_manager+ data.llm = llm+ return data- @property- def repo(self) -> ProjectRepo:- if not self.context.repo:- self.context.repo = ProjectRepo(self.context.git_repo)- return self.context.repo-@propertydef prompt_schema(self):return self.config.prompt_schemadiff --cc metagpt/actions/action_node.pyindex c1de16656,c94349a76..7109f287e--- a/metagpt/actions/action_node.py+++ b/metagpt/actions/action_node.py@@@ -467,11 -481,119 +483,121 @@@ class ActionNodereturn self+ def get_field_name(self):+ """+ Get the field name from the Pydantic model associated with this ActionNode.+ """+ model_class = self.create_class()+ fields = model_class.model_fields++ # Assuming there's only one field in the model+ if len(fields) == 1:+ return next(iter(fields))++ # If there are multiple fields, we might want to use self.key to find the right one+ return self.key++ def get_field_names(self):+ """+ Get the field names associated with this ActionNode's Pydantic model.+ """+ model_class = self.create_class()+ return model_class.model_fields.keys()++ def get_field_types(self):+ """+ Get the field types associated with this ActionNode's Pydantic model.+ """+ model_class = self.create_class()+ return {field_name: field.annotation for field_name, field in model_class.model_fields.items()}++ def xml_compile(self, context):+ """+ Compile the prompt to make it easier for the model to understand the xml format.+ """+ field_names = self.get_field_names()+ # Construct the example using the field names+ examples = []+ for field_name in field_names:+ examples.append(f"<{field_name}>content</{field_name}>")++ # Join all examples into a single string+ example_str = "\n".join(examples)+ # Add the example to the context+ context += f"""+ ### Response format (must be strictly followed): All content must be enclosed in the given XML tags, ensuring each opening <tag> has a corresponding closing </tag>, with no incomplete or self-closing tags allowed.\n+ {example_str}+ """+ return context++ async def code_fill(+ self, context: str, function_name: Optional[str] = None, timeout: int = USE_CONFIG_TIMEOUT+ ) -> Dict[str, str]:+ """+ Fill CodeBlock Using ``` ```+ """+ field_name = self.get_field_name()+ prompt = context+ content = await self.llm.aask(prompt, timeout=timeout)+ extracted_code = sanitize(code=content, entrypoint=function_name)+ result = {field_name: extracted_code}+ return result++ async def single_fill(self, context: str, images: Optional[Union[str, list[str]]] = None) -> Dict[str, str]:+ field_name = self.get_field_name()+ prompt = context+ content = await self.llm.aask(prompt, images=images)+ result = {field_name: content}+ return result++ async def xml_fill(self, context: str, images: Optional[Union[str, list[str]]] = None) -> Dict[str, Any]:+ """+ Fill context with XML tags and convert according to field types, including string, integer, boolean, list and dict types+ """+ field_names = self.get_field_names()+ field_types = self.get_field_types()++ extracted_data: Dict[str, Any] = {}+ content = await self.llm.aask(context, images=images)++ for field_name in field_names:+ pattern = rf"<{field_name}>(.*?)</{field_name}>"+ match = re.search(pattern, content, re.DOTALL)+ if match:+ raw_value = match.group(1).strip()+ field_type = field_types.get(field_name)++ if field_type == str:+ extracted_data[field_name] = raw_value+ elif field_type == int:+ try:+ extracted_data[field_name] = int(raw_value)+ except ValueError:+ extracted_data[field_name] = 0 # 或者其他默认值+ elif field_type == bool:+ extracted_data[field_name] = raw_value.lower() in ("true", "yes", "1", "on", "True")+ elif field_type == list:+ try:+ extracted_data[field_name] = eval(raw_value)+ if not isinstance(extracted_data[field_name], list):+ raise ValueError+ except:+ extracted_data[field_name] = [] # 默认空列表+ elif field_type == dict:+ try:+ extracted_data[field_name] = eval(raw_value)+ if not isinstance(extracted_data[field_name], dict):+ raise ValueError+ except:+ extracted_data[field_name] = {} # 默认空字典++ return extracted_data++ @exp_cache(serializer=ActionNodeSerializer())async def fill(self,- context,+ *,+ req,llm,schema="json",mode="auto",diff --cc metagpt/actions/project_management_an.pyindex 7131a6c99,f53062433..a953feb4c--- a/metagpt/actions/project_management_an.py+++ b/metagpt/actions/project_management_an.py@@@ -9,10 -9,10 +9,10 @@@ from typing import List, Optionafrom metagpt.actions.action_node import ActionNode- REQUIRED_PYTHON_PACKAGES = ActionNode(+ REQUIRED_PACKAGES = ActionNode(key="Required packages",- expected_type=List[str],+ expected_type=Optional[List[str]],- instruction="Provide required third-party packages in requirements.txt format.",+ instruction="Provide required packages The response language should correspond to the context and requirements.",example=["flask==1.1.2", "bcrypt==3.2.0"],)diff --cc metagpt/actions/research.pyindex 99f72b076,5086f10cf..266584485--- a/metagpt/actions/research.py+++ b/metagpt/actions/research.py@@@ -164,11 -159,12 +164,13 @@@ class CollectLinks(Action)Returns:A list of ranked URLs."""- max_results = max(num_results * 2, 6)- results = await self.search_engine.run(query, max_results=max_results, as_string=False)+ max_results = max_num_results or max(num_results * 2, 6)+ results = await self._search_urls(query, max_results=max_results)+ if len(results) == 0:+ return []_results = "\n".join(f"{i}: {j}" for i, j in zip(range(max_results), results))- prompt = COLLECT_AND_RANKURLS_PROMPT.format(topic=topic, query=query, results=_results)+ time_stamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")+ prompt = COLLECT_AND_RANKURLS_PROMPT.format(topic=topic, query=query, results=_results, time_stamp=time_stamp)logger.debug(prompt)indices = await self._aask(prompt)try:diff --cc metagpt/actions/write_code_review.pyindex 6a283f812,c9b494dff..209e4e8ac--- a/metagpt/actions/write_code_review.py+++ b/metagpt/actions/write_code_review.py@@@ -115,9 -118,9 +124,9 @@@ LGT"""REWRITE_CODE_TEMPLATE = """- # Instruction: rewrite code based on the Code Review and Actions+ # Instruction: rewrite the `{filename}` based on the Code Review and Actions-## Rewrite Code: CodeBlock. If it still has some bugs, rewrite {filename} with triple quotes. Do your utmost to optimize THIS SINGLE FILE. Return all completed codes and prohibit the return of unfinished codes.-```Code+## Rewrite Code: CodeBlock. If it still has some bugs, rewrite {filename} using a Markdown code block, with the filename docstring preceding the code block. Do your utmost to optimize THIS SINGLE FILE. Return all completed codes and prohibit the return of unfinished codes.+```python## {filename}...```diff --cc metagpt/config2.pyindex fd0cb0948,27b228b33..1942e5ef5--- a/metagpt/config2.py+++ b/metagpt/config2.py@@@ -86,15 -84,8 +89,14 @@@ class Config(CLIParams, YamlModel)iflytek_api_key: str = ""azure_tts_subscription_key: str = ""azure_tts_region: str = ""+ _extra: dict = dict() # extra config dict+ # Role's custom configuration+ roles: Optional[List[RoleCustomConfig]] = None++ # RoleZero's configuration+ role_zero: RoleZeroConfig = Field(default_factory=RoleZeroConfig)+- omniparse: Optional[OmniParseConfig] = None-@classmethoddef from_home(cls, path):"""Load config from ~/.metagpt/config2.yaml"""@@@ -104,7 -95,7 +106,7 @@@return Config.from_yaml_file(pathname)@classmethod- def default(cls, reload: bool = False, **kwargs):- def default(cls):++ def default(cls, reload: bool = False, **kwargs) -> "Config":"""Load default config- Priority: env < default_config_paths- Inside default_config_paths, the latter one overwrites the former onediff --cc metagpt/configs/llm_config.pyindex 57913956c,ef034ca49..9e27d76a6--- a/metagpt/configs/llm_config.py+++ b/metagpt/configs/llm_config.py@@@ -10,8 -11,7 +11,8 @@@ from typing import Optionafrom pydantic import field_validator+from metagpt.configs.compress_msg_config import CompressType- from metagpt.const import LLM_API_TIMEOUT+ from metagpt.const import CONFIG_ROOT, LLM_API_TIMEOUT, METAGPT_ROOTfrom metagpt.utils.yaml_model import YamlModel@@@ -32,9 -35,9 +36,12 @@@ class LLMType(Enum)MOONSHOT = "moonshot"MISTRAL = "mistral"YI = "yi" # lingyiwanwu+ OPEN_ROUTER = "open_router"+ DEEPSEEK = "deepseek"+ SILICONFLOW = "siliconflow"+ OPENROUTER = "openrouter"+ BEDROCK = "bedrock"+ ARK = "ark" # https://www.volcengine.com/docs/82379/1263482#python-sdkdef __missing__(self, key):return self.OPENAI@@@ -87,9 -97,9 +101,12 @@@ class LLMConfig(YamlModel)# Cost Controlcalc_usage: bool = True+ # Compress request messages under token limit+ compress_type: CompressType = CompressType.NO_COMPRESS++ # For Messages Control+ use_system_prompt: bool = True+@field_validator("api_key")@classmethoddef check_llm_key(cls, v):diff --cc metagpt/configs/omniparse_config.pyindex ecae78697,39742c8a4..8f38f9f51--- a/metagpt/configs/omniparse_config.py+++ b/metagpt/configs/omniparse_config.py@@@ -2,5 -2,5 +2,6 @@@ from metagpt.utils.yaml_model import Yaclass OmniParseConfig(YamlModel):- url: str = ""+ api_key: str = ""+ base_url: str = ""+ timeout: int = 600diff --cc metagpt/environment/werewolf/werewolf_ext_env.pyindex d9644eb9b,a8636536b..c5ecb8b34--- a/metagpt/environment/werewolf/werewolf_ext_env.py+++ b/metagpt/environment/werewolf/werewolf_ext_env.py@@@ -9,8 -8,10 +8,10 @@@ from typing import Any, Callable, Optiofrom pydantic import ConfigDict, Field- from metagpt.base.base_env_space import BaseEnvAction, BaseEnvObsParams++from metagpt.base.base_env_space import BaseEnvObsParamsfrom metagpt.environment.base_env import ExtEnv, mark_as_readable, mark_as_writeable-from metagpt.environment.base_env_space import BaseEnvObsParams+ from metagpt.environment.werewolf.const import STEP_INSTRUCTIONS, RoleState, RoleType+ from metagpt.environment.werewolf.env_space import EnvAction, EnvActionTypefrom metagpt.logs import loggerdiff --cc metagpt/ext/stanford_town/utils/utils.pyindex e09cce8fe,4e81298c9..b4e15f485--- a/metagpt/ext/stanford_town/utils/utils.py+++ b/metagpt/ext/stanford_town/utils/utils.py@@@ -48,8 -48,8 +48,9 @@@ def read_csv_to_list(curr_file: str, hedef get_embedding(text, model: str = "text-embedding-ada-002"):+ config = Config.default()text = text.replace("\n", " ")+ embedding = Noneif not text:text = "this is blank"for idx in range(3):diff --cc metagpt/logs.pyindex 63c10fa2f,7ab51ff8e..ce2f12e4c--- a/metagpt/logs.py+++ b/metagpt/logs.py@@@ -21,19 -13,8 +21,21 @@@ from pydantic import BaseModel, Fielfrom metagpt.const import METAGPT_ROOT+LLM_STREAM_QUEUE: ContextVar[asyncio.Queue] = ContextVar("llm-stream")+++class ToolLogItem(BaseModel):+ type_: str = Field(alias="type", default="str", description="Data type of `value` field.")+ name: str+ value: Any+++TOOL_LOG_END_MARKER = ToolLogItem(+ type="str", name="end_marker", value="\x18\x19\x1B\x18"+) # A special log item to suggest the end of a stream log++ _print_level = "INFO"+def define_log_level(print_level="INFO", logfile_level="DEBUG", name: str = None):"""Adjust the log level to above level"""@@@ -91,53 -43,6 +96,58 @@@ def set_llm_stream_logfunc(func)_llm_stream_log = func+def set_tool_output_logfunc(func):+ global _tool_output_log+ _tool_output_log = func+++async def set_tool_output_logfunc_async(func):+ # async version+ global _tool_output_log_async+ _tool_output_log_async = func+++def set_human_input_func(func):+ global _get_human_input+ _get_human_input = func+++_llm_stream_log = partial(print, end="")+++_tool_output_log = (+ lambda *args, **kwargs: None+) # a dummy function to avoid errors if set_tool_output_logfunc is not called+++async def _tool_output_log_async(*args, **kwargs):+ # async version+ pass+++def create_llm_stream_queue():+ """Creates a new LLM stream queue and sets it in the context variable.++ Returns:+ The newly created asyncio.Queue instance.+ """+ queue = asyncio.Queue()+ LLM_STREAM_QUEUE.set(queue)+ return queue+++def get_llm_stream_queue():+ """Retrieves the current LLM stream queue from the context variable.++ Returns:+ The asyncio.Queue instance if set, otherwise None.+ """+ return LLM_STREAM_QUEUE.get(None)+++_get_human_input = input # get human input from console by default+++++ def _llm_stream_log(msg):+ if _print_level in ["INFO"]:+ print(msg, end="")diff --cc metagpt/provider/base_llm.pyindex f9111ffe0,a95e8dbd3..ec9e7fcf4--- a/metagpt/provider/base_llm.py+++ b/metagpt/provider/base_llm.py@@@ -153,10 -146,10 +153,12 @@@ class BaseLLM(ABC)message.append(self._user_msg(msg, images=images))else:message.extend(msg)+ if stream is None:+ stream = self.config.streamlogger.debug(message)- rsp = await self.acompletion_text(message, stream=stream, timeout=self.get_timeout(timeout))+ compressed_message = self.compress_messages(message, compress_type=self.config.compress_type)+ rsp = await self.acompletion_text(compressed_message, stream=stream, timeout=self.get_timeout(timeout))+ # rsp = await self.acompletion_text(message, stream=stream, timeout=self.get_timeout(timeout))return rspdef _extract_assistant_rsp(self, context):diff --cc metagpt/provider/openai_api.pyindex 8cb503572,bd9c02231..da4278524--- a/metagpt/provider/openai_api.py+++ b/metagpt/provider/openai_api.py@@@ -34,8 -34,8 +34,8 @@@ from metagpt.utils.common import CodePafrom metagpt.utils.cost_manager import CostManagerfrom metagpt.utils.exceptions import handle_exceptionfrom metagpt.utils.token_counter import (- count_input_tokens,+ count_message_tokens,- count_string_tokens,+ count_output_tokens,get_max_completion_tokens,)@@@ -48,9 -48,7 +48,10 @@@LLMType.MOONSHOT,LLMType.MISTRAL,LLMType.YI,+ LLMType.OPEN_ROUTER,+ LLMType.DEEPSEEK,+ LLMType.SILICONFLOW,+ LLMType.OPENROUTER,])class OpenAILLM(BaseLLM):@@@ -253,8 -262,8 +265,8 @@@return usagetry:- usage.prompt_tokens = count_input_tokens(messages, self.pricing_plan)+ usage.prompt_tokens = count_message_tokens(messages, self.pricing_plan)- usage.completion_tokens = count_string_tokens(rsp, self.pricing_plan)+ usage.completion_tokens = count_output_tokens(rsp, self.pricing_plan)except Exception as e:logger.warning(f"usage calculation failed: {e}")diff --cc metagpt/rag/engines/simple.pyindex 8d78fcad7,a03e0149c..61200a295--- a/metagpt/rag/engines/simple.py+++ b/metagpt/rag/engines/simple.py@@@ -38,12 -38,8 +40,13 @@@ from metagpt.rag.factories importget_retriever,)from metagpt.rag.interface import NoEmbedding, RAGObject+ from metagpt.rag.parsers import OmniParse-from metagpt.rag.retrievers.base import ModifiableRAGRetriever, PersistableRAGRetriever+from metagpt.rag.retrievers.base import (+ DeletableRAGRetriever,+ ModifiableRAGRetriever,+ PersistableRAGRetriever,+ QueryableRAGRetriever,+)from metagpt.rag.retrievers.hybrid_retriever import SimpleHybridRetrieverfrom metagpt.rag.schema import (BaseIndexConfig,@@@ -110,7 -106,10 +116,10 @@@ class SimpleEngine(RetrieverQueryEngineif not input_dir and not input_files:raise ValueError("Must provide either `input_dir` or `input_files`.")- documents = SimpleDirectoryReader(input_dir=input_dir, input_files=input_files, fs=fs).load_data()+ file_extractor = cls._get_file_extractor()+ documents = SimpleDirectoryReader(- input_dir=input_dir, input_files=input_files, file_extractor=file_extractor++ input_dir=input_dir, input_files=input_files, file_extractor=file_extractor, fs=fs+ ).load_data()cls._fix_document_metadata(documents)transformations = transformations or cls._default_transformations()@@@ -217,65 -216,29 +226,88 @@@self._persist(str(persist_dir), **kwargs)+ def count(self) -> int:+ """Count."""+ self._ensure_retriever_queryable()++ return self.retriever.query_total_count()++ def clear(self, **kwargs):+ """Clear."""+ self._ensure_retriever_deletable()++ return self.retriever.clear(**kwargs)++ def delete_docs(self, input_files: List[Union[str, Path]]):+ """Delete documents from the index and document store.++ Args:+ input_files (List[Union[str, Path]]): A list of file paths or file names to be deleted.++ Raises:+ NotImplementedError: If the method is not implemented.+ """+ exists_filenames = set()+ filenames = {str(i) for i in input_files}+ for doc_id, info in self.retriever._index.ref_doc_info.items():+ if info.metadata.get("file_path") in filenames:+ exists_filenames.add(doc_id)++ for doc_id in exists_filenames:+ self.retriever._index.delete_ref_doc(doc_id, delete_from_docstore=True)++ @staticmethod+ def get_obj_nodes(objs: Optional[list[RAGObject]] = None) -> list[ObjectNode]:+ """Converts a list of RAGObjects to a list of ObjectNodes."""++ return [ObjectNode(text=obj.rag_key(), metadata=ObjectNode.get_obj_metadata(obj)) for obj in objs]++ @classmethod+ def _from_nodes(+ cls,+ nodes: list[BaseNode],+ transformations: Optional[list[TransformComponent]] = None,+ embed_model: BaseEmbedding = None,+ llm: LLM = None,+ retriever_configs: list[BaseRetrieverConfig] = None,+ ranker_configs: list[BaseRankerConfig] = None,+ ) -> "SimpleEngine":+ embed_model = cls._resolve_embed_model(embed_model, retriever_configs)+ llm = llm or get_rag_llm()++ retriever = get_retriever(configs=retriever_configs, nodes=nodes, embed_model=embed_model)+ rankers = get_rankers(configs=ranker_configs, llm=llm) # Default []++ return cls(+ retriever=retriever,+ node_postprocessors=rankers,+ response_synthesizer=get_response_synthesizer(llm=llm),+ transformations=transformations,+ )++ @classmethod+ def _from_nodes(+ cls,+ nodes: list[BaseNode],+ transformations: Optional[list[TransformComponent]] = None,+ embed_model: BaseEmbedding = None,+ llm: LLM = None,+ retriever_configs: list[BaseRetrieverConfig] = None,+ ranker_configs: list[BaseRankerConfig] = None,+ ) -> "SimpleEngine":+ embed_model = cls._resolve_embed_model(embed_model, retriever_configs)+ llm = llm or get_rag_llm()++ retriever = get_retriever(configs=retriever_configs, nodes=nodes, embed_model=embed_model)+ rankers = get_rankers(configs=ranker_configs, llm=llm) # Default []++ return cls(+ retriever=retriever,+ node_postprocessors=rankers,+ response_synthesizer=get_response_synthesizer(llm=llm),+ transformations=transformations,+ )+@classmethoddef _from_index(cls,@@@ -354,6 -311,22 +386,26 @@@def _default_transformations():return [SentenceSplitter()]+ @property+ def filenames(self) -> Set[str]:+ return self._filenames+++ @staticmethod+ def _get_file_extractor() -> dict[str:BaseReader]:+ """+ Get the file extractor.+ Currently, only PDF use OmniParse. Other document types use the built-in reader from llama_index.++ Returns:+ dict[file_type: BaseReader]+ """+ file_extractor: dict[str:BaseReader] = {}+ if config.omniparse.base_url:+ pdf_parser = OmniParse(+ api_key=config.omniparse.api_key,+ base_url=config.omniparse.base_url,+ parse_options=OmniParseOptions(parse_type=OmniParseType.PDF, result_type=ParseResultType.MD),+ )+ file_extractor[".pdf"] = pdf_parser++ return file_extractordiff --cc metagpt/rag/factories/embedding.pyindex 19b8b36f6,3613fd228..6212aae25--- a/metagpt/rag/factories/embedding.py+++ b/metagpt/rag/factories/embedding.py@@@ -5,8 -5,11 +5,11 @@@ from typing import Any, Optionafrom llama_index.core.embeddings import BaseEmbeddingfrom llama_index.embeddings.azure_openai import AzureOpenAIEmbedding+ from llama_index.embeddings.gemini import GeminiEmbedding+ from llama_index.embeddings.ollama import OllamaEmbedding+ from llama_index.embeddings.openai import OpenAIEmbedding-from metagpt.config2 import config+from metagpt.config2 import Configfrom metagpt.configs.embedding_config import EmbeddingTypefrom metagpt.configs.llm_config import LLMTypefrom metagpt.rag.factories.base import GenericFactorydiff --cc metagpt/rag/schema.pyindex 4180536a3,1e04a546f..12e635a1a--- a/metagpt/rag/schema.py+++ b/metagpt/rag/schema.py@@@ -70,6 -62,36 +70,37 @@@ class BM25RetrieverConfig(IndexRetrieve_no_embedding: bool = PrivateAttr(default=True)+ class MilvusRetrieverConfig(IndexRetrieverConfig):+ """Config for Milvus-based retrievers."""++ uri: str = Field(default="./milvus_local.db", description="The directory to save data.")+ collection_name: str = Field(default="metagpt", description="The name of the collection.")+ token: str = Field(default=None, description="The token for Milvus")+ metadata: Optional[CollectionMetadata] = Field(+ default=None, description="Optional metadata to associate with the collection"+ )+ dimensions: int = Field(default=0, description="Dimensionality of the vectors for Milvus index construction.")++ _embedding_type_to_dimensions: ClassVar[dict[EmbeddingType, int]] = {+ EmbeddingType.GEMINI: 768,+ EmbeddingType.OLLAMA: 4096,+ }++ @model_validator(mode="after")+ def check_dimensions(self):+ if self.dimensions == 0:++ config = Config.default()+ self.dimensions = config.embedding.dimensions or self._embedding_type_to_dimensions.get(+ config.embedding.api_type, 1536+ )+ if not config.embedding.dimensions and config.embedding.api_type not in self._embedding_type_to_dimensions:+ logger.warning(+ f"You didn't set dimensions in config when using {config.embedding.api_type}, default to 1536"+ )++ return self++class ChromaRetrieverConfig(IndexRetrieverConfig):"""Config for Chroma-based retrievers."""diff --cc metagpt/utils/__init__.pyindex 26042eb0e,74652b74b..be911f9dd--- a/metagpt/utils/__init__.py+++ b/metagpt/utils/__init__.py@@@ -10,8 -10,8 +10,8 @@@ from metagpt.utils.read_document imporfrom metagpt.utils.singleton import Singletonfrom metagpt.utils.token_counter import (TOKEN_COSTS,- count_input_tokens,+ count_message_tokens,- count_string_tokens,+ count_output_tokens,)@@@ -19,7 -19,6 +19,8 @@@ __all__ ="read_docx","Singleton","TOKEN_COSTS",- "count_input_tokens",+ "new_transaction_id",+ "count_message_tokens",+ "count_string_tokens",+ "count_output_tokens",]diff --cc metagpt/utils/common.pyindex 90f13da23,47f2768cd..cf2aa58ba--- a/metagpt/utils/common.py+++ b/metagpt/utils/common.py@@@ -603,12 -577,34 +603,36 @@@ def write_json_file(json_file: str, datif not folder_path.exists():folder_path.mkdir(parents=True, exist_ok=True)+ custom_default = partial(to_jsonable_python, fallback=handle_unknown_serialization if use_fallback else None)+with open(json_file, "w", encoding=encoding) as fout:- json.dump(data, fout, ensure_ascii=False, indent=indent, default=to_jsonable_python)+ json.dump(data, fout, ensure_ascii=False, indent=indent, default=custom_default)+ def read_jsonl_file(jsonl_file: str, encoding="utf-8") -> list[dict]:+ if not Path(jsonl_file).exists():+ raise FileNotFoundError(f"json_file: {jsonl_file} not exist, return []")+ datas = []+ with open(jsonl_file, "r", encoding=encoding) as fin:+ try:+ for line in fin:+ data = json.loads(line)+ datas.append(data)+ except Exception:+ raise ValueError(f"read jsonl file: {jsonl_file} failed")+ return datas+++ def add_jsonl_file(jsonl_file: str, data: list[dict], encoding: str = None):+ folder_path = Path(jsonl_file).parent+ if not folder_path.exists():+ folder_path.mkdir(parents=True, exist_ok=True)++ with open(jsonl_file, "a", encoding=encoding) as fout:+ for json_item in data:+ fout.write(json.dumps(json_item) + "\n")++def read_csv_to_list(curr_file: str, header=False, strip_trail=True):"""Reads in a csv file to a list of list. If header is True, it returns a@@@ -931,268 -867,19 +958,286 @@@ def get_markdown_codeblock_type(filenamreturn mappings.get(mime_type, "text")+def get_project_srcs_path(workdir: str | Path) -> Path:+ src_workdir_path = workdir / ".src_workspace"+ if src_workdir_path.exists():+ with open(src_workdir_path, "r") as file:+ src_name = file.read()+ else:+ src_name = Path(workdir).name+ return Path(workdir) / src_name+++async def init_python_folder(workdir: str | Path):+ if not workdir:+ return+ workdir = Path(workdir)+ if not workdir.exists():+ return+ init_filename = Path(workdir) / "__init__.py"+ if init_filename.exists():+ return+ async with aiofiles.open(init_filename, "a"):+ os.utime(init_filename, None)+++def get_markdown_code_block_type(filename: str) -> str:+ if not filename:+ return ""+ ext = Path(filename).suffix+ types = {+ ".py": "python",+ ".js": "javascript",+ ".java": "java",+ ".cpp": "cpp",+ ".c": "c",+ ".html": "html",+ ".css": "css",+ ".xml": "xml",+ ".json": "json",+ ".yaml": "yaml",+ ".md": "markdown",+ ".sql": "sql",+ ".rb": "ruby",+ ".php": "php",+ ".sh": "bash",+ ".swift": "swift",+ ".go": "go",+ ".rs": "rust",+ ".pl": "perl",+ ".asm": "assembly",+ ".r": "r",+ ".scss": "scss",+ ".sass": "sass",+ ".lua": "lua",+ ".ts": "typescript",+ ".tsx": "tsx",+ ".jsx": "jsx",+ ".yml": "yaml",+ ".ini": "ini",+ ".toml": "toml",+ ".svg": "xml", # SVG can often be treated as XML+ # Add more file extensions and corresponding code block types as needed+ }+ return types.get(ext, "")+++def to_markdown_code_block(val: str, type_: str = "") -> str:+ """+ Convert a string to a Markdown code block.++ This function takes a string and wraps it in a Markdown code block.+ If a type is provided, it adds it as a language identifier for syntax highlighting.++ Args:+ val (str): The string to be converted to a Markdown code block.+ type_ (str, optional): The language identifier for syntax highlighting.+ Defaults to an empty string.++ Returns:+ str: The input string wrapped in a Markdown code block.+ If the input string is empty, it returns an empty string.++ Examples:+ >>> to_markdown_code_block("print('Hello, World!')", "python")+ \n```python\nprint('Hello, World!')\n```\n++ >>> to_markdown_code_block("Some text")+ \n```\nSome text\n```\n+ """+ if not val:+ return val or ""+ val = val.replace("```", "\\`\\`\\`")+ return f"\n```{type_}\n{val}\n```\n"+++async def save_json_to_markdown(content: str, output_filename: str | Path):+ """+ Saves the provided JSON content as a Markdown file.++ This function takes a JSON string, converts it to Markdown format,+ and writes it to the specified output file.++ Args:+ content (str): The JSON content to be converted.+ output_filename (str or Path): The path where the output Markdown file will be saved.++ Returns:+ None++ Raises:+ None: Any exceptions are logged and the function returns without raising them.++ Examples:+ >>> await save_json_to_markdown('{"key": "value"}', Path("/path/to/output.md"))+ This will save the Markdown converted JSON to the specified file.++ Notes:+ - This function handles `json.JSONDecodeError` specifically for JSON parsing errors.+ - Any other exceptions during the process are also logged and handled gracefully.+ """+ try:+ m = json.loads(content)+ except json.JSONDecodeError as e:+ logger.warning(f"Failed to decode JSON content: {e}")+ return+ except Exception as e:+ logger.warning(f"An unexpected error occurred: {e}")+ return+ await awrite(filename=output_filename, data=json_to_markdown(m))+++def tool2name(cls, methods: List[str], entry) -> Dict[str, Any]:+ """+ Generates a mapping of class methods to a given entry with class name as a prefix.++ Args:+ cls: The class from which the methods are derived.+ methods (List[str]): A list of method names as strings.+ entry (Any): The entry to be mapped to each method.++ Returns:+ Dict[str, Any]: A dictionary where keys are method names prefixed with the class name and+ values are the given entry. If the number of methods is less than 2,+ the dictionary will contain a single entry with the class name as the key.++ Example:+ >>> class MyClass:+ >>> pass+ >>>+ >>> tool2name(MyClass, ['method1', 'method2'], 'some_entry')+ {'MyClass.method1': 'some_entry', 'MyClass.method2': 'some_entry'}++ >>> tool2name(MyClass, ['method1'], 'some_entry')+ {'MyClass': 'some_entry', 'MyClass.method1': 'some_entry'}+ """+ class_name = cls.__name__+ mappings = {f"{class_name}.{i}": entry for i in methods}+ if len(mappings) < 2:+ mappings[class_name] = entry+ return mappings+++def new_transaction_id(postfix_len=8) -> str:+ """+ Generates a new unique transaction ID based on current timestamp and a random UUID.++ Args:+ postfix_len (int): Length of the random UUID postfix to include in the transaction ID. Default is 8.++ Returns:+ str: A unique transaction ID composed of timestamp and a random UUID.+ """+ return datetime.now().strftime("%Y%m%d%H%M%ST") + uuid.uuid4().hex[0:postfix_len]+++def log_time(method):+ """A time-consuming decorator for printing execution duration."""++ def before_call():+ start_time, cpu_start_time = time.perf_counter(), time.process_time()+ logger.info(f"[{method.__name__}] started at: " f"{datetime.now().strftime('%Y-%m-%d %H:%m:%S')}")+ return start_time, cpu_start_time++ def after_call(start_time, cpu_start_time):+ end_time, cpu_end_time = time.perf_counter(), time.process_time()+ logger.info(+ f"[{method.__name__}] ended. "+ f"Time elapsed: {end_time - start_time:.4} sec, CPU elapsed: {cpu_end_time - cpu_start_time:.4} sec"+ )++ @functools.wraps(method)+ def timeit_wrapper(*args, **kwargs):+ start_time, cpu_start_time = before_call()+ result = method(*args, **kwargs)+ after_call(start_time, cpu_start_time)+ return result++ @functools.wraps(method)+ async def timeit_wrapper_async(*args, **kwargs):+ start_time, cpu_start_time = before_call()+ result = await method(*args, **kwargs)+ after_call(start_time, cpu_start_time)+ return result++ return timeit_wrapper_async if iscoroutinefunction(method) else timeit_wrapper+++async def check_http_endpoint(url: str, timeout: int = 3) -> bool:+ """+ Checks the status of an HTTP endpoint.++ Args:+ url (str): The URL of the HTTP endpoint to check.+ timeout (int, optional): The timeout in seconds for the HTTP request. Defaults to 3.++ Returns:+ bool: True if the endpoint is online and responding with a 200 status code, False otherwise.+ """+ async with aiohttp.ClientSession() as session:+ try:+ async with session.get(url, timeout=timeout) as response:+ return response.status == 200+ except Exception as e:+ print(f"Error accessing the endpoint {url}: {e}")+ return False+++def rectify_pathname(path: Union[str, Path], default_filename: str) -> Path:+ """+ Rectifies the given path to ensure a valid output file path.++ If the given `path` is a directory, it creates the directory (if it doesn't exist) and appends the `default_filename` to it. If the `path` is a file path, it creates the parent directory (if it doesn't exist) and returns the `path`.++ Args:+ path (Union[str, Path]): The input path, which can be a string or a `Path` object.+ default_filename (str): The default filename to use if the `path` is a directory.++ Returns:+ Path: The rectified output path.+ """+ output_pathname = Path(path)+ if output_pathname.is_dir():+ output_pathname.mkdir(parents=True, exist_ok=True)+ output_pathname = output_pathname / default_filename+ else:+ output_pathname.parent.mkdir(parents=True, exist_ok=True)+ return output_pathname+++def generate_fingerprint(text: str) -> str:+ """+ Generate a fingerprint for the given text++ Args:+ text (str): The text for which the fingerprint needs to be generated++ Returns:+ str: The fingerprint value of the text+ """+ text_bytes = text.encode("utf-8")++ # calculate SHA-256 hash+ sha256 = hashlib.sha256()+ sha256.update(text_bytes)+ fingerprint = sha256.hexdigest()++ return fingerprint+++++ def download_model(file_url: str, target_folder: Path) -> Path:+ file_name = file_url.split("/")[-1]+ file_path = target_folder.joinpath(f"{file_name}")+ if not file_path.exists():+ file_path.mkdir(parents=True, exist_ok=True)+ try:+ response = requests.get(file_url, stream=True)+ response.raise_for_status() # 检查请求是否成功+ # 保存文件+ with open(file_path, "wb") as f:+ for chunk in response.iter_content(chunk_size=8192):+ f.write(chunk)+ logger.info(f"权重文件已下载并保存至 {file_path}")+ except requests.exceptions.HTTPError as err:+ logger.info(f"权重文件下载过程中发生错误: {err}")+ return file_pathdiff --cc metagpt/utils/file.pyindex a3f612bcc,f62b44eb8..75107c8be--- a/metagpt/utils/file.py+++ b/metagpt/utils/file.py@@@ -75,128 -68,3 +75,128 @@@ class Filecontent = b"".join(chunks)logger.debug(f"Successfully read file, the path of file: {file_path}")return content++ @staticmethod+ async def is_textual_file(filename: Union[str, Path]) -> bool:+ """Determines if a given file is a textual file.++ A file is considered a textual file if it is plain text or has a+ specific set of MIME types associated with textual formats,+ including PDF and Microsoft Word documents.++ Args:+ filename (Union[str, Path]): The path to the file to be checked.++ Returns:+ bool: True if the file is a textual file, False otherwise.+ """+ is_text, mime_type = await is_text_file(filename)+ if is_text:+ return True+ if mime_type == "application/pdf":+ return True+ if mime_type in {+ "application/msword",+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",+ "application/vnd.ms-word.document.macroEnabled.12",+ "application/vnd.openxmlformats-officedocument.wordprocessingml.template",+ "application/vnd.ms-word.template.macroEnabled.12",+ }:+ return True+ return False++ @staticmethod+ async def read_text_file(filename: Union[str, Path]) -> Optional[str]:+ """Read the whole content of a file. Using absolute paths as the argument for specifying the file location."""+ is_text, mime_type = await is_text_file(filename)+ if is_text:+ return await File._read_text(filename)+ if mime_type == "application/pdf":+ return await File._read_pdf(filename)+ if mime_type in {+ "application/msword",+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",+ "application/vnd.ms-word.document.macroEnabled.12",+ "application/vnd.openxmlformats-officedocument.wordprocessingml.template",+ "application/vnd.ms-word.template.macroEnabled.12",+ }:+ return await File._read_docx(filename)+ return None++ @staticmethod+ async def _read_text(path: Union[str, Path]) -> str:+ return await aread(path)++ @staticmethod+ async def _read_pdf(path: Union[str, Path]) -> str:+ result = await File._omniparse_read_file(path)+ if result:+ return result++ from llama_index.readers.file import PDFReader++ reader = PDFReader()+ lines = reader.load_data(file=Path(path))+ return "\n".join([i.text for i in lines])++ @staticmethod+ async def _read_docx(path: Union[str, Path]) -> str:+ result = await File._omniparse_read_file(path)+ if result:+ return result+ return "\n".join(read_docx(str(path)))++ @staticmethod+ async def _omniparse_read_file(path: Union[str, Path], auto_save_image: bool = False) -> Optional[str]:+ from metagpt.tools.libs import get_env_default+ from metagpt.utils.omniparse_client import OmniParseClient++ env_base_url = await get_env_default(key="base_url", app_name="OmniParse", default_value="")+ env_timeout = await get_env_default(key="timeout", app_name="OmniParse", default_value="")+ conf_base_url, conf_timeout = await File._read_omniparse_config()++ base_url = env_base_url or conf_base_url+ if not base_url:+ return None+ api_key = await get_env_default(key="api_key", app_name="OmniParse", default_value="")+ timeout = env_timeout or conf_timeout or 600+ try:+ timeout = int(timeout)+ except ValueError:+ timeout = 600++ try:+ if not await check_http_endpoint(url=base_url):+ logger.warning(f"{base_url}: NOT AVAILABLE")+ return None+ client = OmniParseClient(api_key=api_key, base_url=base_url, max_timeout=timeout)+ file_data = await aread_bin(filename=path)+ ret = await client.parse_document(file_input=file_data, bytes_filename=str(path))+ except (ValueError, Exception) as e:+ logger.exception(f"{path}: {e}")+ return None+ if not ret.images or not auto_save_image:+ return ret.text++ result = [ret.text]+ img_dir = Path(path).parent / (Path(path).name.replace(".", "_") + "_images")+ img_dir.mkdir(parents=True, exist_ok=True)+ for i in ret.images:+ byte_data = base64.b64decode(i.image)+ filename = img_dir / i.image_name+ await awrite_bin(filename=filename, data=byte_data)+ result.append(f"})")+ return "\n".join(result)++ @staticmethod+ async def _read_omniparse_config() -> Tuple[str, int]:+ config = Config.default()- if config.omniparse and config.omniparse.url:- return config.omniparse.url, config.omniparse.timeout++ if config.omniparse and config.omniparse.base_url:++ return config.omniparse.base_url, config.omniparse.timeout+ return "", 0+++class MemoryFileSystem(_MemoryFileSystem):+ @classmethod+ def _strip_protocol(cls, path):+ return super()._strip_protocol(str(path))diff --cc metagpt/utils/token_counter.pyindex b235ceb7b,1bc6b597d..f710a0e06--- a/metagpt/utils/token_counter.py+++ b/metagpt/utils/token_counter.py@@@ -10,10 -10,15 +10,13 @@@ ref3: https://github.com/Significant-Grref4: https://github.com/hwchase17/langchain/blob/master/langchain/chat_models/openai.pyref5: https://ai.google.dev/models/gemini"""+ import anthropicimport tiktoken-from openai.types import CompletionUsage-from openai.types.chat import ChatCompletionChunk+ from metagpt.logs import logger-from metagpt.utils.ahttp_client import apost+TOKEN_COSTS = {+ "anthropic/claude-3.5-sonnet": {"prompt": 0.003, "completion": 0.015},"gpt-3.5-turbo": {"prompt": 0.0015, "completion": 0.002},"gpt-3.5-turbo-0301": {"prompt": 0.0015, "completion": 0.002},"gpt-3.5-turbo-0613": {"prompt": 0.0015, "completion": 0.002},@@@ -53,21 -69,35 +67,43 @@@"claude-2.0": {"prompt": 0.008, "completion": 0.024},"claude-2.1": {"prompt": 0.008, "completion": 0.024},"claude-3-sonnet-20240229": {"prompt": 0.003, "completion": 0.015},+ "claude-3-5-sonnet": {"prompt": 0.003, "completion": 0.015},+ "claude-3-5-sonnet-v2": {"prompt": 0.003, "completion": 0.015}, # alias of newer 3.5 sonnet+ "claude-3-5-sonnet-20240620": {"prompt": 0.003, "completion": 0.015},"claude-3-opus-20240229": {"prompt": 0.015, "completion": 0.075},+ "claude-3-haiku-20240307": {"prompt": 0.00025, "completion": 0.00125},"yi-34b-chat-0205": {"prompt": 0.0003, "completion": 0.0003},"yi-34b-chat-200k": {"prompt": 0.0017, "completion": 0.0017},+ "openai/gpt-4": {"prompt": 0.03, "completion": 0.06}, # start, for openrouter+ "openai/gpt-4-turbo": {"prompt": 0.01, "completion": 0.03},+ "openai/gpt-4o": {"prompt": 0.005, "completion": 0.015},+ "openai/gpt-4o-2024-05-13": {"prompt": 0.005, "completion": 0.015},+ "openai/gpt-4o-mini": {"prompt": 0.00015, "completion": 0.0006},+ "openai/gpt-4o-mini-2024-07-18": {"prompt": 0.00015, "completion": 0.0006},- "google/gemini-pro-1.5": {"prompt": 0.0025, "completion": 0.0075},+ "google/gemini-flash-1.5": {"prompt": 0.00025, "completion": 0.00075},+ "deepseek/deepseek-coder": {"prompt": 0.00014, "completion": 0.00028},+ "deepseek/deepseek-chat": {"prompt": 0.00014, "completion": 0.00028}, # end, for openrouter+ "yi-large": {"prompt": 0.0028, "completion": 0.0028},+ "microsoft/wizardlm-2-8x22b": {"prompt": 0.00108, "completion": 0.00108}, # for openrouter, start+ "meta-llama/llama-3-70b-instruct": {"prompt": 0.008, "completion": 0.008},+ "llama3-70b-8192": {"prompt": 0.0059, "completion": 0.0079},+ "openai/gpt-3.5-turbo-0125": {"prompt": 0.0005, "completion": 0.0015},+ "openai/gpt-4-turbo-preview": {"prompt": 0.01, "completion": 0.03},+ "openai/o1-preview": {"prompt": 0.015, "completion": 0.06},+ "openai/o1-mini": {"prompt": 0.003, "completion": 0.012},+ "anthropic/claude-3-opus": {"prompt": 0.015, "completion": 0.075},- "anthropic/claude-3.5-sonnet": {"prompt": 0.003, "completion": 0.015},+ "google/gemini-pro-1.5": {"prompt": 0.0025, "completion": 0.0075}, # for openrouter, end"deepseek-chat": {"prompt": 0.00014, "completion": 0.00028},"deepseek-coder": {"prompt": 0.00014, "completion": 0.00028},+ # For ark model https://www.volcengine.com/docs/82379/1099320+ "doubao-lite-4k-240515": {"prompt": 0.000043, "completion": 0.000086},+ "doubao-lite-32k-240515": {"prompt": 0.000043, "completion": 0.000086},+ "doubao-lite-128k-240515": {"prompt": 0.00011, "completion": 0.00014},+ "doubao-pro-4k-240515": {"prompt": 0.00011, "completion": 0.00029},+ "doubao-pro-32k-240515": {"prompt": 0.00011, "completion": 0.00029},+ "doubao-pro-128k-240515": {"prompt": 0.0007, "completion": 0.0013},+ "llama3-70b-llama3-70b-instruct": {"prompt": 0.0, "completion": 0.0},+ "llama3-8b-llama3-8b-instruct": {"prompt": 0.0, "completion": 0.0},}@@@ -200,27 -272,124 +278,134 @@@ TOKEN_MAX ="claude-2.1": 200000,"claude-3-sonnet-20240229": 200000,"claude-3-opus-20240229": 200000,+ "claude-3-5-sonnet-20240620": 200000,+ "claude-3-haiku-20240307": 200000,"yi-34b-chat-0205": 4000,"yi-34b-chat-200k": 200000,+ "openai/gpt-4": 8192, # start, for openrouter+ "openai/gpt-4-turbo": 128000,+ "openai/gpt-4o": 128000,+ "openai/gpt-4o-2024-05-13": 128000,+ "openai/gpt-4o-mini": 128000,+ "openai/gpt-4o-mini-2024-07-18": 128000,- "anthropic/claude-3.5-sonnet": 200000,- "google/gemini-pro-1.5": 2800000,+ "google/gemini-flash-1.5": 2800000,+ "deepseek/deepseek-coder": 128000,+ "deepseek/deepseek-chat": 128000, # end, for openrouter+ "deepseek-chat": 128000,+ "deepseek-coder": 128000,+ "deepseek-ai/DeepSeek-Coder-V2-Instruct": 32000, # siliconflow+ "yi-large": 16385,+ "microsoft/wizardlm-2-8x22b": 65536,+ "meta-llama/llama-3-70b-instruct": 8192,+ "llama3-70b-8192": 8192,+ "openai/gpt-3.5-turbo-0125": 16385,+ "openai/gpt-4-turbo-preview": 128000,+ "openai/o1-preview": 128000,+ "openai/o1-mini": 128000,+ "anthropic/claude-3-opus": 200000,+ "anthropic/claude-3.5-sonnet": 200000,+ "google/gemini-pro-1.5": 4000000,- "deepseek-chat": 32768,- "deepseek-coder": 16385,+ "doubao-lite-4k-240515": 4000,+ "doubao-lite-32k-240515": 32000,+ "doubao-lite-128k-240515": 128000,+ "doubao-pro-4k-240515": 4000,+ "doubao-pro-32k-240515": 32000,+ "doubao-pro-128k-240515": 128000,+ # Qwen https://help.aliyun.com/zh/dashscope/developer-reference/tongyi-qianwen-7b-14b-72b-api-detailes?spm=a2c4g.11186623.0.i20+ "qwen2.5-72b-instruct": 131072,+ "qwen2.5-32b-instruct": 131072,+ "qwen2.5-14b-instruct": 131072,+ "qwen2.5-7b-instruct": 131072,+ "qwen2.5-3b-instruct": 32768,+ "qwen2.5-1.5b-instruct": 32768,+ "qwen2.5-0.5b-instruct": 32768,+ "qwen2-57b-a14b-instruct": 32768,+ "qwen2-72b-instruct": 131072,+ "qwen2-7b-instruct": 32768,+ "qwen2-1.5b-instruct": 32768,+ "qwen2-0.5b-instruct": 32768,+ "qwen1.5-110b-chat": 32000,+ "qwen1.5-72b-chat": 32000,+ "qwen1.5-32b-chat": 32000,+ "qwen1.5-14b-chat": 8000,+ "qwen1.5-7b-chat": 32000,+ "qwen1.5-1.8b-chat": 32000,+ "qwen1.5-0.5b-chat": 32000,+ "codeqwen1.5-7b-chat": 64000,+ "qwen-72b-chat": 32000,+ "qwen-14b-chat": 8000,+ "qwen-7b-chat": 32000,+ "qwen-1.8b-longcontext-chat": 32000,+ "qwen-1.8b-chat": 8000,+ }++ # For Amazon Bedrock US region+ # See https://aws.amazon.com/cn/bedrock/pricing/++ BEDROCK_TOKEN_COSTS = {+ "amazon.titan-tg1-large": {"prompt": 0.0008, "completion": 0.0008},+ "amazon.titan-text-express-v1": {"prompt": 0.0008, "completion": 0.0008},+ "amazon.titan-text-express-v1:0:8k": {"prompt": 0.0008, "completion": 0.0008},+ "amazon.titan-text-lite-v1:0:4k": {"prompt": 0.0003, "completion": 0.0004},+ "amazon.titan-text-lite-v1": {"prompt": 0.0003, "completion": 0.0004},+ "anthropic.claude-instant-v1": {"prompt": 0.0008, "completion": 0.00024},+ "anthropic.claude-instant-v1:2:100k": {"prompt": 0.0008, "completion": 0.00024},+ "anthropic.claude-v1": {"prompt": 0.008, "completion": 0.0024},+ "anthropic.claude-v2": {"prompt": 0.008, "completion": 0.0024},+ "anthropic.claude-v2:1": {"prompt": 0.008, "completion": 0.0024},+ "anthropic.claude-v2:0:18k": {"prompt": 0.008, "completion": 0.0024},+ "anthropic.claude-v2:1:200k": {"prompt": 0.008, "completion": 0.0024},+ "anthropic.claude-3-sonnet-20240229-v1:0": {"prompt": 0.003, "completion": 0.015},+ "anthropic.claude-3-sonnet-20240229-v1:0:28k": {"prompt": 0.003, "completion": 0.015},+ "anthropic.claude-3-sonnet-20240229-v1:0:200k": {"prompt": 0.003, "completion": 0.015},+ "anthropic.claude-3-5-sonnet-20240620-v1:0": {"prompt": 0.003, "completion": 0.015},+ "anthropic.claude-3-haiku-20240307-v1:0": {"prompt": 0.00025, "completion": 0.00125},+ "anthropic.claude-3-haiku-20240307-v1:0:48k": {"prompt": 0.00025, "completion": 0.00125},+ "anthropic.claude-3-haiku-20240307-v1:0:200k": {"prompt": 0.00025, "completion": 0.00125},+ # currently (2024-4-29) only available at US West (Oregon) AWS Region.+ "anthropic.claude-3-opus-20240229-v1:0": {"prompt": 0.015, "completion": 0.075},+ "cohere.command-text-v14": {"prompt": 0.0015, "completion": 0.0015},+ "cohere.command-text-v14:7:4k": {"prompt": 0.0015, "completion": 0.0015},+ "cohere.command-light-text-v14": {"prompt": 0.0003, "completion": 0.0003},+ "cohere.command-light-text-v14:7:4k": {"prompt": 0.0003, "completion": 0.0003},+ "meta.llama2-13b-chat-v1:0:4k": {"prompt": 0.00075, "completion": 0.001},+ "meta.llama2-13b-chat-v1": {"prompt": 0.00075, "completion": 0.001},+ "meta.llama2-70b-v1": {"prompt": 0.00195, "completion": 0.00256},+ "meta.llama2-70b-v1:0:4k": {"prompt": 0.00195, "completion": 0.00256},+ "meta.llama2-70b-chat-v1": {"prompt": 0.00195, "completion": 0.00256},+ "meta.llama2-70b-chat-v1:0:4k": {"prompt": 0.00195, "completion": 0.00256},+ "meta.llama3-8b-instruct-v1:0": {"prompt": 0.0004, "completion": 0.0006},+ "meta.llama3-70b-instruct-v1:0": {"prompt": 0.00265, "completion": 0.0035},+ "mistral.mistral-7b-instruct-v0:2": {"prompt": 0.00015, "completion": 0.0002},+ "mistral.mixtral-8x7b-instruct-v0:1": {"prompt": 0.00045, "completion": 0.0007},+ "mistral.mistral-large-2402-v1:0": {"prompt": 0.008, "completion": 0.024},+ "ai21.j2-grande-instruct": {"prompt": 0.0125, "completion": 0.0125},+ "ai21.j2-jumbo-instruct": {"prompt": 0.0188, "completion": 0.0188},+ "ai21.j2-mid": {"prompt": 0.0125, "completion": 0.0125},+ "ai21.j2-mid-v1": {"prompt": 0.0125, "completion": 0.0125},+ "ai21.j2-ultra": {"prompt": 0.0188, "completion": 0.0188},+ "ai21.j2-ultra-v1": {"prompt": 0.0188, "completion": 0.0188},+ }++ # https://xinghuo.xfyun.cn/sparkapi?scr=price+ SPARK_TOKENS = {+ "general": {"prompt": 0.0, "completion": 0.0}, # Spark-Lite+ "generalv2": {"prompt": 0.0188, "completion": 0.0188}, # Spark V2.0+ "generalv3": {"prompt": 0.0035, "completion": 0.0035}, # Spark Pro+ "generalv3.5": {"prompt": 0.0035, "completion": 0.0035}, # Spark3.5 Max}-def count_input_tokens(messages, model="gpt-3.5-turbo-0125"):+def count_message_tokens(messages, model="gpt-3.5-turbo-0125"):"""Return the number of tokens used by a list of messages."""+ if "claude" in model:+ # rough estimation for models newer than claude-2.1+ vo = anthropic.Client()+ num_tokens = 0+ for message in messages:+ for key, value in message.items():+ num_tokens += vo.count_tokens(str(value))+ return num_tokenstry:encoding = tiktoken.encoding_for_model(model)except KeyError:@@@ -254,11 -430,11 +446,11 @@@tokens_per_message = 4 # every message follows <|start|>{role/name}\n{content}<|end|>\ntokens_per_name = -1 # if there's a name, the role is omittedelif "gpt-3.5-turbo" == model:- print("Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0125.")+ logger.info("Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0125.")- return count_input_tokens(messages, model="gpt-3.5-turbo-0125")+ return count_message_tokens(messages, model="gpt-3.5-turbo-0125")elif "gpt-4" == model:- print("Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.")+ logger.info("Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.")- return count_input_tokens(messages, model="gpt-4-0613")+ return count_message_tokens(messages, model="gpt-4-0613")elif "open-llm-model" == model:"""For self-hosted open_llm api, they include lots of different models. The message tokens calculation isdiff --cc requirements.txtindex 5aa404e7e,51b3d786b..67aabf5dc--- a/requirements.txt+++ b/requirements.txt@@@ -66,15 -71,17 +71,21 @@@ anytreipywidgets==8.1.1Pillowimap_tools==1.5.0 # Used by metagpt/tools/libs/email_login.py- qianfan==0.3.2- dashscope==1.14.1- rank-bm25==0.2.2 # for tool recommendation- gymnasium==0.29.1+pylint~=3.0.3+pygithub~=2.3+htmlmin+fsspec+grep-ast~=0.3.3 # linter- tree-sitter~=0.21.3 # linter+unidiff==0.7.5 # used at metagpt/tools/libs/cr.py+ qianfan~=0.4.4+ dashscope~=1.19.3+ rank-bm25==0.2.2 # for tool recommendation+ jieba==0.42.1 # for tool recommendation+ volcengine-python-sdk[ark]~=1.0.94 # Solution for installation error in Windows: https://github.com/volcengine/volcengine-python-sdk/issues/5-# llama-index-vector-stores-elasticsearch~=0.2.5 # Used by `metagpt/memory/longterm_memory.py`-# llama-index-vector-stores-chroma~=0.1.10 # Used by `metagpt/memory/longterm_memory.py`+ gymnasium==0.29.1+ boto3~=1.34.69+ spark_ai_python~=0.3.30+ agentops+ tree_sitter~=0.23.2+ tree_sitter_python~=0.23.2httpx==0.27.2diff --cc tests/metagpt/rag/engines/test_simple.pyindex e0a174ed2,a10fcbe63..877a789a5--- a/tests/metagpt/rag/engines/test_simple.py+++ b/tests/metagpt/rag/engines/test_simple.py@@@ -75,7 -83,9 +83,9 @@@ class TestSimpleEngine)# Assert- mock_simple_directory_reader.assert_called_once_with(input_dir=input_dir, input_files=input_files, fs=None)+ mock_simple_directory_reader.assert_called_once_with(- input_dir=input_dir, input_files=input_files, file_extractor=file_extractor++ input_dir=input_dir, input_files=input_files, file_extractor=file_extractor, fs=None+ )mock_get_retriever.assert_called_once()mock_get_rankers.assert_called_once()mock_get_response_synthesizer.assert_called_once_with(llm=llm)diff --cc tests/metagpt/roles/test_product_manager.pyindex 2ec6c9baa,143eef2f2..77df9af40--- a/tests/metagpt/roles/test_product_manager.py+++ b/tests/metagpt/roles/test_product_manager.py@@@ -14,8 -15,7 +14,8 @@@ from metagpt.context import Contexfrom metagpt.logs import loggerfrom metagpt.roles import ProductManagerfrom metagpt.utils.common import any_to_str- from tests.metagpt.roles.mock import MockMessages+from metagpt.utils.git_repository import GitRepository+ from tests.metagpt.roles.mock import MockMessages@pytest.mark.asynciodiff --cc tests/metagpt/utils/test_token_counter.pyindex acb99d717,5c8d5e15f..b6eefd61d--- a/tests/metagpt/utils/test_token_counter.py+++ b/tests/metagpt/utils/test_token_counter.py@@@ -7,7 -7,7 +7,7 @@@"""import pytest- from metagpt.utils.token_counter import count_message_tokens, count_string_tokens-from metagpt.utils.token_counter import count_input_tokens, count_output_tokens++from metagpt.utils.token_counter import count_message_tokens, count_output_tokensdef test_count_message_tokens():