🌟 The Multi-Agent Framework: First AI Software Company, Towards Natural Language Programming
tested for embeddings/embed
EvensXiaf2aa4e3f9dc3
diff --git a/examples/llm_vision.py b/examples/llm_vision.pyindex eea6550f6..eff5c4d52 100644--- a/examples/llm_vision.py+++ b/examples/llm_vision.py@@ -14,9 +14,10 @@ async def main():# check if the configured llm supports llm-vision capacity. If not, it will throw a errorinvoice_path = Path(__file__).parent.joinpath("..", "tests", "data", "invoices", "invoice-2.png")- img_base64 = encode_image(invoice_path)- res = await llm.aask(msg="return `True` if this image might be a invoice, or return `False`", images=[img_base64])- assert ("true" in res.lower()) or ("invoice" in res.lower())+ encode_image(invoice_path)+ # res = await llm.aask(msg="return `True` if this image might be a invoice, or return `False`", images=[img_base64])+ await llm.aask(msg="hello")+ # assert ("true" in res.lower()) or ("invoice" in res.lower())if __name__ == "__main__":diff --git a/metagpt/configs/llm_config.py b/metagpt/configs/llm_config.pyindex 34b73d2d5..dbbf5f5d9 100644--- a/metagpt/configs/llm_config.py+++ b/metagpt/configs/llm_config.py@@ -28,7 +28,8 @@ class LLMType(Enum):AZURE = "azure"OLLAMA = "ollama" # /chat at ollama apiOLLAMA_GENERATE = "ollama.generate" # /generate at ollama api- OLLAMA_EMBEDDING = "ollama.embeddings" # /embeddings at ollama api+ OLLAMA_EMBEDDINGS = "ollama.embeddings" # /embeddings at ollama api+ OLLAMA_EMBED = "ollama.embed" # /embeddings at ollama apiQIANFAN = "qianfan" # Baidu BCEDASHSCOPE = "dashscope" # Aliyun LingJi DashScopeMOONSHOT = "moonshot"diff --git a/metagpt/provider/ollama_api.py b/metagpt/provider/ollama_api.pyindex 4fe6be0c2..6a2635b95 100644--- a/metagpt/provider/ollama_api.py+++ b/metagpt/provider/ollama_api.py@@ -20,6 +20,7 @@ class OllamaMessageAPI(Enum):CHAT = auto()GENERATE = auto()EMBED = auto()+ EMBEDDINGS = auto()class OllamaMessageBase:@@ -141,24 +142,50 @@ class OllamaMessageGenerate(OllamaMessageChat, metaclass=OllamaMessageMeta):return to_choice_dict["response"]-class OllamaMessageEmbed(OllamaMessageBase, metaclass=OllamaMessageMeta):- api_type = OllamaMessageAPI.EMBED+class OllamaMessageEmbeddings(OllamaMessageBase, metaclass=OllamaMessageMeta):+ api_type = OllamaMessageAPI.EMBEDDINGS@propertydef api_suffix(self) -> str:return "/embeddings"def apply(self, messages: list[dict]) -> dict:- prompts = []- for msg in messages:- prompt, _ = self._parse_input_msg(msg)- if prompt:- prompts.append(prompt)+ content = messages[0]["content"]+ prompts = [] # NOTE: not support image to embedding+ if isinstance(content, list):+ for msg in content:+ prompt, _ = self._parse_input_msg(msg)+ if prompt:+ prompts.append(prompt)+ else:+ prompts.append(content)sends = {"model": self.model, "prompt": "\n".join(prompts)}sends.update(self.additional_kwargs)return sends+class OllamaMessageEmbed(OllamaMessageEmbeddings, metaclass=OllamaMessageMeta):+ api_type = OllamaMessageAPI.EMBED++ @property+ def api_suffix(self) -> str:+ return "/embed"++ def apply(self, messages: list[dict]) -> dict:+ content = messages[0]["content"]+ prompts = [] # NOTE: not support image to embedding+ if isinstance(content, list):+ for msg in content:+ prompt, _ = self._parse_input_msg(msg)+ if prompt:+ prompts.append(prompt)+ else:+ prompts.append(content)+ sends = {"model": self.model, "input": prompts}+ sends.update(self.additional_kwargs)+ return sends++@register_provider(LLMType.OLLAMA)class OllamaLLM(BaseLLM):"""@@ -263,11 +290,11 @@ class OllamaGenerate(OllamaLLM):return {"options": {"temperature": 0.3}, "stream": self.config.stream}-@register_provider(LLMType.OLLAMA_EMBEDDING)+@register_provider(LLMType.OLLAMA_EMBEDDINGS)class OllamaEmbeddings(OllamaLLM):@propertydef _llama_api_inuse(self) -> OllamaMessageAPI:- return OllamaMessageAPI.EMBED+ return OllamaMessageAPI.EMBEDDINGS@propertydef _llama_api_kwargs(self) -> dict:@@ -277,7 +304,6 @@ class OllamaEmbeddings(OllamaLLM):resp, _, _ = await self.client.arequest(method=self.http_method,url=self.ollama_message.api_suffix,- headers=self.headers,params=self.ollama_message.apply(messages=messages),request_timeout=self.get_timeout(timeout),)@@ -288,3 +314,10 @@ class OllamaEmbeddings(OllamaLLM):def get_choice_text(self, rsp):return rsp+++@register_provider(LLMType.OLLAMA_EMBED)+class OllamaEmbed(OllamaLLM):+ @property+ def _llama_api_inuse(self) -> OllamaMessageAPI:+ return OllamaMessageAPI.EMBED