🌟 The Multi-Agent Framework: First AI Software Company, Towards Natural Language Programming
feat: 优化代码
莘权 马d37954d43a63
diff --git a/metagpt/tools/libs/index_repo.py b/metagpt/tools/libs/index_repo.pyindex a5d9fc6aa..b853b38ac 100644--- a/metagpt/tools/libs/index_repo.py+++ b/metagpt/tools/libs/index_repo.py@@ -93,6 +93,12 @@ class IndexRepo(BaseModel):meta = await self._read_meta()new_files = {}for i in filenames:+ if Path(i).suffix.lower() in {".pdf", ".doc", ".docx"}:+ if str(i) not in self.fingerprints:+ new_files[i] = ""+ logger.warning(f'file: "{i}" not indexed')+ filter_filenames.add(str(i))+ continuecontent = await File.read_text_file(i)token_count = len(encoding.encode(content))if not self._is_buildable(@@ -101,9 +107,7 @@ class IndexRepo(BaseModel):result.append(TextScore(filename=str(i), text=content))continuefile_fingerprint = generate_fingerprint(content)- if str(i) not in self.fingerprints or (- self.fingerprints.get(str(i)) != file_fingerprint and Path(i).suffix.lower() not in {".pdf"}- ):+ if str(i) not in self.fingerprints or (self.fingerprints.get(str(i)) != file_fingerprint):new_files[i] = contentlogger.warning(f'file: "{i}" changed but not indexed')continue@@ -113,6 +117,7 @@ class IndexRepo(BaseModel):filter_filenames.update([str(i) for i in added])for i in others:result.append(TextScore(filename=str(i), text=new_files.get(i)))+ filter_filenames.discard(str(i))nodes = await self._search(query=query, filters=filter_filenames)return result + nodes@@ -142,7 +147,21 @@ class IndexRepo(BaseModel):scores = []query_embedding = await self.embedding.aget_text_embedding(query)for i in flat_nodes:- text_embedding = await self.embedding.aget_text_embedding(i.text)+ try:+ text_embedding = await self.embedding.aget_text_embedding(i.text)+ except Exception as e: # 超过最大长度+ tenth = int(len(i.text) / 10) # DEFAULT_MIN_TOKEN_COUNT = 10000+ logger.warning(+ f"{e}, tenth len={tenth}, pre_part_len={len(i.text[: tenth * 6])}, post_part_len={len(i.text[tenth * 4:])}"+ )+ pre_win_part = await self.embedding.aget_text_embedding(i.text[: tenth * 6])+ post_win_part = await self.embedding.aget_text_embedding(i.text[tenth * 4 :])+ similarity = max(+ self.embedding.similarity(query_embedding, pre_win_part),+ self.embedding.similarity(query_embedding, post_win_part),+ )+ scores.append((similarity, i))+ continuesimilarity = self.embedding.similarity(query_embedding, text_embedding)scores.append((similarity, i))scores.sort(key=lambda x: x[0], reverse=True)@@ -169,6 +188,7 @@ class IndexRepo(BaseModel):file_datas = file_datas or {}for i in filenames:content = file_datas.get(i) or await File.read_text_file(i)+ file_datas[i] = contentif not self._is_fingerprint_changed(filename=i, content=content):continuetoken_count = len(encoding.encode(content))