Open-source, secure environment with real-world tools for enterprise-grade agents.
Python agentaiai-agentai-agentscode-interpretercopilotdevelopmentdevtoolsgptgpt-4javascriptllmnextjsopenaipythonreactsoftwaretypescript
Add (repo) codebase embedding [WIP]
Tomas Valentaca33bcc50053
diff --git a/api-service/agent/basic_agent.py b/api-service/agent/basic_agent.pyindex 22b54c229..46d1ec458 100644--- a/api-service/agent/basic_agent.py+++ b/api-service/agent/basic_agent.py@@ -18,6 +18,7 @@ from langchain.prompts.chat import (ChatPromptTemplate,)+from .memory import get_memoryfrom agent.base import (AgentInteraction,OnLogs,@@ -142,6 +143,7 @@ class BasicAgent(AgentBase):] = [SystemMessage(content=system_template),HumanMessage(content=human_template),+ AIMessagePromptTemplate.from_template("{repo_context}"),AIMessagePromptTemplate.from_template("{agent_scratchpad}"),]return ChatPromptTemplate(@@ -176,14 +178,28 @@ class BasicAgent(AgentBase):tool.callback_manager = callback_manager# ------ # Create LLM from specified model+ # Create LLM from specified models+ prompt = self._create_prompt(self.config, tools, instructions)llm_chain = LLMChain(llm=get_model(self.config, callback_manager),- prompt=self._create_prompt(self.config, tools, instructions),+ prompt=prompt,callback_manager=callback_manager,verbose=True,)+ repo_files = await playground.get_files("/code", ["node_modules"])+ vectordb = await get_memory(self.config, repo_files)++ async def get_repo_context(prompt: str):+ docs = await vectordb.asimilarity_search(prompt, k=4)++ return "Relevant files in the codebase:\n\n" + "\n".join(+ [+ f"{doc.metadata['path']}\n```\n({doc.page_content})\n```\n"+ for doc in docs+ ]+ )+# -----# Create log handlers# Used for sending logs to db/frontend without blocking@@ -236,10 +252,15 @@ class BasicAgent(AgentBase):await self._check_interrupt()# Query the LLM in backgroundscratchpad = get_agent_scratchpad()+ repo_context = await get_repo_context(+ prompt=prompt.format_prompt(scratchpad=scratchpad).to_string(),+ )+self.llm_generation = asyncio.create_task(llm_chain.agenerate([{+ "repo_context": repo_context,"agent_scratchpad": scratchpad,"stop": "Observation:",}diff --git a/api-service/agent/memory.py b/api-service/agent/memory.pyindex df5aeb0a2..8c91df82f 100644--- a/api-service/agent/memory.py+++ b/api-service/agent/memory.py@@ -1,4 +1,25 @@+from typing import AsyncIterator, Callable, List, Tuplefrom langchain.embeddings.openai import OpenAIEmbeddingsfrom langchain.text_splitter import CharacterTextSplitter-from langchain.vectorstores import Chroma-from langchain.document_loaders import TextLoader+from langchain.vectorstores import Chroma, VectorStore++from models.base import ModelConfig+++async def get_memory(config: ModelConfig, files: List[Tuple[str, str]]) -> VectorStore:+ splitter = CharacterTextSplitter()+ documents = splitter.create_documents(+ [file for _, file in files],+ [{"path": path} for path, _ in files],+ )++ embeddings = OpenAIEmbeddings(+ openai_api_key=config.args["openai_api_key"],+ ) # type: ignore++ vectordb = Chroma.from_documents(+ documents,+ embeddings,+ [path for path, _ in files],+ )+ return vectordbdiff --git a/api-service/poetry.lock b/api-service/poetry.lockindex e50c7d802..6f422e25b 100644--- a/api-service/poetry.lock+++ b/api-service/poetry.lock@@ -3198,4 +3198,4 @@ cffi = ["cffi (>=1.11)"][metadata]lock-version = "2.0"python-versions = ">=3.10,<4.0"-content-hash = "027dea841ec4f5b51e33b3abdc93830fb8d79f6ba44b16afbd5b328c0e857c32"+content-hash = "2cc9b3ba06403f81a93c82f621a42c256a7b98a2a67fe3373656fd2f96b2f9f9"diff --git a/api-service/pyproject.toml b/api-service/pyproject.tomlindex e11a7b5ab..83f7a4068 100644--- a/api-service/pyproject.toml+++ b/api-service/pyproject.toml@@ -10,7 +10,6 @@ python-dateutil = "^2.8.2"urllib3 = "^1.26.14"pydantic = "^1.10.6"aenum = "^3.1.11"-openai = "^0.27.0"lxml = "^4.9.2"asyncio = "^3.4.3"postgrest = "^0.10.6"@@ -23,6 +22,7 @@ fastapi = "^0.95.1"langchain = "0.0.141"chevron = "^0.14.0"chromadb = "^0.3.22"+openai = "^0.27.6"[tool.poetry.group.dev]optional = truediff --git a/api-service/session/playground/base.py b/api-service/session/playground/base.pyindex 43836f9c1..3521a5c37 100644--- a/api-service/session/playground/base.py+++ b/api-service/session/playground/base.py@@ -1,8 +1,12 @@import math-from typing import Any+from typing import Any, Coroutine, List, Tuplefrom asyncio import sleep+from pydantic import StrictStr+++from playground_client.models.entry_info import EntryInfofrom playground_client.models.list_filesystem_dir_response import (ListFilesystemDirResponse,)@@ -41,12 +45,33 @@ class Playground(Session):async def checkout_repo(self, repo_url: str):await self.run_command(f"git clone {repo_url} .")- async def push_repo(self, repo_url: str):- await self.run_command(f"git push")+ # async def push_repo(self, repo_url: str):+ # await self.run_command(f"git push")async def change_rootdir(self, rootdir: str):self.rootdir = rootdir+ async def get_files(+ self,+ path: str,+ ignore: List[str] = [],+ ):+ dirs: List[EntryInfo] = [EntryInfo(name=path, isDir=True)]+ files: List[EntryInfo] = []+ while len(dirs) > 0:+ dir = dirs.pop()+ entries = await self.list_dir(dir.name)+ dirs.extend(+ entry for entry in entries if entry.is_dir and not entry.name in ignore+ )+ files.extend(+ entry+ for entry in entries+ if not entry.is_dir and not entry.name in ignore+ )++ return [(entry.name, await self.read_file(entry.name)) for entry in files]+async def run_command(self,cmd: str,diff --git a/components/TitleLink.tsx b/components/TitleLink.tsxindex 864ba05e8..9f65457b5 100644--- a/components/TitleLink.tsx+++ b/components/TitleLink.tsx@@ -48,6 +48,7 @@ function TitleLink({)}+TitleLink.size = Sizeexport default TitleLink\ No newline at end of file