Skip to content

armgit / E2B

Public Imported from GitHub
0

Open-source, secure environment with real-world tools for enterprise-grade agents.

Change backend to support custom models [WIP]

Tomas Valenta0babcdca0237
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index fe628267b..111782438 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -6,7 +6,9 @@
"customizations": {
"vscode": {
"extensions": [
- "ms-python.pylint"
+ "ms-python.pylint",
+ "ms-python.black-formatter",
+ "ms-python.python"
]
}
}
diff --git a/api-service/app.py b/api-service/app.py
index 6ba567c18..4d4b0247f 100644
--- a/api-service/app.py
+++ b/api-service/app.py
@@ -7,12 +7,11 @@ from typing import List
from playground_client.exceptions import NotFoundException
from quart import Quart, request, abort
from quart_cors import cors
-from logging.config import dictConfig
from codegen import Codegen
from codegen.tools.playground import create_playground_tools
-from database import Database, DeploymentState
from codegen.tools.human.tools import create_human_tools
+from database import Database, DeploymentState
from werkzeug.exceptions import HTTPException
load_dotenv()
@@ -38,7 +37,6 @@ def get_request_body_template(blocks: List[dict[str, str]]):
)
return request_body_template
-
@app.errorhandler(TimeoutError)
def timeout_handler(e):
return abort(408, e)
@@ -54,6 +52,7 @@ async def health():
return "OK"
+
# TODO: SECURITY - Check if user invoking this request has permission to generate and deploy to this project
@app.route("/generate", methods=["POST"])
async def generate():
@@ -64,9 +63,9 @@ async def generate():
route_id = body["routeID"]
blocks = body["blocks"]
method = body["method"]
- model_name = body["model"]
route = body["route"]
- max_tokens = body["maxTokens"]
+
+ model_config = body["modelConfig"]
pprint("+++ Blocks:")
pprint(blocks)
@@ -90,8 +89,7 @@ async def generate():
*playground_tools,
*human_tools,
],
- model_name=model_name,
- max_tokens=max_tokens,
+ model_config=model_config,
database=db,
)
diff --git a/api-service/codegen/agent/base.py b/api-service/codegen/agent/base.py
index 6be188d9d..fa4e65aab 100644
--- a/api-service/codegen/agent/base.py
+++ b/api-service/codegen/agent/base.py
@@ -7,6 +7,7 @@ from langchain.agents.chat.base import ChatAgent
from langchain.schema import AgentAction, AgentFinish
from langchain.tools.base import BaseTool
+
from codegen.agent.parsing import parse_thoughts_and_actions, ToolLog
# FINAL_ANSWER_TRIGGERS = ["Final Answer:", "FinalAnswer", "Final Answer"]
diff --git a/api-service/codegen/codegen.py b/api-service/codegen/codegen.py
index 4446a517e..8836cf8ed 100644
--- a/api-service/codegen/codegen.py
+++ b/api-service/codegen/codegen.py
@@ -7,8 +7,8 @@ from typing import (
)
from langchain.agents import AgentExecutor
+from langchain.schema import BaseLanguageModel
from pydantic import BaseModel, PrivateAttr
-from langchain.chat_models import ChatOpenAI
from langchain.callbacks.base import (
AsyncCallbackManager,
BaseCallbackManager,
@@ -16,7 +16,7 @@ from langchain.callbacks.base import (
from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
from langchain.tools import BaseTool
-from session.env import EnvVar
+from models import get_model, ModelConfig
from database import Database
from codegen.agent import CodegenAgent, CodegenAgentExecutor
from codegen.callbacks.logs import LogsCallbackHandler
@@ -55,7 +55,7 @@ class Codegen(BaseModel):
_agent: CodegenAgent = PrivateAttr()
_agent_executor: AgentExecutor = PrivateAttr()
_tools: List[BaseTool] = PrivateAttr()
- _llm: ChatOpenAI = PrivateAttr()
+ _llm: BaseLanguageModel = PrivateAttr()
_database: Database = PrivateAttr()
_callback_manager: BaseCallbackManager = PrivateAttr()
@@ -64,7 +64,7 @@ class Codegen(BaseModel):
database: Database,
callback_manager: BaseCallbackManager,
tools: List[BaseTool],
- llm: ChatOpenAI,
+ llm: BaseLanguageModel,
agent: CodegenAgent,
**kwargs: Any,
) -> None:
@@ -89,8 +89,7 @@ class Codegen(BaseModel):
def from_tools_and_database(
cls,
custom_tools: List[BaseTool],
- model_name: str,
- max_tokens: int,
+ model_config: ModelConfig,
database: Database,
):
callback_manager = AsyncCallbackManager(
@@ -104,14 +103,7 @@ class Codegen(BaseModel):
tool.callback_manager = callback_manager
# Create the LLM
- llm = ChatOpenAI(
- model_name=model_name,
- streaming=True,
- temperature=0,
- max_tokens=max_tokens,
- verbose=True,
- callback_manager=callback_manager,
- )
+ llm = get_model(model_config, callback_manager)
# Create CodegenAgent
agent = CodegenAgent.from_llm_and_tools(
diff --git a/api-service/models/__init__.py b/api-service/models/__init__.py
new file mode 100644
index 000000000..311f515a3
--- /dev/null
+++ b/api-service/models/__init__.py
@@ -0,0 +1 @@
+from .base import get_model, ModelConfig
diff --git a/api-service/models/base.py b/api-service/models/base.py
new file mode 100644
index 000000000..0e8bf1fac
--- /dev/null
+++ b/api-service/models/base.py
@@ -0,0 +1,43 @@
+from typing import TypedDict, Dict, Any
+from enum import Enum
+
+from langchain.llms import Anthropic
+from langchain.chat_models import ChatOpenAI
+from langchain.schema import BaseLanguageModel
+from langchain.callbacks.base import BaseCallbackManager
+
+class ModelProvider(Enum):
+ OpenAI = "OpenAI"
+ Anthropic = "Anthropic"
+
+class ModelConfig(TypedDict):
+ provider: ModelProvider
+ name: str
+ max_tokens: int
+ temperature: float
+ args: Dict[str, Any]
+
+def get_model(
+ config: ModelConfig,
+ callback_manager: BaseCallbackManager,
+ ) -> BaseLanguageModel:
+ match config["provider"]:
+ case ModelProvider.OpenAI:
+ return ChatOpenAI(
+ verbose=True,
+ streaming=True,
+ callback_manager=callback_manager,
+ model_name=config["name"],
+ temperature=config["temperature"],
+ max_tokens=config["max_tokens"],
+ **config["args"],
+ )
+ case ModelProvider.Anthropic:
+ return Anthropic(
+ streaming=True,
+ verbose=True,
+ temperature=config["temperature"],
+ max_tokens=config["max_tokens"],
+ callback_manager=callback_manager,
+ **config["args"],
+ )
diff --git a/api-service/session/playground/nodejs.py b/api-service/session/playground/nodejs.py
index 2d4bfaadd..f2fe569bc 100644
--- a/api-service/session/playground/nodejs.py
+++ b/api-service/session/playground/nodejs.py
@@ -23,6 +23,7 @@ class NodeJSPlayground(Playground):
# )
# # pprint.pprint(f"Result: {result}")
# return result
+
async def run_saved_javascript_code(self):
result = await self.run_command(
f"node {self.default_javascript_code_file}",
diff --git a/pages/index.tsx b/pages/index.tsx
index 1bc3ec599..b12c66a05 100644
--- a/pages/index.tsx
+++ b/pages/index.tsx
@@ -51,7 +51,6 @@ export const getServerSideProps: GetServerSideProps<Props> = async (ctx) => {
}
}
-
const hasDefaultTeam = user?.users_teams.some(t => t.teams.is_default)
if (!hasDefaultTeam) {
// User is one of the old users without default team - create default team.
diff --git a/state/model.ts b/state/model.ts
new file mode 100644
index 000000000..2614395d8
--- /dev/null
+++ b/state/model.ts
@@ -0,0 +1,29 @@
+export enum ModelProvider {
+ OpenAI = 'OpenAI'
+}
+
+export interface ModelConfig {
+ provider: ModelProvider
+ name: string
+ max_tokens: number
+ temperature: number
+ requiredFields?: string[]
+}
+
+const defaultTokens = 2048
+const defaultTemperature = 0
+
+export const models: { [model in keyof typeof ModelProvider]: Omit<ModelConfig, 'provider'>[] } = {
+ [ModelProvider.OpenAI]: [
+ {
+ name: 'gpt-3.5-turbo',
+ max_tokens: defaultTokens,
+ temperature: defaultTemperature,
+ },
+ {
+ name: 'gpt-4',
+ max_tokens: defaultTokens,
+ temperature: defaultTemperature,
+ },
+ ],
+}