Skip to content

armgit / E2B

Public Imported from GitHub
0

Open-source, secure environment with real-world tools for enterprise-grade agents.

Fix Replicate model wrapper [WIP]

Tomas Valenta8bb58f4830f8
diff --git a/api-service/app.py b/api-service/app.py
index 7d003382f..9c6bd3e49 100644
--- a/api-service/app.py
+++ b/api-service/app.py
@@ -16,7 +16,7 @@ from werkzeug.exceptions import HTTPException
load_dotenv()
-url = os.environ.get("SUPABASE_URL")
+url = os.environ.get("NEXT_PUBLIC_SUPABASE_URL")
key = os.environ.get("SUPABASE_SERVICE_ROLE_KEY")
if not url or not key:
@@ -39,14 +39,6 @@ def get_request_body_template(blocks: List[dict[str, str]]):
return request_body_template
-def exception_handler(e):
- print("Handling error", e)
- if isinstance(e, HTTPException):
- return e
- return Response(str(e), status=500)
-
-
@app.route("/health", methods=["GET"])
async def health():
return "OK"
@@ -82,7 +74,7 @@ async def generate():
human_tools = create_human_tools(run_id=run_id, playground=playground)
# Create a new instance of code generator
- cg = Codegen.from_tools_and_database(
+ cg = await Codegen.from_tools_and_database(
# The order in which we pass tools HAS an effect on the LLM behaviour.
custom_tools=[
*playground_tools,
diff --git a/api-service/codegen/codegen.py b/api-service/codegen/codegen.py
index 65c7df92b..1ade8201a 100644
--- a/api-service/codegen/codegen.py
+++ b/api-service/codegen/codegen.py
@@ -86,7 +86,7 @@ class Codegen(BaseModel):
return [tool.name for tool in self._tools]
@classmethod
- def from_tools_and_database(
+ async def from_tools_and_database(
cls,
custom_tools: List[BaseTool],
model_config: ModelConfig,
@@ -103,7 +103,7 @@ class Codegen(BaseModel):
tool.callback_manager = callback_manager
# Create the LLM
- llm = get_model(model_config, callback_manager)
+ llm = await get_model(model_config, callback_manager)
print(
f"Using LLM '{model_config['provider']}' with args:\n{model_config['args']}"
diff --git a/api-service/models/base.py b/api-service/models/base.py
index 0b9c084e2..160fbb4e5 100644
--- a/api-service/models/base.py
+++ b/api-service/models/base.py
@@ -1,15 +1,16 @@
from typing import TypedDict, Dict, Any
from enum import Enum
-
-from langchain.llms import Replicate
from langchain.chat_models import ChatOpenAI
from langchain.schema import BaseLanguageModel
from langchain.callbacks.base import BaseCallbackManager
+from .wrappers.replicate import ReplicateFix
+
class ModelProvider(Enum):
OpenAI = "OpenAI"
Replicate = "Replicate"
+ # HuggingFace = "HuggingFace"
class ModelConfig(TypedDict):
@@ -18,7 +19,7 @@ class ModelConfig(TypedDict):
args: Dict[str, Any]
-def get_model(
+async def get_model(
config: ModelConfig,
callback_manager: BaseCallbackManager,
) -> BaseLanguageModel:
@@ -32,10 +33,17 @@ def get_model(
callback_manager=callback_manager,
)
case ModelProvider.Replicate.value:
- return Replicate(
+ model = ReplicateFix(
**config["args"],
verbose=True,
callback_manager=callback_manager,
)
+ return model
+ # case ModelProvider.HuggingFace.value:
+ # return HuggingFaceEndpointFix(
+ # **config["args"],
+ # verbose=True,
+ # callback_manager=callback_manager,
+ # )
case _:
raise ValueError(f"Provider {config['provider']} no found.")
diff --git a/api-service/models/wrappers/__init__.py b/api-service/models/wrappers/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/api-service/models/wrappers/huggingface_endpoint.py b/api-service/models/wrappers/huggingface_endpoint.py
new file mode 100644
index 000000000..0dccefc74
--- /dev/null
+++ b/api-service/models/wrappers/huggingface_endpoint.py
@@ -0,0 +1,10 @@
+from typing import List, Optional
+from langchain.llms import HuggingFaceEndpoint
+from langchain.schema import LLMResult
+
+
+class HuggingFaceEndpointFix(HuggingFaceEndpoint):
+ def agenerate(
+ self, prompts: List[str], stop: Optional[List[str]] = None
+ ) -> LLMResult:
+ return self.generate(prompts, stop)
diff --git a/api-service/models/wrappers/replicate.py b/api-service/models/wrappers/replicate.py
new file mode 100644
index 000000000..808746410
--- /dev/null
+++ b/api-service/models/wrappers/replicate.py
@@ -0,0 +1,57 @@
+from typing import Any, Dict, List, Optional
+from langchain.llms import Replicate
+from langchain.schema import Generation, LLMResult
+from pydantic import root_validator
+
+
+class ReplicateFix(Replicate):
+ @root_validator()
+ def validate_environment(cls, values: Dict) -> Dict:
+ return values
+
+ async def _acall(self, prompt: str, stop: Optional[List[str]] = None):
+ """Call to replicate endpoint."""
+ print("ACALL")
+ # raise Exception("TT")
+ try:
+ import replicate as replicate_python
+ except ImportError:
+ raise ValueError(
+ "Could not import replicate python package. "
+ "Please install it with `pip install replicate`."
+ )
+ # get the model and version
+ model_str, version_str = self.model.split(":")
+
+ client = replicate_python.Client(self.replicate_api_token)
+ model = client.models.get(model_str)
+ version = model.versions.get(version_str)
+
+ # sort through the openapi schema to get the name of the first input
+ input_properties = sorted(
+ version.openapi_schema["components"]["schemas"]["Input"][
+ "properties"
+ ].items(),
+ key=lambda item: item[1].get("x-order", 0),
+ )
+ first_input_name = input_properties[0][0]
+
+ inputs = {first_input_name: prompt, **self.input}
+ outputs = client.run(self.model, input={**inputs})
+
+ text = ""
+ for token in outputs:
+ print("token", token)
+ text += token
+ if self.callback_manager.is_async:
+ await self.callback_manager.on_llm_new_token(
+ token,
+ verbose=self.verbose,
+ )
+ else:
+ self.callback_manager.on_llm_new_token(
+ token,
+ verbose=self.verbose,
+ )
+
+ return text
diff --git a/api-service/poetry.lock b/api-service/poetry.lock
index 7c2055a1e..085ff3c68 100644
--- a/api-service/poetry.lock
+++ b/api-service/poetry.lock
@@ -401,6 +401,22 @@ files = [
[package.extras]
test = ["pytest (>=6)"]
+[[package]]
+name = "filelock"
+version = "3.11.0"
+description = "A platform independent file lock."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+files = [
+ {file = "filelock-3.11.0-py3-none-any.whl", hash = "sha256:f08a52314748335c6460fc8fe40cd5638b85001225db78c2aa01c8c0db83b318"},
+ {file = "filelock-3.11.0.tar.gz", hash = "sha256:3618c0da67adcc0506b015fd11ef7faf1b493f0b40d87728e19986b536890c37"},
+]
+
+[package.extras]
+docs = ["furo (>=2023.3.27)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"]
+testing = ["covdefaults (>=2.3)", "coverage (>=7.2.2)", "diff-cover (>=7.5)", "pytest (>=7.2.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"]
+
[[package]]
name = "frozenlist"
version = "1.3.3"
@@ -720,6 +736,37 @@ cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10,<13)"]
http2 = ["h2 (>=3,<5)"]
socks = ["socksio (>=1.0.0,<2.0.0)"]
+[[package]]
+name = "huggingface-hub"
+version = "0.13.4"
+description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub"
+category = "main"
+optional = false
+python-versions = ">=3.7.0"
+files = [
+ {file = "huggingface_hub-0.13.4-py3-none-any.whl", hash = "sha256:4d3d40593de6673d624a4baaaf249b9bf5165bfcafd1ad58de361931f0b4fda5"},
+ {file = "huggingface_hub-0.13.4.tar.gz", hash = "sha256:db83d9c2f76aed8cf49893ffadd6be24e82074da2f64b1d36b8ba40eb255e115"},
+]
+
+[package.dependencies]
+filelock = "*"
+packaging = ">=20.9"
+pyyaml = ">=5.1"
+requests = "*"
+tqdm = ">=4.42.1"
+typing-extensions = ">=3.7.4.3"
+
+[package.extras]
+all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "black (>=23.1,<24.0)", "jedi", "mypy (==0.982)", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"]
+cli = ["InquirerPy (==0.3.4)"]
+dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "black (>=23.1,<24.0)", "jedi", "mypy (==0.982)", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"]
+fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"]
+quality = ["black (>=23.1,<24.0)", "mypy (==0.982)", "ruff (>=0.0.241)"]
+tensorflow = ["graphviz", "pydot", "tensorflow"]
+testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "jedi", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "soundfile"]
+torch = ["torch"]
+typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"]
+
[[package]]
name = "hypercorn"
version = "0.14.3"
@@ -2048,4 +2095,4 @@ multidict = ">=4.0"
[metadata]
lock-version = "2.0"
python-versions = ">=3.10,<4.0"
-content-hash = "c05ef11834e390b01bfe57c835f7dc6139d8289921fe5a0f257d81b9bbb17656"
+content-hash = "a23dd82454720cc5fbb8bf48bcb63514f7950463121cb2911d63a9731fcd9a69"
diff --git a/api-service/pyproject.toml b/api-service/pyproject.toml
index bb446e11e..ae3e89855 100644
--- a/api-service/pyproject.toml
+++ b/api-service/pyproject.toml
@@ -18,6 +18,7 @@ quart-cors = "^0.6.0"
langchain = "^0.0.132"
quart = "^0.18.4"
replicate = "^0.8.1"
+huggingface-hub = "^0.13.4"
[tool.poetry.group.dev]
optional = true
diff --git a/state/model.ts b/state/model.ts
index 1b1299a5b..f3d0c50dd 100644
--- a/state/model.ts
+++ b/state/model.ts
@@ -4,6 +4,7 @@ import { SelectedModel } from './store'
export enum ModelProvider {
OpenAI = 'OpenAI',
Replicate = 'Replicate',
+ // HuggingFace = 'HuggingFace',
}
export type ArgValue = string | number
@@ -30,12 +31,32 @@ export interface ModelConfigTemplate {
}
export const models: {
- [model in keyof typeof ModelProvider]: {
+ [provider in keyof typeof ModelProvider]: {
// These creds are merged with the model args then send to the API.
creds?: { [key: string]: Omit<ModelArgTemplate, 'editable' | 'value'> }
models: Omit<ModelConfigTemplate, 'provider'>[]
}
} = {
+ // [ModelProvider.HuggingFace]: {
+ // creds: {
+ // huggingfacehub_api_token: {
+ // label: 'Hugging Face API Key',
+ // type: 'string',
+ // },
+ // },
+ // models: [
+ // {
+ // name: 'Deployed model',
+ // args: {
+ // endpoint_url: {
+ // editable: true,
+ // type: 'string',
+ // label: 'Endpoint URL'
+ // },
+ // },
+ // },
+ // ],
+ // },
[ModelProvider.Replicate]: {
creds: {
replicate_api_token: {
@@ -52,14 +73,6 @@ export const models: {
type: 'string',
label: 'Model name'
},
- max_tokens: {
- type: 'number',
- value: 2048,
- },
- temperature: {
- type: 'number',
- value: 0,
- },
},
},
],