Open-source, secure environment with real-world tools for enterprise-grade agents.
Python agentaiai-agentai-agentscode-interpretercopilotdevelopmentdevtoolsgptgpt-4javascriptllmnextjsopenaipythonreactsoftwaretypescript
Fix Replicate model wrapper [WIP]
Tomas Valenta8bb58f4830f8
diff --git a/api-service/app.py b/api-service/app.pyindex 7d003382f..9c6bd3e49 100644--- a/api-service/app.py+++ b/api-service/app.py@@ -16,7 +16,7 @@ from werkzeug.exceptions import HTTPExceptionload_dotenv()-url = os.environ.get("SUPABASE_URL")+url = os.environ.get("NEXT_PUBLIC_SUPABASE_URL")key = os.environ.get("SUPABASE_SERVICE_ROLE_KEY")if not url or not key:@@ -39,14 +39,6 @@ def get_request_body_template(blocks: List[dict[str, str]]):return request_body_template[email protected](Exception)-def exception_handler(e):- print("Handling error", e)- if isinstance(e, HTTPException):- return e- return Response(str(e), status=500)--@app.route("/health", methods=["GET"])async def health():return "OK"@@ -82,7 +74,7 @@ async def generate():human_tools = create_human_tools(run_id=run_id, playground=playground)# Create a new instance of code generator- cg = Codegen.from_tools_and_database(+ cg = await Codegen.from_tools_and_database(# The order in which we pass tools HAS an effect on the LLM behaviour.custom_tools=[*playground_tools,diff --git a/api-service/codegen/codegen.py b/api-service/codegen/codegen.pyindex 65c7df92b..1ade8201a 100644--- a/api-service/codegen/codegen.py+++ b/api-service/codegen/codegen.py@@ -86,7 +86,7 @@ class Codegen(BaseModel):return [tool.name for tool in self._tools]@classmethod- def from_tools_and_database(+ async def from_tools_and_database(cls,custom_tools: List[BaseTool],model_config: ModelConfig,@@ -103,7 +103,7 @@ class Codegen(BaseModel):tool.callback_manager = callback_manager# Create the LLM- llm = get_model(model_config, callback_manager)+ llm = await get_model(model_config, callback_manager)print(f"Using LLM '{model_config['provider']}' with args:\n{model_config['args']}"diff --git a/api-service/models/base.py b/api-service/models/base.pyindex 0b9c084e2..160fbb4e5 100644--- a/api-service/models/base.py+++ b/api-service/models/base.py@@ -1,15 +1,16 @@from typing import TypedDict, Dict, Anyfrom enum import Enum--from langchain.llms import Replicatefrom langchain.chat_models import ChatOpenAIfrom langchain.schema import BaseLanguageModelfrom langchain.callbacks.base import BaseCallbackManager+from .wrappers.replicate import ReplicateFix+class ModelProvider(Enum):OpenAI = "OpenAI"Replicate = "Replicate"+ # HuggingFace = "HuggingFace"class ModelConfig(TypedDict):@@ -18,7 +19,7 @@ class ModelConfig(TypedDict):args: Dict[str, Any]-def get_model(+async def get_model(config: ModelConfig,callback_manager: BaseCallbackManager,) -> BaseLanguageModel:@@ -32,10 +33,17 @@ def get_model(callback_manager=callback_manager,)case ModelProvider.Replicate.value:- return Replicate(+ model = ReplicateFix(**config["args"],verbose=True,callback_manager=callback_manager,)+ return model+ # case ModelProvider.HuggingFace.value:+ # return HuggingFaceEndpointFix(+ # **config["args"],+ # verbose=True,+ # callback_manager=callback_manager,+ # )case _:raise ValueError(f"Provider {config['provider']} no found.")diff --git a/api-service/models/wrappers/__init__.py b/api-service/models/wrappers/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/api-service/models/wrappers/huggingface_endpoint.py b/api-service/models/wrappers/huggingface_endpoint.pynew file mode 100644index 000000000..0dccefc74--- /dev/null+++ b/api-service/models/wrappers/huggingface_endpoint.py@@ -0,0 +1,10 @@+from typing import List, Optional+from langchain.llms import HuggingFaceEndpoint+from langchain.schema import LLMResult+++class HuggingFaceEndpointFix(HuggingFaceEndpoint):+ def agenerate(+ self, prompts: List[str], stop: Optional[List[str]] = None+ ) -> LLMResult:+ return self.generate(prompts, stop)diff --git a/api-service/models/wrappers/replicate.py b/api-service/models/wrappers/replicate.pynew file mode 100644index 000000000..808746410--- /dev/null+++ b/api-service/models/wrappers/replicate.py@@ -0,0 +1,57 @@+from typing import Any, Dict, List, Optional+from langchain.llms import Replicate+from langchain.schema import Generation, LLMResult+from pydantic import root_validator+++class ReplicateFix(Replicate):+ @root_validator()+ def validate_environment(cls, values: Dict) -> Dict:+ return values++ async def _acall(self, prompt: str, stop: Optional[List[str]] = None):+ """Call to replicate endpoint."""+ print("ACALL")+ # raise Exception("TT")+ try:+ import replicate as replicate_python+ except ImportError:+ raise ValueError(+ "Could not import replicate python package. "+ "Please install it with `pip install replicate`."+ )+ # get the model and version+ model_str, version_str = self.model.split(":")++ client = replicate_python.Client(self.replicate_api_token)+ model = client.models.get(model_str)+ version = model.versions.get(version_str)++ # sort through the openapi schema to get the name of the first input+ input_properties = sorted(+ version.openapi_schema["components"]["schemas"]["Input"][+ "properties"+ ].items(),+ key=lambda item: item[1].get("x-order", 0),+ )+ first_input_name = input_properties[0][0]++ inputs = {first_input_name: prompt, **self.input}+ outputs = client.run(self.model, input={**inputs})++ text = ""+ for token in outputs:+ print("token", token)+ text += token+ if self.callback_manager.is_async:+ await self.callback_manager.on_llm_new_token(+ token,+ verbose=self.verbose,+ )+ else:+ self.callback_manager.on_llm_new_token(+ token,+ verbose=self.verbose,+ )++ return textdiff --git a/api-service/poetry.lock b/api-service/poetry.lockindex 7c2055a1e..085ff3c68 100644--- a/api-service/poetry.lock+++ b/api-service/poetry.lock@@ -401,6 +401,22 @@ files = [[package.extras]test = ["pytest (>=6)"]+[[package]]+name = "filelock"+version = "3.11.0"+description = "A platform independent file lock."+category = "main"+optional = false+python-versions = ">=3.7"+files = [+ {file = "filelock-3.11.0-py3-none-any.whl", hash = "sha256:f08a52314748335c6460fc8fe40cd5638b85001225db78c2aa01c8c0db83b318"},+ {file = "filelock-3.11.0.tar.gz", hash = "sha256:3618c0da67adcc0506b015fd11ef7faf1b493f0b40d87728e19986b536890c37"},+]++[package.extras]+docs = ["furo (>=2023.3.27)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"]+testing = ["covdefaults (>=2.3)", "coverage (>=7.2.2)", "diff-cover (>=7.5)", "pytest (>=7.2.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"]+[[package]]name = "frozenlist"version = "1.3.3"@@ -720,6 +736,37 @@ cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10,<13)"]http2 = ["h2 (>=3,<5)"]socks = ["socksio (>=1.0.0,<2.0.0)"]+[[package]]+name = "huggingface-hub"+version = "0.13.4"+description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub"+category = "main"+optional = false+python-versions = ">=3.7.0"+files = [+ {file = "huggingface_hub-0.13.4-py3-none-any.whl", hash = "sha256:4d3d40593de6673d624a4baaaf249b9bf5165bfcafd1ad58de361931f0b4fda5"},+ {file = "huggingface_hub-0.13.4.tar.gz", hash = "sha256:db83d9c2f76aed8cf49893ffadd6be24e82074da2f64b1d36b8ba40eb255e115"},+]++[package.dependencies]+filelock = "*"+packaging = ">=20.9"+pyyaml = ">=5.1"+requests = "*"+tqdm = ">=4.42.1"+typing-extensions = ">=3.7.4.3"++[package.extras]+all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "black (>=23.1,<24.0)", "jedi", "mypy (==0.982)", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"]+cli = ["InquirerPy (==0.3.4)"]+dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "black (>=23.1,<24.0)", "jedi", "mypy (==0.982)", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"]+fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"]+quality = ["black (>=23.1,<24.0)", "mypy (==0.982)", "ruff (>=0.0.241)"]+tensorflow = ["graphviz", "pydot", "tensorflow"]+testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "jedi", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "soundfile"]+torch = ["torch"]+typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"]+[[package]]name = "hypercorn"version = "0.14.3"@@ -2048,4 +2095,4 @@ multidict = ">=4.0"[metadata]lock-version = "2.0"python-versions = ">=3.10,<4.0"-content-hash = "c05ef11834e390b01bfe57c835f7dc6139d8289921fe5a0f257d81b9bbb17656"+content-hash = "a23dd82454720cc5fbb8bf48bcb63514f7950463121cb2911d63a9731fcd9a69"diff --git a/api-service/pyproject.toml b/api-service/pyproject.tomlindex bb446e11e..ae3e89855 100644--- a/api-service/pyproject.toml+++ b/api-service/pyproject.toml@@ -18,6 +18,7 @@ quart-cors = "^0.6.0"langchain = "^0.0.132"quart = "^0.18.4"replicate = "^0.8.1"+huggingface-hub = "^0.13.4"[tool.poetry.group.dev]optional = truediff --git a/state/model.ts b/state/model.tsindex 1b1299a5b..f3d0c50dd 100644--- a/state/model.ts+++ b/state/model.ts@@ -4,6 +4,7 @@ import { SelectedModel } from './store'export enum ModelProvider {OpenAI = 'OpenAI',Replicate = 'Replicate',+ // HuggingFace = 'HuggingFace',}export type ArgValue = string | number@@ -30,12 +31,32 @@ export interface ModelConfigTemplate {}export const models: {- [model in keyof typeof ModelProvider]: {+ [provider in keyof typeof ModelProvider]: {// These creds are merged with the model args then send to the API.creds?: { [key: string]: Omit<ModelArgTemplate, 'editable' | 'value'> }models: Omit<ModelConfigTemplate, 'provider'>[]}} = {+ // [ModelProvider.HuggingFace]: {+ // creds: {+ // huggingfacehub_api_token: {+ // label: 'Hugging Face API Key',+ // type: 'string',+ // },+ // },+ // models: [+ // {+ // name: 'Deployed model',+ // args: {+ // endpoint_url: {+ // editable: true,+ // type: 'string',+ // label: 'Endpoint URL'+ // },+ // },+ // },+ // ],+ // },[ModelProvider.Replicate]: {creds: {replicate_api_token: {@@ -52,14 +73,6 @@ export const models: {type: 'string',label: 'Model name'},- max_tokens: {- type: 'number',- value: 2048,- },- temperature: {- type: 'number',- value: 0,- },},},],