🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: Support API KEY for OPENAPI functions (#609)
Ziyi Yang477b4ec630ae
diff --git a/camel/functions/__init__.py b/camel/functions/__init__.pyindex bd6e96920..8ef16f47d 100644--- a/camel/functions/__init__.py+++ b/camel/functions/__init__.py@@ -17,6 +17,7 @@ from .openai_function import (get_openai_function_schema,get_openai_tool_schema,)+from .open_api_specs.security_config import openapi_security_configfrom .google_maps_function import MAP_FUNCSfrom .math_functions import MATH_FUNCS@@ -36,6 +37,7 @@ __all__ = ['OpenAIFunction','get_openai_function_schema','get_openai_tool_schema',+ 'openapi_security_config','apinames_filepaths_to_funs_schemas','generate_apinames_filepaths','MAP_FUNCS',diff --git a/camel/functions/open_api_function.py b/camel/functions/open_api_function.pyindex 55d698c1f..b4211553b 100644--- a/camel/functions/open_api_function.py+++ b/camel/functions/open_api_function.py@@ -18,7 +18,7 @@ from typing import Any, Callable, Dict, List, Tupleimport pranceimport requests-from camel.functions import OpenAIFunction+from camel.functions import OpenAIFunction, openapi_security_configfrom camel.types import OpenAPIName@@ -37,8 +37,21 @@ def parse_openapi_file(openapi_spec_path: str) -> Dict[str, Any]:Dict[str, Any]: The parsed OpenAPI specification as a dictionary."""# Load the OpenAPI spec- parser = prance.ResolvingParser(openapi_spec_path)+ parser = prance.ResolvingParser(+ openapi_spec_path, backend="openapi-spec-validator", strict=False+ )openapi_spec = parser.specification+ version = openapi_spec.get('openapi', {})+ if not version:+ raise ValueError(+ "OpenAPI version not specified in the spec. "+ "Only OPENAPI 3.0.x and 3.1.x are supported."+ )+ if not (version.startswith('3.0') or version.startswith('3.1')):+ raise ValueError(+ f"Unsupported OpenAPI version: {version}. "+ f"Only OPENAPI 3.0.x and 3.1.x are supported."+ )return openapi_spec@@ -176,22 +189,33 @@ def openapi_spec_to_openai_schemas(def openapi_function_decorator(- base_url: str, path: str, method: str, operation: Dict[str, Any]+ api_name: str,+ base_url: str,+ path: str,+ method: str,+ openapi_security: List[Dict[str, Any]],+ sec_schemas: Dict[str, Dict[str, Any]],+ operation: Dict[str, Any],) -> Callable:- r"""Decorate a function to make HTTP requests based on OpenAPI operation- details.+ r"""Decorate a function to make HTTP requests based on OpenAPI+ specification details.- This decorator takes the base URL, path, HTTP method, and operation details- from an OpenAPI specification, and returns a decorator. The decorated- function can then be called with keyword arguments corresponding to the- operation's parameters. The decorator handles constructing the request URL,- setting headers, query parameters, and the request body as specified by the- operation details.+ This decorator dynamically constructs and executes an API request based on+ the provided OpenAPI operation specifications, security requirements, and+ parameters. It supports operations secured with `apiKey` type security+ schemes and automatically injects the necessary API keys from environment+ variables. Parameters in `path`, `query`, `header`, and `cookie` are also+ supported.Args:+ api_name (str): The name of the API, used to retrieve API key names+ and URLs from the configuration.base_url (str): The base URL for the API.path (str): The path for the API endpoint, relative to the base URL.method (str): The HTTP method (e.g., 'get', 'post') for the request.+ openapi_security (List[Dict[str, Any]]): The global security+ definitions as specified in the OpenAPI specs.+ sec_schemas (Dict[str, Dict[str, Any]]): Detailed security schemes.operation (Dict[str, Any]): A dictionary containing the OpenAPIoperation details, including parameters and request bodydefinitions.@@ -200,6 +224,12 @@ def openapi_function_decorator(Callable: A decorator that, when applied to a function, enables thefunction to make HTTP requests based on the provided OpenAPIoperation details.++ Raises:+ TypeError: If the security requirements include unsupported types.+ ValueError: If required API keys are missing from environment+ variables or if the content type of the request body is+ unsupported."""def inner_decorator(openapi_function: Callable) -> Callable:@@ -209,6 +239,51 @@ def openapi_function_decorator(params = {}cookies = {}+ # Security definition of operation overrides any declared+ # top-level security.+ sec_requirements = operation.get('security', openapi_security)+ avail_sec_requirement = {}+ # Write to avaliable_security_requirement only if all the+ # security_type are "apiKey"+ for security_requirement in sec_requirements:+ have_unsupported_type = False+ for sec_scheme_name, _ in security_requirement.items():+ sec_type = sec_schemas.get(sec_scheme_name).get('type')+ if sec_type != "apiKey":+ have_unsupported_type = True+ break+ if have_unsupported_type is False:+ avail_sec_requirement = security_requirement+ break++ if sec_requirements and not avail_sec_requirement:+ raise TypeError(+ "Only security schemas of type `apiKey` are supported."+ )++ for sec_scheme_name, _ in avail_sec_requirement.items():+ try:+ API_KEY_NAME = openapi_security_config.get(api_name).get(+ sec_scheme_name+ )+ api_key_value = os.environ[API_KEY_NAME]+ except Exception:+ api_key_url = openapi_security_config.get(api_name).get(+ 'get_api_key_url'+ )+ raise ValueError(+ f"`{API_KEY_NAME}` not found in environment "+ f"variables. Get `{API_KEY_NAME}` here: {api_key_url}"+ )+ request_key_name = sec_schemas.get(sec_scheme_name).get('name')+ request_key_in = sec_schemas.get(sec_scheme_name).get('in')+ if request_key_in == 'query':+ params[request_key_name] = api_key_value+ elif request_key_in == 'header':+ headers[request_key_name] = api_key_value+ elif request_key_in == 'coolie':+ cookies[request_key_name] = api_key_value+# Assign parameters to the correct positionfor param in operation.get('parameters', []):input_param_name = param['name'] + '_in_' + param['in']@@ -307,6 +382,10 @@ def generate_openapi_funcs(raise ValueError("No server information found in OpenAPI spec.")base_url = servers[0].get('url') # Use the first server URL+ # Security requirement objects for all methods+ openapi_security = openapi_spec.get('security', {})+ # Security schemas which can be reused by different methods+ sec_schemas = openapi_spec.get('components', {}).get('securitySchemes', {})functions = []# Traverse paths and methods@@ -321,7 +400,15 @@ def generate_openapi_funcs(sanitized_path = path.replace('/', '_').strip('_')function_name = f"{api_name}_{method}_{sanitized_path}"- @openapi_function_decorator(base_url, path, method, operation)+ @openapi_function_decorator(+ api_name,+ base_url,+ path,+ method,+ openapi_security,+ sec_schemas,+ operation,+ )def openapi_function(**kwargs):passdiff --git a/camel/functions/open_api_specs/nasa_apod/__init__.py b/camel/functions/open_api_specs/nasa_apod/__init__.pynew file mode 100644index 000000000..d963f8cc8--- /dev/null+++ b/camel/functions/open_api_specs/nasa_apod/__init__.py@@ -0,0 +1,13 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the “License”);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an “AS IS” BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========diff --git a/camel/functions/open_api_specs/nasa_apod/openapi.yaml b/camel/functions/open_api_specs/nasa_apod/openapi.yamlnew file mode 100644index 000000000..1d3012e0a--- /dev/null+++ b/camel/functions/open_api_specs/nasa_apod/openapi.yaml@@ -0,0 +1,72 @@+openapi: 3.0.0+servers:+ - url: https://api.nasa.gov/planetary+ - url: http://api.nasa.gov/planetary+info:+ contact:+ email: [email protected]+ description: This endpoint structures the APOD imagery and associated metadata+ so that it can be repurposed for other applications. In addition, if the+ concept_tags parameter is set to True, then keywords derived from the image+ explanation are returned. These keywords could be used as auto-generated+ hashtags for twitter or instagram feeds; but generally help with+ discoverability of relevant imagery+ license:+ name: Apache 2.0+ url: http://www.apache.org/licenses/LICENSE-2.0.html+ title: APOD+ version: 1.0.0+ x-apisguru-categories:+ - media+ - open_data+ x-origin:+ - format: swagger+ url: https://raw.githubusercontent.com/nasa/api-docs/gh-pages/assets/json/APOD+ version: "2.0"+ x-providerName: nasa.gov+ x-serviceName: apod+tags:+ - description: An example tag+ externalDocs:+ description: Here's a link+ url: https://example.com+ name: request tag+paths:+ /apod:+ get:+ description: Returns the picture of the day+ parameters:+ - description: The date of the APOD image to retrieve+ in: query+ name: date+ required: false+ schema:+ type: string+ - description: Retrieve the URL for the high resolution image+ in: query+ name: hd+ required: false+ schema:+ type: boolean+ responses:+ "200":+ content:+ application/json:+ schema:+ items:+ x-thing: ok+ type: array+ description: successful operation+ "400":+ description: Date must be between Jun 16, 1995 and Mar 28, 2019.+ security:+ - api_key: []+ summary: Returns images+ tags:+ - request tag+components:+ securitySchemes:+ api_key:+ in: query+ name: api_key+ type: apiKeydiff --git a/camel/functions/open_api_specs/security_config.py b/camel/functions/open_api_specs/security_config.pynew file mode 100644index 000000000..b19b774d1--- /dev/null+++ b/camel/functions/open_api_specs/security_config.py@@ -0,0 +1,21 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the “License”);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an “AS IS” BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+from camel.types import OpenAPIName++openapi_security_config = {+ OpenAPIName.NASA_APOD.value: {+ "api_key": "NASA_API_KEY",+ "get_api_key_url": "https://api.nasa.gov/",+ },+}diff --git a/camel/types/enums.py b/camel/types/enums.pyindex dd3f1fc93..82a903f49 100644--- a/camel/types/enums.py+++ b/camel/types/enums.py@@ -253,6 +253,7 @@ class OpenAPIName(Enum):COURSERA = "coursera"KLARNA = "klarna"SPEAK = "speak"+ NASA_APOD = "nasa_apod"BIZTOC = "biztoc"CREATE_QR_CODE = "create_qr_code"OUTSCHOOL = "outschool"diff --git a/poetry.lock b/poetry.lockindex bdf984d66..8d5436e00 100644--- a/poetry.lock+++ b/poetry.lock@@ -2450,9 +2450,13 @@ files = [{file = "lxml-5.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:edcfa83e03370032a489430215c1e7783128808fd3e2e0a3225deee278585196"},{file = "lxml-5.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:28bf95177400066596cdbcfc933312493799382879da504633d16cf60bba735b"},{file = "lxml-5.2.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a745cc98d504d5bd2c19b10c79c61c7c3df9222629f1b6210c0368177589fb8"},+ {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b590b39ef90c6b22ec0be925b211298e810b4856909c8ca60d27ffbca6c12e6"},{file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b336b0416828022bfd5a2e3083e7f5ba54b96242159f83c7e3eebaec752f1716"},+ {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:c2faf60c583af0d135e853c86ac2735ce178f0e338a3c7f9ae8f622fd2eb788c"},{file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:4bc6cb140a7a0ad1f7bc37e018d0ed690b7b6520ade518285dc3171f7a117905"},+ {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7ff762670cada8e05b32bf1e4dc50b140790909caa8303cfddc4d702b71ea184"},{file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:57f0a0bbc9868e10ebe874e9f129d2917750adf008fe7b9c1598c0fbbfdde6a6"},+ {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:a6d2092797b388342c1bc932077ad232f914351932353e2e8706851c870bca1f"},{file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:60499fe961b21264e17a471ec296dcbf4365fbea611bf9e303ab69db7159ce61"},{file = "lxml-5.2.2-cp37-cp37m-win32.whl", hash = "sha256:d9b342c76003c6b9336a80efcc766748a333573abf9350f4094ee46b006ec18f"},{file = "lxml-5.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b16db2770517b8799c79aa80f4053cd6f8b716f21f8aca962725a9565ce3ee40"},diff --git a/pyproject.toml b/pyproject.tomlindex 994ec54c3..1e30bf253 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -302,4 +302,4 @@ module = ["pygithub"]-ignore_missing_imports = true+ignore_missing_imports = true\ No newline at end of filediff --git a/test/functions/test_open_api_function.py b/test/functions/test_open_api_function.pyindex 389a25f20..95566e7cb 100644--- a/test/functions/test_open_api_function.py+++ b/test/functions/test_open_api_function.py@@ -345,6 +345,51 @@ def test_speak_explainTask(get_function):assert result == mock_response_data[email protected]('get_function', ['nasa_apod_get_apod'], indirect=True)+def test_nasa_apod_get_apod(get_function, monkeypatch):+ monkeypatch.setenv('NASA_API_KEY', 'fake_api_key')+ mock_response_data = {+ "copyright": "Yann Sainty",+ "date": "2023-08-17",+ "explanation": (+ "Sprawling emission nebulae IC 1396 and Sh2-129 mix glowing "+ "interstellar gas and dark dust clouds in this nearly 12 degree "+ "wide field of view toward the northern constellation Cepheus the "+ "King. Energized by its central star IC 1396 (left), is hundreds "+ "of light-years across and some 3,000 light-years distant. The "+ "nebula's intriguing dark shapes include a winding dark cloud"+ "popularly known as the Elephant's Trunk below and right of "+ "center. Tens of light-years long, it holds the raw material for "+ "star formation and is known to hide protostars within. Located a "+ "similar distance from planet Earth, the bright knots and swept "+ "back ridges of emission of Sh2-129 on the right suggest its "+ "popular name, the Flying Bat Nebula. Within the Flying Bat, "+ "the most recently recognized addition to this royal cosmic zoo "+ "is the faint bluish emission from Ou4, the Giant Squid Nebula. "+ "Near the lower right edge of the frame, the suggestive dark "+ "marking on the sky cataloged as Barnard 150 is also known as the "+ "dark Seahorse Nebula. Notable submissions to APOD: Perseids "+ "Meteor Shower 2023"+ ),+ "hdurl": (+ "https://apod.nasa.gov/apod/image/2308/"+ "ElephantTrunkBatSquidSeahorse.jpg"+ ),+ "media_type": "image",+ "service_version": "v1",+ "title": "A Cosmic Zoo in Cepheus",+ "url": (+ "https://apod.nasa.gov/apod/image/2308/"+ "ElephantTrunkBatSquidSeahorse1024.jpg"+ ),+ }+ mock_response = MagicMock()+ mock_response.json.return_value = mock_response_data+ with patch('requests.request', return_value=mock_response):+ result = get_function(date_in_query='2023-08-17', hd_in_query=True)+ assert result == mock_response_data++@pytest.mark.parametrize('get_function', ['biztoc_getNews'], indirect=True)def test_biztoc_getNews(get_function):mock_response_data = [