š« CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: Integrate Jina Reader into loaders (#656)
Isaac Jin1c6cef0ea709
diff --git a/camel/loaders/__init__.py b/camel/loaders/__init__.pyindex ee476ee28..dced874f5 100644--- a/camel/loaders/__init__.py+++ b/camel/loaders/__init__.py@@ -13,10 +13,12 @@# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========from .base_io import File, read_file+from .jina_url_reader import JinaURLReaderfrom .unstructured_io import UnstructuredIO__all__ = ['File','read_file','UnstructuredIO',+ 'JinaURLReader',]diff --git a/camel/loaders/jina_url_reader.py b/camel/loaders/jina_url_reader.pynew file mode 100644index 000000000..5416212b3--- /dev/null+++ b/camel/loaders/jina_url_reader.py@@ -0,0 +1,99 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========++import os+from typing import Any, Optional++from camel.types.enums import JinaReturnFormat++JINA_ENDPOINT = "https://r.jina.ai/"+++class JinaURLReader:+ r"""URL Reader provided by Jina AI. The output is cleaner and more+ LLM-friendly than the URL Reader of UnstructuredIO. Can be configured to+ replace the UnstructuredIO URL Reader in the pipeline.++ Args:+ api_key (Optional[str], optional): The API key for Jina AI. If not+ provided, the reader will have a lower rate limit. Defaults to+ None.+ return_format (ReturnFormat, optional): The level of detail+ of the returned content, which is optimized for LLMs. For+ now screenshots are not supported. Defaults to+ ReturnFormat.DEFAULT.+ json_response (bool, optional): Whether to return the response+ in JSON format. Defaults to False.+ timeout (int, optional): The maximum time in seconds to wait for+ the page to be rendered. Defaults to 30.+ **kwargs (Any): Additional keyword arguments, including proxies,+ cookies, etc. It should align with the HTTP Header field and+ value pairs listed in the reference.++ References:+ https://jina.ai/reader+ """++ def __init__(+ self,+ api_key: Optional[str] = None,+ return_format: JinaReturnFormat = JinaReturnFormat.DEFAULT,+ json_response: bool = False,+ timeout: int = 30,+ **kwargs: Any,+ ) -> None:+ api_key = api_key or os.getenv('JINA_API_KEY')+ if api_key is None:+ print(+ "[JinaURLReader] JINA_API_KEY not set. This will result in a "+ "low rate limit of Jina URL Reader. Get API key here: "+ "https://jina.ai/reader."+ )++ # if the following field not provided, it will be None+ api_field = f"Bearer {api_key}" if api_key else None+ json_field = "application/json" if json_response else None++ raw_headers = {+ "Authorization": api_field,+ "X-Return-Format": return_format.value,+ "Accept": json_field,+ "X-Timeout": str(timeout),+ **kwargs,+ }++ # eliminate None values+ self._headers = {k: v for k, v in raw_headers.items() if v}++ def read_content(self, url: str) -> str:+ r"""Reads the content of a URL and returns it as a string with+ given form.++ Args:+ url (str): The URL to read.++ Returns:+ str: The content of the URL.+ """++ import requests++ full_url = f"{JINA_ENDPOINT}{url}"+ try:+ resp = requests.get(full_url, headers=self._headers)+ resp.raise_for_status()+ except Exception as e:+ raise Exception(f"Failed to read content from {url}: {e}") from e++ return resp.textdiff --git a/camel/types/enums.py b/camel/types/enums.pyindex f20982553..f3b1328d3 100644--- a/camel/types/enums.py+++ b/camel/types/enums.py@@ -401,3 +401,10 @@ class VoiceType(Enum):VoiceType.NOVA,VoiceType.SHIMMER,}+++class JinaReturnFormat(Enum):+ DEFAULT = None+ MARKDOWN = "markdown"+ HTML = "html"+ TEXT = "text"diff --git a/examples/loaders/jina_url_reader_example.py b/examples/loaders/jina_url_reader_example.pynew file mode 100644index 000000000..78f523e2d--- /dev/null+++ b/examples/loaders/jina_url_reader_example.py@@ -0,0 +1,57 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+++from camel.loaders import JinaURLReader+from camel.types.enums import JinaReturnFormat+++def read_with_different_format(return_format, json_response):+ URL = "https://en.wikipedia.org/wiki/Miss_Meyers"+ jina_url_reader = JinaURLReader(+ return_format=return_format, json_response=json_response+ )+ content = jina_url_reader.read_content(URL)+ print(content)+++def main():+ formats = [+ JinaReturnFormat.DEFAULT,+ JinaReturnFormat.TEXT,+ JinaReturnFormat.HTML,+ JinaReturnFormat.MARKDOWN,+ ]++ print("Choose a return format of read content:")+ print("1. Default, optimized for LLM inputs")+ print("2. Pure Text")+ print("3. HTML")+ print("4. Markdown")+ choice = input("Enter your choice (1-4): ")++ if not choice.isnumeric() or int(choice) < 1 or int(choice) > 4:+ print("Invalid choice. Exiting.")+ return++ return_format = formats[int(choice) - 1]++ json_response = input("Do you want the response in JSON format? (y/N): ")+ json_response = json_response.lower() == "y"++ read_with_different_format(return_format, json_response)+++if __name__ == "__main__":+ main()diff --git a/test/loaders/test_jina_url_reader.py b/test/loaders/test_jina_url_reader.pynew file mode 100644index 000000000..0d209b275--- /dev/null+++ b/test/loaders/test_jina_url_reader.py@@ -0,0 +1,76 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+import json++import pytest++from camel.loaders import JinaURLReader+from camel.types.enums import JinaReturnFormat+++def default_reader():+ return JinaURLReader()+++def test_read_content_default(default_reader: JinaURLReader):+ test_url = "https://en.wikipedia.org/wiki/Hollow_Knight"+ content = default_reader.read_content(test_url)+ assert "Title: Hollow Knight" in content+++def test_read_content_no_protocol(default_reader: JinaURLReader):+ test_url = "en.wikipedia.org/wiki/Hollow_Knight"+ content = default_reader.read_content(test_url)+ assert "Title: Hollow Knight" in content+++def test_read_content_markdown():+ test_url = "https://en.wikipedia.org/wiki/Hollow_Knight"+ markdown_reader = JinaURLReader(return_format=JinaReturnFormat.MARKDOWN)+ content = markdown_reader.read_content(test_url)+ assert "_Hollow Knight_" in content+++def test_read_content_html():+ test_url = "https://en.wikipedia.org/wiki/Hollow_Knight"+ html_reader = JinaURLReader(return_format=JinaReturnFormat.HTML)+ content = html_reader.read_content(test_url)+ assert "<title>Hollow Knight - Wikipedia</title>" in content+++def test_read_content_text():+ test_url = "https://en.wikipedia.org/wiki/Hollow_Knight"+ text_reader = JinaURLReader(return_format=JinaReturnFormat.TEXT)+ content = text_reader.read_content(test_url)+ assert "Hollow Knight" in content+++def test_read_content_json():+ test_url = "https://en.wikipedia.org/wiki/Hollow_Knight"+ json_reader = JinaURLReader(json_response=True)+ content = json_reader.read_content(test_url)+ content_json = json.loads(content)+ assert "data" in content_json+ assert "title" in content_json["data"]+ assert "Hollow Knight" in content_json["data"]["title"]+++def test_read_content_json_webpage():+ test_url = "https://httpbin.org/json"+ json_reader = JinaURLReader(return_format=JinaReturnFormat.TEXT)+ content = json_reader.read_content(test_url)+ content_json = json.loads(content)+ assert "slideshow" in content_json+ assert "slides" in content_json["slideshow"]