Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: Integrate Jina Reader into loaders (#656)

Isaac Jin1c6cef0ea709
diff --git a/camel/loaders/__init__.py b/camel/loaders/__init__.py
index ee476ee28..dced874f5 100644
--- a/camel/loaders/__init__.py
+++ b/camel/loaders/__init__.py
@@ -13,10 +13,12 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
from .base_io import File, read_file
+from .jina_url_reader import JinaURLReader
from .unstructured_io import UnstructuredIO
__all__ = [
'File',
'read_file',
'UnstructuredIO',
+ 'JinaURLReader',
]
diff --git a/camel/loaders/jina_url_reader.py b/camel/loaders/jina_url_reader.py
new file mode 100644
index 000000000..5416212b3
--- /dev/null
+++ b/camel/loaders/jina_url_reader.py
@@ -0,0 +1,99 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+
+import os
+from typing import Any, Optional
+
+from camel.types.enums import JinaReturnFormat
+
+JINA_ENDPOINT = "https://r.jina.ai/"
+
+
+class JinaURLReader:
+ r"""URL Reader provided by Jina AI. The output is cleaner and more
+ LLM-friendly than the URL Reader of UnstructuredIO. Can be configured to
+ replace the UnstructuredIO URL Reader in the pipeline.
+
+ Args:
+ api_key (Optional[str], optional): The API key for Jina AI. If not
+ provided, the reader will have a lower rate limit. Defaults to
+ None.
+ return_format (ReturnFormat, optional): The level of detail
+ of the returned content, which is optimized for LLMs. For
+ now screenshots are not supported. Defaults to
+ ReturnFormat.DEFAULT.
+ json_response (bool, optional): Whether to return the response
+ in JSON format. Defaults to False.
+ timeout (int, optional): The maximum time in seconds to wait for
+ the page to be rendered. Defaults to 30.
+ **kwargs (Any): Additional keyword arguments, including proxies,
+ cookies, etc. It should align with the HTTP Header field and
+ value pairs listed in the reference.
+
+ References:
+ https://jina.ai/reader
+ """
+
+ def __init__(
+ self,
+ api_key: Optional[str] = None,
+ return_format: JinaReturnFormat = JinaReturnFormat.DEFAULT,
+ json_response: bool = False,
+ timeout: int = 30,
+ **kwargs: Any,
+ ) -> None:
+ api_key = api_key or os.getenv('JINA_API_KEY')
+ if api_key is None:
+ print(
+ "[JinaURLReader] JINA_API_KEY not set. This will result in a "
+ "low rate limit of Jina URL Reader. Get API key here: "
+ "https://jina.ai/reader."
+ )
+
+ # if the following field not provided, it will be None
+ api_field = f"Bearer {api_key}" if api_key else None
+ json_field = "application/json" if json_response else None
+
+ raw_headers = {
+ "Authorization": api_field,
+ "X-Return-Format": return_format.value,
+ "Accept": json_field,
+ "X-Timeout": str(timeout),
+ **kwargs,
+ }
+
+ # eliminate None values
+ self._headers = {k: v for k, v in raw_headers.items() if v}
+
+ def read_content(self, url: str) -> str:
+ r"""Reads the content of a URL and returns it as a string with
+ given form.
+
+ Args:
+ url (str): The URL to read.
+
+ Returns:
+ str: The content of the URL.
+ """
+
+ import requests
+
+ full_url = f"{JINA_ENDPOINT}{url}"
+ try:
+ resp = requests.get(full_url, headers=self._headers)
+ resp.raise_for_status()
+ except Exception as e:
+ raise Exception(f"Failed to read content from {url}: {e}") from e
+
+ return resp.text
diff --git a/camel/types/enums.py b/camel/types/enums.py
index f20982553..f3b1328d3 100644
--- a/camel/types/enums.py
+++ b/camel/types/enums.py
@@ -401,3 +401,10 @@ class VoiceType(Enum):
VoiceType.NOVA,
VoiceType.SHIMMER,
}
+
+
+class JinaReturnFormat(Enum):
+ DEFAULT = None
+ MARKDOWN = "markdown"
+ HTML = "html"
+ TEXT = "text"
diff --git a/examples/loaders/jina_url_reader_example.py b/examples/loaders/jina_url_reader_example.py
new file mode 100644
index 000000000..78f523e2d
--- /dev/null
+++ b/examples/loaders/jina_url_reader_example.py
@@ -0,0 +1,57 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+
+
+from camel.loaders import JinaURLReader
+from camel.types.enums import JinaReturnFormat
+
+
+def read_with_different_format(return_format, json_response):
+ URL = "https://en.wikipedia.org/wiki/Miss_Meyers"
+ jina_url_reader = JinaURLReader(
+ return_format=return_format, json_response=json_response
+ )
+ content = jina_url_reader.read_content(URL)
+ print(content)
+
+
+def main():
+ formats = [
+ JinaReturnFormat.DEFAULT,
+ JinaReturnFormat.TEXT,
+ JinaReturnFormat.HTML,
+ JinaReturnFormat.MARKDOWN,
+ ]
+
+ print("Choose a return format of read content:")
+ print("1. Default, optimized for LLM inputs")
+ print("2. Pure Text")
+ print("3. HTML")
+ print("4. Markdown")
+ choice = input("Enter your choice (1-4): ")
+
+ if not choice.isnumeric() or int(choice) < 1 or int(choice) > 4:
+ print("Invalid choice. Exiting.")
+ return
+
+ return_format = formats[int(choice) - 1]
+
+ json_response = input("Do you want the response in JSON format? (y/N): ")
+ json_response = json_response.lower() == "y"
+
+ read_with_different_format(return_format, json_response)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/test/loaders/test_jina_url_reader.py b/test/loaders/test_jina_url_reader.py
new file mode 100644
index 000000000..0d209b275
--- /dev/null
+++ b/test/loaders/test_jina_url_reader.py
@@ -0,0 +1,76 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import json
+
+import pytest
+
+from camel.loaders import JinaURLReader
+from camel.types.enums import JinaReturnFormat
+
+
+def default_reader():
+ return JinaURLReader()
+
+
+def test_read_content_default(default_reader: JinaURLReader):
+ test_url = "https://en.wikipedia.org/wiki/Hollow_Knight"
+ content = default_reader.read_content(test_url)
+ assert "Title: Hollow Knight" in content
+
+
+def test_read_content_no_protocol(default_reader: JinaURLReader):
+ test_url = "en.wikipedia.org/wiki/Hollow_Knight"
+ content = default_reader.read_content(test_url)
+ assert "Title: Hollow Knight" in content
+
+
+def test_read_content_markdown():
+ test_url = "https://en.wikipedia.org/wiki/Hollow_Knight"
+ markdown_reader = JinaURLReader(return_format=JinaReturnFormat.MARKDOWN)
+ content = markdown_reader.read_content(test_url)
+ assert "_Hollow Knight_" in content
+
+
+def test_read_content_html():
+ test_url = "https://en.wikipedia.org/wiki/Hollow_Knight"
+ html_reader = JinaURLReader(return_format=JinaReturnFormat.HTML)
+ content = html_reader.read_content(test_url)
+ assert "<title>Hollow Knight - Wikipedia</title>" in content
+
+
+def test_read_content_text():
+ test_url = "https://en.wikipedia.org/wiki/Hollow_Knight"
+ text_reader = JinaURLReader(return_format=JinaReturnFormat.TEXT)
+ content = text_reader.read_content(test_url)
+ assert "Hollow Knight" in content
+
+
+def test_read_content_json():
+ test_url = "https://en.wikipedia.org/wiki/Hollow_Knight"
+ json_reader = JinaURLReader(json_response=True)
+ content = json_reader.read_content(test_url)
+ content_json = json.loads(content)
+ assert "data" in content_json
+ assert "title" in content_json["data"]
+ assert "Hollow Knight" in content_json["data"]["title"]
+
+
+def test_read_content_json_webpage():
+ test_url = "https://httpbin.org/json"
+ json_reader = JinaURLReader(return_format=JinaReturnFormat.TEXT)
+ content = json_reader.read_content(test_url)
+ content_json = json.loads(content)
+ assert "slideshow" in content_json
+ assert "slides" in content_json["slideshow"]