Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: openai tokencounter disallowed_special setting & google scholar proxy support (#1323)

Wendong-Fan76afef8c1631
diff --git a/camel/toolkits/google_scholar_toolkit.py b/camel/toolkits/google_scholar_toolkit.py
index b59b1dac5..f2158e01f 100644
--- a/camel/toolkits/google_scholar_toolkit.py
+++ b/camel/toolkits/google_scholar_toolkit.py
@@ -33,7 +33,11 @@ class GoogleScholarToolkit(BaseToolkit):
"""
def __init__(
- self, author_identifier: str, is_author_name: bool = False
+ self,
+ author_identifier: str,
+ is_author_name: bool = False,
+ proxy_http: Optional[str] = None,
+ proxy_https: Optional[str] = None,
) -> None:
r"""Initializes the GoogleScholarToolkit with the author's identifier.
@@ -42,8 +46,18 @@ class GoogleScholarToolkit(BaseToolkit):
of the author to search for.
is_author_name (bool): Flag to indicate if the identifier is a
name. (default: :obj:`False`)
+ proxy_http ( Optional[str]): Proxy http address pass to pg.
+ SingleProxy. (default: :obj:`None`)
+ proxy_https ( Optional[str]): Proxy https address pass to pg.
+ SingleProxy. (default: :obj:`None`)
"""
- from scholarly import scholarly
+ from scholarly import ProxyGenerator, scholarly
+
+ # Set Proxy is HTTP or HTTPS provided
+ if proxy_http or proxy_https:
+ pg = ProxyGenerator()
+ pg.SingleProxy(http=proxy_http, https=proxy_https)
+ scholarly.use_proxy(pg)
self.scholarly = scholarly
self.author_identifier = author_identifier
diff --git a/camel/utils/token_counting.py b/camel/utils/token_counting.py
index 2f831e662..3f4751980 100644
--- a/camel/utils/token_counting.py
+++ b/camel/utils/token_counting.py
@@ -144,12 +144,19 @@ class OpenAITokenCounter(BaseTokenCounter):
num_tokens += self.tokens_per_message
for key, value in message.items():
if not isinstance(value, list):
- num_tokens += len(self.encoding.encode(str(value)))
+ num_tokens += len(
+ self.encoding.encode(str(value), disallowed_special=())
+ )
else:
for item in value:
if item["type"] == "text":
num_tokens += len(
- self.encoding.encode(str(item["text"]))
+ self.encoding.encode(
+ str(
+ item["text"],
+ ),
+ disallowed_special=(),
+ )
)
elif item["type"] == "image_url":
image_str: str = item["image_url"]["url"]