🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: openai tokencounter disallowed_special setting & google scholar proxy support (#1323)
Wendong-Fan76afef8c1631
diff --git a/camel/toolkits/google_scholar_toolkit.py b/camel/toolkits/google_scholar_toolkit.pyindex b59b1dac5..f2158e01f 100644--- a/camel/toolkits/google_scholar_toolkit.py+++ b/camel/toolkits/google_scholar_toolkit.py@@ -33,7 +33,11 @@ class GoogleScholarToolkit(BaseToolkit):"""def __init__(- self, author_identifier: str, is_author_name: bool = False+ self,+ author_identifier: str,+ is_author_name: bool = False,+ proxy_http: Optional[str] = None,+ proxy_https: Optional[str] = None,) -> None:r"""Initializes the GoogleScholarToolkit with the author's identifier.@@ -42,8 +46,18 @@ class GoogleScholarToolkit(BaseToolkit):of the author to search for.is_author_name (bool): Flag to indicate if the identifier is aname. (default: :obj:`False`)+ proxy_http ( Optional[str]): Proxy http address pass to pg.+ SingleProxy. (default: :obj:`None`)+ proxy_https ( Optional[str]): Proxy https address pass to pg.+ SingleProxy. (default: :obj:`None`)"""- from scholarly import scholarly+ from scholarly import ProxyGenerator, scholarly++ # Set Proxy is HTTP or HTTPS provided+ if proxy_http or proxy_https:+ pg = ProxyGenerator()+ pg.SingleProxy(http=proxy_http, https=proxy_https)+ scholarly.use_proxy(pg)self.scholarly = scholarlyself.author_identifier = author_identifierdiff --git a/camel/utils/token_counting.py b/camel/utils/token_counting.pyindex 2f831e662..3f4751980 100644--- a/camel/utils/token_counting.py+++ b/camel/utils/token_counting.py@@ -144,12 +144,19 @@ class OpenAITokenCounter(BaseTokenCounter):num_tokens += self.tokens_per_messagefor key, value in message.items():if not isinstance(value, list):- num_tokens += len(self.encoding.encode(str(value)))+ num_tokens += len(+ self.encoding.encode(str(value), disallowed_special=())+ )else:for item in value:if item["type"] == "text":num_tokens += len(- self.encoding.encode(str(item["text"]))+ self.encoding.encode(+ str(+ item["text"],+ ),+ disallowed_special=(),+ ))elif item["type"] == "image_url":image_str: str = item["image_url"]["url"]