The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
community: add trust_env at web_base_loader (#28514)
Dong Shin0b1359801e7a
diff --git a/libs/community/langchain_community/document_loaders/web_base.py b/libs/community/langchain_community/document_loaders/web_base.pyindex 26bc5504d5..94d46c55ea 100644--- a/libs/community/langchain_community/document_loaders/web_base.py+++ b/libs/community/langchain_community/document_loaders/web_base.py@@ -71,6 +71,7 @@ class WebBaseLoader(BaseLoader):# bs_kwargs = None,# session = None,# show_progress = True,+ # trust_env = False,)Lazy load:@@ -128,6 +129,7 @@ class WebBaseLoader(BaseLoader):session: Any = None,*,show_progress: bool = True,+ trust_env: bool = False,) -> None:"""Initialize loader.@@ -140,6 +142,8 @@ class WebBaseLoader(BaseLoader):bs_get_text_kwargs: kwargs for beatifulsoup4 get_textbs_kwargs: kwargs for beatifulsoup4 web page parsingshow_progress: Show progress bar when loading pages.+ trust_env: set to True if using proxy to make web requests, for example+ using http(s)_proxy environment variables. Defaults to False."""# web_path kept for backwards-compatibility.if web_path and web_paths:@@ -189,6 +193,7 @@ class WebBaseLoader(BaseLoader):self.continue_on_failure = continue_on_failureself.autoset_encoding = autoset_encodingself.encoding = encoding+ self.trust_env = trust_env@propertydef web_path(self) -> str:@@ -199,7 +204,7 @@ class WebBaseLoader(BaseLoader):async def _fetch(self, url: str, retries: int = 3, cooldown: int = 2, backoff: float = 1.5) -> str:- async with aiohttp.ClientSession() as session:+ async with aiohttp.ClientSession(trust_env=self.trust_env) as session:for i in range(retries):try:kwargs: Dict = dict(