Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
fix(oxylabs): report scrape failures instead of raising IndexError (#7044)
oxy-giedriusfe62d04cdb5b
diff --git a/docs/edge/ar/tools/web-scraping/oxylabsscraperstool.mdx b/docs/edge/ar/tools/web-scraping/oxylabsscraperstool.mdxindex b83ba8b33..a62f7c18d 100644--- a/docs/edge/ar/tools/web-scraping/oxylabsscraperstool.mdx+++ b/docs/edge/ar/tools/web-scraping/oxylabsscraperstool.mdx@@ -4,7 +4,7 @@ description: >تتيح أدوات استخراج Oxylabs الوصول بسهولة إلى المعلومات من المصادر المعنية. يرجى الاطلاع على قائمة المصادر المتاحة أدناه:- `Amazon Product`- `Amazon Search`- - `Google Seach`+ - `Google Search`- `Universal`icon: globemode: "wide"@@ -87,7 +87,7 @@ print(result)### المعاملات- `query` - مصطلح بحث Amazon.-- `domain` - توطين النطاق لـ Bestbuy.+- `domain` - توطين النطاق لـ Amazon.- `start_page` - رقم صفحة البداية.- `pages` - عدد الصفحات المراد استرجاعها.- `geo_location` - موقع _التوصيل إلى_.diff --git a/docs/edge/en/tools/web-scraping/oxylabsscraperstool.mdx b/docs/edge/en/tools/web-scraping/oxylabsscraperstool.mdxindex 4529a90b7..d69654bac 100644--- a/docs/edge/en/tools/web-scraping/oxylabsscraperstool.mdx+++ b/docs/edge/en/tools/web-scraping/oxylabsscraperstool.mdx@@ -4,7 +4,7 @@ description: >Oxylabs Scrapers allow to easily access the information from the respective sources. Please see the list of available sources below:- `Amazon Product`- `Amazon Search`- - `Google Seach`+ - `Google Search`- `Universal`icon: globemode: "wide"@@ -87,7 +87,7 @@ print(result)### Parameters- `query` - Amazon search term.-- `domain` - Domain localization for Bestbuy.+- `domain` - domain localization for Amazon.- `start_page` - starting page number.- `pages` - number of pages to retrieve.- `geo_location` - the _Deliver to_ location.diff --git a/docs/edge/ko/tools/web-scraping/oxylabsscraperstool.mdx b/docs/edge/ko/tools/web-scraping/oxylabsscraperstool.mdxindex a84058f7e..138869b0b 100644--- a/docs/edge/ko/tools/web-scraping/oxylabsscraperstool.mdx+++ b/docs/edge/ko/tools/web-scraping/oxylabsscraperstool.mdx@@ -4,7 +4,7 @@ description: >Oxylabs 스크래퍼를 사용하면 해당 소스에서 정보를 쉽게 접근할 수 있습니다. 아래에서 사용 가능한 소스 목록을 확인하세요:- `Amazon Product`- `Amazon Search`- - `Google Seach`+ - `Google Search`- `Universal`icon: globemode: "wide"@@ -87,7 +87,7 @@ print(result)### 파라미터- `query` - Amazon 검색어.-- `domain` - Bestbuy의 도메인 로컬라이제이션.+- `domain` - Amazon의 도메인 로컬라이제이션.- `start_page` - 시작 페이지 번호.- `pages` - 가져올 페이지 수.- `geo_location` - _배송지_ 위치.diff --git a/docs/edge/pt-BR/tools/web-scraping/oxylabsscraperstool.mdx b/docs/edge/pt-BR/tools/web-scraping/oxylabsscraperstool.mdxindex db544984e..f5e146ff5 100644--- a/docs/edge/pt-BR/tools/web-scraping/oxylabsscraperstool.mdx+++ b/docs/edge/pt-BR/tools/web-scraping/oxylabsscraperstool.mdx@@ -4,7 +4,7 @@ description: >Os Scrapers da Oxylabs permitem acessar facilmente informações de fontes específicas. Veja abaixo a lista de fontes disponíveis:- `Amazon Product`- `Amazon Search`- - `Google Seach`+ - `Google Search`- `Universal`icon: globemode: "wide"@@ -87,7 +87,7 @@ print(result)### Parâmetros- `query` - termo de busca da Amazon.-- `domain` - Domínio de localização para Bestbuy.+- `domain` - domínio de localização da Amazon.- `start_page` - número da página inicial.- `pages` - quantidade de páginas a ser recuperada.- `geo_location` - local de entrega (_Deliver to_).diff --git a/lib/crewai-tools/src/crewai_tools/tools/oxylabs_amazon_product_scraper_tool/oxylabs_amazon_product_scraper_tool.py b/lib/crewai-tools/src/crewai_tools/tools/oxylabs_amazon_product_scraper_tool/oxylabs_amazon_product_scraper_tool.pyindex 915b462ca..fcfc01169 100644--- a/lib/crewai-tools/src/crewai_tools/tools/oxylabs_amazon_product_scraper_tool/oxylabs_amazon_product_scraper_tool.py+++ b/lib/crewai-tools/src/crewai_tools/tools/oxylabs_amazon_product_scraper_tool/oxylabs_amazon_product_scraper_tool.py@@ -1,25 +1,9 @@-from importlib.metadata import version-import json-import os-from platform import architecture, python_versionfrom typing import Any-from crewai.tools import BaseTool, EnvVar-from pydantic import BaseModel, ConfigDict, Field+from crewai.tools.tool_failure import ToolFailure+from pydantic import BaseModel, Field--try:- from oxylabs import RealtimeClient # type: ignore[import-untyped]- from oxylabs.sources.response import ( # type: ignore[import-untyped]- Response as OxylabsResponse,- )-- OXYLABS_AVAILABLE = True-except ImportError:- RealtimeClient = Any- OxylabsResponse = Any-- OXYLABS_AVAILABLE = False+from crewai_tools.tools.oxylabs_base_tool.oxylabs_base_tool import OxylabsBaseTool__all__ = ["OxylabsAmazonProductScraperConfig", "OxylabsAmazonProductScraperTool"]@@ -51,7 +35,7 @@ class OxylabsAmazonProductScraperConfig(BaseModel):)-class OxylabsAmazonProductScraperTool(BaseTool):+class OxylabsAmazonProductScraperTool(OxylabsBaseTool):"""Scrape Amazon product pages with OxylabsAmazonProductScraperTool.Get Oxylabs account:@@ -63,104 +47,11 @@ class OxylabsAmazonProductScraperTool(BaseTool):config: Configuration options. See ``OxylabsAmazonProductScraperConfig``"""- model_config = ConfigDict(- arbitrary_types_allowed=True,- validate_assignment=True,- )name: str = "Oxylabs Amazon Product Scraper tool"description: str = "Scrape Amazon product pages with Oxylabs Amazon Product Scraper"args_schema: type[BaseModel] = OxylabsAmazonProductScraperArgs- oxylabs_api: Anyconfig: OxylabsAmazonProductScraperConfig- package_dependencies: list[str] = Field(default_factory=lambda: ["oxylabs"])- env_vars: list[EnvVar] = Field(- default_factory=lambda: [- EnvVar(- name="OXYLABS_USERNAME",- description="Username for Oxylabs",- required=True,- ),- EnvVar(- name="OXYLABS_PASSWORD",- description="Password for Oxylabs",- required=True,- ),- ]- )- def __init__(- self,- username: str | None = None,- password: str | None = None,- config: OxylabsAmazonProductScraperConfig | dict[str, Any] | None = None,- **kwargs: Any,- ) -> None:- bits, _ = architecture()- sdk_type = (- f"oxylabs-crewai-sdk-python/"- f"{version('crewai')} "- f"({python_version()}; {bits})"- )-- if username is None or password is None:- username, password = self._get_credentials_from_env()-- if OXYLABS_AVAILABLE:- from oxylabs import RealtimeClient-- kwargs["oxylabs_api"] = RealtimeClient(- username=username,- password=password,- sdk_type=sdk_type,- )- else:- import click-- if click.confirm(- "You are missing the 'oxylabs' package. Would you like to install it?"- ):- import subprocess-- try:- subprocess.run(["uv", "add", "oxylabs"], check=True) # noqa: S607- from oxylabs import RealtimeClient-- kwargs["oxylabs_api"] = RealtimeClient(- username=username,- password=password,- sdk_type=sdk_type,- )- except subprocess.CalledProcessError as e:- raise ImportError("Failed to install oxylabs package") from e- else:- raise ImportError(- "`oxylabs` package not found, please run `uv add oxylabs`"- )-- if config is None:- config = OxylabsAmazonProductScraperConfig()- super().__init__(config=config, **kwargs)-- def _get_credentials_from_env(self) -> tuple[str, str]:- username = os.environ.get("OXYLABS_USERNAME")- password = os.environ.get("OXYLABS_PASSWORD")- if not username or not password:- raise ValueError(- "You must pass oxylabs username and password when instantiating the tool "- "or specify OXYLABS_USERNAME and OXYLABS_PASSWORD environment variables"- )- return username, password-- def _run(self, query: str) -> str:- response = self.oxylabs_api.amazon.scrape_product(- query,- **self.config.model_dump(exclude_none=True),- )-- content = response.results[0].content-- if isinstance(content, dict):- return json.dumps(content)-- return str(content)+ def _run(self, query: str) -> str | ToolFailure:+ return self._scrape(self.oxylabs_api.amazon.scrape_product, query)diff --git a/lib/crewai-tools/src/crewai_tools/tools/oxylabs_amazon_search_scraper_tool/oxylabs_amazon_search_scraper_tool.py b/lib/crewai-tools/src/crewai_tools/tools/oxylabs_amazon_search_scraper_tool/oxylabs_amazon_search_scraper_tool.pyindex 2f693283b..ebd1d2b14 100644--- a/lib/crewai-tools/src/crewai_tools/tools/oxylabs_amazon_search_scraper_tool/oxylabs_amazon_search_scraper_tool.py+++ b/lib/crewai-tools/src/crewai_tools/tools/oxylabs_amazon_search_scraper_tool/oxylabs_amazon_search_scraper_tool.py@@ -1,25 +1,9 @@-from importlib.metadata import version-import json-import os-from platform import architecture, python_versionfrom typing import Any-from crewai.tools import BaseTool, EnvVar-from pydantic import BaseModel, ConfigDict, Field+from crewai.tools.tool_failure import ToolFailure+from pydantic import BaseModel, Field--try:- from oxylabs import RealtimeClient # type: ignore[import-untyped]- from oxylabs.sources.response import ( # type: ignore[import-untyped]- Response as OxylabsResponse,- )-- OXYLABS_AVAILABLE = True-except ImportError:- RealtimeClient = Any- OxylabsResponse = Any-- OXYLABS_AVAILABLE = False+from crewai_tools.tools.oxylabs_base_tool.oxylabs_base_tool import OxylabsBaseTool__all__ = ["OxylabsAmazonSearchScraperConfig", "OxylabsAmazonSearchScraperTool"]@@ -53,7 +37,7 @@ class OxylabsAmazonSearchScraperConfig(BaseModel):)-class OxylabsAmazonSearchScraperTool(BaseTool):+class OxylabsAmazonSearchScraperTool(OxylabsBaseTool):"""Scrape Amazon search results with OxylabsAmazonSearchScraperTool.Get Oxylabs account:@@ -65,104 +49,11 @@ class OxylabsAmazonSearchScraperTool(BaseTool):config: Configuration options. See ``OxylabsAmazonSearchScraperConfig``"""- model_config = ConfigDict(- arbitrary_types_allowed=True,- validate_assignment=True,- )name: str = "Oxylabs Amazon Search Scraper tool"description: str = "Scrape Amazon search results with Oxylabs Amazon Search Scraper"args_schema: type[BaseModel] = OxylabsAmazonSearchScraperArgs- oxylabs_api: Anyconfig: OxylabsAmazonSearchScraperConfig- package_dependencies: list[str] = Field(default_factory=lambda: ["oxylabs"])- env_vars: list[EnvVar] = Field(- default_factory=lambda: [- EnvVar(- name="OXYLABS_USERNAME",- description="Username for Oxylabs",- required=True,- ),- EnvVar(- name="OXYLABS_PASSWORD",- description="Password for Oxylabs",- required=True,- ),- ]- )- def __init__(- self,- username: str | None = None,- password: str | None = None,- config: OxylabsAmazonSearchScraperConfig | dict[str, Any] | None = None,- **kwargs: Any,- ) -> None:- bits, _ = architecture()- sdk_type = (- f"oxylabs-crewai-sdk-python/"- f"{version('crewai')} "- f"({python_version()}; {bits})"- )-- if username is None or password is None:- username, password = self._get_credentials_from_env()-- if OXYLABS_AVAILABLE:- from oxylabs import RealtimeClient-- kwargs["oxylabs_api"] = RealtimeClient(- username=username,- password=password,- sdk_type=sdk_type,- )- else:- import click-- if click.confirm(- "You are missing the 'oxylabs' package. Would you like to install it?"- ):- import subprocess-- try:- subprocess.run(["uv", "add", "oxylabs"], check=True) # noqa: S607- from oxylabs import RealtimeClient-- kwargs["oxylabs_api"] = RealtimeClient(- username=username,- password=password,- sdk_type=sdk_type,- )- except subprocess.CalledProcessError as e:- raise ImportError("Failed to install oxylabs package") from e- else:- raise ImportError(- "`oxylabs` package not found, please run `uv add oxylabs`"- )-- if config is None:- config = OxylabsAmazonSearchScraperConfig()- super().__init__(config=config, **kwargs)-- def _get_credentials_from_env(self) -> tuple[str, str]:- username = os.environ.get("OXYLABS_USERNAME")- password = os.environ.get("OXYLABS_PASSWORD")- if not username or not password:- raise ValueError(- "You must pass oxylabs username and password when instantiating the tool "- "or specify OXYLABS_USERNAME and OXYLABS_PASSWORD environment variables"- )- return username, password-- def _run(self, query: str) -> str:- response = self.oxylabs_api.amazon.scrape_search(- query,- **self.config.model_dump(exclude_none=True),- )-- content = response.results[0].content-- if isinstance(content, dict):- return json.dumps(content)-- return str(content)+ def _run(self, query: str) -> str | ToolFailure:+ return self._scrape(self.oxylabs_api.amazon.scrape_search, query)diff --git a/lib/crewai-tools/src/crewai_tools/tools/oxylabs_base_tool/__init__.py b/lib/crewai-tools/src/crewai_tools/tools/oxylabs_base_tool/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/lib/crewai-tools/src/crewai_tools/tools/oxylabs_base_tool/oxylabs_base_tool.py b/lib/crewai-tools/src/crewai_tools/tools/oxylabs_base_tool/oxylabs_base_tool.pynew file mode 100644index 000000000..664d0b980--- /dev/null+++ b/lib/crewai-tools/src/crewai_tools/tools/oxylabs_base_tool/oxylabs_base_tool.py@@ -0,0 +1,315 @@+from collections.abc import Callable, Iterator+from contextlib import contextmanager+from contextvars import ContextVar+from importlib.metadata import PackageNotFoundError, version+import json+import logging+import os+from platform import architecture, python_version+import re+import threading+from typing import TYPE_CHECKING, Any++from crewai.tools import BaseTool, EnvVar+from crewai.tools.tool_failure import ToolFailure+from pydantic import ConfigDict, Field+++__all__ = ["OxylabsBaseTool"]+++_HTTP_ERROR_PATTERN = re.compile(+ r"(\d{3})\s+(?:Client|Server) Error:\s*(.+?)\s+for url", re.IGNORECASE+)+++_active_capture: ContextVar[list[str] | None] = ContextVar(+ "oxylabs_active_capture", default=None+)+_capture_handler_lock = threading.Lock()+++class _CaptureHandler(logging.Handler):+ """Routes each SDK error to the capture belonging to the call that caused it.++ A single handler serves every concurrent scrape, and the context variable+ keeps one thread's or task's records out of the others' hands. Sharing a+ collector instead would let two simultaneous scrapes each see both errors,+ and a timeout could be reported as the other request's non-retryable 400.+ """++ def emit(self, record: logging.LogRecord) -> None:+ messages = _active_capture.get()+ if messages is not None:+ messages.append(record.getMessage())+++_CAPTURE_HANDLER = _CaptureHandler(level=logging.ERROR)+++@contextmanager+def _captured_sdk_errors() -> Iterator[list[str]]:+ """Collect the error messages the oxylabs SDK only writes to its logger.++ The SDK catches transport and HTTP errors, logs them and hands back an+ empty response, so the cause is absent from the object we get back.+ Listening on its logger is the only way to tell the agent what actually+ went wrong.++ The handler stays attached once installed: it is inert outside a capture,+ and detaching it would race with scrapes running concurrently. No level,+ filter or other handler is touched, so an application that has silenced the+ SDK simply falls back to the generic failure.+ """+ sdk_logger = logging.getLogger("oxylabs")+ with _capture_handler_lock:+ if _CAPTURE_HANDLER not in sdk_logger.handlers:+ sdk_logger.addHandler(_CAPTURE_HANDLER)++ messages: list[str] = []+ token = _active_capture.set(messages)+ try:+ yield messages+ finally:+ _active_capture.reset(token)+++def _api_detail(messages: list[str]) -> str | None:+ """Pull the API's own explanation out of a logged response body."""+ for message in messages:+ try:+ payload = json.loads(message)+ except (TypeError, ValueError):+ continue+ if isinstance(payload, dict):+ detail = payload.get("message")+ if isinstance(detail, str) and detail.strip():+ return detail.strip()+ return None+++def _diagnose(messages: list[str]) -> tuple[str, str | None, bool] | None:+ """Summarize what the SDK logged as (description, code, retryable)."""+ reported = [m.strip() for m in messages if m and m.strip()]+ if not reported:+ return None++ joined = " | ".join(reported)++ http_error = _HTTP_ERROR_PATTERN.search(joined)+ if http_error:+ status = int(http_error.group(1))+ description = f"{status} {http_error.group(2).strip()}"+ detail = _api_detail(reported)+ if detail:+ description = f"{description} - {detail}"+ return description, str(status), status == 429 or status >= 500++ if "timed out" in joined.lower():+ return "the request timed out", "timeout", True++ # Anything else the SDK chose to log, e.g. a connection error.+ return joined[:300], None, False+++class OxylabsBaseTool(BaseTool):+ """Base class for the Oxylabs Web Scraper API tools.++ Holds what every Oxylabs tool shares: the credentialed ``RealtimeClient``,+ and the translation of a Web Scraper API response into either the scraped+ content or a :class:`ToolFailure`.++ Get Oxylabs account:+ https://dashboard.oxylabs.io/en+ """++ model_config = ConfigDict(+ arbitrary_types_allowed=True,+ validate_assignment=True,+ )++ if TYPE_CHECKING:+ # Declared as its own model by every subclass, and enforced in+ # ``__init__``; annotated here only so this class can read it.+ config: Any++ oxylabs_api: Any+ package_dependencies: list[str] = Field(default_factory=lambda: ["oxylabs"])+ env_vars: list[EnvVar] = Field(+ default_factory=lambda: [+ EnvVar(+ name="OXYLABS_USERNAME",+ description="Username for Oxylabs",+ required=True,+ ),+ EnvVar(+ name="OXYLABS_PASSWORD",+ description="Password for Oxylabs",+ required=True,+ ),+ ]+ )++ def __init__(+ self,+ username: str | None = None,+ password: str | None = None,+ config: Any = None,+ **kwargs: Any,+ ) -> None:+ # Resolved before the client is built, so a subclass that forgets the+ # field fails without first opening a session.+ config_field = type(self).model_fields.get("config")+ config_model = config_field.annotation if config_field else None+ if config_model is None:+ raise TypeError(+ f"{type(self).__name__} must declare a 'config' model field"+ )+ if config is None:+ config = config_model()++ if username is None or password is None:+ username, password = self._get_credentials_from_env()++ kwargs["oxylabs_api"] = self._build_client(username, password)++ super().__init__(config=config, **kwargs)++ @staticmethod+ def _get_credentials_from_env() -> tuple[str, str]:+ username = os.environ.get("OXYLABS_USERNAME")+ password = os.environ.get("OXYLABS_PASSWORD")+ if not username or not password:+ raise ValueError(+ "You must pass oxylabs username and password when instantiating the tool "+ "or specify OXYLABS_USERNAME and OXYLABS_PASSWORD environment variables"+ )+ return username, password++ @staticmethod+ def _resolve_realtime_client() -> Any:+ try:+ from oxylabs import RealtimeClient # type: ignore[import-untyped]+ except ImportError:+ import click++ if not click.confirm(+ "You are missing the 'oxylabs' package. Would you like to install it?"+ ):+ raise ImportError(+ "`oxylabs` package not found, please run `uv add oxylabs`"+ ) from None++ import importlib+ import subprocess++ try:+ subprocess.run(["uv", "add", "oxylabs"], check=True) # noqa: S607+ except (subprocess.CalledProcessError, OSError) as e:+ # OSError covers uv itself being absent.+ raise ImportError("Failed to install oxylabs package") from e++ return importlib.import_module("oxylabs").RealtimeClient++ return RealtimeClient++ @classmethod+ def _build_client(cls, username: str, password: str) -> Any:+ realtime_client = cls._resolve_realtime_client()++ try:+ crewai_version = version("crewai")+ except PackageNotFoundError:+ # Only ever reported as telemetry; not worth failing construction.+ crewai_version = "unknown"++ bits, _ = architecture()+ return realtime_client(+ username=username,+ password=password,+ sdk_type=(+ f"oxylabs-crewai-sdk-python/"+ f"{crewai_version} "+ f"({python_version()}; {bits})"+ ),+ )++ def _scrape(self, scraper: Callable[..., Any], target: str) -> str | ToolFailure:+ """Run one scrape and turn the outcome into content or a failure."""+ with _captured_sdk_errors() as sdk_errors:+ response = scraper(target, **self.config.model_dump(exclude_none=True))++ return self._handle_response(response, sdk_errors)++ def _handle_response(+ self, response: Any, sdk_errors: list[str] | None = None+ ) -> str | ToolFailure:+ """Return the scraped content, or report why there is none.++ The oxylabs SDK logs transport and validation errors and hands back an+ empty response rather than raising, so rejected requests -- wrong+ credentials, config the source does not accept, exhausted quota -- have+ to be recognised here instead of reaching the agent as an ``IndexError``+ on ``results[0]``. A non-2xx ``status_code`` on the result is the same+ situation one level down: the job ran, the page did not come back.+ """+ results = getattr(response, "results", None)+ if not results:+ diagnosis = _diagnose(sdk_errors or [])+ if diagnosis:+ description, code, retryable = diagnosis+ return ToolFailure(+ message=(+ f"Oxylabs Web Scraper API rejected the request: {description}"+ ),+ code=code or "request_rejected",+ retryable=retryable,+ )+ return ToolFailure(+ message=(+ "Oxylabs Web Scraper API returned no results and reported no "+ "error, so the request was rejected before any page was "+ "scraped. Check that OXYLABS_USERNAME and OXYLABS_PASSWORD "+ "are valid and that this tool's config options are accepted "+ "for this source."+ ),+ code="empty_response",+ )++ result = results[0]++ try:+ status_code = int(result.status_code)+ except (AttributeError, TypeError, ValueError):+ status_code = None++ if status_code is not None and not 200 <= status_code < 300:+ return ToolFailure(+ message=(+ f"Oxylabs Web Scraper API could not retrieve the page: the "+ f"target responded with status {status_code}."+ ),+ code=str(status_code),+ retryable=status_code == 429 or status_code >= 500,+ )++ content = getattr(result, "content", None)+ if content is None:+ return ToolFailure(+ message=(+ "Oxylabs Web Scraper API returned a result with no content. "+ "The page may be empty, or the parser found nothing to extract."+ ),+ code="empty_content",+ )++ # ``parse``/``parsing_instructions`` results arrive as dicts or lists;+ # only unparsed HTML comes back as a string. ``str()`` on a list would+ # hand the agent a Python repr instead of JSON.+ if isinstance(content, str):+ return content++ try:+ return json.dumps(content)+ except (TypeError, ValueError):+ return str(content)diff --git a/lib/crewai-tools/src/crewai_tools/tools/oxylabs_google_search_scraper_tool/oxylabs_google_search_scraper_tool.py b/lib/crewai-tools/src/crewai_tools/tools/oxylabs_google_search_scraper_tool/oxylabs_google_search_scraper_tool.pyindex 1d29fcd3e..9bff49fe9 100644--- a/lib/crewai-tools/src/crewai_tools/tools/oxylabs_google_search_scraper_tool/oxylabs_google_search_scraper_tool.py+++ b/lib/crewai-tools/src/crewai_tools/tools/oxylabs_google_search_scraper_tool/oxylabs_google_search_scraper_tool.py@@ -1,25 +1,9 @@-from importlib.metadata import version-import json-import os-from platform import architecture, python_versionfrom typing import Any-from crewai.tools import BaseTool, EnvVar-from pydantic import BaseModel, ConfigDict, Field+from crewai.tools.tool_failure import ToolFailure+from pydantic import BaseModel, Field--try:- from oxylabs import RealtimeClient # type: ignore[import-untyped]- from oxylabs.sources.response import ( # type: ignore[import-untyped]- Response as OxylabsResponse,- )-- OXYLABS_AVAILABLE = True-except ImportError:- RealtimeClient = Any- OxylabsResponse = Any-- OXYLABS_AVAILABLE = False+from crewai_tools.tools.oxylabs_base_tool.oxylabs_base_tool import OxylabsBaseTool__all__ = ["OxylabsGoogleSearchScraperConfig", "OxylabsGoogleSearchScraperTool"]@@ -42,6 +26,11 @@ class OxylabsGoogleSearchScraperConfig(BaseModel):limit: int | None = Field(None, description="Number of results to retrieve in each page.")+ locale: str | None = Field(+ None,+ description="`Accept-Language` header value which changes your Google "+ "search page web interface language.",+ )geo_location: str | None = Field(None, description="The Deliver to location.")user_agent_type: str | None = Field(None, description="Device type and browser.")render: str | None = Field(None, description="Enables JavaScript rendering.")@@ -56,7 +45,7 @@ class OxylabsGoogleSearchScraperConfig(BaseModel):)-class OxylabsGoogleSearchScraperTool(BaseTool):+class OxylabsGoogleSearchScraperTool(OxylabsBaseTool):"""Scrape Google Search results with OxylabsGoogleSearchScraperTool.Get Oxylabs account:@@ -68,104 +57,11 @@ class OxylabsGoogleSearchScraperTool(BaseTool):config: Configuration options. See ``OxylabsGoogleSearchScraperConfig``"""- model_config = ConfigDict(- arbitrary_types_allowed=True,- validate_assignment=True,- )name: str = "Oxylabs Google Search Scraper tool"description: str = "Scrape Google Search results with Oxylabs Google Search Scraper"args_schema: type[BaseModel] = OxylabsGoogleSearchScraperArgs- oxylabs_api: Anyconfig: OxylabsGoogleSearchScraperConfig- package_dependencies: list[str] = Field(default_factory=lambda: ["oxylabs"])- env_vars: list[EnvVar] = Field(- default_factory=lambda: [- EnvVar(- name="OXYLABS_USERNAME",- description="Username for Oxylabs",- required=True,- ),- EnvVar(- name="OXYLABS_PASSWORD",- description="Password for Oxylabs",- required=True,- ),- ]- )- def __init__(- self,- username: str | None = None,- password: str | None = None,- config: OxylabsGoogleSearchScraperConfig | dict[str, Any] | None = None,- **kwargs: Any,- ) -> None:- bits, _ = architecture()- sdk_type = (- f"oxylabs-crewai-sdk-python/"- f"{version('crewai')} "- f"({python_version()}; {bits})"- )-- if username is None or password is None:- username, password = self._get_credentials_from_env()-- if OXYLABS_AVAILABLE:- from oxylabs import RealtimeClient-- kwargs["oxylabs_api"] = RealtimeClient(- username=username,- password=password,- sdk_type=sdk_type,- )- else:- import click-- if click.confirm(- "You are missing the 'oxylabs' package. Would you like to install it?"- ):- import subprocess-- try:- subprocess.run(["uv", "add", "oxylabs"], check=True) # noqa: S607- from oxylabs import RealtimeClient-- kwargs["oxylabs_api"] = RealtimeClient(- username=username,- password=password,- sdk_type=sdk_type,- )- except subprocess.CalledProcessError as e:- raise ImportError("Failed to install oxylabs package") from e- else:- raise ImportError(- "`oxylabs` package not found, please run `uv add oxylabs`"- )-- if config is None:- config = OxylabsGoogleSearchScraperConfig()- super().__init__(config=config, **kwargs)-- def _get_credentials_from_env(self) -> tuple[str, str]:- username = os.environ.get("OXYLABS_USERNAME")- password = os.environ.get("OXYLABS_PASSWORD")- if not username or not password:- raise ValueError(- "You must pass oxylabs username and password when instantiating the tool "- "or specify OXYLABS_USERNAME and OXYLABS_PASSWORD environment variables"- )- return username, password-- def _run(self, query: str, **kwargs: Any) -> str:- response = self.oxylabs_api.google.scrape_search(- query,- **self.config.model_dump(exclude_none=True),- )-- content = response.results[0].content-- if isinstance(content, dict):- return json.dumps(content)-- return str(content)+ def _run(self, query: str) -> str | ToolFailure:+ return self._scrape(self.oxylabs_api.google.scrape_search, query)diff --git a/lib/crewai-tools/src/crewai_tools/tools/oxylabs_universal_scraper_tool/oxylabs_universal_scraper_tool.py b/lib/crewai-tools/src/crewai_tools/tools/oxylabs_universal_scraper_tool/oxylabs_universal_scraper_tool.pyindex adb52fbcc..8c2526a8e 100644--- a/lib/crewai-tools/src/crewai_tools/tools/oxylabs_universal_scraper_tool/oxylabs_universal_scraper_tool.py+++ b/lib/crewai-tools/src/crewai_tools/tools/oxylabs_universal_scraper_tool/oxylabs_universal_scraper_tool.py@@ -1,25 +1,10 @@-from importlib.metadata import version-import json-import os-from platform import architecture, python_versionfrom typing import Any-from crewai.tools import BaseTool, EnvVar-from pydantic import BaseModel, ConfigDict, Field+from crewai.tools.tool_failure import ToolFailure+from pydantic import BaseModel, Field+from crewai_tools.tools.oxylabs_base_tool.oxylabs_base_tool import OxylabsBaseTool-try:- from oxylabs import RealtimeClient # type: ignore[import-untyped]- from oxylabs.sources.response import ( # type: ignore[import-untyped]- Response as OxylabsResponse,- )-- OXYLABS_AVAILABLE = True-except ImportError:- RealtimeClient = Any- OxylabsResponse = Any-- OXYLABS_AVAILABLE = False__all__ = ["OxylabsUniversalScraperConfig", "OxylabsUniversalScraperTool"]@@ -47,7 +32,7 @@ class OxylabsUniversalScraperConfig(BaseModel):)-class OxylabsUniversalScraperTool(BaseTool):+class OxylabsUniversalScraperTool(OxylabsBaseTool):"""Scrape any website with OxylabsUniversalScraperTool.Get Oxylabs account:@@ -59,104 +44,11 @@ class OxylabsUniversalScraperTool(BaseTool):config: Configuration options. See ``OxylabsUniversalScraperConfig``"""- model_config = ConfigDict(- arbitrary_types_allowed=True,- validate_assignment=True,- )name: str = "Oxylabs Universal Scraper tool"description: str = "Scrape any url with Oxylabs Universal Scraper"args_schema: type[BaseModel] = OxylabsUniversalScraperArgs- oxylabs_api: Anyconfig: OxylabsUniversalScraperConfig- package_dependencies: list[str] = Field(default_factory=lambda: ["oxylabs"])- env_vars: list[EnvVar] = Field(- default_factory=lambda: [- EnvVar(- name="OXYLABS_USERNAME",- description="Username for Oxylabs",- required=True,- ),- EnvVar(- name="OXYLABS_PASSWORD",- description="Password for Oxylabs",- required=True,- ),- ]- )- def __init__(- self,- username: str | None = None,- password: str | None = None,- config: OxylabsUniversalScraperConfig | dict[str, Any] | None = None,- **kwargs: Any,- ) -> None:- bits, _ = architecture()- sdk_type = (- f"oxylabs-crewai-sdk-python/"- f"{version('crewai')} "- f"({python_version()}; {bits})"- )-- if username is None or password is None:- username, password = self._get_credentials_from_env()-- if OXYLABS_AVAILABLE:- from oxylabs import RealtimeClient-- kwargs["oxylabs_api"] = RealtimeClient(- username=username,- password=password,- sdk_type=sdk_type,- )- else:- import click-- if click.confirm(- "You are missing the 'oxylabs' package. Would you like to install it?"- ):- import subprocess-- try:- subprocess.run(["uv", "add", "oxylabs"], check=True) # noqa: S607- from oxylabs import RealtimeClient-- kwargs["oxylabs_api"] = RealtimeClient(- username=username,- password=password,- sdk_type=sdk_type,- )- except subprocess.CalledProcessError as e:- raise ImportError("Failed to install oxylabs package") from e- else:- raise ImportError(- "`oxylabs` package not found, please run `uv add oxylabs`"- )-- if config is None:- config = OxylabsUniversalScraperConfig()- super().__init__(config=config, **kwargs)-- def _get_credentials_from_env(self) -> tuple[str, str]:- username = os.environ.get("OXYLABS_USERNAME")- password = os.environ.get("OXYLABS_PASSWORD")- if not username or not password:- raise ValueError(- "You must pass oxylabs username and password when instantiating the tool "- "or specify OXYLABS_USERNAME and OXYLABS_PASSWORD environment variables"- )- return username, password-- def _run(self, url: str) -> str:- response = self.oxylabs_api.universal.scrape_url(- url,- **self.config.model_dump(exclude_none=True),- )-- content = response.results[0].content-- if isinstance(content, dict):- return json.dumps(content)-- return str(content)+ def _run(self, url: str) -> str | ToolFailure:+ return self._scrape(self.oxylabs_api.universal.scrape_url, url)diff --git a/lib/crewai-tools/tests/tools/test_oxylabs_tools.py b/lib/crewai-tools/tests/tools/test_oxylabs_tools.pyindex f135e421c..c7b4981df 100644--- a/lib/crewai-tools/tests/tools/test_oxylabs_tools.py+++ b/lib/crewai-tools/tests/tools/test_oxylabs_tools.py@@ -1,8 +1,12 @@+from collections.abc import Callableimport json+import loggingimport os+import threadingfrom unittest.mock import MagicMockfrom crewai.tools.base_tool import BaseTool+from crewai.tools.tool_failure import ToolFailurefrom crewai_tools import (OxylabsAmazonProductScraperTool,OxylabsAmazonSearchScraperTool,@@ -12,9 +16,13 @@ from crewai_tools import (from crewai_tools.tools.oxylabs_amazon_product_scraper_tool.oxylabs_amazon_product_scraper_tool import (OxylabsAmazonProductScraperConfig,)+from crewai_tools.tools.oxylabs_base_tool.oxylabs_base_tool import OxylabsBaseToolfrom crewai_tools.tools.oxylabs_google_search_scraper_tool.oxylabs_google_search_scraper_tool import (OxylabsGoogleSearchScraperConfig,)+from crewai_tools.tools.oxylabs_universal_scraper_tool.oxylabs_universal_scraper_tool import (+ OxylabsUniversalScraperArgs,+)from oxylabs import RealtimeClientfrom oxylabs.sources.response import Response as OxylabsResponsefrom pydantic import BaseModel@@ -156,3 +164,272 @@ def test_tool_invocation(result = tool.run("Scraping Query 2")assert isinstance(result, str)assert "<!DOCTYPE html>" in result+++ALL_TOOL_CLASSES = [+ OxylabsUniversalScraperTool,+ OxylabsAmazonSearchScraperTool,+ OxylabsGoogleSearchScraperTool,+ OxylabsAmazonProductScraperTool,+]+++def build_tool(+ tool_class: type[BaseTool],+ raw_response: dict,+ sdk_logs: list[str] | None = None,+ config: BaseModel | None = None,+) -> BaseTool:+ """Build a tool whose every scrape entrypoint answers with ``raw_response``.++ ``sdk_logs`` reproduces the oxylabs SDK's habit of logging the real cause and+ returning an empty response instead of raising.+ """+ api = MagicMock()+ response = OxylabsResponse(raw_response)++ def scrape(*_args: object, **_kwargs: object) -> OxylabsResponse:+ for line in sdk_logs or []:+ logging.getLogger("oxylabs.internal.api").error(line)+ return response++ api.universal.scrape_url.side_effect = scrape+ api.amazon.scrape_search.side_effect = scrape+ api.amazon.scrape_product.side_effect = scrape+ api.google.scrape_search.side_effect = scrape++ tool = tool_class(username="username", password="password", config=config)+ # setting via __dict__ to bypass pydantic validation+ tool.__dict__["oxylabs_api"] = api+ return tool++[email protected]("tool_class", ALL_TOOL_CLASSES)+def test_rejected_request_reports_failure(tool_class: type[BaseTool]):+ """The SDK logs HTTP errors and returns an empty response instead of raising,+ so a rejected request must be reported rather than indexed into."""+ result = build_tool(tool_class, {}).run("Scraping Query")++ assert isinstance(result, ToolFailure)+ assert result.code == "empty_response"+ assert "OXYLABS_USERNAME" in result.message++[email protected]("tool_class", ALL_TOOL_CLASSES)+ ("status_code", "retryable"),+ [(404, False), (429, True), (500, True), (503, True)],+)+def test_upstream_error_status_reports_failure(+ tool_class: type[BaseTool], status_code: int, retryable: bool+):+ """A non-2xx result carries no page; returning its empty content would hand+ the agent '[]' as though the scrape had succeeded."""+ result = build_tool(+ tool_class, {"results": [{"content": [], "status_code": status_code}]}+ ).run("Scraping Query")++ assert isinstance(result, ToolFailure)+ assert result.code == str(status_code)+ assert result.retryable is retryable++[email protected]("tool_class", ALL_TOOL_CLASSES)+def test_missing_content_reports_failure(tool_class: type[BaseTool]):+ result = build_tool(+ tool_class, {"results": [{"content": None, "status_code": 200}]}+ ).run("Scraping Query")++ assert isinstance(result, ToolFailure)+ assert result.code == "empty_content"++[email protected]("tool_class", ALL_TOOL_CLASSES)+def test_list_content_is_serialized_as_json(tool_class: type[BaseTool]):+ """``parsing_instructions`` can yield a list; str() on it would produce a+ Python repr with single quotes rather than JSON."""+ result = build_tool(+ tool_class,+ {"results": [{"content": [{"title": "Amazing product"}], "status_code": 200}]},+ ).run("Scraping Query")++ assert isinstance(result, str)+ assert json.loads(result) == [{"title": "Amazing product"}]+++def test_subclass_without_config_field_is_reported():+ """The base class defaults ``config`` from the subclass's own model, so a+ subclass that declares none must say so rather than raise ``KeyError``."""++ class MissingConfig(OxylabsBaseTool):+ name: str = "missing config"+ description: str = "declares no config field"+ args_schema: type[BaseModel] = OxylabsUniversalScraperArgs++ def _run(self, url: str) -> str:+ return ""++ with pytest.raises(TypeError, match="must declare a 'config' model field"):+ MissingConfig(username="username", password="password")++[email protected]("tool_class", ALL_TOOL_CLASSES)+def test_rejected_request_names_the_http_cause(tool_class: type[BaseTool]):+ """The agent cannot act on "go read a log", so the status the SDK logged has+ to reach the failure itself."""+ result = build_tool(+ tool_class,+ {},+ sdk_logs=[+ "HTTP error occurred: 401 Client Error: Unauthorized for url: "+ "https://realtime.oxylabs.io/v1/queries",+ "",+ ],+ ).run("Scraping Query")++ assert isinstance(result, ToolFailure)+ assert result.code == "401"+ assert "401 Unauthorized" in result.message+ assert result.retryable is False++[email protected]("tool_class", ALL_TOOL_CLASSES)+def test_rejected_request_includes_the_api_explanation(tool_class: type[BaseTool]):+ """The API explains config it rejects; that explanation is what makes the+ failure actionable."""+ result = build_tool(+ tool_class,+ {},+ sdk_logs=[+ "HTTP error occurred: 400 Client Error: Bad Request for url: "+ "https://realtime.oxylabs.io/v1/queries",+ '{"message": "Parameter `parsing_instructions` can be used just with '+ '`parse` parameter set to `true`."}',+ ],+ ).run("Scraping Query")++ assert isinstance(result, ToolFailure)+ assert result.code == "400"+ assert "400 Bad Request" in result.message+ assert "`parsing_instructions` can be used just with" in result.message++[email protected]("tool_class", ALL_TOOL_CLASSES)+def test_timeout_is_reported_as_retryable(tool_class: type[BaseTool]):+ result = build_tool(+ tool_class,+ {},+ sdk_logs=[+ "Timeout error. The request to https://realtime.oxylabs.io/v1/queries "+ "with method POST has timed out."+ ],+ ).run("Scraping Query")++ assert isinstance(result, ToolFailure)+ assert result.code == "timeout"+ assert result.retryable is True++[email protected]("tool_class", ALL_TOOL_CLASSES)+def test_server_error_is_reported_as_retryable(tool_class: type[BaseTool]):+ result = build_tool(+ tool_class,+ {},+ sdk_logs=[+ "HTTP error occurred: 502 Server Error: Bad Gateway for url: "+ "https://realtime.oxylabs.io/v1/queries"+ ],+ ).run("Scraping Query")++ assert isinstance(result, ToolFailure)+ assert result.code == "502"+ assert result.retryable is True+++def test_google_config_forwards_locale():+ """`locale` is a documented Google Search parameter; the config model used to+ omit it, so it was silently dropped."""+ tool = build_tool(+ OxylabsGoogleSearchScraperTool,+ {"results": [{"content": {"ok": True}, "status_code": 200}]},+ config=OxylabsGoogleSearchScraperConfig(locale="de", limit=2),+ )++ tool.run("iPhone 16")++ _, kwargs = tool.oxylabs_api.google.scrape_search.call_args+ assert kwargs["locale"] == "de"+ assert kwargs["limit"] == 2+++def test_result_without_content_is_reported():+ """A result object missing `content` entirely must not raise AttributeError."""+ tool = build_tool(OxylabsUniversalScraperTool, {"results": [{"status_code": 200}]})++ result = tool.run("https://example.com")++ assert isinstance(result, ToolFailure)+ assert result.code == "empty_content"+++def test_concurrent_scrapes_do_not_share_diagnoses():+ """Two scrapes in flight at once must each be diagnosed from their own error.++ A shared collector would hand both calls both errors, and the timeout below+ would be reported as the other request's non-retryable 400 -- telling the+ agent not to retry something it should.+ """+ both_started = threading.Barrier(2)+ timeout_logged = threading.Event()+ bad_request_logged = threading.Event()+ outcomes: dict[str, ToolFailure] = {}++ def tool_logging(emit: Callable[[], None]) -> BaseTool:+ api = MagicMock()++ def scrape(*_args: object, **_kwargs: object) -> OxylabsResponse:+ both_started.wait(timeout=5)+ emit()+ return OxylabsResponse({})++ api.universal.scrape_url.side_effect = scrape+ tool = OxylabsUniversalScraperTool(username="username", password="password")+ tool.__dict__["oxylabs_api"] = api+ return tool++ sdk_logger = logging.getLogger("oxylabs.internal.api")++ def emit_timeout() -> None:+ sdk_logger.error(+ "Timeout error. The request to https://realtime.oxylabs.io/v1/queries "+ "with method POST has timed out."+ )+ timeout_logged.set()+ # Hold this capture open while the other call logs, which is the window+ # in which the two could bleed into each other.+ bad_request_logged.wait(timeout=5)++ def emit_bad_request() -> None:+ timeout_logged.wait(timeout=5)+ sdk_logger.error(+ "HTTP error occurred: 400 Client Error: Bad Request for url: "+ "https://realtime.oxylabs.io/v1/queries"+ )+ bad_request_logged.set()++ def run(key: str, emit: Callable[[], None]) -> None:+ outcomes[key] = tool_logging(emit).run("https://example.com")++ threads = [+ threading.Thread(target=run, args=("timeout", emit_timeout)),+ threading.Thread(target=run, args=("bad_request", emit_bad_request)),+ ]+ for thread in threads:+ thread.start()+ for thread in threads:+ thread.join(timeout=15)++ assert outcomes["timeout"].code == "timeout"+ assert outcomes["timeout"].retryable is True+ assert outcomes["bad_request"].code == "400"+ assert outcomes["bad_request"].retryable is Falsediff --git a/lib/crewai-tools/tool.specs.json b/lib/crewai-tools/tool.specs.jsonindex 2b614ba12..dfcb43d4a 100644--- a/lib/crewai-tools/tool.specs.json+++ b/lib/crewai-tools/tool.specs.json@@ -17466,6 +17466,19 @@"description": "Number of results to retrieve in each page.","title": "Limit"},+ "locale": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "default": null,+ "description": "`Accept-Language` header value which changes your Google search page web interface language.",+ "title": "Locale"+ },"pages": {"anyOf": [{