Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
import json
from collections.abc import Generator
from typing import Any, TypedDict
from urllib.parse import urljoin
import httpx
from httpx import Response
from core.helper import ssrf_proxy
from core.rag.extractor.watercrawl.exceptions import (
WaterCrawlAuthenticationError,
WaterCrawlBadRequestError,
WaterCrawlPermissionError,
)
WATERCRAWL_REQUEST_TIMEOUT: httpx.Timeout = httpx.Timeout(30.0, connect=5.0)
# The crawl-status stream is a long-lived SSE connection that can stay open for
# the whole duration of a crawl, so it keeps an unbounded read while still
# capping the initial connection. Regular requests use WATERCRAWL_REQUEST_TIMEOUT
# so a stalled endpoint can't hang a worker forever.
_STREAM_TIMEOUT = httpx.Timeout(None, connect=10.0)
class SpiderOptions(TypedDict):
max_depth: int
page_limit: int
allowed_domains: list[str]
exclude_paths: list[str]
include_paths: list[str]
class PageOptions(TypedDict):
exclude_tags: list[str]
include_tags: list[str]
wait_time: int
include_html: bool
only_main_content: bool
include_links: bool
timeout: int
accept_cookies_selector: str
locale: str
actions: list[Any]
class BaseAPIClient:
def __init__(self, api_key, base_url):
self.api_key = api_key
self.base_url = base_url
self.headers = {
"X-API-Key": self.api_key,
"Content-Type": "application/json",
"Accept": "application/json",
"User-Agent": "WaterCrawl-Plugin",
"Accept-Language": "en-US",
}
def _request(
self,
method: str,
endpoint: str,
query_params: dict[str, Any] | None = None,
data: dict[str, Any] | None = None,
**kwargs,
) -> Response:
stream = kwargs.pop("stream", False)
url = urljoin(self.base_url, endpoint)
headers = httpx.Headers(self.headers)
headers.update(kwargs.pop("headers", {}))
kwargs.setdefault("timeout", _STREAM_TIMEOUT if stream else WATERCRAWL_REQUEST_TIMEOUT)
return ssrf_proxy.make_request(
method,
url,
headers=dict(headers),
params=query_params,
json=data,
max_retries=0,
stream_response=stream,
**kwargs,
)
def _get(self, endpoint: str, query_params: dict[str, Any] | None = None, **kwargs):
return self._request("GET", endpoint, query_params=query_params, **kwargs)
def _post(
self, endpoint: str, query_params: dict[str, Any] | None = None, data: dict[str, Any] | None = None, **kwargs
):
return self._request("POST", endpoint, query_params=query_params, data=data, **kwargs)
def _put(
self, endpoint: str, query_params: dict[str, Any] | None = None, data: dict[str, Any] | None = None, **kwargs
):
return self._request("PUT", endpoint, query_params=query_params, data=data, **kwargs)
def _delete(self, endpoint: str, query_params: dict[str, Any] | None = None, **kwargs):
return self._request("DELETE", endpoint, query_params=query_params, **kwargs)
def _patch(
self, endpoint: str, query_params: dict[str, Any] | None = None, data: dict[str, Any] | None = None, **kwargs
):
return self._request("PATCH", endpoint, query_params=query_params, data=data, **kwargs)
class WaterCrawlAPIClient(BaseAPIClient):
def __init__(self, api_key, base_url: str | None = "https://app.watercrawl.dev/"):
super().__init__(api_key, base_url)
def process_eventstream(self, response: Response, download: bool = False) -> Generator:
try:
for raw_line in response.iter_lines():
line = raw_line.decode("utf-8") if isinstance(raw_line, bytes) else raw_line
if line.startswith("data:"):
line = line[5:].strip()
data = json.loads(line)
if data["type"] == "result" and download:
data["data"] = self.download_result(data["data"])
yield data
finally:
response.close()
def process_response(self, response: Response) -> dict[str, Any] | bytes | list[Any] | None | Generator:
if response.status_code == 401:
raise WaterCrawlAuthenticationError(response)
if response.status_code == 403:
raise WaterCrawlPermissionError(response)
if 400 <= response.status_code < 500:
raise WaterCrawlBadRequestError(response)
response.raise_for_status()
if response.status_code == 204:
return None
content_type = response.headers.get("Content-Type", "")
media_type = content_type.split(";", 1)[0].strip().lower()
if media_type == "application/json":
try:
return response.json() or {}
except ValueError as exc:
raise ValueError("Invalid JSON response from WaterCrawl") from exc
if media_type == "application/octet-stream":
return response.content
if media_type == "text/event-stream":
return self.process_eventstream(response)
raise Exception(f"Unknown response type: {content_type}")
def get_crawl_requests_list(self, page: int | None = None, page_size: int | None = None):
query_params = {"page": page or 1, "page_size": page_size or 10}
return self.process_response(
self._get(
"/api/v1/core/crawl-requests/",
query_params=query_params,
)
)
def get_crawl_request(self, item_id: str):
return self.process_response(
self._get(
f"/api/v1/core/crawl-requests/{item_id}/",
)
)
def create_crawl_request(
self,
url: list | str | None = None,
spider_options: SpiderOptions | None = None,
page_options: PageOptions | None = None,
plugin_options: dict[str, Any] | None = None,
):
data = {
# 'urls': url if isinstance(url, list) else [url],
"url": url,
"options": {
"spider_options": spider_options or {},
"page_options": page_options or {},
"plugin_options": plugin_options or {},
},
}
return self.process_response(
self._post(
"/api/v1/core/crawl-requests/",
data=data,
)
)
def stop_crawl_request(self, item_id: str):
return self.process_response(
self._delete(
f"/api/v1/core/crawl-requests/{item_id}/",
)
)
def download_crawl_request(self, item_id: str):
return self.process_response(
self._get(
f"/api/v1/core/crawl-requests/{item_id}/download/",
)
)
def monitor_crawl_request(self, item_id: str, prefetched=False) -> Generator:
query_params = {"prefetched": str(prefetched).lower()}
generator = self.process_response(
self._get(f"/api/v1/core/crawl-requests/{item_id}/status/", stream=True, query_params=query_params),
)
if not isinstance(generator, Generator):
raise ValueError("Generator expected")
yield from generator
def get_crawl_request_results(
self, item_id: str, page: int = 1, page_size: int = 25, query_params: dict[str, Any] | None = None
):
query_params = query_params or {}
query_params.update({"page": page or 1, "page_size": page_size or 25})
return self.process_response(
self._get(f"/api/v1/core/crawl-requests/{item_id}/results/", query_params=query_params)
)
def scrape_url(
self,
url: str,
page_options: PageOptions | None = None,
plugin_options: dict[str, Any] | None = None,
sync: bool = True,
prefetched: bool = True,
):
response_result = self.create_crawl_request(url=url, page_options=page_options, plugin_options=plugin_options)
if not sync:
return response_result
for event_data in self.monitor_crawl_request(response_result["uuid"], prefetched):
if event_data["type"] == "result":
return event_data["data"]
def download_result(self, result_object: dict[str, Any]):
response = ssrf_proxy.get(result_object["result"], timeout=WATERCRAWL_REQUEST_TIMEOUT, max_retries=0)
try:
response.raise_for_status()
result_object["result"] = response.json()
finally:
response.close()
return result_object