🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(StealthyFetcher): Wait for the real response status code
Karim shoair94745b155533
diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.pyindex 1eb9976..2c0ff3d 100644--- a/scrapling/engines/camo.py+++ b/scrapling/engines/camo.py@@ -84,6 +84,14 @@ class CamoufoxEngine::return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""addons = [] if self.disable_ads else [DefaultAddons.UBO]+ # Store the final response+ final_response = None++ def handle_response(finished_response):+ nonlocal final_response+ if finished_response.request.resource_type == "document":+ final_response = finished_response+with Camoufox(geoip=self.geoip,proxy=self.proxy,@@ -100,13 +108,15 @@ class CamoufoxEngine:page = browser.new_page()page.set_default_navigation_timeout(self.timeout)page.set_default_timeout(self.timeout)+ # Listen for all responses+ page.on("response", handle_response)if self.disable_resources:page.route("**/*", intercept_route)if self.extra_headers:page.set_extra_http_headers(self.extra_headers)- res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)+ first_response = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)page.wait_for_load_state(state="domcontentloaded")if self.network_idle:page.wait_for_load_state('networkidle')@@ -123,21 +133,24 @@ class CamoufoxEngine:if self.network_idle:page.wait_for_load_state('networkidle')+ response_bytes = final_response.body() if final_response else page.content().encode('utf-8')+ # In case we didn't catch a document type somehow+ final_response = final_response if final_response else first_response# This will be parsed inside `Response`- encoding = res.headers.get('content-type', '') or 'utf-8' # default encoding+ encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding# PlayWright API sometimes give empty status text for some reason!- status_text = res.status_text or StatusText.get(res.status)+ status_text = final_response.status_text or StatusText.get(final_response.status)response = Response(- url=res.url,+ url=final_response.url,text=page.content(),- body=page.content().encode('utf-8'),- status=res.status,+ body=response_bytes,+ status=final_response.status,reason=status_text,encoding=encoding,cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},- headers=res.all_headers(),- request_headers=res.request.all_headers(),+ headers=final_response.all_headers(),+ request_headers=final_response.request.all_headers(),**self.adaptor_arguments)page.close()@@ -151,6 +164,14 @@ class CamoufoxEngine::return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""addons = [] if self.disable_ads else [DefaultAddons.UBO]+ # Store the final response+ final_response = None++ async def handle_response(finished_response):+ nonlocal final_response+ if finished_response.request.resource_type == "document":+ final_response = finished_response+async with AsyncCamoufox(geoip=self.geoip,proxy=self.proxy,@@ -167,13 +188,15 @@ class CamoufoxEngine:page = await browser.new_page()page.set_default_navigation_timeout(self.timeout)page.set_default_timeout(self.timeout)+ # Listen for all responses+ page.on("response", handle_response)if self.disable_resources:await page.route("**/*", async_intercept_route)if self.extra_headers:await page.set_extra_http_headers(self.extra_headers)- res = await page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)+ first_response = await page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)await page.wait_for_load_state(state="domcontentloaded")if self.network_idle:await page.wait_for_load_state('networkidle')@@ -190,21 +213,24 @@ class CamoufoxEngine:if self.network_idle:await page.wait_for_load_state('networkidle')+ response_bytes = await final_response.body() if final_response else (await page.content()).encode('utf-8')+ # In case we didn't catch a document type somehow+ final_response = final_response if final_response else first_response# This will be parsed inside `Response`- encoding = res.headers.get('content-type', '') or 'utf-8' # default encoding+ encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding# PlayWright API sometimes give empty status text for some reason!- status_text = res.status_text or StatusText.get(res.status)+ status_text = final_response.status_text or StatusText.get(final_response.status)response = Response(- url=res.url,+ url=final_response.url,text=await page.content(),- body=(await page.content()).encode('utf-8'),- status=res.status,+ body=response_bytes,+ status=final_response.status,reason=status_text,encoding=encoding,cookies={cookie['name']: cookie['value'] for cookie in await page.context.cookies()},- headers=await res.all_headers(),- request_headers=await res.request.all_headers(),+ headers=await final_response.all_headers(),+ request_headers=await final_response.request.all_headers(),**self.adaptor_arguments)await page.close()