🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix: fixing the response logic
Karim shoaird018d6e4386d
diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.pyindex 7a2e50e..ef28acf 100644--- a/scrapling/engines/toolbelt/convertor.py+++ b/scrapling/engines/toolbelt/convertor.py@@ -115,10 +115,10 @@ class ResponseFactory:history = cls._process_response_history(first_response, parser_arguments)try:- if "html" in final_response.headers.get("content-type", ""):+ if "html" in final_response.all_headers().get("content-type", ""):page_content = cls._get_page_content(page)else:- page_content = final_response.text()+ page_content = final_response.body()except Exception as e: # pragma: no coverlog.error(f"Error getting page content: {e}")page_content = ""@@ -250,10 +250,10 @@ class ResponseFactory:history = await cls._async_process_response_history(first_response, parser_arguments)try:- if "html" in final_response.headers.get("content-type", ""):+ if "html" in (await final_response.all_headers()).get("content-type", ""):page_content = await cls._get_async_page_content(page)else:- page_content = await final_response.text()+ page_content = await final_response.body()except Exception as e: # pragma: no coverlog.error(f"Error getting page content in async: {e}")page_content = ""