Skip to content

A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.

feat(enrichers): to_website additional infos

dextmorgn20357cd319dd
diff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_website.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_website.py
index 73dffb9f..c2c9d627 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_website.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_website.py
@@ -1,5 +1,6 @@
-from typing import List, Union
+from typing import Dict, List, Union
import requests
+from bs4 import BeautifulSoup
from flowsint_core.utils import is_valid_domain
from flowsint_core.core.enricher_base import Enricher
from flowsint_enrichers.registry import flowsint_enricher
@@ -28,18 +29,98 @@ class DomainToWebsiteEnricher(Enricher):
def key(cls) -> str:
return "domain"
+ def _extract_page_info(self, html_content: str) -> Dict[str, any]:
+ """Extract title, description, content and technologies from HTML."""
+ soup = BeautifulSoup(html_content, 'html.parser')
+
+ # Extract title
+ title = None
+ title_tag = soup.find('title')
+ if title_tag:
+ title = title_tag.get_text().strip()
+
+ # Extract meta description
+ description = None
+ meta_desc = soup.find('meta', attrs={'name': 'description'})
+ if meta_desc and meta_desc.get('content'):
+ description = meta_desc.get('content').strip()
+
+ # Extract text content (remove scripts and styles)
+ for script in soup(['script', 'style']):
+ script.decompose()
+ content = soup.get_text(separator=' ', strip=True)
+ # Limit content to first 5000 characters
+ if len(content) > 5000:
+ content = content[:5000] + "..."
+
+ # Detect technologies
+ technologies = []
+
+ # Check for common frameworks and libraries
+ if soup.find('meta', attrs={'name': 'generator'}):
+ generator = soup.find('meta', attrs={'name': 'generator'}).get('content')
+ if generator:
+ technologies.append(generator)
+
+ # Check for React
+ if soup.find('div', id='root') or soup.find('div', id='react-root'):
+ technologies.append('React')
+
+ # Check for Vue.js
+ if soup.find(attrs={'data-v-'}):
+ technologies.append('Vue.js')
+
+ # Check for Angular
+ if soup.find(attrs={'ng-app'}) or soup.find(attrs={'ng-version'}):
+ technologies.append('Angular')
+
+ # Check for WordPress
+ if soup.find('meta', attrs={'name': 'generator', 'content': lambda x: x and 'WordPress' in x}):
+ technologies.append('WordPress')
+
+ return {
+ 'title': title,
+ 'description': description,
+ 'content': content,
+ 'technologies': technologies
+ }
+
async def scan(self, data: List[InputType]) -> List[OutputType]:
results: List[OutputType] = []
for domain in data:
try:
+ website_data = {
+ 'url': f"https://{domain.domain}",
+ 'domain': domain,
+ 'active': False
+ }
+
# Try HTTPS first
try:
https_url = f"https://{domain.domain}"
- response = requests.head(
+ response = requests.get(
https_url, timeout=10, allow_redirects=True
)
+
if response.status_code < 400:
- results.append(Website(url=https_url, domain=domain, active=True))
+ website_data['url'] = https_url
+ website_data['active'] = True
+ website_data['status_code'] = response.status_code
+
+ # Extract relevant headers
+ website_data['headers'] = {
+ 'content-type': response.headers.get('content-type'),
+ 'server': response.headers.get('server'),
+ 'x-powered-by': response.headers.get('x-powered-by')
+ }
+ # Remove None values
+ website_data['headers'] = {k: v for k, v in website_data['headers'].items() if v}
+
+ # Extract page information
+ page_info = self._extract_page_info(response.text)
+ website_data.update(page_info)
+
+ results.append(Website(**website_data))
continue
except requests.RequestException:
pass
@@ -47,15 +128,33 @@ class DomainToWebsiteEnricher(Enricher):
# Try HTTP if HTTPS fails
try:
http_url = f"http://{domain.domain}"
- response = requests.head(http_url, timeout=10, allow_redirects=True)
+ response = requests.get(http_url, timeout=10, allow_redirects=True)
+
if response.status_code < 400:
- results.append(Website(url=http_url, domain=domain, active=True))
+ website_data['url'] = http_url
+ website_data['active'] = True
+ website_data['status_code'] = response.status_code
+
+ # Extract relevant headers
+ website_data['headers'] = {
+ 'content-type': response.headers.get('content-type'),
+ 'server': response.headers.get('server'),
+ 'x-powered-by': response.headers.get('x-powered-by')
+ }
+ # Remove None values
+ website_data['headers'] = {k: v for k, v in website_data['headers'].items() if v}
+
+ # Extract page information
+ page_info = self._extract_page_info(response.text)
+ website_data.update(page_info)
+
+ results.append(Website(**website_data))
continue
except requests.RequestException:
pass
# If both fail, still add HTTPS URL as default
- results.append(Website(url=f"https://{domain.domain}", domain=domain, active=False))
+ results.append(Website(**website_data))
except Exception as e:
Logger.error(