A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
fix(transforms): org_to_domains using whoxy
dextmorgndbb292d5e99a
diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_history.py b/flowsint-transforms/src/flowsint_transforms/domain/to_history.pyindex deddfd59..1cef979d 100644--- a/flowsint-transforms/src/flowsint_transforms/domain/to_history.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_history.py@@ -15,9 +15,6 @@ from dotenv import load_dotenvload_dotenv()-WHOXY_API_KEY = os.getenv("WHOXY_API_KEY")--class DomainToHistoryTransform(Transform):"""[WHOXY] Takes a domain and returns history infos about it (history, organization, owners, emails, etc.)."""diff --git a/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py b/flowsint-transforms/src/flowsint_transforms/organization/to_domains.pyindex d4cd1b70..9a8a0fcc 100644--- a/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py+++ b/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py@@ -1,21 +1,19 @@import osimport re-from typing import Any, List, Union, Dict, Set+from typing import Any, List, Union, Dict, Set, Optionalfrom flowsint_core.core.transform_base import Transformfrom flowsint_types.domain import Domainfrom flowsint_types.organization import Organizationfrom flowsint_types.individual import Individual-from flowsint_types.email import Email-from flowsint_types.phone import Phonefrom flowsint_types.address import Locationfrom flowsint_core.core.logger import Logger+from flowsint_core.core.graph_db import Neo4jConnectionfrom tools.network.whoxy import WhoxyTool+from flowsint_core.utils import is_valid_domain, is_root_domainfrom dotenv import load_dotenvload_dotenv()-WHOXY_API_KEY = os.getenv("WHOXY_API_KEY")-class OrgToDomainsTransform(Transform):"""[WHOXY] Takes an organization and returns the domains it registered."""@@ -24,6 +22,39 @@ class OrgToDomainsTransform(Transform):InputType = List[Organization]OutputType = List[Domain]+ def __init__(+ self,+ sketch_id: Optional[str] = None,+ scan_id: Optional[str] = None,+ neo4j_conn: Optional[Neo4jConnection] = None,+ vault=None,+ params: Optional[Dict[str, Any]] = None,+ ):+ super().__init__(+ sketch_id=sketch_id,+ scan_id=scan_id,+ neo4j_conn=neo4j_conn,+ params_schema=self.get_params_schema(),+ vault=vault,+ params=params,+ )++ @classmethod+ def required_params(cls) -> bool:+ return True++ @classmethod+ def get_params_schema(cls) -> List[Dict[str, Any]]:+ """Declare required parameters for this transform"""+ return [+ {+ "name": "WHOXY_API_KEY",+ "type": "vaultSecret",+ "description": "The Whoxy API key to use for domain lookups.",+ "required": True,+ },+ ]+@classmethoddef name(cls) -> str:return "org_to_domains"@@ -54,10 +85,20 @@ class OrgToDomainsTransform(Transform):"""Find domains related to organizations using whoxy api."""domains: OutputType = []self._extracted_data = [] # Store all extracted data for postprocess+ self._extracted_individuals = [] # Store extracted individuals for testing+ self._extracted_organizations = [] # Store extracted organizations for testing+ api_key = self.get_secret("WHOXY_API_KEY", os.getenv("WHOXY_API_KEY"))for org in data:- infos_data = self.__get_infos_from_whoxy(org.name)+ infos_data = self.__get_infos_from_whoxy(org.name, api_key)if infos_data and "search_result" in infos_data:+ Logger.info(+ self.sketch_id,+ {+ "message": f"[WHOXY] Found {len(infos_data['search_result'])} domains for organization {org.name}"+ },+ )+# Process each domain resultfor result in infos_data["search_result"]:if self.__is_valid_domain_result(result):@@ -65,7 +106,7 @@ class OrgToDomainsTransform(Transform):if domain_name:domain = Domain(domain=domain_name, root=True)domains.append(domain)-+# Store extracted data for postprocessextracted_info = {'org': org,@@ -79,6 +120,16 @@ class OrgToDomainsTransform(Transform):}}self._extracted_data.append(extracted_info)++ Logger.info(+ self.sketch_id,+ {+ "message": f"[WHOXY] Processing domain {domain_name} for organization {org.name}"+ },+ )++ # Process contacts and extract individuals/organizations during scan+ self.__process_contacts_during_scan(extracted_info)else:Logger.info(self.sketch_id,@@ -86,12 +137,82 @@ class OrgToDomainsTransform(Transform):)return domains- def __get_infos_from_whoxy(self, org_name: str) -> Dict[str, Any]:+ def __process_contacts_during_scan(self, extracted_info: Dict[str, Any]):+ """Process contacts and extract individuals and organizations during scan method."""+ org_name = extracted_info["org"].name+ domain_name = extracted_info["domain"].domain+ contacts = extracted_info["contacts"]++ for contact_type, contact in contacts.items():+ if contact:+ Logger.info(+ self.sketch_id,+ {+ "message": f"[WHOXY] Processing {contact_type} contact for {domain_name}"+ },+ )++ # Extract individual (if name is not redacted)+ individual = self.__extract_individual_from_contact(+ contact, contact_type+ )+ if individual:+ # Store the extracted individual for testing/debugging+ individual_info = {+ "individual": individual,+ "contact_type": contact_type,+ "domain_name": domain_name,+ "org_name": org_name,+ "contact_data": contact,+ }+ self._extracted_individuals.append(individual_info)++ Logger.info(+ self.sketch_id,+ {+ "message": f"[WHOXY] Extracted individual: {individual.full_name} ({contact_type}) for {domain_name}"+ },+ )++ # Extract organization (if company name is not redacted)+ organization = self.__extract_organization_from_contact(+ contact, contact_type+ )+ if organization:+ # Store the extracted organization for testing/debugging+ organization_info = {+ "organization": organization,+ "contact_type": contact_type,+ "domain_name": domain_name,+ "org_name": org_name,+ "contact_data": contact,+ }+ self._extracted_organizations.append(organization_info)++ Logger.info(+ self.sketch_id,+ {+ "message": f"[WHOXY] Extracted organization: {organization.name} ({contact_type}) for {domain_name}"+ },+ )++ # Extract other non-redacted information (country, email, etc.)+ self.__extract_additional_info_from_contact(contact, contact_type, domain_name, org_name)+ else:+ Logger.info(+ self.sketch_id,+ {+ "message": f"[WHOXY] No contact data for {contact_type} contact for {domain_name}"+ },+ )++ def __get_infos_from_whoxy(self, org_name: str, api_key: str) -> Dict[str, Any]:+ """Get domain information from Whoxy API or test data."""infos: Dict[str, Any] = {}whoxy = WhoxyTool()try:params = {- "key": WHOXY_API_KEY,+ "key": api_key,"reverse": "whois","company": org_name,}@@ -108,16 +229,27 @@ class OrgToDomainsTransform(Transform):domain_name = result.get("domain_name")if not domain_name:return False+ # A result is valid if it has a domain name - we'll filter contacts individually laterreturn True+ def __is_redacted(self, value: str) -> bool:+ """Check if a value is redacted."""+ if not value:+ return True+ return "REDACTED FOR PRIVACY" in value.upper() or "PRIVACY" in value.upper()+def __extract_individual_from_contact(self, contact: Dict[str, Any], contact_type: str) -> Individual:"""Extract individual information from contact data."""full_name = contact.get("full_name", "")- # Skip if name is redacted- if "REDACTED" in full_name or "REDACTED FOR PRIVACY" in full_name:+ # Skip if name is redacted - we can't create an individual without a name+ if self.__is_redacted(full_name) or not full_name:+ Logger.info(+ self.sketch_id,+ {"message": f"[WHOXY] Skipping contact with redacted/empty name: {full_name}"},+ )return None# Parse full name into first and last name@@ -126,19 +258,20 @@ class OrgToDomainsTransform(Transform):last_name = " ".join(name_parts[1:]) if len(name_parts) > 1 else ""# Extract email and phone- email = contact.get("email_address", "")+ email_raw = contact.get("email_address", "")phone = contact.get("phone_number", "")- # Skip if email is redacted or invalid- if (- "REDACTED" in email- or "REDACTED FOR PRIVACY" in email- or not self.__is_valid_email(email)- ):- email = ""+ # Handle comma-separated emails+ emails = []+ if email_raw and not self.__is_redacted(email_raw):+ # Split by comma and clean up each email+ email_list = [e.strip() for e in email_raw.split(",")]+ for email in email_list:+ if email and self.__is_valid_email(email):+ emails.append(email)# Skip if phone is redacted- if "REDACTED" in phone or "REDACTED FOR PRIVACY" in phone:+ if self.__is_redacted(phone):phone = ""# Extract address information@@ -148,13 +281,13 @@ class OrgToDomainsTransform(Transform):country = contact.get("country_name", "")# Skip if address is redacted- if "REDACTED" in address or "REDACTED FOR PRIVACY" in address:+ if self.__is_redacted(address):address = ""- if "REDACTED" in city or "REDACTED FOR PRIVACY" in city:+ if self.__is_redacted(city):city = ""- if "REDACTED" in zip_code or "REDACTED FOR PRIVACY" in zip_code:+ if self.__is_redacted(zip_code):zip_code = ""- if "REDACTED" in country or "REDACTED FOR PRIVACY" in country:+ if self.__is_redacted(country):country = ""# Create individual object@@ -162,10 +295,17 @@ class OrgToDomainsTransform(Transform):first_name=first_name,last_name=last_name,full_name=full_name,- email_addresses=[email] if email else None,+ email_addresses=emails if emails else None,phone_numbers=[phone] if phone else None,)+ Logger.info(+ self.sketch_id,+ {+ "message": f"[WHOXY] Extracted individual: {full_name} ({contact_type}) with {len(emails)} emails"+ },+ )+return individualdef __is_valid_email(self, email: str) -> bool:@@ -184,7 +324,9 @@ class OrgToDomainsTransform(Transform):country = contact.get("country_name", "")# Skip if any part is redacted- if any("REDACTED" in field for field in [address, city, zip_code, country]):+ if any(+ self.__is_redacted(field) for field in [address, city, zip_code, country]+ ):return Noneif not all([address, city, zip_code, country]):@@ -194,217 +336,379 @@ class OrgToDomainsTransform(Transform):address=address, city=city, zip=zip_code, country=country)+ def __extract_organization_from_contact(+ self, contact: Dict[str, Any], contact_type: str+ ) -> Organization:+ """Extract organization information from contact data."""+ company_name = contact.get("company_name", "")++ # Skip if company name is redacted or empty+ if not company_name or self.__is_redacted(company_name):+ return None++ # Create organization object+ organization = Organization(name=company_name)++ Logger.info(+ self.sketch_id,+ {+ "message": f"[WHOXY] Extracted organization: {company_name} ({contact_type})"+ },+ )++ return organization++ def __extract_additional_info_from_contact(+ self, contact: Dict[str, Any], contact_type: str, domain_name: str, org_name: str+ ):+ """Extract additional non-redacted information from contact data."""+ # Extract country information+ country_name = contact.get("country_name", "")+ country_code = contact.get("country_code", "")++ if country_name and not self.__is_redacted(country_name):+ Logger.info(+ self.sketch_id,+ {+ "message": f"[WHOXY] Found country: {country_name} ({contact_type}) for {domain_name}"+ },+ )++ if country_code and not self.__is_redacted(country_code):+ Logger.info(+ self.sketch_id,+ {+ "message": f"[WHOXY] Found country code: {country_code} ({contact_type}) for {domain_name}"+ },+ )++ # Extract email (even if individual name is redacted)+ email_raw = contact.get("email_address", "")+ if email_raw and not self.__is_redacted(email_raw):+ emails = []+ email_list = [e.strip() for e in email_raw.split(",")]+ for email in email_list:+ if email and self.__is_valid_email(email):+ emails.append(email)++ if emails:+ Logger.info(+ self.sketch_id,+ {+ "message": f"[WHOXY] Found emails: {emails} ({contact_type}) for {domain_name}"+ },+ )+def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:"""Create Neo4j nodes and relationships from extracted data."""if not self.neo4j_conn:+ Logger.info(+ self.sketch_id,+ {"message": "[WHOXY] No Neo4j connection, skipping postprocess"},+ )return results+ Logger.info(+ self.sketch_id,+ {+ "message": f"[WHOXY] Starting postprocess with {len(self._extracted_individuals)} individuals and {len(self._extracted_organizations)} organizations"+ },+ )+# Track processed entities to avoid duplicatesprocessed_domains: Set[str] = set()processed_individuals: Set[str] = set()+ processed_organizations: Set[str] = set()processed_emails: Set[str] = set()processed_phones: Set[str] = set()processed_addresses: Set[str] = set()- for extracted_info in self._extracted_data:- org = extracted_info['org']- domain = extracted_info['domain']- domain_data = extracted_info['domain_data']- contacts = extracted_info['contacts']-- domain_name = domain.domain- if domain_name in processed_domains:- continue- processed_domains.add(domain_name)-- # Create organization node- self.create_node(- "organization",- "name",- org.name,- caption=org.name,- type="organization",- )+ # Track processed input organizations to ensure they're created+ processed_input_orgs: Set[str] = set()- # Create domain node- self.create_node(- "domain",- "domain",- domain_name,- label=domain_name,- caption=domain_name,- type="domain",- )+ # Process extracted individuals (already filtered and extracted during scan)+ for individual_info in self._extracted_individuals:+ individual = individual_info["individual"]+ contact_type = individual_info["contact_type"]+ domain_name = individual_info["domain_name"]+ org_name = individual_info["org_name"]- # Create relationship between organization and domain- self.create_relationship(- "organization",- "name",- org.name,- "domain",- "domain",- domain_name,- "HAS_REGISTERED_DOMAIN",- )-- # Process all contact types- for contact_type, contact in contacts.items():- if contact:- self.__process_contact(- contact,- contact_type.upper(),- domain_name,- org.name,- processed_individuals,- processed_emails,- processed_phones,- processed_addresses,- )-- self.log_graph_message(- f"Processed domain {domain_name} for organization {org.name}"+ Logger.info(+ self.sketch_id,+ {+ "message": f"[WHOXY] Processing individual: {individual.full_name} ({contact_type}) for {domain_name}"+ },)- return results-- def __process_contact(- self,- contact: Dict[str, Any],- contact_type: str,- domain_name: str,- org_name: str,- processed_individuals: Set[str],- processed_emails: Set[str],- processed_phones: Set[str],- processed_addresses: Set[str],- ):- """Process a contact and create all related entities and relationships."""+ # Create organization node if not already processed+ if org_name not in processed_input_orgs:+ processed_input_orgs.add(org_name)+ Logger.info(+ self.sketch_id,+ {"message": f"[WHOXY] Creating organization node: {org_name}"},+ )+ self.create_node(+ "organization",+ "name",+ org_name,+ caption=org_name,+ type="organization",+ )- # Extract individual- individual = self.__extract_individual_from_contact(contact, contact_type)- if not individual:- return+ # Create domain node if not already processed+ if domain_name not in processed_domains:+ processed_domains.add(domain_name)+ Logger.info(+ self.sketch_id,+ {"message": f"[WHOXY] Creating domain node: {domain_name}"},+ )+ self.create_node(+ "domain",+ "domain",+ domain_name,+ label=domain_name,+ caption=domain_name,+ type="domain",+ )- individual_id = (- f"{individual.first_name}_{individual.last_name}_{individual.full_name}"- )- if individual_id in processed_individuals:- return-- processed_individuals.add(individual_id)-- # Create individual node- self.create_node(- "individual",- "full_name",- individual.full_name,- caption=individual.full_name,- type="individual",- )+ # Create relationship between organization and domain+ self.create_relationship(+ "organization",+ "name",+ org_name,+ "domain",+ "domain",+ domain_name,+ "HAS_REGISTERED_DOMAIN",+ )- # Create relationship between individual and domain- self.create_relationship(- "individual",- "full_name",- individual.full_name,- "domain",- "domain",- domain_name,- f"IS_{contact_type}_CONTACT",- )+ # Create individual node if not already processed+ individual_id = (+ f"{individual.first_name}_{individual.last_name}_{individual.full_name}"+ )+ if individual_id not in processed_individuals:+ processed_individuals.add(individual_id)+ Logger.info(+ self.sketch_id,+ {+ "message": f"[WHOXY] Creating individual node: {individual.full_name}"+ },+ )+ self.create_node(+ "individual",+ "full_name",+ individual.full_name,+ caption=individual.full_name,+ type="individual",+ )- # Create relationship between individual and organization- self.create_relationship(- "individual",- "full_name",- individual.full_name,- "organization",- "name",- org_name,- f"WORKS_FOR",- )+ # Create relationship between individual and domain+ self.create_relationship(+ "individual",+ "full_name",+ individual.full_name,+ "domain",+ "domain",+ domain_name,+ f"IS_{contact_type.upper()}_CONTACT",+ )- # Process email addresses- if individual.email_addresses:- for email in individual.email_addresses:- if email and email not in processed_emails:- processed_emails.add(email)+ # Create relationship between individual and organization+ self.create_relationship(+ "individual",+ "full_name",+ individual.full_name,+ "organization",+ "name",+ org_name,+ "WORKS_FOR",+ )- # Create email node+ # Process email addresses+ if individual.email_addresses:+ for email in individual.email_addresses:+ if email and email not in processed_emails:+ processed_emails.add(email)+ Logger.info(+ self.sketch_id,+ {"message": f"[WHOXY] Creating email node: {email}"},+ )+ self.create_node(+ "email",+ "email",+ email,+ caption=email,+ type="email",+ )+ self.create_relationship(+ "individual",+ "full_name",+ individual.full_name,+ "email",+ "email",+ email,+ "HAS_EMAIL",+ )++ # Process phone numbers+ if individual.phone_numbers:+ for phone in individual.phone_numbers:+ if phone and phone not in processed_phones:+ processed_phones.add(phone)+ Logger.info(+ self.sketch_id,+ {"message": f"[WHOXY] Creating phone node: {phone}"},+ )+ self.create_node(+ "phone",+ "number",+ phone,+ caption=phone,+ type="phone",+ )+ self.create_relationship(+ "individual",+ "full_name",+ individual.full_name,+ "phone",+ "number",+ phone,+ "HAS_PHONE",+ )++ # Process physical address from contact data+ contact_data = individual_info["contact_data"]+ address = self.__extract_physical_address(contact_data)+ if address:+ address_id = (+ f"{address.address}_{address.city}_{address.zip}_{address.country}"+ )+ if address_id not in processed_addresses:+ processed_addresses.add(address_id)+ Logger.info(+ self.sketch_id,+ {+ "message": f"[WHOXY] Creating address node: {address.address}"+ },+ )self.create_node(- "email",- "email",- email,- caption=email,- type="email",+ "location",+ "address",+ address.address,+ caption=f"{address.address}, {address.city}",+ type="location",)-- # Create relationship between individual and emailself.create_relationship("individual","full_name",individual.full_name,- "email",- "email",- email,- "HAS_EMAIL",+ "location",+ "address",+ address.address,+ "LIVES_AT",)- # Process phone numbers- if individual.phone_numbers:- for phone in individual.phone_numbers:- if phone and phone not in processed_phones:- processed_phones.add(phone)-- # Create phone node- self.create_node(- "phone",- "number",- phone,- caption=phone,- type="phone",- )+ self.log_graph_message(+ f"Processed individual {individual.full_name} ({contact_type}) for domain {domain_name}"+ )- # Create relationship between individual and phone- self.create_relationship(- "individual",- "full_name",- individual.full_name,- "phone",- "number",- phone,- "HAS_PHONE",- )+ # Process extracted organizations+ for organization_info in self._extracted_organizations:+ organization = organization_info["organization"]+ contact_type = organization_info["contact_type"]+ domain_name = organization_info["domain_name"]+ org_name = organization_info["org_name"]- # Process physical address- address = self.__extract_physical_address(contact)- if address:- address_id = (- f"{address.address}_{address.city}_{address.zip}_{address.country}"+ Logger.info(+ self.sketch_id,+ {+ "message": f"[WHOXY] Processing organization: {organization.name} ({contact_type}) for {domain_name}"+ },)- if address_id not in processed_addresses:- processed_addresses.add(address_id)- # Create address node+ # Create input organization node if not already processed+ if org_name not in processed_input_orgs:+ processed_input_orgs.add(org_name)+ Logger.info(+ self.sketch_id,+ {"message": f"[WHOXY] Creating organization node: {org_name}"},+ )+ self.create_node(+ "organization",+ "name",+ org_name,+ caption=org_name,+ type="organization",+ )++ # Create domain node if not already processed+ if domain_name not in processed_domains:+ processed_domains.add(domain_name)+ Logger.info(+ self.sketch_id,+ {"message": f"[WHOXY] Creating domain node: {domain_name}"},+ )+ self.create_node(+ "domain",+ "domain",+ domain_name,+ label=domain_name,+ caption=domain_name,+ type="domain",+ )++ # Create relationship between input organization and domain+ self.create_relationship(+ "organization",+ "name",+ org_name,+ "domain",+ "domain",+ domain_name,+ "HAS_REGISTERED_DOMAIN",+ )++ # Create extracted organization node if not already processed+ if organization.name not in processed_organizations:+ processed_organizations.add(organization.name)+ Logger.info(+ self.sketch_id,+ {+ "message": f"[WHOXY] Creating organization node: {organization.name}"+ },+ )self.create_node(- "location",- "address",- address.address,- caption=f"{address.address}, {address.city}",- type="location",+ "organization",+ "name",+ organization.name,+ caption=organization.name,+ type="organization",)- # Create relationship between individual and address+ # Create relationship between extracted organization and domainself.create_relationship(- "individual",- "full_name",- individual.full_name,- "location",- "address",- address.address,- "LIVES_AT",+ "organization",+ "name",+ organization.name,+ "domain",+ "domain",+ domain_name,+ f"IS_{contact_type.upper()}_CONTACT",)+ self.log_graph_message(+ f"Processed organization {organization.name} ({contact_type}) for domain {domain_name}"+ )++ Logger.info(+ self.sketch_id,+ {+ "message": f"[WHOXY] Postprocess completed. Processed {len(processed_individuals)} individuals and {len(processed_organizations)} organizations"+ },+ )++ return results+InputType = OrgToDomainsTransform.InputTypeOutputType = OrgToDomainsTransform.OutputType