A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat(enricher): email to domain + fix empty values
dextmorgn14f79fecc538
diff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_history.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_history.pyindex d7af1906..77db477e 100644--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_history.py+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_history.py@@ -1,22 +1,25 @@+import jsonimport osimport re-import json-from typing import Any, List, Union, Dict, Set, Optional+from typing import Any, Dict, List, Optional, Set, Union++from dotenv import load_dotenvfrom flowsint_core.core.enricher_base import Enricher-from flowsint_enrichers.registry import flowsint_enricher+from flowsint_core.core.logger import Logger+from flowsint_core.utils import is_root_domain, is_valid_domain+from flowsint_types.address import Locationfrom flowsint_types.domain import Domain+from flowsint_types.email import Emailfrom flowsint_types.individual import Individualfrom flowsint_types.organization import Organization-from flowsint_types.email import Emailfrom flowsint_types.phone import Phone-from flowsint_core.utils import is_valid_domain, is_root_domain-from flowsint_types.address import Location-from flowsint_core.core.logger import Logger++from flowsint_enrichers.registry import flowsint_enricherfrom tools.network.whoxy import WhoxyTool-from dotenv import load_dotenvload_dotenv()+@flowsint_enricherclass DomainToHistoryEnricher(Enricher):"""[WHOXY] Takes a domain and returns history infos about it (history, organization, owners, emails, etc.)."""@@ -185,7 +188,12 @@ class DomainToHistoryEnricher(Enricher):)# Extract other non-redacted information (country, email, etc.)- self.__extract_additional_info_from_contact(contact, contact_type, domain_name, extracted_info["original_domain"].domain)+ self.__extract_additional_info_from_contact(+ contact,+ contact_type,+ domain_name,+ extracted_info["original_domain"].domain,+ )else:Logger.info(self.sketch_id,@@ -220,8 +228,6 @@ class DomainToHistoryEnricher(Enricher):# A record is valid if it has a domain name - we'll filter contacts individually laterreturn True--def __is_redacted(self, value: str) -> bool:"""Check if a value is redacted."""if not value:@@ -238,14 +244,16 @@ class DomainToHistoryEnricher(Enricher):if self.__is_redacted(full_name) or not full_name:Logger.info(self.sketch_id,- {"message": f"[WHOXY] Skipping contact with redacted/empty name: {full_name}"},+ {+ "message": f"[WHOXY] Skipping contact with redacted/empty name: {full_name}"+ },)return None# Parse full name into first and last namename_parts = full_name.strip().split()- first_name = name_parts[0] if name_parts else ""- last_name = " ".join(name_parts[1:]) if len(name_parts) > 1 else ""+ first_name = name_parts[0] if name_parts else "N/A"+ last_name = " ".join(name_parts[1:]) if len(name_parts) > 1 else "N/A"# Extract email and phoneemail_raw = contact.get("email_address", "")@@ -322,9 +330,7 @@ class DomainToHistoryEnricher(Enricher):if not all([address, city, zip_code, country]):return None- return Location(- address=address, city=city, zip=zip_code, country=country- )+ return Location(address=address, city=city, zip=zip_code, country=country)def __extract_organization_from_contact(self, contact: Dict[str, Any], contact_type: str@@ -349,13 +355,17 @@ class DomainToHistoryEnricher(Enricher):return organizationdef __extract_additional_info_from_contact(- self, contact: Dict[str, Any], contact_type: str, domain_name: str, original_domain: str+ self,+ contact: Dict[str, Any],+ contact_type: str,+ domain_name: str,+ original_domain: str,):"""Extract additional non-redacted information from contact data."""# Extract country informationcountry_name = contact.get("country_name", "")country_code = contact.get("country_code", "")-+if country_name and not self.__is_redacted(country_name):Logger.info(self.sketch_id,@@ -363,7 +373,7 @@ class DomainToHistoryEnricher(Enricher):"message": f"[WHOXY] Found country: {country_name} ({contact_type}) for {domain_name}"},)-+if country_code and not self.__is_redacted(country_code):Logger.info(self.sketch_id,@@ -380,7 +390,7 @@ class DomainToHistoryEnricher(Enricher):for email in email_list:if email and self.__is_valid_email(email):emails.append(email)-+if emails:Logger.info(self.sketch_id,@@ -389,7 +399,9 @@ class DomainToHistoryEnricher(Enricher):},)- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:+ def postprocess(+ self, results: List[OutputType], original_input: List[InputType]+ ) -> List[OutputType]:"""Create Neo4j nodes and relationships from extracted data."""if not self._graph_service:Logger.info(@@ -440,7 +452,9 @@ class DomainToHistoryEnricher(Enricher):# Create relationship between original domain and found domainoriginal_domain_obj = Domain(domain=original_domain_name)domain_obj_rel = Domain(domain=domain_name)- self.create_relationship(original_domain_obj, domain_obj_rel, "HAS_RELATED_DOMAIN")+ self.create_relationship(+ original_domain_obj, domain_obj_rel, "HAS_RELATED_DOMAIN"+ )# Create individual node if not already processedindividual_id = (@@ -458,7 +472,9 @@ class DomainToHistoryEnricher(Enricher):# Create relationship between individual and domaindomain_obj_contact = Domain(domain=domain_name)- self.create_relationship(individual, domain_obj_contact, f"IS_{contact_type.upper()}_CONTACT")+ self.create_relationship(+ individual, domain_obj_contact, f"IS_{contact_type.upper()}_CONTACT"+ )# Process email addressesif individual.email_addresses:@@ -537,7 +553,9 @@ class DomainToHistoryEnricher(Enricher):# Create relationship between original domain and found domainoriginal_domain_obj3 = Domain(domain=original_domain_name)domain_obj_rel3 = Domain(domain=domain_name)- self.create_relationship(original_domain_obj3, domain_obj_rel3, "HAS_RELATED_DOMAIN")+ self.create_relationship(+ original_domain_obj3, domain_obj_rel3, "HAS_RELATED_DOMAIN"+ )# Create organization node if not already processedif organization.name not in processed_organizations:@@ -552,7 +570,9 @@ class DomainToHistoryEnricher(Enricher):# Create relationship between organization and domaindomain_obj_org = Domain(domain=domain_name)- self.create_relationship(organization, domain_obj_org, f"IS_{contact_type.upper()}_CONTACT")+ self.create_relationship(+ organization, domain_obj_org, f"IS_{contact_type.upper()}_CONTACT"+ )self.log_graph_message(f"Processed organization {organization.name} ({contact_type}) for domain {domain_name}"diff --git a/flowsint-enrichers/src/flowsint_enrichers/email/to_domain.py b/flowsint-enrichers/src/flowsint_enrichers/email/to_domain.pynew file mode 100644index 00000000..d90984ed--- /dev/null+++ b/flowsint-enrichers/src/flowsint_enrichers/email/to_domain.py@@ -0,0 +1,60 @@+from typing import List++from flowsint_core.core.enricher_base import Enricher+from flowsint_types.domain import Domain+from flowsint_types.email import Email++from flowsint_enrichers.registry import flowsint_enricher+++@flowsint_enricher+class EmailToDomainEnricher(Enricher):+ """From email to domain."""++ InputType = Email+ OutputType = Domain++ @classmethod+ def name(cls) -> str:+ return "email_to_domain"++ @classmethod+ def category(cls) -> str:+ return "Email"++ @classmethod+ def key(cls) -> str:+ return "email"++ async def scan(self, data: List[InputType]) -> List[OutputType]:+ results: List[OutputType] = []++ for email in data:+ splitted = email.email.split("@")+ domain = splitted[1]+ results.append(Domain(domain=domain))++ return results++ def postprocess(+ self, results: List[OutputType], original_input: List[InputType]+ ) -> List[OutputType]:+ for email_obj, domain_obj in zip(original_input, results):+ if not self._graph_service:+ continue+ # Create email node+ self.create_node(email_obj)+ self.create_node(domain_obj)+ # Create relationship between email and gravatar+ self.create_relationship(email_obj, domain_obj, "HAS_DOMAIN")++ self.log_graph_message(+ f"Exctracted domain for {email_obj.email} -> domain: {domain_obj.domain}"+ )++ return results+++# Make types available at module level for easy access+InputType = EmailToDomainEnricher.InputType+OutputType = EmailToDomainEnricher.OutputTypediff --git a/flowsint-enrichers/src/flowsint_enrichers/email/to_domains.py b/flowsint-enrichers/src/flowsint_enrichers/email/to_domains.pyindex d26eca92..29f257fb 100644--- a/flowsint-enrichers/src/flowsint_enrichers/email/to_domains.py+++ b/flowsint-enrichers/src/flowsint_enrichers/email/to_domains.py@@ -1,16 +1,18 @@import osimport re-from typing import Any, List, Dict, Set, Optional+from typing import Any, Dict, List, Optional, Set++from dotenv import load_dotenvfrom flowsint_core.core.enricher_base import Enricher-from flowsint_enrichers.registry import flowsint_enricher+from flowsint_core.core.logger import Logger+from flowsint_types.address import Locationfrom flowsint_types.domain import Domain-from flowsint_types.individual import Individualfrom flowsint_types.email import Email+from flowsint_types.individual import Individualfrom flowsint_types.phone import Phone-from flowsint_types.address import Location-from flowsint_core.core.logger import Logger++from flowsint_enrichers.registry import flowsint_enricherfrom tools.network.whoxy import WhoxyTool-from dotenv import load_dotenvload_dotenv()@@ -142,8 +144,8 @@ class EmailToDomainsEnricher(Enricher):# Parse full name into first and last namename_parts = full_name.strip().split()- first_name = name_parts[0] if name_parts else ""- last_name = " ".join(name_parts[1:]) if len(name_parts) > 1 else ""+ first_name = name_parts[0] if name_parts else "N/A"+ last_name = " ".join(name_parts[1:]) if len(name_parts) > 1 else "N/A"# Extract email and phoneemail = contact.get("email_address", "")@@ -210,11 +212,11 @@ class EmailToDomainsEnricher(Enricher):if not all([address, city, zip_code, country]):return None- return Location(- address=address, city=city, zip=zip_code, country=country- )+ return Location(address=address, city=city, zip=zip_code, country=country)- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:+ def postprocess(+ self, results: List[OutputType], original_input: List[InputType]+ ) -> List[OutputType]:"""Create Neo4j nodes and relationships from extracted data."""if not self._graph_service:return results@@ -299,7 +301,9 @@ class EmailToDomainsEnricher(Enricher):# Create relationship between individual and domaindomain_obj_ind = Domain(domain=domain_name)- self.create_relationship(individual, domain_obj_ind, f"IS_{contact_type}_CONTACT")+ self.create_relationship(+ individual, domain_obj_ind, f"IS_{contact_type}_CONTACT"+ )# Create relationship between individual and emailemail_obj_ind = Email(email=email_address)