A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat(enrichers): remove heavy logging on whoxy enrichers
dextmorgn609c95767759
diff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_history.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_history.pyindex fd900d32..3dfe7466 100644--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_history.py+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_history.py@@ -1,13 +1,11 @@-import jsonimport osimport reimport time-from typing import Any, Dict, List, Optional, Set, Union+from typing import Any, Dict, List, Optional, Setfrom dotenv import load_dotenvfrom flowsint_core.core.enricher_base import Enricherfrom flowsint_core.core.logger import Logger-from flowsint_core.utils import is_root_domain, is_valid_domainfrom flowsint_types.address import Locationfrom flowsint_types.domain import Domainfrom flowsint_types.email import Email@@ -85,13 +83,6 @@ class DomainToHistoryEnricher(Enricher):time.sleep(1)infos_data = self.__get_infos_from_whoxy(domain.domain, api_key)if infos_data and "whois_records" in infos_data:- Logger.info(- self.sketch_id,- {- "message": f"[WHOXY] Found {len(infos_data['whois_records'])} WHOIS records for {domain.domain}"- },- )-# Process each WHOIS recordfor record in infos_data["whois_records"]:if self.__is_valid_whois_record(record):@@ -116,20 +107,8 @@ class DomainToHistoryEnricher(Enricher):}self._extracted_data.append(extracted_info)- Logger.info(- self.sketch_id,- {- "message": f"[WHOXY] Processing WHOIS record {record.get('num', 'unknown')} for {domain_name}"- },- )-# Process contacts and extract individuals during scanself.__process_contacts_during_scan(extracted_info)- else:- Logger.info(- self.sketch_id,- {"message": f"[WHOXY] No info found for domain {domain.domain}."},- )return domainsdef __process_contacts_during_scan(self, extracted_info: Dict[str, Any]):@@ -139,13 +118,6 @@ class DomainToHistoryEnricher(Enricher):for contact_type, contact in contacts.items():if contact:- Logger.info(- self.sketch_id,- {- "message": f"[WHOXY] Processing {contact_type} contact for {domain_name}"- },- )-# Extract individual (if name is not redacted)individual = self.__extract_individual_from_contact(contact, contact_type@@ -161,13 +133,6 @@ class DomainToHistoryEnricher(Enricher):}self._extracted_individuals.append(individual_info)- Logger.info(- self.sketch_id,- {- "message": f"[WHOXY] Extracted individual: {individual.full_name} ({contact_type}) for {domain_name}"- },- )-# Extract organization (if company name is not redacted)organization = self.__extract_organization_from_contact(contact, contact_type@@ -183,27 +148,6 @@ class DomainToHistoryEnricher(Enricher):}self._extracted_organizations.append(organization_info)- Logger.info(- self.sketch_id,- {- "message": f"[WHOXY] Extracted organization: {organization.name} ({contact_type}) for {domain_name}"- },- )-- # Extract other non-redacted information (country, email, etc.)- self.__extract_additional_info_from_contact(- contact,- contact_type,- domain_name,- extracted_info["original_domain"].domain,- )- else:- Logger.info(- self.sketch_id,- {- "message": f"[WHOXY] No contact data for {contact_type} contact for {domain_name}"- },- )def __get_infos_from_whoxy(self, domain: str, api_key: str) -> Dict[str, Any]:"""Get WHOIS history information from Whoxy API or test data."""@@ -245,12 +189,6 @@ class DomainToHistoryEnricher(Enricher):# Skip if name is redacted - we can't create an individual without a nameif self.__is_redacted(full_name) or not full_name:- Logger.info(- self.sketch_id,- {- "message": f"[WHOXY] Skipping contact with redacted/empty name: {full_name}"- },- )return None# Parse full name into first and last name@@ -300,13 +238,6 @@ class DomainToHistoryEnricher(Enricher):phone_numbers=[phone] if phone else None,)- Logger.info(- self.sketch_id,- {- "message": f"[WHOXY] Extracted individual: {full_name} ({contact_type}) with {len(emails)} emails"- },- )-return individualdef __is_valid_email(self, email: str) -> bool:@@ -348,78 +279,15 @@ class DomainToHistoryEnricher(Enricher):# Create organization objectorganization = Organization(name=company_name)- Logger.info(- self.sketch_id,- {- "message": f"[WHOXY] Extracted organization: {company_name} ({contact_type})"- },- )-return organization- def __extract_additional_info_from_contact(- self,- contact: Dict[str, Any],- contact_type: str,- domain_name: str,- original_domain: str,- ):- """Extract additional non-redacted information from contact data."""- # Extract country information- country_name = contact.get("country_name", "")- country_code = contact.get("country_code", "")-- if country_name and not self.__is_redacted(country_name):- Logger.info(- self.sketch_id,- {- "message": f"[WHOXY] Found country: {country_name} ({contact_type}) for {domain_name}"- },- )-- if country_code and not self.__is_redacted(country_code):- Logger.info(- self.sketch_id,- {- "message": f"[WHOXY] Found country code: {country_code} ({contact_type}) for {domain_name}"- },- )-- # Extract email (even if individual name is redacted)- email_raw = contact.get("email_address", "")- if email_raw and not self.__is_redacted(email_raw):- emails = []- email_list = [e.strip() for e in email_raw.split(",")]- for email in email_list:- if email and self.__is_valid_email(email):- emails.append(email)-- if emails:- Logger.info(- self.sketch_id,- {- "message": f"[WHOXY] Found emails: {emails} ({contact_type}) for {domain_name}"- },- )-def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:"""Create Neo4j nodes and relationships from extracted data."""if not self._graph_service:- Logger.info(- self.sketch_id,- {"message": "[WHOXY] No Neo4j connection, skipping postprocess"},- )return results- Logger.info(- self.sketch_id,- {- "message": f"[WHOXY] Starting postprocess with {len(self._extracted_individuals)} individuals"- },- )-# Track processed entities to avoid duplicatesprocessed_domains: Set[str] = set()processed_individuals: Set[str] = set()@@ -435,20 +303,9 @@ class DomainToHistoryEnricher(Enricher):domain_name = individual_info["domain_name"]original_domain_name = individual_info["original_domain"]- Logger.info(- self.sketch_id,- {- "message": f"[WHOXY] Processing individual: {individual.full_name} ({contact_type}) for {domain_name}"- },- )-# Create domain node if not already processedif domain_name not in processed_domains:processed_domains.add(domain_name)- Logger.info(- self.sketch_id,- {"message": f"[WHOXY] Creating domain node: {domain_name}"},- )domain_obj = Domain(domain=domain_name)self.create_node(domain_obj)@@ -465,12 +322,6 @@ class DomainToHistoryEnricher(Enricher):)if individual_id not in processed_individuals:processed_individuals.add(individual_id)- Logger.info(- self.sketch_id,- {- "message": f"[WHOXY] Creating individual node: {individual.full_name}"- },- )self.create_node(individual)# Create relationship between individual and domain@@ -485,10 +336,6 @@ class DomainToHistoryEnricher(Enricher):email_str = email_obj.emailif email_str and email_str not in processed_emails:processed_emails.add(email_str)- Logger.info(- self.sketch_id,- {"message": f"[WHOXY] Creating email node: {email_str}"},- )email_node = Email(email=email_str)self.create_node(email_node)self.create_relationship(individual, email_node, "HAS_EMAIL")@@ -499,10 +346,6 @@ class DomainToHistoryEnricher(Enricher):phone_str = phone_obj.numberif phone_str and phone_str not in processed_phones:processed_phones.add(phone_str)- Logger.info(- self.sketch_id,- {"message": f"[WHOXY] Creating phone node: {phone_str}"},- )phone_node = Phone(number=phone_str)self.create_node(phone_node)self.create_relationship(individual, phone_node, "HAS_PHONE")@@ -516,12 +359,6 @@ class DomainToHistoryEnricher(Enricher):)if address_id not in processed_addresses:processed_addresses.add(address_id)- Logger.info(- self.sketch_id,- {- "message": f"[WHOXY] Creating address node: {address.address}"- },- )self.create_node(address)self.create_relationship(individual, address, "LIVES_AT")@@ -534,41 +371,16 @@ class DomainToHistoryEnricher(Enricher):organization = organization_info["organization"]contact_type = organization_info["contact_type"]domain_name = organization_info["domain_name"]- original_domain_name = organization_info["original_domain"]-- Logger.info(- self.sketch_id,- {- "message": f"[WHOXY] Processing organization: {organization.name} ({contact_type}) for {domain_name}"- },- )# Create domain node if not already processedif domain_name not in processed_domains:processed_domains.add(domain_name)- Logger.info(- self.sketch_id,- {"message": f"[WHOXY] Creating domain node: {domain_name}"},- )domain_obj = Domain(domain=domain_name)self.create_node(domain_obj)- # Create relationship between original domain and found domain- original_domain_obj3 = Domain(domain=original_domain_name)- domain_obj_rel3 = Domain(domain=domain_name)- self.create_relationship(- original_domain_obj3, domain_obj_rel3, "HAS_RELATED_DOMAIN"- )-# Create organization node if not already processedif organization.name not in processed_organizations:processed_organizations.add(organization.name)- Logger.info(- self.sketch_id,- {- "message": f"[WHOXY] Creating organization node: {organization.name}"- },- )self.create_node(organization)# Create relationship between organization and domain@@ -581,13 +393,6 @@ class DomainToHistoryEnricher(Enricher):f"Processed organization {organization.name} ({contact_type}) for domain {domain_name}")- Logger.info(- self.sketch_id,- {- "message": f"[WHOXY] Postprocess completed. Processed {len(processed_individuals)} individuals and {len(processed_organizations)} organizations"- },- )-return resultsdiff --git a/flowsint-enrichers/src/flowsint_enrichers/individual/to_domains.py b/flowsint-enrichers/src/flowsint_enrichers/individual/to_domains.pyindex 07a14a92..2480ad3a 100644--- a/flowsint-enrichers/src/flowsint_enrichers/individual/to_domains.py+++ b/flowsint-enrichers/src/flowsint_enrichers/individual/to_domains.py@@ -130,13 +130,6 @@ class IndividualToDomainsEnricher(Enricher):},}self._extracted_data.append(extracted_info)- else:- Logger.info(- self.sketch_id,- {- "message": f"[WHOXY] No domain found for individual {individual.full_name}."- },- )return domainsdef __get_infos_from_whoxy(